diff --git a/.github/scripts/smoke_test.sh b/.github/scripts/smoke_test.sh index fa7ad8d..c9ccf20 100755 --- a/.github/scripts/smoke_test.sh +++ b/.github/scripts/smoke_test.sh @@ -67,20 +67,40 @@ cli list >/dev/null || fail "list failed on an empty data directory" # 4. Offline benchmark round-trip against a stub llama-bench. The stub answers # --help with the feature flags the adapter probes for and otherwise prints -# canned rows for the requested pp/tg sweep, exactly as tests/runtime_flow.rs -# does. The "model" is a GGUF header: magic, version 3, zero counts. +# one row per requested workload. Decode starts with one seed token; the +# headline PP512/TG128 pair must run before the remaining PP128 sweep. +# The "model" is a GGUF header: magic, version 3, zero counts. MODEL="$WORK/Qwen3-4B.gguf" { printf 'GGUF\003'; head -c 19 /dev/zero; } > "$MODEL" -ROWS="$(printf '[{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"%s","model_size":24,"model_n_params":4000000000,"n_prompt":128,"n_gen":0,"n_depth":0,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]},{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"%s","model_size":24,"model_n_params":4000000000,"n_prompt":512,"n_gen":0,"n_depth":0,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]},{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"%s","model_size":24,"model_n_params":4000000000,"n_prompt":0,"n_gen":128,"n_depth":0,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]}]' "$MODEL" "$MODEL" "$MODEL")" STUB="$WORK/llama-bench" -{ - printf '#!/bin/sh\n' - printf 'if [ "$1" = --help ]; then\n' - printf " printf '%%s\\\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json'\n" - printf 'else\n' - printf " cat <<'JSON'\n%s\nJSON\n" "$ROWS" - printf 'fi\n' -} > "$STUB" +cat > "$STUB" <<'SH' +#!/bin/sh +set -eu +if [ "${1:-}" = --help ]; then + printf '%s\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json' + exit 0 +fi +pp= tg= depth= reps= format= model= +while [ "$#" -gt 0 ]; do + case "$1" in + -m) model="$2"; shift 2 ;; + -p) pp="$2"; shift 2 ;; + -n) tg="$2"; shift 2 ;; + -d) depth="$2"; shift 2 ;; + -r) reps="$2"; shift 2 ;; + -o) format="$2"; shift 2 ;; + --no-warmup) shift ;; + *) printf 'Unexpected fixture argument: %s\n' "$1" >&2; exit 2 ;; + esac +done +[ -f "$model" ] && [ "$reps" = 2 ] && [ "$format" = json ] || exit 2 +case "$pp:$tg:$depth" in + 512:0:0|0:128:1|128:0:0) ;; + *) printf 'Unexpected fixture workload: %s\n' "$pp:$tg:$depth" >&2; exit 2 ;; +esac +printf '%s\n' "$pp:$tg:$depth" >> "$0.calls" +printf '[{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"Qwen3-4B.gguf","model_size":24,"model_n_params":4000000000,"n_prompt":%s,"n_gen":%s,"n_depth":%s,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]}]\n' "$pp" "$tg" "$depth" +SH chmod 0755 "$STUB" REPORT="$WORK/report.json" @@ -88,6 +108,9 @@ if ! cli llama-cpp --runtime-path "$STUB" run "$MODEL" --pp 128,512 --reps 2 --y cat "$WORK/run.log" >&2 fail "offline llama-cpp benchmark run failed" fi +EXPECTED_CALLS="$(printf '%s\n' '512:0:0' '0:128:1' '128:0:0')" +[ "$(cat "$STUB.calls")" = "$EXPECTED_CALLS" ] || fail "expected PP512, TG128 (depth 1), then PP128, exactly once each" +say "headline-first workload order and decode seed depth verified" [ -f "$REPORT" ] || fail "run did not write $REPORT" grep -q '"name": *"llama-cpp"' "$REPORT" || fail "report does not record the llama-cpp runtime" grep -q "\"computearena_version\": *\"$EXPECTED\"" "$REPORT" || fail "report does not record computearena_version $EXPECTED" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e61e3c7..4f3b2fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,3 +24,14 @@ jobs: - run: cargo clippy --workspace --all-targets -- -D warnings - run: cargo test --workspace --all-targets - run: cargo build --workspace --release + - name: Smoke test the packaged binary (offline) + # Exercise the same fixture as staging/release on every PR, so protocol + # changes cannot pass unit tests while breaking the release smoke test. + shell: bash + run: | + PKGID="$(cargo pkgid --locked -p computearena-cli)" + VERSION="${PKGID##*#}" + VERSION="${VERSION##*@}" + SMOKE_DIR="$(mktemp -d)" + .github/scripts/package.sh target/release/computearena "computearena-smoke-${VERSION}" "$SMOKE_DIR" + sh .github/scripts/smoke_test.sh "$SMOKE_DIR/computearena-smoke-${VERSION}.tar.gz" "$VERSION" diff --git a/Cargo.lock b/Cargo.lock index 8a499a4..e70214f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "computearena-cli" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index ebc2594..b3b8621 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/computearena-cli"] [workspace.package] -version = "0.1.1" +version = "0.1.2" edition = "2021" license = "Apache-2.0" rust-version = "1.85" diff --git a/README.md b/README.md index 09c4ff2..7f2ffb2 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,12 @@ computearena basert run model.base --pp 512,2048 --tg 128 --reps 5 computearena llama-cpp run model.gguf --yes --output ./report.json ``` +Only a run with the full default sweep (PP128 to PP16384 and TG128) can be +submitted, so every model and chip is comparable at every size. A custom `--pp` +or `--tg` still produces a valid signed report for local use: the client says +it will be local only before the run starts, shows such reports as `LOCAL ONLY` +in its lists, and leaves them out of a submission with what is missing. + Two profiles are offered before a run starts. Standard runs without an external cooldown wait. With the currently released BaseRT harness, thermally controlled (`--cooldown`) waits once before the complete harness run; llama.cpp waits before @@ -155,12 +161,16 @@ and piped input must use `--yes`. Details are in [docs/benchmark-profiles.md](docs/benchmark-profiles.md). Telemetry is automatic for both runtimes and needs no flag, credential, or -sudo. ComputeArena observes the single process it launches for current BaseRT -and llama.cpp builds: resident memory, operating-system temperature sensors, -power state, and NVIDIA or ROCm device snapshots where those vendor tools exist. -A future BaseRT harness can advertise native same-run telemetry, which the CLI -will use without a CLI release or version-string rule. Coverage and limitations -are in [docs/telemetry.md](docs/telemetry.md). +sudo. ComputeArena observes the processes it launches: resident memory, +operating-system temperature sensors, power state, memory pressure and swap, +plus NVIDIA or ROCm device snapshots where those vendor tools exist. Signed +environment boundaries also record the OS/kernel, CPU layout, host memory, and +available GPU configuration; static macOS display configuration is cached so +`system_profiler` is not rerun at every boundary. A BaseRT harness can +advertise native same-run telemetry, which the CLI uses without a CLI release +or version-string rule. Runtime-native detail is preferred when it is more +accurate; portable host boundaries remain available for cross-runtime analysis. +Coverage and limitations are in [docs/telemetry.md](docs/telemetry.md). ## Runtimes @@ -204,6 +214,21 @@ of `--runtime-path`. BaseRT 0.2.4 and newer can also start this client with BaseRT remains responsible for choosing a compatible backend artifact, downloading split files, conversion, and writing `hub.json` provenance. +ComputeArena says when the BaseRT it found is worth updating, and never +refuses to run an older one. A harness that does not advertise the +headline-first protocol (BaseRT 0.2.4 and older) is named before the benchmark +plan, with what its report will be signed as; this needs no network, because +the harness describes itself. A newer BaseRT release is mentioned the same way. +That lookup asks GitHub for BaseRT's latest release in the background, keeps +the answer for 24 hours in `basert-update-check.json`, and never delays or +fails a run; when it has not answered before a benchmark starts, the notice +follows the run instead. `computearena basert install` installs the latest +release where the official installer does, and the full-screen interface +offers the same with `u` on its menu, asking for a second press before it +replaces anything. A harness chosen with `--runtime-path` or +`COMPUTEARENA_BASERT_HARNESS` is yours to update, and on platforms without a +prebuilt BaseRT the notice points at the release to build from instead. + ### llama.cpp The adapter asks for a GGUF file rather than scanning the disk, and lists the @@ -214,10 +239,19 @@ reads its GGUF header only. The history lives in `recent-gguf.json` in the data directory, is never part of a report, and can be deleted to reset the list; a corrupt or unwritable history never blocks a benchmark. -llama.cpp measurements carry their own protocol identifiers and record native -warmup, zero context depth, and the exclusion of sampling and tokenization, so -they are never presented as BaseRT numbers. The measurement contract is in -[docs/runtime-adapters.md](docs/runtime-adapters.md). +The standard headline is PP512 followed by TG128 **before** the remaining PP +sweep. PP starts empty; TG starts with one untimed seed token. A capable BaseRT +harness reserves 4K for the headline (`computearena-throughput/3`), without +changing its existing warmup, repetitions, timing or telemetry. Older harnesses +retain their existing invocation and recorded protocol. + +ComputeArena uses the user's unmodified llama-bench build. Its native capacity +is retained: stock llama-bench has no independent 4K reservation option, and +`-d 4096` would add real history instead. This difference, actual workload order, +warmup and context requests are signed in the JSON. There is no claim of exact +cross-runtime equivalence. Existing reports remain verifiable and submittable; +protocol differences are not a new leaderboard filter. +The measurement contract is in [docs/runtime-adapters.md](docs/runtime-adapters.md). ## Reports and signatures @@ -300,7 +334,14 @@ refused. The server compares the signed runtime checksum with its catalogue of official builds. An unrecognized or custom build is still accepted and shown with download guidance; only a report whose signature does not verify is rejected. -Submitting the same report again succeeds rather than failing. +Submitting a report that is still published succeeds as an existing duplicate. +If you deleted it on the website, re-submitting that saved report fails instead, +including through **Select all**. The CLI labels it **Failed: previously deleted** +and continues attempting the other valid reports. Successful uploads remain +saved; the final summary counts the failures and the command exits nonzero. +Local files are unchanged. You can rerun the same model with the same settings +and submit the newly generated report as a separate benchmark. There is no need +to choose a different configuration. The client talks to `https://computearena.ai/api/v1`. `--api-url` or `COMPUTEARENA_API_URL` point it at another deployment, such as a local @@ -320,6 +361,7 @@ minimumClientVersion for an actionable upgrade message. | `COMPUTEARENA_API_URL` | API base URL, same as `--api-url` | | `COMPUTEARENA_BASERT_HARNESS` | Path to `basert-benchmark-harness`, same as `--runtime-path` for BaseRT | | `BASERT_INSTALL_DIR` | Where BaseRT is looked for and installed; `~/.basert` by default | +| `COMPUTEARENA_BASERT_RELEASE_API` | Where BaseRT's latest release is looked up, for mirrors and tests; GitHub's API for `basecompute/baseRT` by default. Only a version number is read from the answer | | `BASERT_MODELS_DIR` | Where installed BaseRT models are listed from; BaseRT's own model cache by default | | `CUDA_VISIBLE_DEVICES` | Respected by the CUDA chip fallback; a mask leaves the chip unresolved | | `NO_COLOR` | Plain output | @@ -334,12 +376,19 @@ reports, sessions, and the signing key. - Report envelope: `computearena-benchmark/1` - BaseRT harness output: `basert-benchmark-harness/1`; the older `basert-harness/1` is still accepted by the server -- llama.cpp measurements: `computearena-measurements/1`, executed as - `llama-bench-independent-pp-tg/1` or, with cooldown, - `llama-bench-conditioned-pp-tg/1` +- llama.cpp measurements: `computearena-measurements/1` +- Comparable throughput semantics: `computearena-throughput/2`, with native + evidence retained as `basert-throughput-protocol/1` or + `llama-bench-json/1`. Older BaseRT results use + `computearena-throughput-legacy/1` and are marked non-comparable. +- Headline-first BaseRT capacity: `computearena-throughput/3`, native evidence + `basert-throughput-protocol/2` / `basert-bench-capacity/1`. The historical + `comparable` metadata is not a guarantee of identical measured performance + and does not exclude older reports from the website. - Telemetry: `computearena-telemetry/1` for externally observed BaseRT and llama.cpp runs. A BaseRT harness advertising `features.same_run_telemetry` uses native `basert-telemetry/4` instead. +- Host environment boundaries: `computearena-environment/1` - Signing: Ed25519 over `computearena-json-v1` canonical JSON ## Development diff --git a/crates/computearena-cli/src/adapters/basert.rs b/crates/computearena-cli/src/adapters/basert.rs index 5261fc9..ecabf77 100644 --- a/crates/computearena-cli/src/adapters/basert.rs +++ b/crates/computearena-cli/src/adapters/basert.rs @@ -1,5 +1,8 @@ use super::{BenchmarkRequest, RuntimeAdapter, RuntimeOutput}; -use crate::protocol::BASERT_SAME_RUN_TELEMETRY_SCHEMA; +use crate::protocol::{ + BASERT_ISOLATED_PROTOCOL_SCHEMA, BASERT_SAME_RUN_TELEMETRY_SCHEMA, + DECODE_INITIAL_CONTEXT_TOKENS, THROUGHPUT_PROTOCOL_ID, +}; use crate::ui::TerminalUi; use anyhow::{bail, Context, Result}; use serde_json::{json, Value}; @@ -14,6 +17,27 @@ enum TelemetryMode { NativeSameRun, } +pub(crate) fn supports_isolated_workloads(descriptor: &Value) -> bool { + descriptor + .pointer("/features/isolated_workload_contexts") + .and_then(Value::as_bool) + == Some(true) +} + +pub(crate) fn supports_headline_capacity(descriptor: &Value) -> Result { + let supported = descriptor + .pointer("/features/headline_context_capacity") + .and_then(Value::as_bool) + == Some(true); + if supported + && descriptor["capacity_protocol_schema"].as_str() + != Some(crate::protocol::BASERT_CAPACITY_PROTOCOL_SCHEMA) + { + bail!("BaseRT advertises an unsupported headline context protocol"); + } + Ok(supported) +} + fn telemetry_mode(descriptor: &Value) -> Result { if descriptor .pointer("/features/same_run_telemetry") @@ -81,6 +105,8 @@ impl RuntimeAdapter for BaseRtAdapter { ) -> Result { let ui = TerminalUi::detect(); let mode = telemetry_mode(descriptor)?; + let isolated_workloads = supports_isolated_workloads(descriptor); + let headline_capacity = supports_headline_capacity(descriptor)?; let suite_conditioning = if mode == TelemetryMode::ExternalWholeProcess && r.cooldown { let mut cooldown = crate::conditioning::Cooldown::new(); Some(cooldown.prepare("BaseRT benchmark suite")) @@ -97,6 +123,13 @@ impl RuntimeAdapter for BaseRtAdapter { .arg(r.reps.to_string()) .arg("-w") .arg(r.warmup.to_string()); + if headline_capacity { + command.arg("--headline-first"); + } else if isolated_workloads { + command.arg("--isolated-workloads"); + } + let native_environment_before = + (mode == TelemetryMode::NativeSameRun).then(crate::telemetry::capture_environment); let (output, external_telemetry) = match mode { TelemetryMode::ExternalWholeProcess => { println!("{}", ui.neutral("Telemetry: observing the BaseRT harness process and available device sensors (whole run, 1-second sampling; no telemetry replays).")); @@ -119,6 +152,10 @@ impl RuntimeAdapter for BaseRtAdapter { if !output.status.success() { bail!("BaseRT benchmark exited with {}", output.status); } + let native_environment = native_environment_before.map(|before| { + json!({"schema":"computearena-environment/1","measurement_relation":"outside_runtime_execution", + "before":before,"after":crate::telemetry::capture_environment()}) + }); let mut benchmark: Value = serde_json::from_slice(&output.stdout).context("BaseRT returned invalid JSON")?; let expected_schema = match mode { @@ -127,6 +164,10 @@ impl RuntimeAdapter for BaseRtAdapter { }; crate::benchmark::validate_harness_result(&benchmark, expected_schema)?; validate_requested_workloads(&benchmark, r)?; + normalize_protocol(&mut benchmark, r, isolated_workloads, headline_capacity)?; + if let Some(environment) = native_environment { + crate::telemetry::attach_environment(&mut benchmark, environment); + } if let Some(mut telemetry) = external_telemetry { let mut policy = if r.cooldown { crate::conditioning::before_suite_policy() @@ -162,6 +203,127 @@ impl RuntimeAdapter for BaseRtAdapter { } } +fn normalize_protocol( + benchmark: &mut Value, + request: &BenchmarkRequest<'_>, + isolated_workloads: bool, + headline_capacity: bool, +) -> Result<()> { + let runtime_protocol = benchmark.get("protocol").cloned().unwrap_or(Value::Null); + benchmark["params"]["decode_context_tokens"] = json!(DECODE_INITIAL_CONTEXT_TOKENS); + if headline_capacity { + return normalize_headline_protocol(benchmark, request, runtime_protocol); + } + if !isolated_workloads { + benchmark["protocol"] = json!({ + "id":"computearena-throughput-legacy/1", + "comparable":false, + "reason":"The installed BaseRT harness does not advertise isolated workload contexts", + "context_isolation":"shared_sweep_context", + "decode":{"initial_context_tokens":DECODE_INITIAL_CONTEXT_TOKENS}, + "runtime_protocol":runtime_protocol + }); + return Ok(()); + } + + if runtime_protocol["schema"].as_str() != Some(BASERT_ISOLATED_PROTOCOL_SCHEMA) + || runtime_protocol["context_isolation"].as_str() != Some("per_workload") + || runtime_protocol["context_capacity_policy"].as_str() != Some("minimum_required") + || runtime_protocol["model_load_in_timing"].as_bool() != Some(false) + { + bail!("BaseRT advertised isolated contexts but returned incompatible protocol metadata"); + } + for prompt in request.pp.split(',') { + let tokens = prompt.parse::()?; + let workload = &runtime_protocol["prefill"][prompt]; + if workload["initial_context_tokens"].as_u64() != Some(0) + || workload["context_capacity_tokens"].as_u64() != Some(tokens) + { + bail!("BaseRT returned incompatible PP{tokens} context metadata"); + } + } + if runtime_protocol["decode"]["initial_context_tokens"].as_u64() + != Some(DECODE_INITIAL_CONTEXT_TOKENS) + || runtime_protocol["decode"]["context_capacity_tokens"].as_u64() + != Some(u64::from(request.tg) + DECODE_INITIAL_CONTEXT_TOKENS) + { + bail!( + "BaseRT returned incompatible TG{} context metadata", + request.tg + ); + } + benchmark["protocol"] = json!({ + "id":THROUGHPUT_PROTOCOL_ID, + "comparable":true, + "context_isolation":"per_workload", + "context_capacity_policy":"minimum_required", + "model_load_in_timing":false, + "prefill":{"initial_context_tokens":0}, + "decode":{"initial_context_tokens":DECODE_INITIAL_CONTEXT_TOKENS}, + "tokenization_timed":false, + "sampling_timed":false, + "runtime_protocol":runtime_protocol + }); + Ok(()) +} + +fn normalize_headline_protocol( + benchmark: &mut Value, + request: &BenchmarkRequest<'_>, + runtime: Value, +) -> Result<()> { + use crate::protocol::{ + headline_order, headline_prefill, BASERT_CAPACITY_PROTOCOL_SCHEMA, + HEADLINE_CONTEXT_CAPACITY, HEADLINE_PROTOCOL_ID, + }; + let headline = headline_prefill(request.pp).parse::()?; + let capacity = (headline + u64::from(request.tg)).max(HEADLINE_CONTEXT_CAPACITY); + if runtime["schema"] != BASERT_CAPACITY_PROTOCOL_SCHEMA + || runtime["profile"] != "basert-bench-capacity/1" + || runtime["context_isolation"] != "headline_then_per_prefill" + || runtime["context_capacity_policy"] != "basert_bench_default" + || runtime["model_load_in_timing"] != false + || runtime["execution_layout"] != "headline_then_prefill_processes" + || runtime["execution_order"] != json!(headline_order(request.pp, request.tg)) + { + bail!("BaseRT returned incompatible headline capacity/order metadata"); + } + for prompt in request.pp.split(',') { + let tokens = prompt.parse::()?; + let expected = if tokens == headline { + capacity + } else { + tokens.max(HEADLINE_CONTEXT_CAPACITY) + }; + if runtime["prefill"][prompt]["initial_context_tokens"] != 0 + || runtime["prefill"][prompt]["context_capacity_tokens"].as_u64() != Some(expected) + { + bail!("BaseRT returned incompatible PP{tokens} capacity metadata"); + } + } + if runtime["decode"]["initial_context_tokens"] != DECODE_INITIAL_CONTEXT_TOKENS + || runtime["decode"]["context_capacity_tokens"] != capacity + || runtime["decode"]["seed_prefill_in_timing"] != false + || runtime["measurement"]["timed_repetitions"] != request.reps + || runtime["measurement"]["requested_warmup_repetitions"] != request.warmup + || benchmark["params"]["ctx"] != capacity + { + bail!("BaseRT returned incompatible headline measurement metadata"); + } + benchmark["protocol"] = json!({ + "id":HEADLINE_PROTOCOL_ID, "comparable":true, + "profile":"headline-first-capacity/1", "context_isolation":"headline_then_per_prefill", + "context_capacity_policy":"basert_bench_default", "model_load_in_timing":false, + "prefill":{"initial_context_tokens":0}, + "decode":{"initial_context_tokens":DECODE_INITIAL_CONTEXT_TOKENS,"context_capacity_tokens":capacity}, + "execution_order":runtime["execution_order"], "execution_layout":runtime["execution_layout"], + "measurement":runtime["measurement"], "cooldown_enabled":request.cooldown, + "tokenization_timed":false, "sampling_timed":false, + "runtime_protocol":runtime + }); + Ok(()) +} + /// A compatible harness must also have completed the work the user requested. /// Keep raw samples unchanged; reject partial runs instead of signing them. fn validate_requested_workloads(value: &Value, r: &BenchmarkRequest<'_>) -> Result<()> { @@ -219,6 +381,70 @@ fn validate_requested_workloads(value: &Value, r: &BenchmarkRequest<'_>) -> Resu mod tests { use super::*; + #[test] + fn headline_capacity_is_capability_gated_and_does_not_change_warmup() { + assert!(!supports_headline_capacity(&json!({})).unwrap()); + assert!(supports_headline_capacity( + &json!({"features":{"headline_context_capacity":true}}) + ) + .is_err()); + assert!( + supports_headline_capacity(&json!({"features":{"headline_context_capacity":true}, + "capacity_protocol_schema":crate::protocol::BASERT_CAPACITY_PROTOCOL_SCHEMA})) + .unwrap() + ); + let request = BenchmarkRequest { + model: Path::new("test.base"), + pp: "128,512", + tg: 128, + reps: 3, + warmup: 0, + cooldown: false, + }; + let original = json!({"params":{"ctx":4096},"protocol":{ + "schema":"basert-throughput-protocol/2","profile":"basert-bench-capacity/1", + "context_isolation":"headline_then_per_prefill","context_capacity_policy":"basert_bench_default", + "model_load_in_timing":false,"execution_layout":"headline_then_prefill_processes", + "execution_order":["pp512","tg128","pp128"], + "prefill":{"128":{"initial_context_tokens":0,"context_capacity_tokens":4096}, + "512":{"initial_context_tokens":0,"context_capacity_tokens":4096}}, + "decode":{"initial_context_tokens":1,"context_capacity_tokens":4096,"seed_prefill_in_timing":false}, + "measurement":{"timed_repetitions":3,"requested_warmup_repetitions":0} + }}); + let mut report = original.clone(); + normalize_protocol(&mut report, &request, true, true).unwrap(); + assert_eq!( + report["protocol"]["id"], + crate::protocol::HEADLINE_PROTOCOL_ID + ); + assert_eq!( + report["protocol"]["measurement"]["requested_warmup_repetitions"], + 0 + ); + assert_eq!(report["protocol"]["runtime_protocol"], original["protocol"]); + for (pointer, value) in [ + ("/protocol/decode/initial_context_tokens", json!(4096)), + ("/protocol/decode/context_capacity_tokens", json!(129)), + ("/protocol/prefill/128/context_capacity_tokens", json!(128)), + ("/protocol/measurement/timed_repetitions", json!(5)), + ( + "/protocol/measurement/requested_warmup_repetitions", + json!(12), + ), + ( + "/protocol/execution_order", + json!(["pp128", "pp512", "tg128"]), + ), + ] { + let mut invalid = original.clone(); + *invalid.pointer_mut(pointer).unwrap() = value; + assert!( + normalize_protocol(&mut invalid, &request, true, true).is_err(), + "{pointer}" + ); + } + } + #[test] fn replay_based_harnesses_use_external_observation() { let descriptor = json!({"telemetry_schema":"basert-telemetry/3", @@ -245,4 +471,37 @@ mod tests { "features":{"same_run_telemetry":true}}); assert!(telemetry_mode(&descriptor).is_err()); } + #[test] + fn isolated_harness_metadata_normalizes_to_the_shared_protocol() { + let request = BenchmarkRequest { + model: Path::new("model.base"), + pp: "128,512", + tg: 128, + reps: 3, + warmup: 3, + cooldown: false, + }; + let mut benchmark = json!({ + "params":{}, + "protocol":{ + "schema":BASERT_ISOLATED_PROTOCOL_SCHEMA, + "context_isolation":"per_workload", + "context_capacity_policy":"minimum_required", + "model_load_in_timing":false, + "prefill":{ + "128":{"initial_context_tokens":0,"context_capacity_tokens":128}, + "512":{"initial_context_tokens":0,"context_capacity_tokens":512} + }, + "decode":{"initial_context_tokens":1,"context_capacity_tokens":129} + } + }); + normalize_protocol(&mut benchmark, &request, true, false).unwrap(); + assert_eq!(benchmark["protocol"]["id"], THROUGHPUT_PROTOCOL_ID); + assert_eq!(benchmark["protocol"]["comparable"], true); + assert_eq!(benchmark["params"]["decode_context_tokens"], 1); + assert_eq!( + benchmark["protocol"]["runtime_protocol"]["decode"]["context_capacity_tokens"], + 129 + ); + } } diff --git a/crates/computearena-cli/src/adapters/llama_cpp.rs b/crates/computearena-cli/src/adapters/llama_cpp.rs index a842966..554319e 100644 --- a/crates/computearena-cli/src/adapters/llama_cpp.rs +++ b/crates/computearena-cli/src/adapters/llama_cpp.rs @@ -85,12 +85,12 @@ impl RuntimeAdapter for LlamaCppAdapter { ) -> Result { validate_model(r.model)?; let ui = TerminalUi::detect(); - println!("{}", ui.neutral("Telemetry: observing process memory and available device sensors (whole run, 1-second sampling).")); + println!("{}", ui.neutral("Telemetry: observing each runtime process and available device sensors (1-second sampling).")); let (rows, telemetry) = if r.cooldown { let mut cooldown = crate::conditioning::Cooldown::new(); run_conditioned(executable, r, |label| cooldown.prepare(label))? } else { - run_native(executable, r, r.pp, r.tg)? + run_unconditioned(executable, r)? }; let mut result = normalize(&rows, r)?; ui.section("Benchmark results"); @@ -111,8 +111,7 @@ impl RuntimeAdapter for LlamaCppAdapter { ); result.benchmark["protocol"]["telemetry_available"] = json!(telemetry.get("observer").is_some() || telemetry.get("workloads").is_some()); - result.benchmark["protocol"]["measurement_observer"] = - json!("external_whole_process_sampler"); + result.benchmark["protocol"]["measurement_observer"] = json!("external_process_sampler"); if let Some(peak) = crate::telemetry::attach_whole_process(&mut result.benchmark, telemetry) { println!("{}", ui.neutral(format!("Telemetry: observed peak process memory {peak:.0} MiB (includes loading and warmup)."))); @@ -120,7 +119,6 @@ impl RuntimeAdapter for LlamaCppAdapter { println!("{}", ui.neutral("Telemetry: process memory unavailable; see sensor coverage in the saved report.")); } if r.cooldown { - result.benchmark["protocol"]["id"] = json!("llama-bench-conditioned-pp-tg/1"); result.benchmark["protocol"]["cooldown_enabled"] = json!(true); result.benchmark["protocol"]["execution_layout"] = json!("one_process_per_workload"); result.benchmark["protocol"]["conditioning"] = @@ -144,6 +142,7 @@ fn run_native( r: &BenchmarkRequest<'_>, pp: &str, tg: u32, + depth: u32, ) -> Result<(Value, Value)> { let mut command = Command::new(executable); command @@ -151,7 +150,9 @@ fn run_native( .arg(r.model) .args(["-p", pp, "-n"]) .arg(tg.to_string()) - .args(["-d", "0", "-r"]) + .arg("-d") + .arg(depth.to_string()) + .arg("-r") .arg(r.reps.to_string()) .args(["-o", "json"]); if r.warmup == 0 { @@ -170,22 +171,128 @@ fn run_native( Ok((rows, telemetry)) } +fn aggregate_observations( + scope: &str, + ordered: Vec<(String, Value)>, + conditioning: Option>, +) -> Value { + let mut observations = Map::new(); + let mut peak: Option = None; + let mut before = None; + let mut after = None; + for (label, mut telemetry) in ordered { + if before.is_none() { + before = telemetry.pointer("/boundaries/before").cloned(); + } + after = telemetry.pointer("/boundaries/after").cloned(); + if let Some(value) = telemetry + .pointer("/process_memory/statistics/peak") + .and_then(Value::as_f64) + { + peak = Some(peak.map_or(value, |current| current.max(value))); + } + telemetry["scope"] = json!("one_runtime_process"); + telemetry["includes"] = json!([ + "model_loading", + "warmup", + "requested_workloads_for_this_process", + "runtime_teardown" + ]); + observations.insert(label, telemetry); + } + let mut telemetry = json!({ + "schema":"computearena-telemetry/1", + "coverage":"basic", + "scope":scope, + "measurement_relation":"concurrent_observer", + "workloads":observations, + "process_memory":{ + "metric":"resident_set_size", + "unit":"MiB", + "statistics":{"available":peak.is_some(),"peak":peak}, + "note":"Maximum observed peak across separate process windows including loading and warmup" + }, + "boundaries":{ + "schema":"computearena-environment/1", + "measurement_relation":"outside_runtime_execution", + "before":before, + "after":after + } + }); + if let Some(conditioning) = conditioning { + telemetry["conditioning"] = crate::conditioning::policy(); + telemetry["conditioning_workloads"] = Value::Object(conditioning); + } + telemetry +} + +fn run_unconditioned(executable: &Path, r: &BenchmarkRequest<'_>) -> Result<(Value, Value)> { + let headline = crate::protocol::headline_prefill(r.pp); + let (prefill, prefill_telemetry) = run_native(executable, r, headline, 0, 0)?; + let (decode, decode_telemetry) = run_native( + executable, + r, + "0", + r.tg, + crate::protocol::DECODE_INITIAL_CONTEXT_TOKENS as u32, + )?; + let mut rows = prefill + .as_array() + .context("llama.cpp prefill output must be an array")? + .clone(); + rows.extend( + decode + .as_array() + .context("llama.cpp decode output must be an array")? + .iter() + .cloned(), + ); + let mut observations = vec![ + (format!("pp{headline}"), prefill_telemetry), + (format!("tg{}", r.tg), decode_telemetry), + ]; + let remaining = + r.pp.split(',') + .filter(|pp| *pp != headline) + .collect::>() + .join(","); + if !remaining.is_empty() { + let (sweep, telemetry) = run_native(executable, r, &remaining, 0, 0)?; + rows.extend( + sweep + .as_array() + .context("llama.cpp sweep output must be an array")? + .iter() + .cloned(), + ); + observations.push(("remaining_prefill_sweep".into(), telemetry)); + } + let telemetry = aggregate_observations("headline_then_prefill_processes", observations, None); + Ok((Value::Array(rows), telemetry)) +} + fn run_conditioned( executable: &Path, r: &BenchmarkRequest<'_>, mut prepare: impl FnMut(&str) -> Value, ) -> Result<(Value, Value)> { - let schedule: Vec<(u32, u32)> = - r.pp.split(',') - .map(|v| v.parse::().map(|pp| (pp, 0))) - .chain(std::iter::once(Ok((0, r.tg)))) - .collect::>()?; + let headline = crate::protocol::headline_prefill(r.pp); + let mut schedule = vec![ + (headline.parse::()?, 0, 0), + ( + 0, + r.tg, + crate::protocol::DECODE_INITIAL_CONTEXT_TOKENS as u32, + ), + ]; + for pp in r.pp.split(',').filter(|pp| *pp != headline) { + schedule.push((pp.parse::()?, 0, 0)); + } let mut rows = Vec::new(); - let mut observations = Map::new(); + let mut observations = Vec::new(); let mut conditioning = Map::new(); - let mut peak: Option = None; let ui = TerminalUi::detect(); - for (index, (pp, tg)) in schedule.iter().enumerate() { + for (index, (pp, tg, depth)) in schedule.iter().enumerate() { let label = if *pp > 0 { format!("pp{pp}") } else { @@ -208,33 +315,27 @@ fn run_conditioned( r.reps )) ); - let (native, telemetry) = run_native(executable, r, &pp.to_string(), *tg)?; + let (native, telemetry) = run_native(executable, r, &pp.to_string(), *tg, *depth)?; let native = native .as_array() .context("llama.cpp benchmark output must be an array")?; if native.len() != 1 || native[0]["n_prompt"].as_u64() != Some(u64::from(*pp)) || native[0]["n_gen"].as_u64() != Some(u64::from(*tg)) + || native[0]["n_depth"].as_u64() != Some(u64::from(*depth)) { bail!("llama.cpp returned an unexpected workload for {label}; no report was signed"); } rows.extend(native.iter().cloned()); - if let Some(value) = telemetry - .pointer("/process_memory/statistics/peak") - .and_then(Value::as_f64) - { - peak = Some(peak.map_or(value, |p| p.max(value))); - } - observations.insert(label, telemetry); + observations.push((label, telemetry)); } Ok(( Value::Array(rows), - json!({"schema":"computearena-telemetry/1","coverage":"basic", - "scope":"separate_workload_processes","measurement_relation":"concurrent_observer", - "conditioning":crate::conditioning::policy(),"conditioning_workloads":conditioning, - "workloads":observations,"process_memory":{"metric":"resident_set_size","unit":"MiB", - "statistics":{"available":peak.is_some(),"peak":peak}, - "note":"Maximum observed peak across separate process windows including loading and warmup"}}), + aggregate_observations( + "separate_workload_processes", + observations, + Some(conditioning), + ), )) } @@ -297,6 +398,8 @@ pub(crate) fn normalize(value: &Value, r: &BenchmarkRequest<'_>) -> Result) -> Result 0 && tg == 0; + let label = if is_pp { + format!("pp{pp}") + } else { + format!("tg{tg}") + }; + execution_order.push(label.clone()); + let expected_depth = if is_pp { + 0 + } else { + crate::protocol::DECODE_INITIAL_CONTEXT_TOKENS + }; + if row["n_depth"].as_u64() != Some(expected_depth) { + bail!("llama.cpp returned an unexpected context depth"); + } + // Stock llama-bench derives the requested context from PP+TG+depth. + // The engine may pad allocations; never claim unreported physical capacity. + context_requests.insert( + label, + json!({ + "initial_context_tokens":expected_depth, + "requested_context_capacity_tokens":pp+tg+expected_depth, + "capacity_source":"llama_bench_native_workload_parameters" + }), + ); if !(is_pp && expected.contains(&pp) || pp == 0 && tg == u64::from(r.tg)) { bail!("llama.cpp returned an unexpected workload PP{pp}/TG{tg}"); } @@ -411,12 +535,24 @@ pub(crate) fn normalize(value: &Value, r: &BenchmarkRequest<'_>) -> Result Option { /// retire an old client by returning HTTP 426 (or the matching error code) /// without making older clients fail with an unexplained generic status. pub(crate) fn server_error(status: reqwest::StatusCode, body: &str) -> String { + if status == reqwest::StatusCode::CONFLICT + && error_code(body).as_deref() == Some("submission_deleted") + { + return "This benchmark was previously deleted from ComputeArena. This saved report cannot be submitted again, including through Select all. You can rerun the same model with the same settings and submit the newly generated report as a separate benchmark. Your local report is unchanged.".to_string(); + } let message = error_message(body).unwrap_or_else(|| format!("server returned HTTP {}", status.as_u16())); let upgrade_required = status == reqwest::StatusCode::UPGRADE_REQUIRED @@ -93,4 +98,24 @@ mod tests { "Invalid report" ); } + + #[test] + fn deleted_reports_explain_why_select_all_cannot_restore_them() { + for body in [ + r#"{"error":{"code":"submission_deleted"}}"#, + r#"{"error":{"code":"submission_deleted","message":"Report deleted"}}"#, + ] { + let message = server_error(reqwest::StatusCode::CONFLICT, body); + assert!(message.contains("previously deleted")); + assert!(message.contains("Select all")); + assert!(message.contains("rerun the same model with the same settings")); + assert!(message.contains("newly generated report as a separate benchmark")); + assert!(message.contains("local report is unchanged")); + } + let body = r#"{"error":{"code":"submission_conflict","message":"Different report"}}"#; + assert_eq!( + server_error(reqwest::StatusCode::CONFLICT, body), + "Different report" + ); + } } diff --git a/crates/computearena-cli/src/basert_updates.rs b/crates/computearena-cli/src/basert_updates.rs new file mode 100644 index 0000000..a4cc5a7 --- /dev/null +++ b/crates/computearena-cli/src/basert_updates.rs @@ -0,0 +1,437 @@ +//! What ComputeArena says about the BaseRT it found. +//! +//! Two things are worth a sentence before a benchmark is spent on them: a +//! newer BaseRT release exists, or the installed harness predates the +//! benchmark protocol current reports use. The second needs no network, because +//! the harness says what it supports. Neither ever stops a run: an older BaseRT +//! keeps working exactly as before, and its report records what was used. + +use crate::adapters::basert::{supports_headline_capacity, supports_isolated_workloads}; +use crate::reports::Paths; +use crate::runtimes::{Source, BASERT_INSTALL_SCRIPT, BASERT_RELEASES}; +use crate::ui::TerminalUi; +use crate::updates::{LatestRelease, ReleaseCheck}; +use semver::Version; +use serde_json::Value; + +/// The first BaseRT release whose harness runs the headline-first protocol. +const HEADLINE_FIRST_SINCE: Version = Version::new(0, 2, 5); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Protocol { + /// PP512 and TG128 first, in a freshly loaded model with a 4K reservation. + HeadlineFirst, + /// One context per workload, without the headline order. + IsolatedWorkloads, + /// One context shared by the whole sweep; signed as not comparable. + SharedContext, +} + +fn protocol(descriptor: &Value) -> Protocol { + // A harness advertising a capacity protocol this client cannot read is + // newer than the client, not older: the run itself reports that. + if supports_headline_capacity(descriptor).unwrap_or(true) { + Protocol::HeadlineFirst + } else if supports_isolated_workloads(descriptor) { + Protocol::IsolatedWorkloads + } else { + Protocol::SharedContext + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Advice { + /// Stands on its own: a status line, a plan row. + pub(crate) summary: String, + /// What an older protocol means for the report; empty for a plain update. + pub(crate) consequence: Option, + /// What to do about it. + pub(crate) action: String, + /// Whether `computearena basert install` is that action. + pub(crate) installable: bool, + /// The older protocol in a few words, for a benchmark plan. + pub(crate) plan_note: Option<&'static str>, +} + +/// What to say about this BaseRT, if anything. `latest` is the newest release +/// when that is known; an older protocol is reported either way. +pub(crate) fn advice( + descriptor: &Value, + latest: Option<&Version>, + source: Source, + prebuilt_for_platform: bool, +) -> Option { + let installed_text = descriptor + .pointer("/runtime/version") + .and_then(Value::as_str) + .filter(|version| !version.is_empty()); + let installed = installed_text.and_then(|version| Version::parse(version).ok()); + let newer = match (&installed, latest) { + (Some(installed), Some(latest)) if latest > installed => Some(latest), + _ => None, + }; + let protocol = protocol(descriptor); + if protocol == Protocol::HeadlineFirst && newer.is_none() { + return None; + } + + let name = match installed_text { + Some(version) => format!("BaseRT {version}"), + None => "This BaseRT".to_string(), + }; + // The release worth moving to: the newest when it is known to carry the + // protocol, otherwise the first one that does. + let target = match latest { + Some(latest) if *latest >= HEADLINE_FIRST_SINCE => format!("BaseRT {latest}"), + _ if protocol != Protocol::HeadlineFirst => { + format!("BaseRT {HEADLINE_FIRST_SINCE} or newer") + } + _ => "the newer release".to_string(), + }; + let (summary, consequence, plan_note) = match protocol { + Protocol::HeadlineFirst => ( + format!( + "{target} is available (installed: {}).", + installed_text.unwrap_or("unknown") + ), + None, + None, + ), + Protocol::IsolatedWorkloads => ( + format!("{name} predates the current benchmark protocol."), + Some(format!( + "Its runs are signed without the headline-first order. {target} measures PP512 and TG128 first, in a freshly loaded model with a 4K reservation, and reports telemetry from the timed repetitions." + )), + Some("Older BaseRT protocol: no headline-first order"), + ), + Protocol::SharedContext => ( + format!("{name} predates the current benchmark protocol."), + Some(format!( + "Its runs share one context across the sweep and are signed as computearena-throughput-legacy/1, marked not comparable. {target} measures PP512 and TG128 first, in a freshly loaded model with a 4K reservation, and reports telemetry from the timed repetitions." + )), + Some("Older BaseRT protocol: signed as not comparable"), + ), + }; + + let chosen_by_hand = match source { + Source::Override => Some("--runtime-path".to_string()), + Source::Environment(variable) => Some(variable.to_string()), + Source::Managed | Source::Path | Source::KnownLocation => None, + }; + let (action, installable) = match (chosen_by_hand, prebuilt_for_platform) { + (Some(how), true) => ( + format!( + "This harness was chosen with {how}: point that at a newer build, or leave it out and run `computearena basert install`." + ), + false, + ), + (Some(how), false) => ( + format!( + "This harness was chosen with {how}: point that at a build of the newer release ({BASERT_RELEASES})." + ), + false, + ), + (None, true) => ( + format!( + "Update with `computearena basert install`, or the official installer: {BASERT_INSTALL_SCRIPT}" + ), + true, + ), + (None, false) => ( + format!( + "No prebuilt BaseRT is published for this platform; build the harness from the newer release: {BASERT_RELEASES}" + ), + false, + ), + }; + Some(Advice { + summary, + consequence, + action, + installable, + plan_note, + }) +} + +/// Whether `computearena basert install` can fetch a bundle on this machine. +pub(crate) fn prebuilt_for_this_platform() -> bool { + crate::runtimes::asset_rule( + crate::adapters::Runtime::Basert, + std::env::consts::OS, + std::env::consts::ARCH, + ) + .is_ok() +} + +pub(crate) fn print_advice(ui: TerminalUi, advice: &Advice) { + println!("{} {}", ui.warning("!"), ui.strong(&advice.summary)); + if let Some(consequence) = &advice.consequence { + println!(" {}", ui.neutral(consequence)); + } + println!(" {}", ui.neutral(&advice.action)); +} + +/// The release lookup for one command: what the last lookup found, and the +/// one in progress. Nothing here waits for the network. +pub(crate) struct Watch { + latest: Option, + check: Option, +} + +impl Watch { + pub(crate) fn start(paths: &Paths) -> Self { + let (latest, check) = crate::updates::BASERT.start(paths); + Self { latest, check } + } + + #[cfg(test)] + pub(crate) fn known(latest: Option) -> Self { + Self { + latest, + check: None, + } + } + + /// Takes in the lookup's answer if it has arrived; true when it changed + /// what is known. + pub(crate) fn refresh(&mut self) -> bool { + let Some(answer) = self.check.as_ref().and_then(ReleaseCheck::poll) else { + return false; + }; + self.check = None; + match answer { + Some(release) if self.latest.as_ref() != Some(&release) => { + self.latest = Some(release); + true + } + _ => false, + } + } + + pub(crate) fn latest(&self) -> Option<&Version> { + self.latest.as_ref().map(|release| &release.version) + } + + pub(crate) fn advice(&self, descriptor: &Value, source: Source) -> Option { + advice( + descriptor, + self.latest(), + source, + prebuilt_for_this_platform(), + ) + } +} + +/// One `run`: says what is known before the benchmark starts, and afterwards +/// only what the lookup learned in the meantime, so nothing is said twice. +pub(crate) struct RunNotice { + watch: Watch, + descriptor: Option, + source: Source, + said: bool, +} + +impl RunNotice { + /// A harness that cannot be probed says nothing here; the run itself + /// explains what is wrong with it. + pub(crate) fn start(paths: &Paths, harness: &std::path::Path, source: Source) -> Self { + use crate::adapters::Runtime; + Self { + watch: Watch::start(paths), + descriptor: Runtime::Basert.adapter().probe(harness).ok(), + source, + said: false, + } + } + + fn say(&mut self, ui: TerminalUi) { + self.watch.refresh(); + let Some(descriptor) = &self.descriptor else { + return; + }; + if let Some(advice) = self.watch.advice(descriptor, self.source) { + print_advice(ui, &advice); + self.said = true; + } + } + + pub(crate) fn before_run(&mut self, ui: TerminalUi) { + self.say(ui); + } + + /// A benchmark takes minutes, so a lookup that was still running when it + /// started has answered by now. + pub(crate) fn after_run(&mut self, ui: TerminalUi) { + if !self.said { + self.say(ui); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn harness(version: &str, headline: bool, isolated: bool) -> Value { + json!({ + "runtime": {"name": "basert", "version": version}, + "capacity_protocol_schema": "basert-throughput-protocol/2", + "features": { + "headline_context_capacity": headline, + "isolated_workload_contexts": isolated + } + }) + } + + fn version(text: &str) -> Version { + Version::parse(text).unwrap() + } + + #[test] + fn a_current_basert_with_nothing_newer_says_nothing() { + let current = harness("0.2.5", true, true); + assert_eq!(advice(¤t, None, Source::Managed, true), None); + assert_eq!( + advice(¤t, Some(&version("0.2.5")), Source::Managed, true), + None + ); + // An older release on the feed is not an update. + assert_eq!( + advice(¤t, Some(&version("0.2.4")), Source::Managed, true), + None + ); + } + + #[test] + fn a_newer_release_is_offered_with_the_command_that_installs_it() { + let advice = advice( + &harness("0.2.5", true, true), + Some(&version("0.2.6")), + Source::KnownLocation, + true, + ) + .unwrap(); + assert_eq!( + advice.summary, + "BaseRT 0.2.6 is available (installed: 0.2.5)." + ); + assert_eq!(advice.consequence, None); + assert_eq!(advice.plan_note, None); + assert!(advice.installable); + assert!(advice.action.contains("`computearena basert install`")); + assert!(advice.action.contains(BASERT_INSTALL_SCRIPT)); + } + + #[test] + fn an_older_protocol_is_reported_offline_and_names_what_the_report_will_say() { + // 0.2.4: neither isolated contexts nor the headline order. + let offline = advice(&harness("0.2.4", false, false), None, Source::Path, true).unwrap(); + assert_eq!( + offline.summary, + "BaseRT 0.2.4 predates the current benchmark protocol." + ); + let consequence = offline.consequence.as_deref().unwrap(); + assert!(consequence.contains("computearena-throughput-legacy/1")); + assert!(consequence.contains("not comparable")); + assert!(consequence.contains("BaseRT 0.2.5 or newer measures PP512 and TG128 first")); + assert_eq!( + offline.plan_note, + Some("Older BaseRT protocol: signed as not comparable") + ); + assert!(offline.installable); + + // With the feed's answer the advice names the release to move to. + let online = advice( + &harness("0.2.4", false, false), + Some(&version("0.2.6")), + Source::Path, + true, + ) + .unwrap(); + assert!(online + .consequence + .unwrap() + .contains("BaseRT 0.2.6 measures PP512 and TG128 first")); + + // Isolated contexts without the headline order are still comparable. + let isolated = advice(&harness("0.2.4", false, true), None, Source::Path, true).unwrap(); + assert_eq!( + isolated.plan_note, + Some("Older BaseRT protocol: no headline-first order") + ); + let consequence = isolated.consequence.unwrap(); + assert!(consequence.contains("without the headline-first order")); + assert!(!consequence.contains("legacy")); + } + + #[test] + fn the_action_fits_how_the_harness_was_found_and_what_the_platform_offers() { + let old = harness("0.2.4", false, false); + let by_flag = advice(&old, None, Source::Override, true).unwrap(); + assert!(by_flag.action.contains("chosen with --runtime-path")); + assert!(!by_flag.installable); + + let by_variable = advice( + &old, + None, + Source::Environment("COMPUTEARENA_BASERT_HARNESS"), + true, + ) + .unwrap(); + assert!(by_variable + .action + .contains("chosen with COMPUTEARENA_BASERT_HARNESS")); + + let no_bundle = advice(&old, None, Source::Path, false).unwrap(); + assert!(no_bundle.action.contains("No prebuilt BaseRT is published")); + assert!(no_bundle.action.contains(BASERT_RELEASES)); + assert!(!no_bundle.installable); + assert!(!no_bundle.action.contains("basert install")); + } + + #[test] + fn versions_that_cannot_be_compared_never_invent_an_update() { + let unversioned = json!({ + "runtime": {"name": "basert"}, + "capacity_protocol_schema": "basert-throughput-protocol/2", + "features": {"headline_context_capacity": true} + }); + assert_eq!( + advice(&unversioned, Some(&version("9.9.9")), Source::Managed, true), + None + ); + let development = harness("main-abc123", true, true); + assert_eq!( + advice(&development, Some(&version("9.9.9")), Source::Managed, true), + None + ); + // An unversioned harness that lacks the protocol is still told so. + let old = json!({"runtime": {"name": "basert"}, "features": {}}); + let advice = advice(&old, None, Source::Managed, true).unwrap(); + assert_eq!( + advice.summary, + "This BaseRT predates the current benchmark protocol." + ); + + // A harness newer than this client understands is not "older". + let future = json!({ + "runtime": {"name": "basert", "version": "0.9.0"}, + "capacity_protocol_schema": "basert-throughput-protocol/9", + "features": {"headline_context_capacity": true} + }); + assert_eq!(super::advice(&future, None, Source::Managed, true), None); + } + + #[test] + fn a_lookup_that_finishes_later_updates_what_is_known_once() { + let mut watch = Watch::known(Some(crate::updates::release_for_tests("0.2.6"))); + assert_eq!(watch.latest(), Some(&version("0.2.6"))); + // Nothing in progress: nothing changes. + assert!(!watch.refresh()); + let advice = watch + .advice(&harness("0.2.5", true, true), Source::Managed) + .unwrap(); + assert!(advice.summary.contains("BaseRT 0.2.6 is available")); + } +} diff --git a/crates/computearena-cli/src/benchmark.rs b/crates/computearena-cli/src/benchmark.rs index c0ee364..840bdb5 100644 --- a/crates/computearena-cli/src/benchmark.rs +++ b/crates/computearena-cli/src/benchmark.rs @@ -70,7 +70,7 @@ pub(crate) fn plan_rows( .map(|n| format!("PP{n}")) .collect::>() .join(", "); - Ok(vec![ + let mut rows = vec![ ("Runtime", runtime.adapter().display_name().to_string()), // The resolved, absolute path: the plan's job is to say exactly which // file will be read, so this is never shortened. @@ -82,7 +82,19 @@ pub(crate) fn plan_rows( "Output", "Signed JSON report saved locally\nNothing is uploaded automatically".to_string(), ), - ]) + ]; + // Only a custom sweep gets this row: the default plan stays as short as + // it was, and a run that cannot be uploaded says so where it is decided. + if crate::sweep::requested_gap(r.pp, r.tg).is_some() { + rows.push(( + "Submission", + format!( + "Local only: a partial run cannot be submitted\nComputeArena accepts the full default sweep ({})", + crate::sweep::required_summary() + ), + )); + } + Ok(rows) } pub(crate) const LOAD_WARNING: [&str; 2] = [ @@ -100,6 +112,12 @@ pub(crate) fn print_benchmark_plan(runtime: Runtime, r: &BenchmarkRequest<'_>) - println!(); println!("{} {}", ui.warning("!"), LOAD_WARNING[0]); println!(" {}", LOAD_WARNING[1]); + // Last thing before the run is confirmed: a custom sweep can take as long + // as the default one, and its report cannot be uploaded. + if let Some(gap) = crate::sweep::requested_gap(r.pp, r.tg) { + println!(); + crate::sweep::print_local_only_notice(ui, &gap); + } Ok(()) } @@ -181,7 +199,7 @@ pub(crate) fn benchmark_details( } rows.push(( "Context", - "Independent PP and TG tests; initial context depth 0".to_string(), + "Headline PP512/TG first when requested, before the larger PP sweep. PP starts empty; TG starts with one untimed seed token. Stock llama-bench controls reserved capacity; no runtime modifications.".to_string(), )); rows.push(( "Telemetry", diff --git a/crates/computearena-cli/src/main.rs b/crates/computearena-cli/src/main.rs index 90c6374..93ccbbf 100644 --- a/crates/computearena-cli/src/main.rs +++ b/crates/computearena-cli/src/main.rs @@ -3,6 +3,7 @@ mod api; use adapters::{BenchmarkRequest, Runtime}; mod auth; mod basert_models; +mod basert_updates; mod benchmark; mod conditioning; mod config; @@ -14,6 +15,7 @@ mod recent_gguf; mod reports; mod runtimes; mod submission; +mod sweep; mod telemetry; mod theme; #[cfg(unix)] @@ -116,10 +118,11 @@ enum Action { Run { /// Local model file (.base for BaseRT, .gguf for llama.cpp). model: Option, - /// Comma-separated prefill token counts. + /// Comma-separated prefill token counts. Only a run that includes + /// every default size can be submitted; anything less is local only. #[arg(long, default_value = DEFAULT_PREFILL_TOKENS)] pp: String, - /// Decode token count per repetition. + /// Decode token count per repetition. Only the default can be submitted. #[arg(long, default_value_t = DEFAULT_DECODE_TOKENS)] tg: u32, /// Recorded repetitions. @@ -222,6 +225,13 @@ fn run() -> Result<()> { /// get it onto ComputeArena: the command to run, and the sign-in it needs. fn print_submission_hint(paths: &Paths, api_url: &str, report: &std::path::Path) -> Result<()> { let ui = TerminalUi::detect(); + if let Some(gap) = read_report(report) + .ok() + .and_then(|value| sweep::report_gap(&value)) + { + println!("{} {}", ui.neutral("Local only:"), gap.submission_blocker()); + return Ok(()); + } let submit = format!("computearena submit {}", report.display()); match load_api_session(paths, api_url)? { Some(session) => println!( @@ -360,9 +370,21 @@ fn execute( None => return Ok(()), }, }; + let chosen_by_flag = harness.is_some(); let (harness, model) = benchmark::identify_benchmark_paths(runtime, harness, &model, paths)?; benchmark::print_resolved_paths(runtime, &harness)?; + // Before the plan: an older BaseRT decides what the report will + // say, and updating it takes less time than the run does. + let mut basert_notice = if runtime == Runtime::Basert { + let source = runtimes::source_of(runtime, chosen_by_flag, paths)?; + Some(basert_updates::RunNotice::start(paths, &harness, source)) + } else { + None + }; + if let Some(notice) = basert_notice.as_mut() { + notice.before_run(TerminalUi::detect()); + } let Some(cooldown_enabled) = runtime.adapter().confirm( &BenchmarkRequest { model: &model, @@ -391,6 +413,9 @@ fn execute( output, )?; print_submission_hint(paths, api_url, &report)?; + if let Some(notice) = basert_notice.as_mut() { + notice.after_run(TerminalUi::detect()); + } Ok(()) } Action::List { json } => list_reports(paths, json), @@ -726,14 +751,23 @@ mod tests { "benchmark": { "schema": HARNESS_SCHEMA, "mode": "text", - "raw_samples": {"prefill": {"128": []}, "decode": []} + "raw_samples": { + "prefill": { + "128": [], "256": [], "512": [], "1024": [], + "2048": [], "4096": [], "8192": [], "16384": [] + }, + "decode": [{"generated_tokens": 128, "elapsed_ns": 1}] + } } }) } fn signed_sample_report() -> Value { + sign_sample(sample_report()) + } + + fn sign_sample(mut report: Value) -> Value { let key = SigningKey::generate(&mut OsRng); - let mut report = sample_report(); let public = key.verifying_key().to_bytes(); report["installation"] = json!({ "key_id": sha256_hex(&public), @@ -800,17 +834,25 @@ mod tests { let valid_path = temporary.path().join("valid.json"); let tampered_path = temporary.path().join("tampered.json"); let malformed_path = temporary.path().join("malformed.json"); + let partial_path = temporary.path().join("partial.json"); let valid = signed_sample_report(); let mut tampered = valid.clone(); tampered["model"]["size_bytes"] = json!(2); + // A correctly signed PP2048-only run: nothing is wrong with it, and it + // still cannot be submitted. + let mut partial = sample_report(); + partial["benchmark"]["raw_samples"]["prefill"] = json!({"2048": []}); + let partial = sign_sample(partial); fs::write(&valid_path, serde_json::to_vec(&valid).unwrap()).unwrap(); fs::write(&tampered_path, serde_json::to_vec(&tampered).unwrap()).unwrap(); fs::write(&malformed_path, b"{not-json").unwrap(); + fs::write(&partial_path, serde_json::to_vec(&partial).unwrap()).unwrap(); let preflight = preflight_submissions(&[ valid_path.clone(), tampered_path.clone(), malformed_path.clone(), + partial_path.clone(), ]); assert_eq!(preflight.ready.len(), 1); @@ -822,6 +864,12 @@ mod tests { .contains("signature verification failed")); assert_eq!(preflight.invalid[1].path, malformed_path); assert!(preflight.invalid[1].reason.starts_with("Invalid JSON:")); + assert_eq!(preflight.local_only.len(), 1); + assert_eq!(preflight.local_only[0].path, partial_path); + assert_eq!( + preflight.local_only[0].reason, + "Partial run, not submittable: it is missing PP128, PP256, PP512, PP1024, PP4096, PP8192 and PP16384. ComputeArena accepts only runs with the full default sweep (PP128 to PP16384 and TG128). Run the benchmark again without --pp and --tg to get a submittable report." + ); } #[test] @@ -897,6 +945,7 @@ mod tests { #[test] fn submission_stops_only_for_systemic_http_errors() { + assert!(!should_stop_submission(reqwest::StatusCode::CONFLICT)); assert!(!should_stop_submission(reqwest::StatusCode::BAD_REQUEST)); assert!(!should_stop_submission( reqwest::StatusCode::UNPROCESSABLE_ENTITY diff --git a/crates/computearena-cli/src/model_identity.rs b/crates/computearena-cli/src/model_identity.rs index b7bee5d..d102a3d 100644 --- a/crates/computearena-cli/src/model_identity.rs +++ b/crates/computearena-cli/src/model_identity.rs @@ -539,7 +539,9 @@ pub(crate) fn report_identity_notice(model: &Value) -> String { (false, _, Some(repository)) => format!( "The model bytes were recorded from {repository} without an exact file; the server will match their hash against artifacts it has already verified." ), - (false, _, None) => "Model identity is unresolved. The signed report remains submittable and will be labelled unverified; use computearena identify to bind manually acquired bytes to an exact Hugging Face file.".to_string(), + // Says only what identity decides: whether the run itself can be + // submitted depends on its sweep, and is reported separately. + (false, _, None) => "Model identity is unresolved. That alone does not block a submission: the run is labelled unverified. Use computearena identify to bind manually acquired bytes to an exact Hugging Face file.".to_string(), } } diff --git a/crates/computearena-cli/src/protocol.rs b/crates/computearena-cli/src/protocol.rs index a09c1f8..deb9a1e 100644 --- a/crates/computearena-cli/src/protocol.rs +++ b/crates/computearena-cli/src/protocol.rs @@ -7,6 +7,31 @@ pub(crate) const TELEMETRY_SCHEMA: &str = "basert-telemetry/3"; /// collection to it. Older replay-based harnesses remain on the external /// observer path. pub(crate) const BASERT_SAME_RUN_TELEMETRY_SCHEMA: &str = "basert-telemetry/4"; +pub(crate) const BASERT_ISOLATED_PROTOCOL_SCHEMA: &str = "basert-throughput-protocol/1"; +pub(crate) const THROUGHPUT_PROTOCOL_ID: &str = "computearena-throughput/2"; +pub(crate) const HEADLINE_PROTOCOL_ID: &str = "computearena-throughput/3"; +pub(crate) const BASERT_CAPACITY_PROTOCOL_SCHEMA: &str = "basert-throughput-protocol/2"; +pub(crate) const HEADLINE_CONTEXT_CAPACITY: u64 = 4096; + +/// Inputs are validated before execution. Custom sweeps without PP512 keep +/// their first requested PP size instead of introducing an unrequested test. +pub(crate) fn headline_prefill(pp: &str) -> &str { + pp.split(',') + .find(|size| *size == "512") + .unwrap_or_else(|| pp.split(',').next().unwrap_or("512")) +} + +pub(crate) fn headline_order(pp: &str, tg: u32) -> Vec { + let headline = headline_prefill(pp); + let mut order = vec![format!("pp{headline}"), format!("tg{tg}")]; + order.extend( + pp.split(',') + .filter(|size| *size != headline) + .map(|size| format!("pp{size}")), + ); + order +} +pub(crate) const DECODE_INITIAL_CONTEXT_TOKENS: u64 = 1; pub(crate) const SIGNATURE_DOMAIN: &[u8] = b"computearena-benchmark/1\0"; pub(crate) const SIGNATURE_ALGORITHM: &str = "ed25519"; diff --git a/crates/computearena-cli/src/reports.rs b/crates/computearena-cli/src/reports.rs index 087195d..64877ac 100644 --- a/crates/computearena-cli/src/reports.rs +++ b/crates/computearena-cli/src/reports.rs @@ -337,8 +337,10 @@ pub(crate) fn list_reports(paths: &Paths, as_json: bool) -> Result<()> { // `--json` keeps the newest-first order machines and the pickers use. for (index, report) in reports.iter().rev().enumerate() { let status = report["status"].as_str().unwrap_or("invalid"); - let status_label = if status == "valid" { + let status_label = if report["submittable"].as_bool() == Some(true) { ui.success("VALID") + } else if status == "valid" { + ui.warning("LOCAL ONLY") } else { ui.error("INVALID") }; @@ -368,6 +370,9 @@ pub(crate) fn list_reports(paths: &Paths, as_json: bool) -> Result<()> { .as_str() .unwrap_or("unknown quantization") ); + if let Some(blocker) = report["submission_blocker"].as_str() { + println!(" {}", ui.warning(blocker)); + } println!(" Throughput:"); let prefill = report["prefill"].as_array().cloned().unwrap_or_default(); @@ -436,6 +441,12 @@ pub(crate) fn report_summaries(paths: &Paths) -> Result> { Ok(_) => "valid".to_string(), Err(error) => format!("invalid: {error}"), }; + // Valid and submittable are different things: a partial run + // verifies, but only the full default sweep is accepted. + let submission_blocker = (status == "valid") + .then(|| crate::sweep::report_gap(&value)) + .flatten() + .map(|gap| gap.submission_blocker()); let (model_id, variant) = model_identity_for_report(&value); let model = match variant.as_deref() { Some(variant) => format!("{model_id} ({variant})"), @@ -469,6 +480,8 @@ pub(crate) fn report_summaries(paths: &Paths) -> Result> { "decode": decode, "peak_memory_mb": peak_memory_for_report(&value), "ending_temperature_c": value.pointer("/benchmark/thermal/die_end_c").and_then(Value::as_f64), + "submittable": status == "valid" && submission_blocker.is_none(), + "submission_blocker": submission_blocker, "status": status, "path": path })); diff --git a/crates/computearena-cli/src/runtimes.rs b/crates/computearena-cli/src/runtimes.rs index 69c9d09..3876d24 100644 --- a/crates/computearena-cli/src/runtimes.rs +++ b/crates/computearena-cli/src/runtimes.rs @@ -78,6 +78,14 @@ pub(crate) struct Located { pub(crate) source: Source, } +/// How the executable a command is about to run was chosen. +pub(crate) fn source_of(runtime: Runtime, chosen_by_flag: bool, paths: &Paths) -> Result { + if chosen_by_flag { + return Ok(Source::Override); + } + locate(runtime, None, paths).map(|located| located.source) +} + pub(crate) fn executable_on_path(name: &str) -> Option { std::env::var_os("PATH").and_then(|path| { std::env::split_paths(&path) @@ -933,7 +941,7 @@ pub(crate) fn install( path: executable, source: Source::Managed, }; - report_found(ui, runtime, &located)?; + report_found(ui, runtime, &located, paths)?; Ok(Some(located)) } @@ -941,7 +949,12 @@ pub(crate) fn install( /// Print the executable that will run, and check it is usable before anyone /// picks a model. Returns the version when the runtime reports one. -pub(crate) fn report_found(ui: TerminalUi, runtime: Runtime, located: &Located) -> Result<()> { +pub(crate) fn report_found( + ui: TerminalUi, + runtime: Runtime, + located: &Located, + paths: &Paths, +) -> Result<()> { let adapter = runtime.adapter(); let capabilities = adapter.probe(&located.path).with_context(|| { format!( @@ -963,6 +976,14 @@ pub(crate) fn report_found(ui: TerminalUi, runtime: Runtime, located: &Located) ui.muted(located.source.describe()), ); println!(" {}", ui.neutral(compact_path(&located.path))); + // From what the last lookup found; the one started here is for next time, + // so finding a runtime never waits for the network. + if runtime == Runtime::Basert { + let watch = crate::basert_updates::Watch::start(paths); + if let Some(advice) = watch.advice(&capabilities, located.source) { + crate::basert_updates::print_advice(ui, &advice); + } + } Ok(()) } @@ -984,7 +1005,7 @@ pub(crate) fn ensure_runtime( let adapter = runtime.adapter(); loop { let problem = match locate(runtime, override_path.clone(), paths) { - Ok(located) => match report_found(ui, runtime, &located) { + Ok(located) => match report_found(ui, runtime, &located, paths) { Ok(()) => return Ok(RuntimeSetup::Ready(located.path)), Err(error) => format!("{error:#}"), }, diff --git a/crates/computearena-cli/src/submission.rs b/crates/computearena-cli/src/submission.rs index 8f247b7..43437bf 100644 --- a/crates/computearena-cli/src/submission.rs +++ b/crates/computearena-cli/src/submission.rs @@ -1,4 +1,6 @@ -use crate::api::{client as api_client, server_error as api_server_error}; +use crate::api::{ + client as api_client, error_code as api_error_code, server_error as api_server_error, +}; use crate::auth::load_api_session; use crate::config::SUBMISSION_HTTP_TIMEOUT; use crate::model_identity::{verify_submission_model, SubmissionModelVerification}; @@ -16,6 +18,8 @@ use std::fs; use std::io::{self, IsTerminal}; use std::path::{Path, PathBuf}; +pub(crate) const DELETED_SUBMISSION_NOTICE: &str = "Previously deleted reports will fail to submit.\nSelect all does not restore them. Other valid reports\nwill still be attempted. Local files stay unchanged.\nYou can rerun the same model with the same settings,\nthen submit the newly generated report as a separate\nbenchmark."; + #[derive(Debug)] pub(crate) struct PreparedSubmission { pub(crate) path: PathBuf, @@ -35,6 +39,9 @@ pub(crate) struct InvalidSubmission { pub(crate) struct SubmissionPreflight { pub(crate) ready: Vec, pub(crate) invalid: Vec, + /// Valid reports ComputeArena does not accept: partial runs. They are kept + /// apart from the invalid ones because nothing is wrong with them. + pub(crate) local_only: Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -42,6 +49,7 @@ enum SubmissionOutcomeKind { Submitted, Duplicate, Rejected, + PreviouslyDeleted, NotAttempted, } @@ -59,6 +67,8 @@ pub(crate) struct SubmissionSummary { pub(crate) duplicates: usize, /// Reports left out because they failed the preflight checks. pub(crate) skipped: usize, + /// Partial runs left out: valid, but only the full default sweep is accepted. + pub(crate) local_only: usize, } impl SubmissionSummary { @@ -70,6 +80,13 @@ impl SubmissionSummary { if self.skipped > 0 { parts.push(format!("{} skipped as invalid", self.skipped)); } + if self.local_only > 0 { + parts.push(format!( + "{} partial run(s) kept local: only the full default sweep ({}) is accepted", + self.local_only, + crate::sweep::required_summary() + )); + } parts.join("; ") } } @@ -105,8 +122,10 @@ pub(crate) fn select_reports_for_submission( let items: Vec = reports .iter() .map(|report| { - let status = if report["status"].as_str() == Some("valid") { + let status = if report["submittable"].as_bool() == Some(true) { ui.success("VALID") + } else if report["status"].as_str() == Some("valid") { + ui.warning("LOCAL ONLY") } else { ui.error("INVALID") }; @@ -122,11 +141,12 @@ pub(crate) fn select_reports_for_submission( )) }) .collect(); - // Valid benchmarks start ticked: submitting everything submittable is the - // usual intent, so most people only press Enter. + // Submittable benchmarks start ticked: submitting everything that can be + // is the usual intent, so most people only press Enter. A partial run is + // valid but local only, and says why if it is chosen anyway. let preselected: Vec = reports .iter() - .map(|report| report["status"].as_str() == Some("valid")) + .map(|report| report["submittable"].as_bool() == Some(true)) .collect(); let indexes = match choose_many(ui, "Benchmarks to submit: ", &items, &preselected)? { @@ -221,11 +241,37 @@ pub(crate) fn submit_reports( } print_submission_preflight(ui, &preflight); + if preflight.ready.is_empty() && preflight.invalid.is_empty() { + bail!( + "nothing was uploaded: {}. ComputeArena accepts only runs with the full default sweep ({}). Run the benchmark again without --pp and --tg to get a submittable report; the saved reports are unchanged and stay valid for local use.", + if preflight.local_only.len() == 1 { + "the selected benchmark is a partial run".to_string() + } else { + format!( + "all {} selected benchmarks are partial runs", + preflight.local_only.len() + ) + }, + crate::sweep::required_summary() + ); + } if assume_yes && !skip_invalid && !preflight.invalid.is_empty() { bail!( "refusing a partial non-interactive submission; review the invalid reports or pass --yes --skip-invalid" ); } + if assume_yes && !skip_invalid && !preflight.local_only.is_empty() { + bail!( + "refusing to upload only part of the selection: {} of the selected benchmarks {} and cannot be submitted (see above). Pass --yes --skip-invalid to upload the other {}, or run them again without --pp and --tg.", + preflight.local_only.len(), + if preflight.local_only.len() == 1 { + "is a partial run" + } else { + "are partial runs" + }, + preflight.ready.len() + ); + } if preflight.ready.is_empty() { bail!("no valid benchmarks were selected; nothing was uploaded"); } @@ -254,6 +300,7 @@ pub(crate) fn submit_reports( if prompt_yes_no("Preview the JSON data before submitting?", true)? { print_submission_preview(ui, &preflight.ready)?; } + println!("{}", ui.neutral(DELETED_SUBMISSION_NOTICE)); if !prompt_yes_no( &format!( "Submit the {} valid benchmark(s) now?", @@ -267,15 +314,19 @@ pub(crate) fn submit_reports( ); return Ok(SubmissionSummary { skipped: preflight.invalid.len(), + local_only: preflight.local_only.len(), ..SubmissionSummary::default() }); } + } else { + println!("{}", ui.neutral(DELETED_SUBMISSION_NOTICE)); } let endpoint = format!("{api_url}/submissions"); let client = api_client(SUBMISSION_HTTP_TIMEOUT)?; let report_count = preflight.ready.len(); let skipped = preflight.invalid.len(); + let local_only = preflight.local_only.len(); let mut outcomes = Vec::with_capacity(report_count); let mut queue = preflight.ready.into_iter().enumerate(); while let Some((index, report)) = queue.next() { @@ -363,7 +414,13 @@ pub(crate) fn submit_reports( eprintln!("{} {message}", ui.error("✗")); outcomes.push(SubmissionOutcome { label, - kind: SubmissionOutcomeKind::Rejected, + kind: if status == reqwest::StatusCode::CONFLICT + && api_error_code(&body).as_deref() == Some("submission_deleted") + { + SubmissionOutcomeKind::PreviouslyDeleted + } else { + SubmissionOutcomeKind::Rejected + }, detail: Some(message.clone()), }); if should_stop_submission(status) { @@ -413,23 +470,37 @@ pub(crate) fn submit_reports( .filter(|outcome| { matches!( outcome.kind, - SubmissionOutcomeKind::Rejected | SubmissionOutcomeKind::NotAttempted + SubmissionOutcomeKind::Rejected + | SubmissionOutcomeKind::PreviouslyDeleted + | SubmissionOutcomeKind::NotAttempted ) }) .count(); if failures > 0 { + let deleted = outcomes + .iter() + .filter(|outcome| outcome.kind == SubmissionOutcomeKind::PreviouslyDeleted) + .count(); bail!( - "{failures} of {report_count} eligible benchmark(s) were not submitted; successful submissions remain saved" + "{submitted} uploaded, {duplicates} already present; {failures} of {report_count} eligible benchmark(s) were not submitted ({deleted} previously deleted). Successful uploads remain saved; local files are unchanged." ); } println!( "{} Submission complete: {submitted} uploaded, {duplicates} already present.", ui.success("✓"), ); + if local_only > 0 { + println!( + "{} {local_only} partial run(s) were not uploaded and stay local: only the full default sweep ({}) is accepted.", + ui.warning("!"), + crate::sweep::required_summary() + ); + } Ok(SubmissionSummary { submitted, duplicates, skipped, + local_only, }) } @@ -466,6 +537,16 @@ pub(crate) fn preflight_submissions(reports: &[PathBuf]) -> SubmissionPreflight }); continue; } + // A partial run is a valid report that the server refuses; saying so + // here, with what is missing, beats a rejection after the upload. + if let Some(gap) = crate::sweep::report_gap(&value) { + preflight.local_only.push(InvalidSubmission { + path: path.clone(), + label: submission_label(&value, path), + reason: gap.submission_blocker(), + }); + continue; + } preflight.ready.push(PreparedSubmission { path: path.clone(), value, @@ -549,6 +630,17 @@ fn print_submission_preflight(ui: TerminalUi, preflight: &SubmissionPreflight) { ui.neutral("— will not be uploaded") ); + println!( + " {:<22} {} {}", + "Partial runs", + if preflight.local_only.is_empty() { + ui.neutral(0) + } else { + ui.warning(preflight.local_only.len()) + }, + ui.neutral("— local only, will not be uploaded") + ); + if !preflight.invalid.is_empty() { println!("\n{} Invalid reports:", ui.warning("!")); for invalid in &preflight.invalid { @@ -557,6 +649,17 @@ fn print_submission_preflight(ui: TerminalUi, preflight: &SubmissionPreflight) { println!(" {}", ui.muted(invalid.path.display())); } } + if !preflight.local_only.is_empty() { + println!( + "\n{} Partial runs (valid reports, local only):", + ui.warning("!") + ); + for partial in &preflight.local_only { + println!(" {} {}", ui.warning("✗"), partial.label); + println!(" {}", ui.neutral(&partial.reason)); + println!(" {}", ui.muted(partial.path.display())); + } + } println!(); println!("{}", ui.neutral(crate::reports::SIGNATURE_SCOPE_NOTICE)); } @@ -603,6 +706,9 @@ fn print_submission_results(ui: TerminalUi, outcomes: &[SubmissionOutcome]) { SubmissionOutcomeKind::Submitted => (ui.success("✓"), "Submitted"), SubmissionOutcomeKind::Duplicate => (ui.neutral("="), "Already submitted"), SubmissionOutcomeKind::Rejected => (ui.error("✗"), "Rejected"), + SubmissionOutcomeKind::PreviouslyDeleted => { + (ui.error("✗"), "Failed: previously deleted") + } SubmissionOutcomeKind::NotAttempted => (ui.warning("—"), "Not attempted"), }; println!(" {marker} {} — {status}", outcome.label); diff --git a/crates/computearena-cli/src/sweep.rs b/crates/computearena-cli/src/sweep.rs new file mode 100644 index 0000000..f48c835 --- /dev/null +++ b/crates/computearena-cli/src/sweep.rs @@ -0,0 +1,240 @@ +//! Which benchmark reports ComputeArena accepts. +//! +//! Only complete runs are published: every default prefill size and the +//! default decode length, so every model and chip can be compared at every +//! size. A custom `--pp` or `--tg` still produces a valid signed report for +//! local use; it is just not submittable, and everything that touches such a +//! report says so, says why, and says how to get a submittable one. + +use crate::config::{DEFAULT_DECODE_TOKENS, DEFAULT_PREFILL_TOKENS}; +use crate::ui::TerminalUi; +use serde_json::Value; + +/// What a run lacks against the full default sweep. Extra sizes are fine. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct SweepGap { + pub(crate) missing_prefill: Vec, + /// The decode length that was measured instead of the default; zero when + /// the report carries no decode measurement at all. + pub(crate) decode: Option, +} + +pub(crate) fn required_prefill() -> Vec { + DEFAULT_PREFILL_TOKENS + .split(',') + .filter_map(|size| size.trim().parse().ok()) + .collect() +} + +/// "PP128 to PP16384 and TG128" +pub(crate) fn required_summary() -> String { + let required = required_prefill(); + format!( + "PP{} to PP{} and TG{DEFAULT_DECODE_TOKENS}", + required.first().copied().unwrap_or_default(), + required.last().copied().unwrap_or_default() + ) +} + +pub(crate) fn sweep_gap(prefill: &[u32], decode: Option) -> Option { + let missing_prefill: Vec = required_prefill() + .into_iter() + .filter(|size| !prefill.contains(size)) + .collect(); + let decode = match decode { + Some(tokens) if tokens == DEFAULT_DECODE_TOKENS => None, + other => Some(other.unwrap_or(0)), + }; + (!missing_prefill.is_empty() || decode.is_some()).then_some(SweepGap { + missing_prefill, + decode, + }) +} + +/// The gap a `run` invocation is about to produce, from its flags. +pub(crate) fn requested_gap(pp: &str, tg: u32) -> Option { + let prefill: Vec = pp + .split(',') + .filter_map(|size| size.trim().parse().ok()) + .collect(); + sweep_gap(&prefill, Some(tg)) +} + +/// The gap in a saved report, read from the signed samples the server +/// itself recomputes throughput from, with the request parameters as a +/// fallback for a report that records no samples. +pub(crate) fn report_gap(report: &Value) -> Option { + let mut prefill: Vec = report + .pointer("/benchmark/raw_samples/prefill") + .and_then(Value::as_object) + .map(|groups| groups.keys().filter_map(|size| size.parse().ok()).collect()) + .unwrap_or_default(); + if prefill.is_empty() { + prefill = report + .pointer("/benchmark/params/pp") + .and_then(Value::as_str) + .map(|pp| { + pp.split(',') + .filter_map(|size| size.trim().parse().ok()) + .collect() + }) + .unwrap_or_default(); + } + let decode = report + .pointer("/benchmark/raw_samples/decode/0/generated_tokens") + .and_then(Value::as_u64) + .or_else(|| { + report + .pointer("/benchmark/params/tg") + .and_then(Value::as_u64) + }) + .and_then(|tokens| u32::try_from(tokens).ok()); + sweep_gap(&prefill, decode) +} + +impl SweepGap { + /// "it is missing PP128, PP256 and PP512 and it measured TG64 instead of TG128" + pub(crate) fn describe(&self) -> String { + let mut parts = Vec::new(); + if !self.missing_prefill.is_empty() { + let sizes: Vec = self + .missing_prefill + .iter() + .map(|size| format!("PP{size}")) + .collect(); + parts.push(format!("it is missing {}", join_list(&sizes))); + } + match self.decode { + Some(0) => parts.push(format!( + "it has no TG{DEFAULT_DECODE_TOKENS} decode measurement" + )), + Some(tokens) => parts.push(format!( + "it measured TG{tokens} instead of TG{DEFAULT_DECODE_TOKENS}" + )), + None => {} + } + parts.join(" and ") + } + + /// Why a saved report cannot be uploaded, in words a person can act on. + pub(crate) fn submission_blocker(&self) -> String { + format!( + "Partial run, not submittable: {}. ComputeArena accepts only runs with the full default sweep ({}). Run the benchmark again without --pp and --tg to get a submittable report.", + self.describe(), + required_summary() + ) + } +} + +/// Shown before a custom sweep starts, so nobody spends a long run on a +/// report they expected to upload. +pub(crate) fn print_local_only_notice(ui: TerminalUi, gap: &SweepGap) { + println!( + "{} {}", + ui.warning("!"), + ui.strong("This will be a local-only run.") + ); + println!( + " {}", + ui.neutral(format!( + "Compared with the default sweep, {}.", + gap.describe() + )) + ); + println!( + " {}", + ui.neutral(format!( + "ComputeArena accepts only runs with the full default sweep ({}), so this report can be saved, inspected and verified, but not submitted.", + required_summary() + )) + ); + println!( + " {}", + ui.neutral("Omit --pp and --tg to run the default sweep instead.") + ); +} + +fn join_list(items: &[String]) -> String { + match items { + [] => String::new(), + [only] => only.clone(), + [head @ .., last] => format!("{} and {last}", head.join(", ")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn the_default_sweep_is_complete_and_extra_sizes_are_fine() { + assert_eq!( + requested_gap(DEFAULT_PREFILL_TOKENS, DEFAULT_DECODE_TOKENS), + None + ); + assert_eq!( + requested_gap( + &format!("{DEFAULT_PREFILL_TOKENS},32768"), + DEFAULT_DECODE_TOKENS + ), + None + ); + assert_eq!(required_summary(), "PP128 to PP16384 and TG128"); + } + + #[test] + fn a_custom_sweep_names_exactly_what_is_missing() { + let gap = requested_gap("2048", 128).unwrap(); + assert_eq!( + gap.missing_prefill, + vec![128, 256, 512, 1024, 4096, 8192, 16384] + ); + assert_eq!(gap.decode, None); + assert_eq!( + gap.describe(), + "it is missing PP128, PP256, PP512, PP1024, PP4096, PP8192 and PP16384" + ); + let blocker = gap.submission_blocker(); + assert!(blocker.starts_with("Partial run, not submittable:")); + assert!(blocker.contains("PP128 to PP16384 and TG128")); + assert!(blocker.contains("without --pp and --tg")); + + let decode = requested_gap(DEFAULT_PREFILL_TOKENS, 64).unwrap(); + assert_eq!(decode.describe(), "it measured TG64 instead of TG128"); + let both = requested_gap("128,256,512,1024,2048,4096,8192", 256).unwrap(); + assert_eq!( + both.describe(), + "it is missing PP16384 and it measured TG256 instead of TG128" + ); + } + + #[test] + fn saved_reports_are_judged_by_their_signed_samples() { + let groups: serde_json::Map = required_prefill() + .into_iter() + .map(|size| (size.to_string(), json!([{"tokens": size, "elapsed_ns": 1}]))) + .collect(); + let complete = json!({"benchmark": {"raw_samples": { + "prefill": groups, + "decode": [{"generated_tokens": 128, "elapsed_ns": 1}] + }}}); + assert_eq!(report_gap(&complete), None); + + let partial = json!({"benchmark": {"raw_samples": { + "prefill": {"2048": [{"tokens": 2048, "elapsed_ns": 1}]}, + "decode": [{"generated_tokens": 128, "elapsed_ns": 1}] + }}}); + assert_eq!(report_gap(&partial).unwrap().missing_prefill.len(), 7); + + let from_params = json!({"benchmark": {"params": {"pp": "512", "tg": 128}}}); + assert_eq!(report_gap(&from_params).unwrap().missing_prefill.len(), 7); + + let nothing = json!({"benchmark": {}}); + let gap = report_gap(¬hing).unwrap(); + assert_eq!(gap.decode, Some(0)); + assert!(gap + .describe() + .contains("it has no TG128 decode measurement")); + } +} diff --git a/crates/computearena-cli/src/telemetry.rs b/crates/computearena-cli/src/telemetry.rs index 84aa7c6..3c06a02 100644 --- a/crates/computearena-cli/src/telemetry.rs +++ b/crates/computearena-cli/src/telemetry.rs @@ -166,7 +166,7 @@ impl Sampler { eprintln!( "{}", crate::ui::TerminalUi::detect().neutral(format!( - "Benchmark running — {}s elapsed; {memory} (whole run)", + "Benchmark running — {}s elapsed; {memory} (runtime process)", started.elapsed().as_secs() )) ); @@ -231,13 +231,32 @@ fn unavailable(reason: &str) -> Value { /// Attach runtime-neutral whole-process telemetry and retain the compatibility /// memory field consumed by existing local summaries and the web projection. /// Detailed scope and limitations remain on the telemetry object. +pub(crate) fn capture_environment() -> Value { + snapshots::capture() +} + +pub(crate) fn attach_environment(benchmark: &mut Value, environment: Value) { + benchmark["environment"] = environment; +} + pub(crate) fn attach_whole_process(benchmark: &mut Value, telemetry: Value) -> Option { + if let Some(environment) = telemetry.get("boundaries").cloned() { + attach_environment(benchmark, environment); + } + let memory_scope = if telemetry["scope"] + .as_str() + .is_some_and(|scope| scope.starts_with("separate_")) + { + "all_runtime_processes" + } else { + "whole_runtime_process" + }; let peak = telemetry .pointer("/process_memory/statistics/peak") .and_then(Value::as_f64); if let Some(peak) = peak { benchmark["memory"] = json!({"process_peak_rss_mb":peak, - "measurement_relation":"concurrent_observer","scope":"whole_runtime_process", + "measurement_relation":"concurrent_observer","scope":memory_scope, "unit":"MiB","note":"Observed sampled peak, including loading and warmup; not a kernel high-water mark"}); } benchmark["telemetry"] = telemetry; @@ -264,7 +283,8 @@ pub(crate) fn run_observed(command: &mut Command) -> Result<(Output, Value)> { Ok(sampler) => sampler.finish(), Err(error) => unavailable(&format!("Could not start telemetry worker: {error}")), }; - telemetry["boundaries"] = json!({"measurement_relation":"outside_runtime_execution", + telemetry["boundaries"] = json!({"schema":"computearena-environment/1", + "measurement_relation":"outside_runtime_execution", "before":before,"after":snapshots::capture()}); Ok((output?, telemetry)) } diff --git a/crates/computearena-cli/src/telemetry/snapshots.rs b/crates/computearena-cli/src/telemetry/snapshots.rs index ab33439..096d1bf 100644 --- a/crates/computearena-cli/src/telemetry/snapshots.rs +++ b/crates/computearena-cli/src/telemetry/snapshots.rs @@ -5,8 +5,11 @@ use std::fs; use std::io::{Read, Seek, SeekFrom}; use std::path::Path; use std::process::{Command, Stdio}; +#[cfg(target_os = "macos")] +use std::sync::OnceLock; use std::thread; use std::time::{Duration, Instant}; +use sysinfo::System; const PROBE_TIMEOUT: Duration = Duration::from_secs(2); const MAX_OUTPUT: u64 = 65_536; @@ -55,6 +58,132 @@ fn number(value: &str) -> Option { .filter(|v| v.is_finite() && *v >= 0.0) } +fn mebibytes(bytes: u64) -> f64 { + bytes as f64 / 1_048_576.0 +} + +fn operating_system() -> Value { + json!({ + "family": std::env::consts::OS, + "architecture": std::env::consts::ARCH, + "version": System::long_os_version(), + "kernel_version": System::kernel_version() + }) +} + +fn host_resources() -> Value { + let system = System::new_all(); + let total_memory = system.total_memory(); + let available_memory = system.available_memory(); + json!({ + "logical_cpu_count": system.cpus().len(), + "physical_cpu_count": System::physical_core_count(), + "memory": { + "unit": "MiB", + "total": mebibytes(total_memory), + "available": mebibytes(available_memory), + "available_percent": if total_memory > 0 { + Some(available_memory as f64 * 100.0 / total_memory as f64) + } else { + None + } + }, + "swap": { + "unit": "MiB", + "total": mebibytes(system.total_swap()), + "used": mebibytes(system.used_swap()), + "active": system.used_swap() > 0 + } + }) +} + +#[cfg(target_os = "macos")] +fn memory_pressure() -> Value { + let text = query( + Path::new("/usr/bin/memory_pressure"), + &["-Q"], + PROBE_TIMEOUT, + ); + let free_percent = text.as_deref().and_then(|text| { + text.lines().find_map(|line| { + line.strip_prefix("System-wide memory free percentage:") + .and_then(|value| number(value.trim_end_matches('%'))) + }) + }); + json!({"provider":"memory_pressure","available":free_percent.is_some(), + "system_free_percent":free_percent, + "reason_if_unavailable":"memory_pressure -Q failed or returned unsupported output"}) +} + +#[cfg(target_os = "linux")] +fn memory_pressure() -> Value { + let text = fs::read_to_string("/proc/pressure/memory").ok(); + let mut classes = Map::new(); + if let Some(text) = text.as_deref() { + for line in text.lines() { + let mut fields = line.split_whitespace(); + let Some(class) = fields.next() else { + continue; + }; + let mut values = Map::new(); + for field in fields { + let Some((key, value)) = field.split_once('=') else { + continue; + }; + if let Some(value) = number(value) { + values.insert(key.to_owned(), json!(value)); + } + } + classes.insert(class.to_owned(), Value::Object(values)); + } + } + json!({"provider":"linux_psi","available":!classes.is_empty(),"classes":classes, + "reason_if_unavailable":"/proc/pressure/memory is unavailable"}) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn memory_pressure() -> Value { + json!({"available":false,"reason":"No memory-pressure provider for this OS yet"}) +} + +#[cfg(target_os = "macos")] +fn apple_accelerators() -> Vec { + static DEVICES: OnceLock> = OnceLock::new(); + DEVICES + .get_or_init(|| { + let Some(text) = query( + Path::new("/usr/sbin/system_profiler"), + &["SPDisplaysDataType", "-json"], + PROBE_TIMEOUT, + ) else { + return Vec::new(); + }; + let Ok(value) = serde_json::from_str::(&text) else { + return Vec::new(); + }; + value["SPDisplaysDataType"] + .as_array() + .into_iter() + .flatten() + .take(32) + .map(|display| { + json!({ + "name": display["sppci_model"].as_str().or_else(|| display["_name"].as_str()), + "core_count": display["sppci_cores"].as_str().and_then(|value| value.parse::().ok()), + "device_type": display["sppci_device_type"].as_str(), + "metal_support": display["spdisplays_metal"].as_str() + }) + }) + .collect() + }) + .clone() +} + +#[cfg(not(target_os = "macos"))] +fn apple_accelerators() -> Vec { + Vec::new() +} + fn nvidia(text: &str) -> Vec { text.lines() .filter_map(|line| { @@ -114,6 +243,11 @@ fn rocm(text: &str) -> Vec { fn accelerators() -> Value { let mut providers = Vec::new(); + let apple_devices = apple_accelerators(); + if !apple_devices.is_empty() { + providers + .push(json!({"provider":"system_profiler","available":true,"devices":apple_devices})); + } for (program, args) in [ ("nvidia-smi", vec!["--query-gpu=index,name,temperature.gpu,power.draw,memory.used,memory.total,utilization.gpu","--format=csv,noheader,nounits"]), ("rocm-smi", vec!["--showtemp","--showpower","--showuse","--showmeminfo","vram","--json"]), @@ -142,20 +276,39 @@ fn power_state() -> Value { } }); let settings = query(Path::new("/usr/bin/pmset"), &["-g"], PROBE_TIMEOUT); - let low_power = settings.as_deref().and_then(|text| { - text.lines().find_map(|line| { - let mut fields = line.split_whitespace(); - if fields.next()? != "lowpowermode" { - return None; - } - match fields.next()? { - "1" => Some(true), - "0" => Some(false), - _ => None, - } + let setting = |name: &str| { + settings.as_deref().and_then(|text| { + text.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next()? == name).then(|| fields.next()).flatten() + }) }) + }; + let low_power = setting("lowpowermode").and_then(|value| match value { + "1" => Some(true), + "0" => Some(false), + _ => None, + }); + let power_mode = setting("powermode").and_then(|value| value.parse::().ok()); + let high_power = setting("highpowermode").and_then(|value| match value { + "1" => Some(true), + "0" => Some(false), + _ => None, }); - json!({"provider":"pmset","power_source":source,"low_power_mode":low_power}) + let performance_mode = if low_power == Some(true) || power_mode == Some(1) { + Some("low_power") + } else if high_power == Some(true) || power_mode == Some(2) { + Some("high_power") + } else if power_mode == Some(0) { + Some("automatic") + } else if low_power == Some(false) { + Some("standard_or_automatic") + } else { + None + }; + json!({"provider":"pmset","power_source":source,"low_power_mode":low_power, + "performance_mode":performance_mode,"performance_mode_raw":power_mode, + "high_power_mode":high_power}) } #[cfg(target_os = "linux")] @@ -188,9 +341,14 @@ fn power_state() -> Value { pub(super) fn capture() -> Value { let started = Instant::now(); + let operating_system = operating_system(); + let host = host_resources(); + let pressure = memory_pressure(); let power = power_state(); let accelerators = accelerators(); - json!({"power_state":power,"accelerators":accelerators,"probe_elapsed_ms":started.elapsed().as_secs_f64()*1000.0}) + json!({"operating_system":operating_system,"host":host,"memory_pressure":pressure, + "power_state":power,"accelerators":accelerators, + "probe_elapsed_ms":started.elapsed().as_secs_f64()*1000.0}) } #[cfg(test)] @@ -217,4 +375,15 @@ mod tests { assert!(query(Path::new("/bin/sleep"), &["2"], Duration::from_millis(30)).is_none()); assert!(start.elapsed() < Duration::from_secs(1)); } + #[test] + fn host_snapshot_reports_os_memory_swap_and_cpu_shape() { + let os = operating_system(); + let host = host_resources(); + assert_eq!(os["family"], std::env::consts::OS); + assert_eq!(os["architecture"], std::env::consts::ARCH); + assert!(host["logical_cpu_count"].as_u64().unwrap_or(0) > 0); + assert!(host["memory"]["total"].as_f64().unwrap_or(0.0) > 0.0); + assert_eq!(host["swap"]["unit"], "MiB"); + assert!(host["swap"]["active"].is_boolean()); + } } diff --git a/crates/computearena-cli/src/tui/app.rs b/crates/computearena-cli/src/tui/app.rs index f596cf8..0fcc5cc 100644 --- a/crates/computearena-cli/src/tui/app.rs +++ b/crates/computearena-cli/src/tui/app.rs @@ -52,6 +52,31 @@ pub(crate) struct ReportRow { pub(crate) detail: String, pub(crate) path: PathBuf, pub(crate) valid: bool, + /// Why a valid report cannot be uploaded: a partial run is local only. + pub(crate) submission_blocker: Option, +} + +impl ReportRow { + pub(crate) fn submittable(&self) -> bool { + self.valid && self.submission_blocker.is_none() + } +} + +/// Where a run is decided, what an older BaseRT will sign gets one row. +fn protocol_row(advice: Option<&crate::basert_updates::Advice>) -> Option<(&'static str, String)> { + let advice = advice?; + let note = advice.plan_note?; + Some(( + "Protocol", + format!( + "{note}\n{}", + if advice.installable { + "Update BaseRT from the menu (Esc, then u) for the current one" + } else { + "A newer BaseRT harness signs the current one" + } + ), + )) } #[derive(Clone, Copy, PartialEq, Eq)] @@ -198,6 +223,17 @@ pub(crate) struct App { /// offered for submission. pub(crate) pending_submission: Option, update_check: Option, + /// What the runtime in use says it supports and how it was found, kept + /// from the probe that made it usable. + runtime_descriptor: Option<(serde_json::Value, runtimes::Source)>, + /// BaseRT's newest release; looked up the first time BaseRT is entered. + basert_watch: Option, + /// Set by the first `u` on the menu: updating replaces an installation, + /// so it takes a second press. + update_armed: bool, + /// Whether a prebuilt BaseRT exists for this machine, so an update can be + /// installed from here. + prebuilt_basert: bool, } impl App { @@ -227,6 +263,10 @@ impl App { completed_report: Arc::new(Mutex::new(None)), pending_submission: None, update_check, + runtime_descriptor: None, + basert_watch: None, + update_armed: false, + prebuilt_basert: crate::basert_updates::prebuilt_for_this_platform(), }; app.refresh_account(); // The same rule as the printed session: only ask which runtime to use @@ -273,15 +313,24 @@ impl App { /// screen explaining how to obtain it. fn enter_runtime(&mut self) -> Result<()> { let adapter = self.runtime.adapter(); - match runtimes::locate(self.runtime, self.harness_override.clone(), &self.paths) - .and_then(|located| adapter.probe(&located.path).map(|_| located.path)) - { - Ok(path) => { - self.executable = Some(path); + match runtimes::locate(self.runtime, self.harness_override.clone(), &self.paths).and_then( + |located| { + adapter + .probe(&located.path) + .map(|descriptor| (located, descriptor)) + }, + ) { + Ok((located, descriptor)) => { + self.executable = Some(located.path); + self.runtime_descriptor = Some((descriptor, located.source)); + if self.runtime == Runtime::Basert && self.basert_watch.is_none() { + self.basert_watch = Some(crate::basert_updates::Watch::start(&self.paths)); + } self.screens.push(Screen::Menu { cursor: 0 }); } Err(error) => { self.executable = None; + self.runtime_descriptor = None; self.screens.push(Screen::Setup { problem: format!("{error:#}"), instructions: manual_instructions(self.runtime), @@ -292,6 +341,30 @@ impl App { Ok(()) } + /// What is worth saying about the BaseRT in use: a newer release, or a + /// harness that predates the current benchmark protocol. + pub(crate) fn basert_advice(&self) -> Option { + if self.runtime != Runtime::Basert { + return None; + } + let (descriptor, source) = self.runtime_descriptor.as_ref()?; + crate::basert_updates::advice( + descriptor, + self.basert_watch.as_ref().and_then(|watch| watch.latest()), + *source, + self.prebuilt_basert, + ) + } + + /// Whether the menu offers `u`: there is something to gain, and installing + /// the latest release is what gains it. + pub(crate) fn update_offered(&self) -> bool { + matches!(self.screen(), Screen::Menu { .. }) + && self + .basert_advice() + .is_some_and(|advice| advice.installable) + } + pub(crate) fn runtime_label(&self) -> String { match &self.executable { Some(path) => format!("{} · {}", self.runtime.adapter().name(), compact_path(path)), @@ -330,6 +403,9 @@ impl App { changed = true; } } + if let Some(watch) = self.basert_watch.as_mut() { + changed |= watch.refresh(); + } if let Some(pending) = self.pending.as_ref() { match pending.receiver.try_recv() { Ok(loaded) => { @@ -404,7 +480,8 @@ impl App { ) .map(|(_, model)| model)?; let request = self.benchmark_request(&model); - let rows = plan_rows(self.runtime, &request)?; + let mut rows = plan_rows(self.runtime, &request)?; + rows.extend(protocol_row(self.basert_advice().as_ref())); let options = profile_options(self.runtime, &request)?; self.screens.push(Screen::Plan { model, @@ -514,7 +591,7 @@ impl App { // intent. let marks = rows .iter() - .map(|row| mode == ReportMode::Submit && row.valid) + .map(|row| mode == ReportMode::Submit && row.submittable()) .collect(); self.screens.push(Screen::Reports { rows, @@ -544,6 +621,11 @@ impl App { "The fields above will be publicly accessible on ComputeArena. Local file paths are not sent." .to_string(), ); + lines.extend( + crate::submission::DELETED_SUBMISSION_NOTICE + .lines() + .map(str::to_string), + ); self.screens.push(Screen::Preview { lines, scroll: 0, @@ -563,7 +645,9 @@ impl App { // A report that fails its checks is listed and left out; it // must not stop the others from being uploaded. let summary = submit_reports(&paths, &reports, &api_url, true, true)?; - Ok(if summary.skipped == 0 && summary.duplicates == 0 { + // Anything short of every selected report being uploaded + // (a duplicate, an invalid report, a partial run) is spelled out. + Ok(if summary.submitted == count { format!("Submitted {count} benchmark(s)") } else { summary.describe() @@ -843,6 +927,12 @@ impl App { return Ok(()); } self.status.clear(); + // Armed only until the next key, whatever that key is. + let update_armed = std::mem::take(&mut self.update_armed); + if matches!(key.code, KeyCode::Char('u')) && self.update_offered() { + self.confirm_or_start_update(update_armed); + return Ok(()); + } match key.code { KeyCode::Char(character) => self.on_char(character)?, KeyCode::Up => self.move_cursor(-1), @@ -859,6 +949,26 @@ impl App { Ok(()) } + /// Updating downloads a release and replaces the installed bundle, so the + /// first `u` says what will happen and the second one does it. + fn confirm_or_start_update(&mut self, armed: bool) { + if armed { + self.start_install(); + return; + } + self.update_armed = true; + let release = match self.basert_watch.as_ref().and_then(|watch| watch.latest()) { + Some(latest) => format!("BaseRT {latest}"), + None => "the latest BaseRT".to_string(), + }; + let directory = runtimes::basert_install_dir() + .map(|directory| compact_path(&directory)) + .unwrap_or_else(|| "its install directory".to_string()); + self.status = format!( + "Press u again to download {release} into {directory}, replacing the BaseRT files there" + ); + } + fn on_char(&mut self, character: char) -> Result<()> { // Typing filters and path entry take precedence over shortcuts. match self.screen_mut() { @@ -1041,8 +1151,15 @@ impl App { mode, } = self.screen_mut() { - if *mode == ReportMode::Submit && rows[*cursor].valid { + if *mode == ReportMode::Submit && rows[*cursor].submittable() { marks[*cursor] = !marks[*cursor]; + return; + } + let blocker = (*mode == ReportMode::Submit) + .then(|| rows[*cursor].submission_blocker.clone()) + .flatten(); + if let Some(blocker) = blocker { + self.status = blocker; } } } @@ -1053,9 +1170,15 @@ impl App { } = self.screen_mut() { if *mode == ReportMode::Submit { - let target = !marks.iter().all(|marked| *marked); + // Rows that can never be ticked (invalid reports, partial + // runs) must not keep "all" from ever being reached. + let target = !marks + .iter() + .zip(rows.iter()) + .filter(|(_, row)| row.submittable()) + .all(|(marked, _)| *marked); for (mark, row) in marks.iter_mut().zip(rows.iter()) { - *mark = target && row.valid; + *mark = target && row.submittable(); } } } @@ -1240,7 +1363,12 @@ impl App { .map(|(row, _)| row.path.clone()) .collect(); if selected.is_empty() { - self.status = "Select at least one valid benchmark with Space".to_string(); + // Pressing Enter on a partial run explains that row + // rather than asking for a selection it cannot join. + self.status = + rows[*cursor].submission_blocker.clone().unwrap_or_else(|| { + "Select at least one valid benchmark with Space".to_string() + }); } else { self.open_preview(selected)?; } @@ -1390,6 +1518,7 @@ fn report_rows(paths: &Paths) -> Result> { ), path: PathBuf::from(report["path"].as_str().unwrap_or_default()), valid: report["status"].as_str() == Some("valid"), + submission_blocker: report["submission_blocker"].as_str().map(str::to_string), }) .collect()) } @@ -1419,6 +1548,10 @@ mod tests { completed_report: Arc::new(Mutex::new(None)), pending_submission: None, update_check: None, + runtime_descriptor: None, + basert_watch: None, + update_armed: false, + prebuilt_basert: true, } } @@ -1436,6 +1569,164 @@ mod tests { job } + fn report_row(name: &str, submission_blocker: Option<&str>) -> ReportRow { + ReportRow { + label: name.to_string(), + detail: String::new(), + path: PathBuf::from(format!("{name}.json")), + valid: true, + submission_blocker: submission_blocker.map(str::to_string), + } + } + + fn old_basert() -> serde_json::Value { + serde_json::json!({"runtime": {"name": "basert", "version": "0.2.4"}, "features": {}}) + } + + fn current_basert() -> serde_json::Value { + serde_json::json!({ + "runtime": {"name": "basert", "version": "0.2.5"}, + "capacity_protocol_schema": "basert-throughput-protocol/2", + "features": {"headline_context_capacity": true} + }) + } + + fn press(app: &mut App, character: char) { + use ratatui::crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; + app.on_key(KeyEvent::new(KeyCode::Char(character), KeyModifiers::NONE)) + .unwrap(); + } + + #[test] + fn an_older_basert_is_named_on_the_menu_and_updating_takes_two_presses() { + let dir = tempfile::tempdir().unwrap(); + let mut app = app(dir.path(), None); + app.runtime = Runtime::Basert; + app.screens.push(Screen::Menu { cursor: 0 }); + assert_eq!(app.basert_advice(), None, "nothing is known yet"); + + app.runtime_descriptor = Some((old_basert(), runtimes::Source::KnownLocation)); + let advice = app.basert_advice().unwrap(); + assert_eq!( + advice.summary, + "BaseRT 0.2.4 predates the current benchmark protocol." + ); + let row = protocol_row(Some(&advice)).unwrap(); + assert_eq!(row.0, "Protocol"); + assert!(row + .1 + .starts_with("Older BaseRT protocol: signed as not comparable\n")); + assert!(row.1.lines().all(|line| line.chars().count() <= 64)); + + // The first press only says what a second one would do. + assert!(app.update_offered()); + press(&mut app, 'u'); + assert!(app.update_armed); + assert!(app.job.is_none()); + assert!(app + .status + .starts_with("Press u again to download the latest BaseRT into ")); + assert!(app.status.ends_with("replacing the BaseRT files there")); + // Any other key stands it down. + press(&mut app, 'j'); + assert!(!app.update_armed); + assert!(app.job.is_none()); + + // With the feed's answer the confirmation names the release. + app.basert_watch = Some(crate::basert_updates::Watch::known(Some( + crate::updates::release_for_tests("0.2.6"), + ))); + press(&mut app, 'u'); + assert!(app + .status + .starts_with("Press u again to download BaseRT 0.2.6 into ")); + } + + #[test] + fn updating_is_offered_only_where_it_helps() { + let dir = tempfile::tempdir().unwrap(); + let mut app = app(dir.path(), None); + app.runtime = Runtime::Basert; + app.screens.push(Screen::Menu { cursor: 0 }); + + // A current BaseRT with nothing newer: no notice, and `u` is inert. + app.runtime_descriptor = Some((current_basert(), runtimes::Source::Managed)); + app.basert_watch = Some(crate::basert_updates::Watch::known(Some( + crate::updates::release_for_tests("0.2.5"), + ))); + assert_eq!(app.basert_advice(), None); + assert!(protocol_row(app.basert_advice().as_ref()).is_none()); + press(&mut app, 'u'); + assert!(!app.update_armed); + + // A newer release is an update, but says nothing about the protocol. + app.basert_watch = Some(crate::basert_updates::Watch::known(Some( + crate::updates::release_for_tests("0.2.6"), + ))); + assert!(app.update_offered()); + assert!(protocol_row(app.basert_advice().as_ref()).is_none()); + + // A harness chosen by hand is not replaced by an install. + app.runtime_descriptor = Some((old_basert(), runtimes::Source::Override)); + assert!(app.basert_advice().is_some()); + assert!(!app.update_offered()); + let row = protocol_row(app.basert_advice().as_ref()).unwrap(); + assert!(row + .1 + .ends_with("A newer BaseRT harness signs the current one")); + + // Nor on a platform without a prebuilt BaseRT. + app.runtime_descriptor = Some((old_basert(), runtimes::Source::Path)); + app.prebuilt_basert = false; + assert!(!app.update_offered()); + + // Only on the menu, and never for llama.cpp. + app.prebuilt_basert = true; + assert!(app.update_offered()); + app.screens.push(Screen::Account { cursor: 0 }); + assert!(!app.update_offered()); + app.screens.pop(); + app.runtime = Runtime::LlamaCpp; + assert_eq!(app.basert_advice(), None); + } + + #[test] + fn a_partial_run_cannot_be_ticked_and_says_why() { + let dir = tempfile::tempdir().unwrap(); + let mut app = app(dir.path(), Some("isu")); + let blocker = "Partial run, not submittable: it is missing PP128."; + app.screens.push(Screen::Reports { + rows: vec![ + report_row("full", None), + report_row("partial", Some(blocker)), + ], + marks: vec![false, false], + cursor: 1, + mode: ReportMode::Submit, + }); + let marks = |app: &App| match app.screen() { + Screen::Reports { marks, .. } => marks.clone(), + _ => unreachable!(), + }; + + // Space on the partial run explains it instead of ticking it. + app.toggle_mark(); + assert_eq!(marks(&app), [false, false]); + assert_eq!(app.status, blocker); + + // So does Enter, when nothing else is selected. + app.status.clear(); + app.activate().unwrap(); + assert_eq!(app.status, blocker); + assert!(matches!(app.screen(), Screen::Reports { .. })); + + // "All" means every report that can be submitted, and still toggles. + app.mark_all(); + assert_eq!(marks(&app), [true, false]); + app.mark_all(); + assert_eq!(marks(&app), [false, false]); + } + #[test] fn a_saved_benchmark_opens_its_preview_when_signed_in() { let dir = tempfile::tempdir().unwrap(); @@ -1453,7 +1744,9 @@ mod tests { app.activate().unwrap(); assert!(matches!( app.screen(), - Screen::Preview { reports, .. } if *reports == vec![report.clone()] + Screen::Preview { reports, lines, .. } if *reports == vec![report.clone()] + && lines.iter().any(|line| line.contains("Previously deleted reports will fail")) + && lines.iter().any(|line| line.contains("Select all does not restore them")) )); // The picker that started the run is gone: Esc lands on the menu. assert!(matches!( diff --git a/crates/computearena-cli/src/tui/draw.rs b/crates/computearena-cli/src/tui/draw.rs index 05836fb..52dabb5 100644 --- a/crates/computearena-cli/src/tui/draw.rs +++ b/crates/computearena-cli/src/tui/draw.rs @@ -215,6 +215,9 @@ fn footer(frame: &mut Frame, area: Rect, app: &App) { Screen::HubModels { .. } => "↑/↓ move · Enter list files · Esc back", Screen::HubFiles { .. } => "↑/↓ move · Enter download · Esc back", Screen::BaseRtModels { .. } => "type to filter · ↑/↓ move · Enter download · Esc back", + Screen::Menu { .. } if app.update_offered() => { + "↑/↓ move · Enter select · u update BaseRT · Esc back · Ctrl+C quit" + } _ => "↑/↓ move · Enter select · Esc back · Ctrl+C quit", }; let status = if app.status.is_empty() { @@ -245,7 +248,7 @@ fn body(frame: &mut Frame, area: Rect, app: &mut App) { instructions, cursor, } => setup_screen(frame, area, problem, instructions, *cursor), - Screen::Menu { cursor } => menu_screen(frame, area, *cursor), + Screen::Menu { cursor } => menu_screen(frame, area, *cursor, app.basert_advice()), Screen::Models { rows, filter, @@ -371,13 +374,106 @@ fn setup_screen( render_list(frame, areas[1], "What next", items, cursor); } -fn menu_screen(frame: &mut Frame, area: Rect, cursor: usize) { +/// Greedy word wrapping, done here rather than by the widget so the panel +/// can be given exactly the rows its text needs. +fn wrap_words(text: &str, width: usize) -> Vec { + let width = width.max(8); + let mut lines = Vec::new(); + let mut line = String::new(); + for word in text.split_whitespace() { + let needed = line.chars().count() + usize::from(!line.is_empty()) + word.chars().count(); + if needed > width && !line.is_empty() { + lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + +/// The notice above the menu: everything when there is room for it and the +/// whole menu, otherwise just what it is and what to press. +fn basert_notice( + advice: &crate::basert_updates::Advice, + width: usize, + rows_to_spare: usize, +) -> Vec> { + let action = if advice.installable { + "Press u to update BaseRT now.".to_string() + } else { + advice.action.clone() + }; + let mut summary = wrap_words(&advice.summary, width.saturating_sub(2)).into_iter(); + let mut lines = vec![Line::from(vec![ + Span::styled("! ", Style::default().fg(danger())), + Span::styled( + summary.next().unwrap_or_default(), + Style::default().add_modifier(Modifier::BOLD), + ), + ])]; + lines.extend(summary.map(|rest| { + Line::from(Span::styled( + format!(" {rest}"), + Style::default().add_modifier(Modifier::BOLD), + )) + })); + let plain = |text: &str| -> Vec> { + wrap_words(text, width) + .into_iter() + .map(|line| Line::from(Span::styled(line, Style::default().fg(neutral())))) + .collect() + }; + let consequence = advice.consequence.as_deref().map(plain).unwrap_or_default(); + let action = plain(&action); + if lines.len() + consequence.len() + action.len() <= rows_to_spare { + lines.extend(consequence); + } + lines.extend(action); + lines +} + +fn menu_screen( + frame: &mut Frame, + area: Rect, + cursor: usize, + advice: Option, +) { let items = MENU_ITEMS .iter() .enumerate() .map(|(index, (label, detail))| item(*label, *detail, index == cursor)) .collect(); - render_list(frame, area, "ComputeArena", items, cursor); + let Some(advice) = advice else { + render_list(frame, area, "ComputeArena", items, cursor); + return; + }; + + // Above the menu rather than in the status line, which the next key + // clears: this stays true until BaseRT is updated. + let whole_menu = MENU_ITEMS.len() * 2 + 2; + let rows_to_spare = usize::from(area.height).saturating_sub(whole_menu + 2); + let lines = basert_notice( + &advice, + usize::from(area.width.saturating_sub(2)), + rows_to_spare, + ); + let height = lines.len() as u16 + 2; + // A terminal too short for even the short form keeps its menu. + if area.height < height + 6 { + render_list(frame, area, "ComputeArena", items, cursor); + return; + } + let areas = Layout::vertical([Constraint::Length(height), Constraint::Min(6)]).split(area); + frame.render_widget( + Paragraph::new(lines).block(focus_panel("BaseRT", false)), + areas[0], + ); + render_list(frame, areas[1], "ComputeArena", items, cursor); } fn models_screen(frame: &mut Frame, area: Rect, rows: &[ModelRow], filter: &str, cursor: usize) { @@ -534,13 +630,19 @@ fn reports_screen( .zip(marks) .enumerate() .map(|(index, (row, marked))| { - let tick = match (mode, marked, row.valid) { + let tick = match (mode, marked, row.submittable()) { (ReportMode::Submit, true, _) => "[x] ", (ReportMode::Submit, false, true) => "[ ] ", (ReportMode::Submit, false, false) => "[-] ", (ReportMode::Verify, _, _) => "", }; - let status = if row.valid { "VALID" } else { "INVALID" }; + let status = if row.submittable() { + "VALID" + } else if row.valid { + "LOCAL ONLY" + } else { + "INVALID" + }; item( format!("{tick}{} [{status}]", row.label), row.detail.clone(), diff --git a/crates/computearena-cli/src/updates.rs b/crates/computearena-cli/src/updates.rs index 487a723..aed3e62 100644 --- a/crates/computearena-cli/src/updates.rs +++ b/crates/computearena-cli/src/updates.rs @@ -1,4 +1,4 @@ -//! A quiet, cached update hint for the interactive client. +//! Quiet, cached release hints: one for the client itself and one for BaseRT. //! //! The GitHub request runs on a worker thread and failures are deliberately //! ignored: running and retaining benchmarks must keep working offline. A @@ -12,12 +12,93 @@ use std::fs; use std::sync::mpsc::{self, Receiver, TryRecvError}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -const LATEST_RELEASE_API: &str = - "https://api.github.com/repos/basecompute/computearena-cli/releases/latest"; -const UPDATE_CACHE_FILE: &str = "update-check.json"; const UPDATE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60); const UPDATE_HTTP_TIMEOUT: Duration = Duration::from_secs(3); +/// Where the newest release of something is published, and where the answer +/// is kept between sessions. +pub(crate) struct ReleaseFeed { + latest_release_api: &'static str, + /// Names a different endpoint, for tests and mirrors. Only a version is + /// ever read from the answer, so it cannot put words on the screen. + api_override: Option<&'static str>, + cache_file: &'static str, + fallback_url: &'static str, +} + +const COMPUTEARENA: ReleaseFeed = ReleaseFeed { + latest_release_api: "https://api.github.com/repos/basecompute/computearena-cli/releases/latest", + api_override: None, + cache_file: "update-check.json", + fallback_url: COMPUTEARENA_QUICKSTART, +}; + +pub(crate) const BASERT: ReleaseFeed = ReleaseFeed { + latest_release_api: "https://api.github.com/repos/basecompute/baseRT/releases/latest", + api_override: Some("COMPUTEARENA_BASERT_RELEASE_API"), + cache_file: "basert-update-check.json", + fallback_url: crate::runtimes::BASERT_RELEASES, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LatestRelease { + checked_at: u64, + pub(crate) version: Version, + release_url: String, +} + +/// A release lookup running on a worker thread. +pub(crate) struct ReleaseCheck { + receiver: Receiver>, +} + +impl ReleaseCheck { + /// `None` means still running; `Some(None)` means the check completed + /// without an answer (including an offline/network failure). + pub(crate) fn poll(&self) -> Option> { + match self.receiver.try_recv() { + Ok(release) => Some(release), + Err(TryRecvError::Empty) => None, + Err(TryRecvError::Disconnected) => Some(None), + } + } +} + +impl ReleaseFeed { + /// What the last lookup found, and whether it is recent enough to reuse. + fn known(&self, paths: &Paths) -> (Option, bool) { + let cached = read_cache(&paths.root.join(self.cache_file)); + let fresh = cached.as_ref().is_some_and(cache_is_fresh); + (cached, fresh) + } + + /// What is already known, and a lookup in progress when that is missing + /// or more than a day old. Never waits for the network. + pub(crate) fn start(&self, paths: &Paths) -> (Option, Option) { + let (cached, fresh) = self.known(paths); + if fresh { + return (cached, None); + } + + let file = paths.root.join(self.cache_file); + let api = self + .api_override + .and_then(|variable| std::env::var(variable).ok()) + .filter(|url| !url.is_empty()) + .unwrap_or_else(|| self.latest_release_api.to_string()); + let fallback_url = self.fallback_url; + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + let result = fetch_latest(&api, fallback_url); + if let Some(release) = result.as_ref() { + let _ = write_cache(&file, release); + } + let _ = sender.send(result); + }); + (cached, Some(ReleaseCheck { receiver })) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct UpdateNotice { latest: Version, @@ -32,63 +113,42 @@ impl UpdateNotice { } } -#[derive(Clone, Debug)] -struct CachedRelease { - checked_at: u64, - version: Version, - release_url: String, -} - -pub(crate) struct UpdateCheck { - receiver: Receiver>, -} +pub(crate) struct UpdateCheck(ReleaseCheck); impl UpdateCheck { /// `None` means still running; `Some(None)` means the check completed with /// no update (including an offline/network failure). pub(crate) fn poll(&self) -> Option> { - match self.receiver.try_recv() { - Ok(notice) => Some(notice), - Err(TryRecvError::Empty) => None, - Err(TryRecvError::Disconnected) => Some(None), - } + self.0 + .poll() + .map(|release| release.as_ref().and_then(update_notice)) } } +/// The client's own update hint. pub(crate) fn start(paths: &Paths) -> (Option, Option) { - let file = paths.root.join(UPDATE_CACHE_FILE); - let cached = read_cache(&file); - let initial = cached.as_ref().and_then(update_notice); - if cached.as_ref().is_some_and(cache_is_fresh) { - return (initial, None); - } - - let (sender, receiver) = mpsc::channel(); - std::thread::spawn(move || { - let result = fetch_latest(); - if let Some(release) = result.as_ref() { - let _ = write_cache(&file, release); - } - let _ = sender.send(result.as_ref().and_then(update_notice)); - }); - (initial, Some(UpdateCheck { receiver })) + let (known, check) = COMPUTEARENA.start(paths); + ( + known.as_ref().and_then(update_notice), + check.map(UpdateCheck), + ) } fn current_version() -> Option { Version::parse(env!("CARGO_PKG_VERSION")).ok() } -fn update_notice(release: &CachedRelease) -> Option { +fn update_notice(release: &LatestRelease) -> Option { (release.version > current_version()?).then(|| UpdateNotice { latest: release.version.clone(), }) } -fn cache_is_fresh(release: &CachedRelease) -> bool { +fn cache_is_fresh(release: &LatestRelease) -> bool { unix_seconds().saturating_sub(release.checked_at) < UPDATE_CACHE_TTL.as_secs() } -fn fetch_latest() -> Option { +fn fetch_latest(api: &str, fallback_url: &str) -> Option { let client = reqwest::blocking::Client::builder() .connect_timeout(UPDATE_HTTP_TIMEOUT) .timeout(UPDATE_HTTP_TIMEOUT) @@ -96,7 +156,7 @@ fn fetch_latest() -> Option { .build() .ok()?; let response = client - .get(LATEST_RELEASE_API) + .get(api) .header("Accept", "application/vnd.github+json") .send() .ok()? @@ -105,27 +165,27 @@ fn fetch_latest() -> Option { let body = response.text().ok()?; let value: Value = serde_json::from_str(&body).ok()?; let tag = value.get("tag_name")?.as_str()?.trim_start_matches('v'); - Some(CachedRelease { + Some(LatestRelease { checked_at: unix_seconds(), version: Version::parse(tag).ok()?, release_url: value .get("html_url") .and_then(Value::as_str) - .unwrap_or(COMPUTEARENA_QUICKSTART) + .unwrap_or(fallback_url) .to_string(), }) } -fn read_cache(file: &std::path::Path) -> Option { +fn read_cache(file: &std::path::Path) -> Option { let value: Value = serde_json::from_slice(&fs::read(file).ok()?).ok()?; - Some(CachedRelease { + Some(LatestRelease { checked_at: value.get("checkedAtUnixSeconds")?.as_u64()?, version: Version::parse(value.get("latestVersion")?.as_str()?).ok()?, release_url: value.get("releaseUrl")?.as_str()?.to_string(), }) } -fn write_cache(file: &std::path::Path, release: &CachedRelease) -> anyhow::Result<()> { +fn write_cache(file: &std::path::Path, release: &LatestRelease) -> anyhow::Result<()> { if let Some(parent) = file.parent() { fs::create_dir_all(parent)?; } @@ -147,25 +207,28 @@ fn unix_seconds() -> u64 { .as_secs() } +#[cfg(test)] +pub(crate) fn release_for_tests(version: &str) -> LatestRelease { + LatestRelease { + checked_at: unix_seconds(), + version: Version::parse(version).unwrap(), + release_url: format!("https://example.test/v{version}"), + } +} + #[cfg(test)] mod tests { use super::*; - fn release(version: &str) -> CachedRelease { - CachedRelease { - checked_at: unix_seconds(), - version: Version::parse(version).unwrap(), - release_url: format!("https://example.test/v{version}"), - } - } - #[test] fn only_newer_semantic_versions_create_a_notice() { let current = current_version().unwrap(); - assert!(update_notice(&release(¤t.to_string())).is_none()); + assert!(update_notice(&release_for_tests(¤t.to_string())).is_none()); let newer = Version::new(current.major, current.minor, current.patch + 1); assert_eq!( - update_notice(&release(&newer.to_string())).unwrap().latest, + update_notice(&release_for_tests(&newer.to_string())) + .unwrap() + .latest, newer ); } @@ -173,12 +236,42 @@ mod tests { #[test] fn cache_round_trips() { let directory = tempfile::tempdir().unwrap(); - let file = directory.path().join(UPDATE_CACHE_FILE); - let expected = release("9.8.7"); + let file = directory.path().join(COMPUTEARENA.cache_file); + let expected = release_for_tests("9.8.7"); write_cache(&file, &expected).unwrap(); let actual = read_cache(&file).unwrap(); assert_eq!(actual.version, expected.version); assert_eq!(actual.release_url, expected.release_url); assert!(cache_is_fresh(&actual)); } + + #[test] + fn a_fresh_answer_is_reused_without_a_lookup_and_feeds_do_not_share_it() { + let directory = tempfile::tempdir().unwrap(); + let paths = Paths::resolve(Some(directory.path().to_path_buf())).unwrap(); + fs::create_dir_all(&paths.root).unwrap(); + write_cache( + &paths.root.join(BASERT.cache_file), + &release_for_tests("0.2.5"), + ) + .unwrap(); + + let (known, check) = BASERT.start(&paths); + assert_eq!(known.unwrap().version, Version::new(0, 2, 5)); + assert!(check.is_none(), "a fresh answer must not start a lookup"); + // BaseRT's newest release says nothing about the client's own. + assert!(read_cache(&paths.root.join(COMPUTEARENA.cache_file)).is_none()); + } + + #[test] + fn a_stale_answer_is_still_offered_while_it_is_refreshed() { + let directory = tempfile::tempdir().unwrap(); + let paths = Paths::resolve(Some(directory.path().to_path_buf())).unwrap(); + fs::create_dir_all(&paths.root).unwrap(); + assert_eq!(BASERT.known(&paths), (None, false)); + let mut stale = release_for_tests("0.2.5"); + stale.checked_at -= UPDATE_CACHE_TTL.as_secs() + 1; + write_cache(&paths.root.join(BASERT.cache_file), &stale).unwrap(); + assert_eq!(BASERT.known(&paths), (Some(stale), false)); + } } diff --git a/crates/computearena-cli/tests/adapter_contract.rs b/crates/computearena-cli/tests/adapter_contract.rs index 997f5e9..e48286b 100644 --- a/crates/computearena-cli/tests/adapter_contract.rs +++ b/crates/computearena-cli/tests/adapter_contract.rs @@ -68,9 +68,10 @@ fn telemetry_is_collected_for_the_child_summarized_and_signature_protected() { let mut report = f.signed(); let telemetry = &report["benchmark"]["telemetry"]; assert_eq!(telemetry["schema"], "computearena-telemetry/1"); - assert_eq!(telemetry["observer"]["requested_interval_ms"], 1000); + let observed = &telemetry["workloads"]["pp512"]; + assert_eq!(observed["observer"]["requested_interval_ms"], 1000); assert!( - telemetry["process_memory"]["statistics"]["sample_count"] + observed["process_memory"]["statistics"]["sample_count"] .as_u64() .unwrap() >= 1 @@ -79,13 +80,14 @@ fn telemetry_is_collected_for_the_child_summarized_and_signature_protected() { report["benchmark"]["memory"]["process_peak_rss_mb"], telemetry["process_memory"]["statistics"]["peak"] ); - assert_eq!(telemetry["energy"]["available"], false); - assert_eq!(telemetry["per_workload"]["available"], false); + assert_eq!(observed["energy"]["available"], false); + assert_eq!(observed["per_workload"]["available"], false); success(&f.verify()); if let Some(path) = std::env::var_os("COMPUTEARENA_TELEMETRY_TEST_REPORT") { fs::copy(&f.report, path).unwrap(); } - report["benchmark"]["telemetry"]["observer"]["requested_interval_ms"] = json!(25); + report["benchmark"]["telemetry"]["workloads"]["pp512"]["observer"]["requested_interval_ms"] = + json!(25); fs::write(&f.report, serde_json::to_vec_pretty(&report).unwrap()).unwrap(); failure(&f.verify(), "signature verification failed"); } @@ -188,10 +190,69 @@ fn repeated_runs_have_unique_ids_but_keep_the_installation_identity() { } } +#[test] +fn select_all_reports_deleted_failures_and_still_uploads_other_benchmarks() { + for statuses in [vec![409, 201, 200], vec![409]] { + let f = Fixture::full_sweep("llama-cpp"); + let reports = f.dir.path().join("data/reports"); + fs::create_dir_all(&reports).unwrap(); + let originals: Vec<_> = (0..statuses.len()) + .map(|index| { + let report = f.signed(); + let path = reports.join(format!("saved-{index}.json")); + fs::rename(&f.report, &path).unwrap(); + (path, report) + }) + .collect(); + let (url, received) = server(&f, statuses); + let mut child = f + .command() + .args(["--api-url", &url, "submit", "--yes"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"all\n").unwrap(); + let output = child.wait_with_output().unwrap(); + failure(&output, "1 previously deleted"); + let printed = text(&output); + for hint in [ + "Failed: previously deleted", + "Select all does not restore", + "rerun the same model with the same settings", + "newly generated report as a separate benchmark", + "local files are unchanged", + ] { + assert!(printed.contains(hint), "missing {hint}: {printed}"); + } + if originals.len() == 3 { + assert!( + printed.contains("1 uploaded, 1 already present"), + "{printed}" + ); + } else { + assert!( + printed.contains("0 uploaded, 0 already present"), + "{printed}" + ); + } + let attempted = received.join().unwrap(); + assert_eq!(attempted.len(), originals.len()); + for (path, report) in originals { + assert!(attempted.contains(&report)); + assert_eq!( + serde_json::from_slice::(&fs::read(path).unwrap()).unwrap(), + report + ); + } + } +} + #[test] fn systemic_submission_failure_stops_the_queue_but_report_rejection_does_not() { for status in [422, 429, 500] { - let f = Fixture::new("basert"); + let f = Fixture::full_sweep("basert"); let first = f.signed(); let first_path = f.dir.path().join("first.json"); fs::rename(&f.report, &first_path).unwrap(); @@ -327,16 +388,37 @@ use std::time::{Duration, Instant}; const RUNTIMES: [&str; 2] = ["basert", "llama-cpp"]; +/// The default sweep. ComputeArena accepts only runs that contain all of it. +const DEFAULT_SWEEP: [u64; 8] = [128, 256, 512, 1024, 2048, 4096, 8192, 16384]; + struct Fixture { dir: tempfile::TempDir, runtime: &'static str, executable: PathBuf, model: PathBuf, report: PathBuf, + /// Prefill sizes the fake runtime measures. + sizes: Vec, } impl Fixture { + /// A short custom sweep (PP128 and PP512): quick to reason about in + /// protocol tests, and a local-only run as far as submission goes. fn new(runtime: &'static str) -> Self { + Self::with_sizes(runtime, &[128, 512]) + } + + /// The full default sweep, run without `--pp` and `--tg` exactly as the + /// CLI tells people to: the only kind of report that can be submitted. + fn full_sweep(runtime: &'static str) -> Self { + Self::with_sizes(runtime, &DEFAULT_SWEEP) + } + + fn is_default_sweep(&self) -> bool { + self.sizes == DEFAULT_SWEEP + } + + fn with_sizes(runtime: &'static str, sizes: &[u64]) -> Self { // Spaces exercise command argument handling rather than shell interpolation. let dir = tempfile::Builder::new() .prefix("arena contract ") @@ -375,6 +457,7 @@ impl Fixture { executable, model, report, + sizes: sizes.to_vec(), }; fixture.install(&fixture.result(), ""); fixture @@ -382,18 +465,35 @@ impl Fixture { fn result(&self) -> Value { if self.runtime == "basert" { + let pp = self + .sizes + .iter() + .map(u64::to_string) + .collect::>() + .join(","); + // Every workload takes 100 ms and then 200 ms, so its mean rate is + // 7.5 tokens per token of workload: PP128 is 960 tok/s, PP512 3840. + let mut metrics = serde_json::Map::new(); + let mut prefill = serde_json::Map::new(); + for size in &self.sizes { + metrics.insert(format!("pp{size}_t_s"), json!(*size as f64 * 7.5)); + prefill.insert( + size.to_string(), + json!([{"tokens":size,"elapsed_ns":100000000},{"tokens":size,"elapsed_ns":200000000}]), + ); + } + metrics.insert("decode_t_s".into(), json!(960.0)); json!({"schema":"basert-benchmark-harness/1","mode":"text","runtime_version":"0.2.4", - "chip":"Test CPU","backend":"CPU","params":{"pp":"128,512","tg":128,"reps":2}, - "metrics":{"pp128_t_s":960.0,"pp512_t_s":3840.0,"decode_t_s":960.0}, - "raw_samples":{"prefill":{ - "128":[{"tokens":128,"elapsed_ns":100000000},{"tokens":128,"elapsed_ns":200000000}], - "512":[{"tokens":512,"elapsed_ns":100000000},{"tokens":512,"elapsed_ns":200000000}]}, + "chip":"Test CPU","backend":"CPU","params":{"pp":pp,"tg":128,"reps":2}, + "metrics":metrics, + "raw_samples":{"prefill":prefill, "decode":[{"generated_tokens":128,"elapsed_ns":100000000},{"generated_tokens":128,"elapsed_ns":200000000}]}}) } else { - Value::Array([(128,0),(512,0),(0,128)].into_iter().map(|(pp,tg)| json!({ + let workloads = self.sizes.iter().map(|pp| (*pp, 0)).chain([(0, 128)]); + Value::Array(workloads.map(|(pp,tg)| json!({ "build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M", "model_filename":self.model,"model_size":24,"model_n_params":4000000000u64, - "n_prompt":pp,"n_gen":tg,"n_depth":0,"n_gpu_layers":0,"backends":"CPU", + "n_prompt":pp,"n_gen":tg,"n_depth":if pp == 0 { 1 } else { 0 },"n_gpu_layers":0,"backends":"CPU", "cpu_info":"Test CPU","gpu_info":"","samples_ns":[100000000,200000000], // Reported aggregates are deliberately bogus: native samples are authoritative. "avg_ts":999999.0,"avg_ns":1 @@ -412,8 +512,50 @@ impl Fixture { let descriptor = json!({"schema":"basert-benchmark-harness-descriptor/1", "runtime":{"name":"basert","version":"0.2.4"},"result_schema":"basert-benchmark-harness/1", "telemetry_schema":telemetry_schema, - "features":{"telemetry":true,"same_run_telemetry":native_same_run}}); - let script = format!("#!/bin/sh\ncase \"$1\" in\n describe) printf '%s\\n' '{descriptor}';;\n --help) printf '%s\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json';;\n *) printf '%s\\n' \"$@\" > \"$ARENA_TEST_ARGS\"\n{before_result}\n/bin/cat <<'RESULT'\n{result}\nRESULT\n;;\nesac\n"); + "capacity_protocol_schema":"basert-throughput-protocol/2", + "features":{"telemetry":true,"same_run_telemetry":native_same_run, + "headline_context_capacity":result.pointer("/protocol/schema").and_then(Value::as_str)==Some("basert-throughput-protocol/2")}}); + let result_script = if self.runtime == "llama-cpp" { + let rows = result.as_array().unwrap(); + let prefill: Value = rows + .iter() + .filter(|row| row["n_prompt"].as_u64().unwrap_or(0) > 0) + .cloned() + .collect(); + let decode: Value = rows + .iter() + .filter(|row| row["n_gen"].as_u64().unwrap_or(0) > 0) + .cloned() + .collect(); + // The headline workload (PP512) runs first and on its own; the + // other sizes follow in one sweep. Rows are chosen by the position + // the fixture gave them, so they keep their place even when a test + // deliberately corrupts a token count. + let headline_index = self.sizes.iter().position(|size| *size == 512).unwrap_or(0); + let headline_size = self.sizes[headline_index]; + let headline: Value = prefill + .as_array() + .unwrap() + .iter() + .skip(headline_index) + .take(1) + .cloned() + .collect(); + let remaining: Value = prefill + .as_array() + .unwrap() + .iter() + .enumerate() + .filter(|(index, _)| *index != headline_index) + .map(|(_, row)| row.clone()) + .collect(); + format!( + "tg=0\npp=0\nwhile [ \"$#\" -gt 0 ]; do\ncase \"$1\" in\n-n) shift; tg=\"$1\";;\n-p) shift; pp=\"$1\";;\nesac\nshift\ndone\nif [ \"$tg\" != 0 ]; then\n/bin/cat <<'RESULT'\n{decode}\nRESULT\nelif [ \"$pp\" = {headline_size} ]; then\n/bin/cat <<'RESULT'\n{headline}\nRESULT\nelse\n/bin/cat <<'RESULT'\n{remaining}\nRESULT\nfi" + ) + } else { + format!("/bin/cat <<'RESULT'\n{result}\nRESULT") + }; + let script = format!("#!/bin/sh\ncase \"$1\" in\n describe) printf '%s\\n' '{descriptor}';;\n --help) printf '%s\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json';;\n *) printf '%s\\n' \"$@\" >> \"$ARENA_TEST_ARGS\"\n{before_result}\n{result_script}\n;;\nesac\n"); fs::write(&self.executable, script).unwrap(); fs::set_permissions(&self.executable, fs::Permissions::from_mode(0o755)).unwrap(); } @@ -429,22 +571,74 @@ impl Fixture { .args(["--data-dir"]) .arg(self.dir.path().join("data")) .env("COMPUTEARENA_API_URL", "http://127.0.0.1:1/api/v1") + // No test may ask GitHub which BaseRT is newest: the lookup is + // pointed at a closed port unless a test serves its own answer. + .env( + "COMPUTEARENA_BASERT_RELEASE_API", + "http://127.0.0.1:1/latest", + ) .stdin(Stdio::null()); cmd } fn run(&self, extra: &[&str]) -> Output { - self.command() - .arg(self.runtime) - .arg("run") - .arg(&self.model) - .args([ - "--pp", "128,512", "--tg", "128", "--reps", "2", "--yes", "--output", - ]) - .arg(&self.report) - .args(extra) - .output() + self.run_command(extra).output().unwrap() + } + + /// What the last BaseRT release lookup is remembered to have found. + fn remember_latest_basert(&self, version: &str) { + let data = self.dir.path().join("data"); + fs::create_dir_all(&data).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) .unwrap() + .as_secs(); + fs::write( + data.join("basert-update-check.json"), + serde_json::to_vec(&json!({ + "checkedAtUnixSeconds": now, + "latestVersion": version, + "releaseUrl": "https://example.test/release" + })) + .unwrap(), + ) + .unwrap(); + } + + /// A BaseRT harness that advertises the headline-first protocol. + fn install_headline_capable(&self) { + let mut result = self.result(); + result["params"]["ctx"] = json!(4096); + result["protocol"] = json!({"schema":"basert-throughput-protocol/2","profile":"basert-bench-capacity/1", + "context_isolation":"headline_then_per_prefill","context_capacity_policy":"basert_bench_default", + "model_load_in_timing":false,"execution_layout":"headline_then_prefill_processes", + "execution_order":["pp512","tg128","pp128"], + "prefill":{"128":{"initial_context_tokens":0,"context_capacity_tokens":4096}, + "512":{"initial_context_tokens":0,"context_capacity_tokens":4096}}, + "decode":{"initial_context_tokens":1,"context_capacity_tokens":4096,"seed_prefill_in_timing":false}, + "measurement":{"timed_repetitions":2,"requested_warmup_repetitions":3, + "warmup_policy":"fixed_repetitions","minimum_warmup_s":0, + "telemetry":"disabled","cooldown":false,"timing":"harness_existing_token_operations"}}); + self.install(&result, ""); + } + + fn run_command(&self, extra: &[&str]) -> Command { + let mut command = self.command(); + command.arg(self.runtime).arg("run").arg(&self.model); + if !self.is_default_sweep() { + let pp = self + .sizes + .iter() + .map(u64::to_string) + .collect::>() + .join(","); + command.args(["--pp", &pp, "--tg", "128"]); + } + command + .args(["--reps", "2", "--yes", "--output"]) + .arg(&self.report) + .args(extra); + command } fn signed(&self) -> Value { @@ -499,11 +693,11 @@ fn runtimes_agree_on_samples_units_and_rates_without_faking_protocol_equivalence } assert_eq!(b["benchmark"]["metrics"]["pp512_t_s"], 3840.0); assert_eq!(b["benchmark"]["protocol"]["warmup"], "runtime_native"); - assert_eq!(b["benchmark"]["params"]["decode_context_tokens"], 0); + assert_eq!(b["benchmark"]["params"]["decode_context_tokens"], 1); assert_eq!(b["benchmark"]["protocol"]["telemetry_available"], true); assert_eq!( b["benchmark"]["telemetry"]["scope"], - "whole_runtime_process" + "headline_then_prefill_processes" ); assert!(b["benchmark"]["telemetry"].get("memory_replay").is_none()); assert_eq!( @@ -538,12 +732,26 @@ fn runtimes_agree_on_samples_units_and_rates_without_faking_protocol_equivalence .contains(fixture.dir.path().to_str().unwrap())); success(&fixture.verify()); let args = fs::read_to_string(fixture.dir.path().join("args")).unwrap(); - assert!(args.contains("\n128,512\n")); + if fixture.runtime == "basert" { + assert!(args.contains("\n128,512\n")); + } else { + let prompts: Vec<_> = args + .lines() + .collect::>() + .windows(2) + .filter(|pair| pair[0] == "-p") + .map(|pair| pair[1]) + .collect(); + assert_eq!(prompts, ["512", "0", "128"]); + assert!(!args.contains("--ctx")); + assert!(!args.contains("-d\n4096\n")); + } assert!(args.contains(fixture.model.to_str().unwrap())); assert!(!args.contains("--cooldown")); assert!(!args.contains("--telemetry")); if fixture.runtime == "llama-cpp" { assert!(args.contains("-d\n0\n")); + assert!(args.contains("-d\n1\n")); assert!(args.contains("-o\njson\n")); } } @@ -570,6 +778,27 @@ fn basert_native_same_run_telemetry_is_selected_by_capability_not_version() { assert!(args.contains("--telemetry")); } +#[test] +fn headline_capable_basert_signs_new_metadata_without_changing_requested_repetitions() { + let f = Fixture::new("basert"); + f.install_headline_capable(); + let result = f.result(); + let signed = f.signed(); + assert_eq!( + signed["benchmark"]["protocol"]["id"], + "computearena-throughput/3" + ); + assert_eq!(signed["benchmark"]["raw_samples"], result["raw_samples"]); + let args = fs::read_to_string(f.dir.path().join("args")).unwrap(); + assert!(args.contains("--headline-first")); + assert!(!args.contains("--isolated-workloads")); + assert!(args.contains("-r\n2\n-w\n3\n")); + success(&f.verify()); + if let Some(path) = std::env::var_os("COMPUTEARENA_HEADLINE_TEST_REPORT") { + fs::copy(&f.report, path).unwrap(); + } +} + #[test] fn signed_fields_cannot_be_tampered_with_in_either_runtime() { for runtime in RUNTIMES { @@ -823,9 +1052,14 @@ fn server(f: &Fixture, statuses: Vec) -> (String, thread::JoinHandle) -> (String, thread::JoinHandle = report["benchmark"]["raw_samples"]["prefill"] + .as_object() + .unwrap() + .keys() + .map(|size| size.parse().unwrap()) + .collect(); + for size in DEFAULT_SWEEP { + assert!( + measured.contains(&size), + "PP{size} missing from {measured:?}" + ); + } + success(&f.verify()); + } +} + +#[test] +fn a_custom_sweep_is_a_local_only_run_and_says_so_before_and_after_it_runs() { + for runtime in RUNTIMES { + let f = Fixture::new(runtime); + let output = f.run(&[]); + success(&output); + let printed = text(&output); + let missing = "it is missing PP256, PP1024, PP2048, PP4096, PP8192 and PP16384"; + // In the plan, before the run starts: what it will lack, and how to + // get a submittable one. + let notice = printed + .find("This will be a local-only run.") + .expect(&printed); + assert!( + printed.find("Benchmark plan").unwrap() < notice, + "{printed}" + ); + assert!( + notice < printed.find("Running the benchmark").unwrap(), + "{printed}" + ); + assert!( + printed.contains("Local only: a partial run cannot be submitted"), + "{printed}" + ); + assert!(printed.contains(missing), "{printed}"); + assert!(printed.contains("PP128 to PP16384 and TG128"), "{printed}"); + assert!(printed.contains("Omit --pp and --tg"), "{printed}"); + // After the run: no invitation to submit a report that would be refused. + assert!( + printed.contains("Local only: Partial run, not submittable:"), + "{printed}" + ); + assert!(!printed.contains("computearena submit"), "{printed}"); + // The report itself is a good signed report. + success(&f.verify()); + } +} + +#[test] +fn partial_runs_are_refused_at_submission_with_what_is_missing_and_how_to_fix_it() { + for runtime in RUNTIMES { + let f = Fixture::new(runtime); + let original = f.signed(); + let (url, received) = server(&f, vec![]); + let output = f + .command() + .args(["--api-url", &url, "submit", "--yes"]) + .arg(&f.report) + .output() + .unwrap(); + failure( + &output, + "nothing was uploaded: the selected benchmark is a partial run", + ); + let printed = text(&output); + for expected in [ + "Partial runs (valid reports, local only)", + "Partial run, not submittable: it is missing PP256, PP1024, PP2048, PP4096, PP8192 and PP16384", + "ComputeArena accepts only runs with the full default sweep (PP128 to PP16384 and TG128)", + "Run the benchmark again without --pp and --tg", + ] { + assert!(printed.contains(expected), "missing {expected:?}: {printed}"); + } + // Refused before login is even asked for, and never sent. + assert!(!printed.contains("Login is required"), "{printed}"); + assert!(!printed.contains("Submitting report"), "{printed}"); + assert!(!printed.contains("Invalid reports:"), "{printed}"); + assert!(received.join().unwrap().is_empty()); + assert_eq!( + serde_json::from_slice::(&fs::read(&f.report).unwrap()).unwrap(), + original + ); + success(&f.verify()); + } +} + +#[test] +fn a_partial_run_in_a_batch_is_left_local_while_complete_runs_upload() { + let partial = Fixture::new("llama-cpp"); + partial.signed(); + let f = Fixture::full_sweep("llama-cpp"); + let complete = f.signed(); + let partial_path = f.dir.path().join("partial.json"); + fs::copy(&partial.report, &partial_path).unwrap(); + + // Non-interactive and not told to skip: nothing is sent. + let output = f + .command() + .args(["submit", "--yes"]) + .arg(&f.report) + .arg(&partial_path) + .output() + .unwrap(); + failure(&output, "1 of the selected benchmarks is a partial run"); + assert!(!text(&output).contains("Submitting report")); + + let (url, received) = server(&f, vec![201]); + let output = f + .command() + .args(["--api-url", &url, "submit", "--yes", "--skip-invalid"]) + .arg(&f.report) + .arg(&partial_path) + .output() + .unwrap(); + success(&output); + let printed = text(&output); + assert!( + printed.contains("Partial run, not submittable"), + "{printed}" + ); + assert!( + printed.contains("1 partial run(s) were not uploaded and stay local"), + "{printed}" + ); + assert_eq!(received.join().unwrap(), vec![complete]); +} + +#[test] +fn saved_report_lists_mark_partial_runs_as_local_only() { let f = Fixture::new("basert"); + success(&f.run(&[])); + let reports = f.dir.path().join("data/reports"); + fs::create_dir_all(&reports).unwrap(); + fs::copy(&f.report, reports.join("partial.json")).unwrap(); + let listed = f.command().args(["list"]).output().unwrap(); + success(&listed); + let printed = text(&listed); + assert!(printed.contains("LOCAL ONLY"), "{printed}"); + assert!( + printed.contains("Partial run, not submittable"), + "{printed}" + ); + let json = f.command().args(["list", "--json"]).output().unwrap(); + success(&json); + let summaries: Value = serde_json::from_slice(&json.stdout).unwrap(); + assert_eq!(summaries[0]["status"], "valid"); + assert_eq!(summaries[0]["submittable"], false); + assert!(summaries[0]["submission_blocker"] + .as_str() + .unwrap() + .starts_with("Partial run, not submittable")); +} + +/// Serves one "latest release" answer the way GitHub does, on a loopback port. +fn release_feed(tag: &str) -> (String, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/latest", listener.local_addr().unwrap()); + let body = json!({"tag_name": tag, "html_url": "https://example.test/release"}).to_string(); + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = stream.read(&mut buffer).unwrap(); + assert!(read > 0); + request.extend_from_slice(&buffer[..read]); + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + }); + (url, handle) +} + +/// How the advice ends depends on whether BaseRT publishes a bundle for the +/// machine running the tests; both endings are correct. +fn names_a_way_to_update(printed: &str) -> bool { + printed.contains("Update with `computearena basert install`") + || printed.contains("No prebuilt BaseRT is published for this platform") +} + +#[test] +fn an_older_basert_is_named_before_the_plan_with_what_it_signs_and_how_to_update() { + let f = Fixture::new("basert"); + let output = f.run(&[]); + success(&output); + let printed = text(&output); + let summary = "BaseRT 0.2.4 predates the current benchmark protocol."; + let notice = printed.find(summary).expect(&printed); + assert!( + notice < printed.find("Benchmark plan").unwrap(), + "{printed}" + ); + assert!( + printed.contains("signed as computearena-throughput-legacy/1, marked not comparable"), + "{printed}" + ); + // Offline, the release to move to is the first one with the protocol. + assert!( + printed.contains("BaseRT 0.2.5 or newer measures PP512 and TG128 first"), + "{printed}" + ); + assert!(names_a_way_to_update(&printed), "{printed}"); + // Said once: the end of the run does not repeat it. + assert_eq!(printed.matches(summary).count(), 1, "{printed}"); + // It is advice, not a gate: the report is signed as before. + let report: Value = serde_json::from_slice(&fs::read(&f.report).unwrap()).unwrap(); + assert_eq!( + report["benchmark"]["protocol"]["id"], + "computearena-throughput-legacy/1" + ); + success(&f.verify()); + + // With the newest release known, the advice names it. + let f = Fixture::new("basert"); + f.remember_latest_basert("0.2.6"); + let printed = text(&f.run(&[])); + assert!( + printed.contains("BaseRT 0.2.6 measures PP512 and TG128 first"), + "{printed}" + ); + + // llama.cpp has nothing to do with any of this. + let llama = Fixture::new("llama-cpp"); + llama.remember_latest_basert("9.9.9"); + let printed = text(&llama.run(&[])); + assert!(!printed.contains("BaseRT"), "{printed}"); +} + +#[test] +fn a_current_basert_is_told_about_a_newer_release_and_nothing_else() { + let f = Fixture::new("basert"); + f.install_headline_capable(); + f.remember_latest_basert("0.2.4"); + let output = f.run(&[]); + success(&output); + let printed = text(&output); + assert!(!printed.contains("is available"), "{printed}"); + assert!(!printed.contains("predates"), "{printed}"); + + fs::remove_file(&f.report).unwrap(); + f.remember_latest_basert("9.9.9"); + let output = f.run(&[]); + success(&output); + let printed = text(&output); + let notice = printed + .find("BaseRT 9.9.9 is available (installed: 0.2.4).") + .expect(&printed); + assert!( + notice < printed.find("Benchmark plan").unwrap(), + "{printed}" + ); + assert!(!printed.contains("predates"), "{printed}"); + assert!(names_a_way_to_update(&printed), "{printed}"); + + // A harness named by hand is not something an install would replace. + fs::remove_file(&f.report).unwrap(); + let output = f + .run_command(&["--runtime-path", f.executable.to_str().unwrap()]) + .output() + .unwrap(); + success(&output); + let printed = text(&output); + assert!( + printed.contains("This harness was chosen with --runtime-path"), + "{printed}" + ); +} + +#[test] +fn the_release_lookup_runs_beside_the_benchmark_and_is_remembered() { + let f = Fixture::new("basert"); + f.install_headline_capable(); + let (feed, served) = release_feed("v9.9.9"); + let output = f + .run_command(&[]) + .env("COMPUTEARENA_BASERT_RELEASE_API", &feed) + .output() + .unwrap(); + success(&output); + served.join().unwrap(); + let printed = text(&output); + // Whether the answer arrived before the plan or during the run, it is + // said exactly once. + assert_eq!( + printed + .matches("BaseRT 9.9.9 is available (installed: 0.2.4).") + .count(), + 1, + "{printed}" + ); + let remembered: Value = serde_json::from_slice( + &fs::read(f.dir.path().join("data/basert-update-check.json")).unwrap(), + ) + .unwrap(); + assert_eq!(remembered["latestVersion"], "9.9.9"); + + // The next run answers from that, without a lookup: the feed is gone. + fs::remove_file(&f.report).unwrap(); + let output = f + .run_command(&[]) + .env("COMPUTEARENA_BASERT_RELEASE_API", &feed) + .output() + .unwrap(); + success(&output); + let printed = text(&output); + let notice = printed + .find("BaseRT 9.9.9 is available (installed: 0.2.4).") + .expect(&printed); + assert!( + notice < printed.find("Benchmark plan").unwrap(), + "{printed}" + ); +} + +#[test] +fn an_unreachable_release_feed_never_delays_or_fails_a_run() { + let f = Fixture::new("basert"); + f.install_headline_capable(); + let started = Instant::now(); + let output = f.run(&[]); + success(&output); + assert!(started.elapsed() < Duration::from_secs(20)); + let printed = text(&output); + assert!(!printed.contains("is available"), "{printed}"); + assert!(!f.dir.path().join("data/basert-update-check.json").exists()); +} + +#[test] +fn the_printed_session_names_an_older_basert_when_it_finds_it() { + let f = Fixture::new("basert"); + f.remember_latest_basert("0.2.6"); + let mut child = f + .command() + .arg("basert") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"6\n").unwrap(); + let output = child.wait_with_output().unwrap(); + success(&output); + let printed = text(&output); + let found = printed.find("Found BaseRT 0.2.4").expect(&printed); + let notice = printed + .find("BaseRT 0.2.4 predates the current benchmark protocol.") + .expect(&printed); + assert!(found < notice, "{printed}"); + assert!( + notice < printed.find("Run benchmarks").unwrap(), + "{printed}" + ); + assert!( + printed.contains("BaseRT 0.2.6 measures PP512 and TG128 first"), + "{printed}" + ); +} + +#[test] +fn offline_report_cannot_be_uploaded_without_login() { + let f = Fixture::full_sweep("basert"); let original = f.signed(); let output = f .command() diff --git a/crates/computearena-cli/tests/runtime_flow.rs b/crates/computearena-cli/tests/runtime_flow.rs index 6455be6..8ad3b72 100644 --- a/crates/computearena-cli/tests/runtime_flow.rs +++ b/crates/computearena-cli/tests/runtime_flow.rs @@ -14,7 +14,7 @@ fn llama_cpp_offline_run_signs_runtime_identity_and_remains_verifiable_after_upg header.extend_from_slice(&0_u64.to_le_bytes()); fs::write(&model, header).unwrap(); - let rows: Vec = [(128, 0), (512, 0), (0, 128)] + let pp_rows: Vec = [(128, 0), (512, 0)] .into_iter() .map(|(pp, tg)| { json!({ @@ -26,10 +26,20 @@ fn llama_cpp_offline_run_signs_runtime_identity_and_remains_verifiable_after_upg }) }) .collect(); + let tg_rows = vec![json!({ + "build_commit":"abc123", "build_number":123, "model_type":"Qwen3 Q4_K_M", + "model_filename":model, "model_size":24, "model_n_params":4000000000u64, + "n_prompt":0, "n_gen":128, "n_depth":1, "n_gpu_layers":99, + "gpu_info":"Apple M5 Pro", "cpu_info":"Apple M5 Pro", "backends":"Metal", + "samples_ns":[100000000,200000000] + })]; + let executable = dir.path().join("llama-bench"); fs::write(&executable, format!( - "#!/bin/sh\nif [ \"$1\" = --help ]; then\nprintf '%s\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json'\nelse\ncat <<'JSON'\n{}\nJSON\nfi\n", - serde_json::to_string(&rows).unwrap() + "#!/bin/sh\nif [ \"$1\" = --help ]; then\nprintf '%s\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json'\nelse\ncase \" $* \" in\n *\" -d 1 \"*) printf '%s\\n' '{}' ;;\n *\" -p 512 \"*) printf '%s\\n' '{}' ;;\n *) printf '%s\\n' '{}' ;;\nesac\nfi\n", + serde_json::to_string(&tg_rows).unwrap(), + serde_json::to_string(&pp_rows[1..]).unwrap(), + serde_json::to_string(&pp_rows[..1]).unwrap() )).unwrap(); fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); let report = dir.path().join("report.json"); @@ -113,6 +123,11 @@ fn basert_uses_the_same_signed_binary_identity_flow() { fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); let report = dir.path().join("report.json"); let output = Command::new(env!("CARGO_BIN_EXE_computearena")) + // Tests never ask GitHub which BaseRT release is newest. + .env( + "COMPUTEARENA_BASERT_RELEASE_API", + "http://127.0.0.1:1/latest", + ) .args(["basert", "--runtime-path"]) .arg(&executable) .arg("--data-dir") diff --git a/crates/computearena-cli/tests/runtime_setup.rs b/crates/computearena-cli/tests/runtime_setup.rs index d4faee2..4648d2f 100644 --- a/crates/computearena-cli/tests/runtime_setup.rs +++ b/crates/computearena-cli/tests/runtime_setup.rs @@ -33,7 +33,7 @@ impl Sandbox { fn fake_runtime(&self, at: &Path) { let descriptor = json!({"schema":"basert-benchmark-harness-descriptor/1", "runtime":{"name":"basert","version":"0.2.4"},"result_schema":"basert-benchmark-harness/1"}); - let rows: Vec = [(128, 0), (512, 0), (0, 128)] + let pp_rows: Vec = [(128, 0), (512, 0)] .into_iter() .map(|(pp, tg)| { json!({"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M", @@ -42,9 +42,16 @@ impl Sandbox { "cpu_info":"Test CPU","gpu_info":"","samples_ns":[100000000,200000000]}) }) .collect(); + let tg_rows = vec![json!({"build_commit":"abc123","build_number":123, + "model_type":"Qwen3 Q4_K_M","model_filename":"model.gguf","model_size":24, + "model_n_params":4000000000u64,"n_prompt":0,"n_gen":128,"n_depth":1, + "n_gpu_layers":0,"backends":"CPU","cpu_info":"Test CPU","gpu_info":"", + "samples_ns":[100000000,200000000]})]; let script = format!( - "#!/bin/sh\ncase \"$1\" in\n describe) printf '%s\\n' '{descriptor}';;\n --help) printf '%s\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json';;\n *) /bin/cat <<'RESULT'\n{}\nRESULT\n;;\nesac\n", - Value::Array(rows) + "#!/bin/sh\ncase \"$1\" in\n describe) printf '%s\\n' '{descriptor}';;\n --help) printf '%s\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json';;\n *) case \" $* \" in\n *\" -d 1 \"*) printf '%s\\n' '{}' ;;\n *\" -p 512 \"*) printf '%s\\n' '{}' ;;\n *) printf '%s\\n' '{}' ;;\n esac;;\nesac\n", + Value::Array(tg_rows), + json!(&pp_rows[1..]), + json!(&pp_rows[..1]) ); fs::create_dir_all(at.parent().unwrap()).unwrap(); fs::write(at, script).unwrap(); @@ -83,6 +90,11 @@ impl Sandbox { .env("NO_PROXY", "*") .env("BASERT_INSTALL_DIR", self.path().join("basert-home")) .env("COMPUTEARENA_API_URL", "http://127.0.0.1:1/api/v1") + // Tests never ask GitHub which BaseRT release is newest. + .env( + "COMPUTEARENA_BASERT_RELEASE_API", + "http://127.0.0.1:1/latest", + ) .arg("--data-dir") .arg(self.path().join("data")); cmd diff --git a/docs/benchmark-profiles.md b/docs/benchmark-profiles.md index 5a96b49..d812112 100644 --- a/docs/benchmark-profiles.md +++ b/docs/benchmark-profiles.md @@ -26,6 +26,11 @@ belongs to the harness's multi-pass telemetry protocol. The wait adds 10 seconds A future harness can provide native per-workload conditioning together with native same-run telemetry. ComputeArena uses that path only when the descriptor advertises `features.same_run_telemetry: true` and `telemetry_schema: basert-telemetry/4`. +`features.headline_context_capacity: true` plus the supported capacity schema +enables `--headline-first`: PP512/TG128 at 4K reservation before the larger sweep. +The native harness's warmup, repetition counts, telemetry and optional cooldown +remain unchanged. Older `isolated_workload_contexts` support and legacy paths +remain available and are identified in each report. ## llama.cpp cooldown semantics @@ -33,7 +38,16 @@ llama-bench has no external pause hook between workload phases in a running swee ComputeArena launches each requested PP size with TG0 and the requested TG workload with PP0 in a **fresh process**. It waits before each launch, then loading and native warmup occur. This is not equivalent to BaseRT's one pre-suite wait or a reset immediately -before a measured repetition. Standard mode still uses one process. +before a measured repetition. Both modes run the headline PP512, then TG128, +then remaining PP sizes. Standard mode groups the remaining PP sizes into one +process. PP starts empty; TG has one untimed seed token. Custom sweeps without +PP512 use the first requested PP size as headline. + +The user's unmodified llama-bench controls reservation. Native context requests +are PP+TG+depth; there is no independent 4K capacity flag. The signed report +explicitly records that the 4K target was not applied. `-d 4096` is never used +as a substitute because it would add actual history. Warmup and timers are +unchanged; no fork or replacement llama.cpp build is required. The controller uses a fixed set of initially readable CPU/GPU/die-labelled temperature sensors. It excludes battery/storage/ambient labels. The first stable window establishes @@ -57,10 +71,12 @@ The CLI calculates these wait estimates from the selected sweep. There is no cal total runtime prediction yet. It says so instead of reusing BaseRT's replay-duration estimate, which would be incorrect for llama.cpp. -Conditioned reports use `llama-bench-conditioned-pp-tg/1` with -`execution_layout: one_process_per_workload`. The private backend must include support -for this protocol before accepting those submissions. Existing standard reports still -use `llama-bench-independent-pp-tg/1`. No database migration is needed. +llama.cpp profiles normalize to `computearena-throughput/2`. Conditioned reports use +`execution_layout: one_process_per_workload`; new standard reports use +`headline_then_prefill_processes`. BaseRT's new capacity profile uses `/3`. +The runtime-specific protocol evidence and +cooldown metadata remain signed. The private backend must accept the normalized +protocol before these submissions are published. No database migration is required. ## Audit of remaining differences diff --git a/docs/release-notes/0.1.2.md b/docs/release-notes/0.1.2.md new file mode 100644 index 0000000..61b3786 --- /dev/null +++ b/docs/release-notes/0.1.2.md @@ -0,0 +1,148 @@ +# ComputeArena CLI 0.1.2 + +This release changes how a benchmark runs and which benchmarks can be +published. The headline workloads, PP512 and TG128, now run before the rest of +the prefill sweep, reports sign a description of the host they ran on, and +computearena.ai accepts only complete runs, so every published benchmark can be +compared with every other at every size. It also says when the installed +BaseRT is worth updating, and explains what happens when a benchmark you +deleted on the website is submitted again. + +## Install or upgrade + +On macOS or Linux: + +```sh +curl -LsSf https://computearena.ai/install.sh | sh +``` + +The same command upgrades an existing installation. Saved reports, the +installation signing key, login sessions, model receipts, and installed +runtimes are kept. Release archives and checksums are also available below for +manual installation. + +BaseRT users should also upgrade BaseRT to 0.2.5 or newer, with +`computearena basert install` or the official BaseRT installer. ComputeArena +selects the new BaseRT behaviour below by what the installed harness +advertises: with BaseRT 0.2.4 it keeps working exactly as before, the report +records that the older protocol was used, and the client now says so before +the run. + +## Highlights + +- **Only complete runs are published.** A run is submittable when it contains + the full default sweep: every default prefill size, PP128 to PP16384, and + TG128. That is what `run` does when `--pp` and `--tg` are left out. Extra + prefill sizes are fine, and the number of repetitions is not part of the + rule. A run with a custom `--pp` or `--tg` is still a valid signed report + that you can save, inspect, and verify, but it is local only, and the client + says so at every step: in the benchmark plan before the run is confirmed, + with the workloads it will lack; after the run, in place of the submit hint; + as `LOCAL ONLY` in `list` and in the submission pickers, where such reports + are not pre-selected; and at `submit`, where partial runs are listed apart + from invalid reports and refused before login or any upload, while complete + runs in the same batch still go through. computearena.ai enforces the same + rule, so earlier client releases receive the same explanation from the + server. +- **Headline first.** PP512 runs first, then TG128, then the remaining prefill + sizes; a custom sweep without PP512 uses its first size as the headline. + Prefill starts from an empty context and decode starts from one untimed seed + token. Warmup, repetitions, timers, and telemetry are unchanged for both + runtimes. +- **BaseRT reserves 4K for the headline.** A BaseRT harness that advertises + `features.headline_context_capacity`, which BaseRT 0.2.5 is the first + release to do, is run with `--headline-first`: the headline is measured in a + freshly loaded model with a 4K context reservation, and the client checks + the returned capacity, history, order, and repetitions before signing. These + reports normalize to `computearena-throughput/3`. The same harness reports + power, energy, temperature, and memory from the timed repetitions + themselves, and the client uses that same-run telemetry in place of + observing the process from outside. Older harnesses keep their existing + invocation and recorded protocol, with no new flags passed to them. +- **BaseRT update notices.** ComputeArena says when the BaseRT it found is + worth updating, and never refuses to run an older one. A harness that + predates the headline-first protocol is named before the benchmark plan, + with what its report will be signed as; that needs no network, because the + harness describes itself. A newer BaseRT release is mentioned the same way, + from a background lookup that is remembered for a day and never delays a + run. The full-screen interface shows the notice above its menu and offers + `u` to install the latest release, asking for a second press before it + replaces anything. A harness chosen with `--runtime-path` or + `COMPUTEARENA_BASERT_HARNESS` is left for you to update, and platforms + without a prebuilt BaseRT are pointed at the release to build from. +- **Stock llama-bench, differences recorded.** ComputeArena still runs your + unmodified `llama-bench`. It has no independent context reservation option, + so its native PP+TG+depth request is kept and the report records that the 4K + target was not applied; `-d 4096` is never used as a substitute, because it + would add real token history. Standard runs use one process for the + headline, one for decode, and one for the remaining prefill sizes; thermally + controlled runs still use one process per workload. The workload order, + warmup, and context requests are part of the signed report. +- **The host is part of the signed report.** Signed before-and-after + environment boundaries (`computearena-environment/1`) record the OS family, + version and kernel, logical and physical CPU counts, total and available + memory, swap use, macOS memory pressure or Linux PSI, the power or + performance mode, and the available GPU configuration. Slow probes run only + outside the measured process windows, and static macOS GPU details are read + once rather than at every boundary. +- **Deleted benchmarks say so.** A benchmark deleted on the website cannot be + published again from the same saved report. The client says this before a + submission, labels such a report **Failed: previously deleted**, keeps + uploading the other selected reports, and ends with the uploaded, duplicate, + and failed counts and a nonzero exit status. Local files are unchanged. To + publish that result again, run the benchmark again with the same settings + and submit the new report; it counts as a separate benchmark. + +## Reports and compatibility + +Reports saved by 0.1.0 and 0.1.1 remain verifiable. They remain submittable +when they contain the full default sweep, which is what those releases produce +unless `--pp` or `--tg` was passed. Benchmarks already published on +computearena.ai are not changed by this release. + +Matching workload labels do not promise identical conditions across runtimes. +BaseRT and llama.cpp differ in warmup, context reservation, and process layout; +those differences are recorded in each signed report rather than hidden, and +computearena.ai does not split rankings or add a filter because of them. + +## Supported systems + +Prebuilt ComputeArena binaries are available for: + +- macOS on Apple Silicon; +- Linux x86_64 with glibc 2.31 or newer; and +- Linux arm64 with glibc 2.31 or newer. + +Windows and Intel Macs are not supported in this release. ComputeArena does not +bundle an inference runtime or model; install BaseRT or llama.cpp and obtain a +compatible model before benchmarking. Prebuilt BaseRT runtimes exist for macOS +on Apple Silicon and for Linux arm64 with CUDA; on Linux x86-64, use llama.cpp +or a BaseRT benchmark harness you built yourself. + +The macOS binary is not signed with an Apple Developer ID. The release archive +is checksum-verified and carries a Sigstore keyless signature from the release +workflow. Browser-downloaded copies may need their quarantine attribute removed +as described in the README. + +## Trust and limits + +A valid signature shows that a report was not changed after it was signed. It +does not attest that the runtime executed the benchmark honestly or that the +recorded host description is truthful; computearena.ai recomputes every rate +from the signed raw samples and repeats the model identity checks on its own, +because a public client is not a trust boundary. Unless a harness reports its +own same-run telemetry, telemetry is observed from outside the runtime at +one-second intervals, so short peaks can be missed, and a process-window +reading is not a per-token metric. + +Accuracy benchmarks, MLX, vLLM, Windows, and Intel macOS support are not part +of v0.1.2. + +For usage and measurement details, see the +[README](https://github.com/basecompute/computearena-cli#readme), +[docs/benchmark-profiles.md](https://github.com/basecompute/computearena-cli/blob/main/docs/benchmark-profiles.md), +[docs/runtime-adapters.md](https://github.com/basecompute/computearena-cli/blob/main/docs/runtime-adapters.md), +and +[docs/telemetry.md](https://github.com/basecompute/computearena-cli/blob/main/docs/telemetry.md). +Questions and feedback are welcome in the +[ComputeArena Discord](https://discord.gg/CCT24GWhPG). diff --git a/docs/runtime-adapters.md b/docs/runtime-adapters.md index 511ae8f..850d39b 100644 --- a/docs/runtime-adapters.md +++ b/docs/runtime-adapters.md @@ -3,8 +3,10 @@ ComputeArena owns menus, authentication, signing, report storage, and submission. RuntimeAdapter implementations own executable discovery, capabilities, model selection, benchmark execution, and translation into the shared raw sample/metric structure. -BaseRT keeps its native harness schema; llama.cpp uses computearena-measurements/1. -Both retain the computearena-benchmark/1 signed envelope, so old reports remain readable. +BaseRT keeps its native harness evidence; llama.cpp uses computearena-measurements/1. +Adapters retain versioned timing/context semantics in their protocol metadata while +retaining their runtime-specific evidence. Both retain the computearena-benchmark/1 +signed envelope, so old reports remain readable. ## Commands @@ -37,24 +39,36 @@ Both retain the computearena-benchmark/1 signed envelope, so old reports remain All adapters store positive per-repetition token counts and elapsed nanoseconds. The server recomputes arithmetic mean throughput from these samples. llama.cpp must return every requested independent PP/TG workload exactly once, -with the expected repetition count and zero initial context depth. Mixed model, -build, device, and runtime settings in one run are rejected. +with the expected repetition count. PP runs with `n_depth=0`; TG runs in a +separate process with `n_prompt=0` and `n_depth=1`, creating one untimed seed +token before timed generation. Mixed model, build, device, and runtime settings +in one report are rejected. Build identity comes from llama-bench JSON's build_number and build_commit. Capability probing uses --help because --version is not implemented consistently. The initial llama.cpp adapter uses native warmup and records it as runtime_native. --warmup 0 disables it; any positive value enables native warmup, not that number -of repetitions. The plan states this before execution. Automatic external telemetry -is collected over the whole process (see telemetry.md). Optional cooldown runs each -workload in a fresh process using llama-bench-conditioned-pp-tg/1 (see benchmark-profiles.md). -Native effective settings are preserved. +of repetitions. The plan states this before execution. Standard mode uses a PP512 +process, then a TG process, then a remaining-PP-sweep process; optional cooldown uses one fresh process per +workload (see benchmark-profiles.md). Automatic external telemetry is collected +for each process and native effective settings are preserved. No equivalence between BaseRT and GGUF quantization names is assumed. Both adapters emit the runtime-neutral `computearena-model/1` identity described in model-identity.md. -The PP/TG counts alone do not establish equivalent timing semantics. llama.cpp -records a distinct protocol ID and its exclusion of sampling/tokenization. -Consumers must retain runtime, quantization, context depth, and protocol when -comparing results. Existing BaseRT measurements are not retroactively relabelled. +The PP/TG counts alone do not establish equivalent timing semantics. BaseRT's +new `features.headline_context_capacity` selects `--headline-first`: 4K reserved +capacity for PP512/TG128 first, then the remaining PP sweep. It produces +`computearena-throughput/3` with nested `basert-throughput-protocol/2` evidence. +Only capacity and order change, not the harness's warmup or timed operations. +Older isolated harnesses still use `/2`; legacy harnesses retain their old path. + +llama.cpp remains `/2` with additional order and context-request evidence. Its +unmodified native capacity request is PP+TG+initial-depth, not forced to 4K. +Physical allocation padding is not inferred. No patched runtime, extra depth, +or unsupported flag is introduced. Both runtimes run the requested headline +first when supported, and use their existing native warmup/timing policies. +Consumers retain runtime, quantization, protocol and signed runtime evidence. +Existing measurements are not retroactively relabelled or excluded from rankings. ## Binary provenance diff --git a/docs/telemetry.md b/docs/telemetry.md index 035aec8..3d8d3b1 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -22,30 +22,42 @@ therefore record `computearena-telemetry/1` under `benchmark.telemetry`: - Observer timing: requested interval, attempts, missed deadlines, elapsed window, and time spent reading sensors. Aggregate storage is bounded (up to 128 temperature sensors), rather than accumulating an unlimited time series. +- Signed before/after `computearena-environment/1` boundaries: OS family, + architecture, version and kernel; logical/physical CPU count; total and + available host memory; swap use; macOS memory pressure or Linux PSI; + normalized power/performance mode; and available accelerator configuration. + Apple GPU name/core count and Metal capability come from `system_profiler`; + that static configuration is cached once per CLI process. Dynamic memory, + pressure, swap and power fields are refreshed at each boundary. ## Measurement boundaries The worker is initialized before launching the runtime and samples every second until -it exits. For a standard BaseRT or llama.cpp run it covers model loading, runtime -warmup, the entire PP/TG sweep, and teardown. These are **concurrently observed -whole-run measurements**, not per-PP diagnostic results. One-second sampling can miss -short peaks or a very short-lived process. +it exits. A legacy BaseRT suite is one process. Standard llama.cpp uses one PP-sweep +process plus one TG process; thermally controlled llama.cpp uses one process per +workload. Each observation covers that process's model loading, native warmup, +measured work, and teardown. These are **concurrently observed process-window +measurements**, not runtime allocator counters. One-second sampling can miss short +peaks or a very short-lived process. The existing `benchmark.memory.process_peak_rss_mb` compatibility field carries the observed peak alongside its scope, units, and caveat, so existing CLI/backend memory summaries can consume it. It is not a process-lifetime kernel high-water mark. Detailed temperature and power data remain in the signed telemetry object. -Vendor tools run only outside the runtime process window. Each invocation has a +Vendor tools and other slow environment probes run only outside a runtime process +window. Static macOS GPU configuration is cached. Every command invocation has a two-second timeout, a 64-KiB output cap checked while running, and is killed/reaped on timeout. Temporary output files are removed automatically. The worker owns its OS sensor handles and is stopped/joined on completion or failure. Energy, runtime allocator/KV-cache queries, and per-workload attribution are explicitly unavailable in the external-observer path. Power snapshots are not energy measurements, -and a whole-run sensor reading is not a per-token metric. Optional llama.cpp cooldown -groups telemetry by isolated workload process, still including loading/warmup. Current -BaseRT cooldown performs one external wait before the suite. See benchmark-profiles.md. +and a process-window sensor reading is not a per-token metric. llama.cpp groups +telemetry by the PP/TG process layout, still including loading and warmup. Current +BaseRT cooldown performs one external wait before the suite. Native BaseRT fields +remain available when its harness reports them from the recorded runs. See +benchmark-profiles.md. ## BaseRT capability transition @@ -57,11 +69,16 @@ Runtime selection is capability-based, not tied to a BaseRT version string: `telemetry_schema: basert-telemetry/4` selects the native `--telemetry` path. - Advertising native same-run telemetry with an unknown or missing schema fails closed instead of silently changing measurement semantics. +- `features.isolated_workload_contexts: true` selects `--isolated-workloads`. + The CLI validates the returned `basert-throughput-protocol/1` capacities and + initial-context metadata before marking the result comparable. Missing support + retains the legacy report but marks its normalized protocol non-comparable. -The future native schema may contain energy, KV-cache, allocator, and per-workload -data, but only if those values are collected during the same runs that produce the -signed throughput samples. This descriptor contract lets a new BaseRT release plug -in without requiring a corresponding ComputeArena release. + +The native schema may contain energy, KV-cache, allocator, and per-workload data, +but only when those values are collected during the same runs that produce the +signed throughput samples. These descriptor contracts let a BaseRT release plug in +without requiring a corresponding ComputeArena release. ## Overhead and validation diff --git a/docs/testing.md b/docs/testing.md index d9d83e4..802c44b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -35,17 +35,19 @@ They do not run GPU workloads. See [telemetry.md](telemetry.md) for collector co | CLI UX | Runtime-scoped help, no nested runtime selectors, one-per-line runtime chooser skipped when a single runtime is installed, menu exit, common list/inspect/verify commands, NO_COLOR output, explicit install/capability guidance | | Discovery | PATH discovery, explicit executable path override, paths containing spaces, missing/incompatible executables, executable feedback on selection, ComputeArena-installed copies, BaseRT's default install location | | Installation | Plan shown before installing, local bundle unpacking, install records, replacement of earlier copies, refusal without a terminal or --yes, non-interactive runs pointing at install | -| Runtime invocation | Requested PP sweep forwarded, separate PP/TG samples, llama.cpp depth zero and JSON output, no replay telemetry flag for current BaseRT, no default cooldown, native warmup disablement | +| Runtime invocation | Requested PP sweep forwarded, separate PP/TG processes, llama.cpp PP depth zero and TG depth one, JSON output, no replay telemetry flag for current BaseRT, no default cooldown, native warmup disablement | | Result consistency | Identical raw measurements produce identical token/second metrics and units across adapters; bogus llama.cpp aggregate rates are ignored | -| Protocol differences | Current BaseRT and llama.cpp use concurrent whole-run telemetry; future BaseRT native same-run telemetry is selected only by an advertised capability; llama.cpp conditioned runs use a distinct per-workload-process protocol | +| Protocol compatibility | Isolated BaseRT metadata and llama.cpp normalize to computearena-throughput/2; legacy BaseRT remains readable but non-comparable; native same-run telemetry is selected only by an advertised capability | | Runtime failures | Nonzero exit, malformed JSON, executable mutation during a benchmark: no report signed | -| Measurement validation | Missing workloads, token-count mismatches, repetition mismatches, invalid/unsafe durations, nonzero llama.cpp depth, inconsistent build identity | +| Measurement validation | Missing workloads, token-count mismatches, repetition mismatches, invalid/unsafe durations, incorrect llama.cpp PP/TG depth, incompatible BaseRT context metadata, inconsistent build identity | | Binary identity | Signed SHA-256 equals the executable bytes; platform identity recorded; report remains verifiable after the executable is removed or upgraded | | Report integrity | Changes to runtime identity, digest, version, timing, rates, token sizes, model, timestamp, or run ID invalidate the signature | | Signature format | Unsigned/malformed reports, bad algorithm/canonicalization/key ID/signature rejected; whitespace and key-order changes accepted | | Run identity | New runs get distinct IDs while preserving the installation key | | Submission | Actual JSON upload body equals the signed report; anonymous submission; public-access notice; informational checksum mismatch/download guidance; duplicate HTTP 200 is successful | | Batch handling | Invalid reports require explicit skip for noninteractive partial uploads; only valid reports sent; all-invalid batches rejected locally | +| BaseRT updates | An older harness is named before the plan, once, with what its report is signed as and how to update; a current harness hears only about a newer release; the release lookup runs beside the benchmark, is remembered for the next run, and an unreachable feed neither delays nor fails a run; a harness chosen by hand is not offered an install; llama.cpp is unaffected. No test contacts GitHub: the lookup is pointed at a closed port or a loopback feed | +| Full sweep only | The default run is submittable; a custom `--pp`/`--tg` run is announced as local only in the plan and after the run, listed as `LOCAL ONLY`, and refused at submission with the missing workloads before login or any upload; in a batch it is left local while complete runs upload | | Failure recovery | HTTP 422 allows the next report; HTTP 429/500 stop the queue; saved reports remain available | ## What passing tests do not prove