From cd368a268b974b02eeeb918a17aa15d034737e50 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 03:06:25 +1000 Subject: [PATCH 01/10] chore: bump version to 0.1.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea2b9fc..14145a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "computearena-cli" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index fe81271..dfbf149 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/computearena-cli"] [workspace.package] -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "Apache-2.0" rust-version = "1.85" From 33b120ea3d3c7ebe93e6e2b625d9143d006144b6 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 13:22:17 +1000 Subject: [PATCH 02/10] feat(models): unify acquisition and provenance --- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 51 +- crates/computearena-cli/Cargo.toml | 1 + .../computearena-cli/src/adapters/basert.rs | 4 +- crates/computearena-cli/src/adapters/gguf.rs | 10 +- crates/computearena-cli/src/basert_models.rs | 314 +++++++++++ crates/computearena-cli/src/benchmark.rs | 7 +- crates/computearena-cli/src/huggingface.rs | 387 +++++++++++++- crates/computearena-cli/src/main.rs | 21 + crates/computearena-cli/src/model_identity.rs | 489 ++++++++++++++++++ crates/computearena-cli/src/models.rs | 95 +++- crates/computearena-cli/src/tui/app.rs | 162 +++++- crates/computearena-cli/src/tui/draw.rs | 37 +- docs/model-identity.md | 53 ++ docs/runtime-adapters.md | 13 +- 16 files changed, 1570 insertions(+), 76 deletions(-) create mode 100644 crates/computearena-cli/src/basert_models.rs create mode 100644 crates/computearena-cli/src/model_identity.rs create mode 100644 docs/model-identity.md diff --git a/Cargo.lock b/Cargo.lock index 14145a2..316ad63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,6 +339,7 @@ dependencies = [ "ed25519-dalek", "flate2", "libc", + "percent-encoding", "rand_core 0.6.4", "ratatui", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index dfbf149..37de675 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ base64 = "0.22" clap = { version = "4", features = ["derive"] } dialoguer = { version = "0.12", default-features = false, features = ["fuzzy-select"] } dirs = "5" +percent-encoding = "2" libc = "0.2" ed25519-dalek = { version = "2", features = ["rand_core"] } flate2 = "1" diff --git a/README.md b/README.md index ca5fde8..8e9eb09 100644 --- a/README.md +++ b/README.md @@ -93,11 +93,11 @@ or, from a checkout, `cargo build --release`, which writes `brew install llama.cpp` or its [releases](https://github.com/ggml-org/llama.cpp/releases). The client can also install either one for you (see [Runtimes](#runtimes)). -2. Have a model on disk: a `.base` bundle for BaseRT, for example from - `basert pull Qwen/Qwen3-0.6B`, or a `.gguf` file for llama.cpp. For - llama.cpp the model picker can also search the Hugging Face Hub and - download a GGUF for you, smallest quantization first; set `HF_TOKEN` in - your environment for gated repositories. +2. Choose a model. The BaseRT picker can browse BaseRT's public catalogue and + run its backend-aware `basert pull` flow; the llama.cpp picker can search + Hugging Face and download a GGUF, smallest file first. Existing `.base` and + `.gguf` files can still be entered directly. Set `HF_TOKEN` for gated + Hugging Face repositories. 3. Run `computearena`. In a terminal this opens the full-screen interface: arrow keys move, Enter selects, Esc goes back, Ctrl+C leaves, and the wheel or PgUp/PgDn scrolls long output. It asks which @@ -123,11 +123,12 @@ computearena llama-cpp run model.gguf # benchmark and save a signed report computearena list # saved reports (--json for machines) computearena inspect # print one report computearena verify # check its signature +computearena identify model.gguf # verify a manually downloaded model computearena login # connect this installation to your account computearena submit # upload chosen reports ``` -`run`, `list`, `inspect`, `verify`, `login`, `logout`, `install`, and `submit` +`run`, `list`, `inspect`, `verify`, `identify`, `login`, `logout`, `install`, and `submit` work under either runtime selector. Without a selector they act as BaseRT commands, so scripts written for earlier versions keep working. `--data-dir`, `--runtime-path` (alias `--harness`), and `--api-url` are accepted anywhere. @@ -196,7 +197,10 @@ The harness must advertise the `basert-benchmark-harness/1` protocol through `describe --json`. `COMPUTEARENA_BASERT_HARNESS` is the environment equivalent of `--runtime-path`. BaseRT 0.2.4 and newer can also start this client with `basert computearena`, provided `computearena` is beside `basert` or on -`PATH`. +`PATH`. The model picker uses the `basert` command beside the harness, on +`PATH`, or in BaseRT's install directory to list and pull catalogue models. +BaseRT remains responsible for choosing a compatible backend artifact, +downloading split files, conversion, and writing `hub.json` provenance. ### llama.cpp @@ -222,9 +226,10 @@ under a `basert/computearena` directory instead; the first run moves that directory whole, reports, key, sessions, and installed runtimes included, and says so. Inside it, `reports/` holds one JSON file per run, `keys/installation.ed25519` is the private signing key -created on first use, `auth.json` holds login sessions, and -`runtimes/` holds llama.cpp builds the client installed. `--output` writes a -report elsewhere instead. +created on first use, `auth.json` holds login sessions, `runtimes/` holds +llama.cpp builds the client installed, and `model-provenance/` holds +content-addressed receipts for models acquired or identified through +ComputeArena. `--output` writes a report elsewhere instead. Each report is a `computearena-benchmark/1` envelope: a run ID, a timestamp, the client version, the runtime (name, version, adapter descriptor, and the @@ -244,10 +249,28 @@ sign invented numbers. The executable and model hashes identify what was claimed to run; they do not attest the process, its libraries, or GPU kernels, and they cannot see a file swapped and restored between the two hashes. -Model identity is deliberately left unresolved. The name and quantization -embedded in the file are kept as display metadata and marked `unverified`. No -filename catalogue or override decides what a file really is, and instruct, -MoE, revision, and fine-tune variants are never assumed equivalent. +Model identity separates three things in `computearena-model/1`: the canonical +Hugging Face model class, the converted artifact repository/revision/path, and +the SHA-256 of the exact local bytes. Quantization is namespaced by format, so +BaseRT Q4 and GGUF Q4_K_M remain distinct variants of a model class. Downloads +are pinned to immutable Hugging Face revisions and checked against published +LFS SHA-256 values when available. BaseRT `hub.json`, standard Hugging Face +cache paths, and GGUF source metadata provide conservative fallback evidence. +Ambiguous merges and files with no evidence stay unresolved rather than being +grouped by filename. + +For a file copied or downloaded outside either model picker, bind it to the +exact Hub object before benchmarking: + +```sh +computearena identify ./model.gguf \ + https://huggingface.co/owner/repository/blob//path/to/model.gguf +``` + +This succeeds only when the local SHA-256 matches the Hub's file object. It +does not let a user assign a model name without evidence. See +[docs/model-identity.md](docs/model-identity.md) for the report fields and +trust limits. Chip names pass through one normalization before signing, so known aliases (M5Pro and Apple M5 Pro, or GB10 and NVIDIA GB10) receive one name while the diff --git a/crates/computearena-cli/Cargo.toml b/crates/computearena-cli/Cargo.toml index e9afe28..6268e1f 100644 --- a/crates/computearena-cli/Cargo.toml +++ b/crates/computearena-cli/Cargo.toml @@ -16,6 +16,7 @@ base64.workspace = true clap.workspace = true dialoguer.workspace = true dirs.workspace = true +percent-encoding.workspace = true ed25519-dalek.workspace = true flate2.workspace = true rand_core.workspace = true diff --git a/crates/computearena-cli/src/adapters/basert.rs b/crates/computearena-cli/src/adapters/basert.rs index cf9d758..5261fc9 100644 --- a/crates/computearena-cli/src/adapters/basert.rs +++ b/crates/computearena-cli/src/adapters/basert.rs @@ -63,8 +63,8 @@ impl RuntimeAdapter for BaseRtAdapter { serde_json::from_slice(&output.stdout).context("reading BaseRT benchmark capabilities") } - fn select_model(&self, _paths: &crate::reports::Paths) -> Result> { - crate::models::prompt_model_path() + fn select_model(&self, paths: &crate::reports::Paths) -> Result> { + crate::models::prompt_model_path(paths) } fn confirm(&self, r: &BenchmarkRequest<'_>, yes: bool) -> Result> { diff --git a/crates/computearena-cli/src/adapters/gguf.rs b/crates/computearena-cli/src/adapters/gguf.rs index 486c10b..4be159e 100644 --- a/crates/computearena-cli/src/adapters/gguf.rs +++ b/crates/computearena-cli/src/adapters/gguf.rs @@ -75,7 +75,12 @@ pub(crate) fn inspect(path: &Path) -> Result { let item = value(&mut file, kind, 0)?; if matches!( key.as_str(), - "general.name" | "general.architecture" | "general.file_type" + "general.name" + | "general.architecture" + | "general.file_type" + | "general.repo_url" + | "general.source.repo_url" + | "general.base_model.0.repo_url" ) { metadata.insert(key, item); } @@ -110,6 +115,9 @@ pub(crate) fn inspect(path: &Path) -> Result { "architecture": metadata.get("general.architecture"), "quantization": quant, "gguf_file_type": file_type, "format": "gguf", "format_version": version, + "repo_url": metadata.get("general.repo_url"), + "source_repo_url": metadata.get("general.source.repo_url"), + "base_model_repo_url": metadata.get("general.base_model.0.repo_url"), "file_name": path.file_name().and_then(|s| s.to_str()), "size_bytes": file.metadata()?.len() })) diff --git a/crates/computearena-cli/src/basert_models.rs b/crates/computearena-cli/src/basert_models.rs new file mode 100644 index 0000000..d3df0e4 --- /dev/null +++ b/crates/computearena-cli/src/basert_models.rs @@ -0,0 +1,314 @@ +//! Browse and pull BaseRT's public model catalogue through the installed +//! `basert` command. BaseRT remains responsible for backend-aware artifact +//! selection, split downloads, conversion, and its `hub.json` provenance. + +use anyhow::{bail, Context, Result}; +use serde_json::Value; +use std::collections::{btree_map::Entry, BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +#[derive(Clone, Debug)] +pub(crate) struct RemoteModel { + pub(crate) id: String, + pub(crate) variant: String, + pub(crate) architecture: String, + pub(crate) size_bytes: Option, + pub(crate) pull_target: String, +} + +pub(crate) fn locate_cli(harness: Option<&Path>) -> Result { + let sibling = harness + .and_then(Path::parent) + .map(|directory| directory.join("basert")); + sibling + .filter(|candidate| candidate.is_file()) + .or_else(|| crate::runtimes::executable_on_path("basert")) + .or_else(|| { + crate::runtimes::basert_install_dir() + .map(|directory| directory.join("basert")) + .filter(|candidate| candidate.is_file()) + }) + .context("the BaseRT model tool was not found beside basert-benchmark-harness or on PATH") +} + +fn list(cli: &Path, remote: bool) -> Result> { + let mut command = Command::new(cli); + command.arg("list"); + if remote { + command.arg("--remote"); + } + let output = command + .arg("--json") + .output() + .with_context(|| format!("running {} list", cli.display()))?; + if !output.status.success() { + bail!( + "BaseRT could not list {}models: {}", + if remote { "remote " } else { "installed " }, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + serde_json::from_slice(&output.stdout).context("BaseRT returned an invalid model list") +} + +pub(crate) fn available(harness: Option<&Path>) -> Result> { + let cli = locate_cli(harness)?; + Ok(normalize_available(list(&cli, true)?)) +} + +fn normalize_available(rows: Vec) -> Vec { + let mut models = BTreeMap::new(); + let mut installed = BTreeSet::new(); + for row in rows { + let Some(model) = remote_model(&row) else { + continue; + }; + // `basert pull --target` chooses a backend-compatible artifact. Several + // catalogue rows can therefore represent one actionable choice; show + // the choice once instead of promising a specific internal variant. + let key = (model.id.to_ascii_lowercase(), model.pull_target.clone()); + if row.get("installed").and_then(Value::as_bool) == Some(true) { + installed.insert(key); + continue; + } + match models.entry(key) { + Entry::Vacant(entry) => { + entry.insert(model); + } + Entry::Occupied(mut entry) => { + // Multiple backend variants can have different sizes. The + // generic pull target cannot promise which one BaseRT picks. + if entry.get().size_bytes != model.size_bytes { + entry.get_mut().size_bytes = None; + } + } + } + } + models.retain(|key, _| !installed.contains(key)); + let mut models: Vec<_> = models.into_values().collect(); + models.sort_by(|left, right| { + left.id + .to_ascii_lowercase() + .cmp(&right.id.to_ascii_lowercase()) + .then_with(|| left.pull_target.cmp(&right.pull_target)) + }); + models +} + +fn remote_model(row: &Value) -> Option { + let variant = row.get("variant")?.as_str()?.to_string(); + let pull_target = target_for(&variant).ok()?; + let architecture = row.get("arch").and_then(Value::as_str).unwrap_or("unknown"); + if architecture == "whisper" { + return None; + } + Some(RemoteModel { + id: row.get("id")?.as_str()?.to_string(), + variant, + architecture: architecture.to_string(), + size_bytes: row.get("size_bytes").and_then(Value::as_u64), + pull_target, + }) +} + +pub(crate) fn display_pull_target(target: &str) -> String { + let scheme = target.strip_prefix("base-").unwrap_or(target); + format!("BaseRT {}", scheme.to_ascii_uppercase()) +} + +fn target_for(variant: &str) -> Result { + let lower = variant.to_ascii_lowercase(); + for bits in ["2", "3", "4", "5", "6", "8"] { + if lower + .as_bytes() + .windows(bits.len() + 1) + .any(|window| window == format!("q{bits}").as_bytes()) + { + return Ok(format!("base-q{bits}")); + } + } + for target in ["bf16", "mxfp4", "nvfp4"] { + if lower.contains(target) { + return Ok(target.to_string()); + } + } + bail!("BaseRT cannot select the {variant} variant through `basert pull`") +} + +fn same_quantization(left: &str, right: &str) -> bool { + target_for(left).ok() == target_for(right).ok() +} + +pub(crate) fn pull(cli: &Path, model: &RemoteModel) -> Result { + println!( + "Downloading {} ({}) with BaseRT…", + model.id, model.pull_target + ); + let status = Command::new(cli) + .arg("pull") + .arg(&model.id) + .args(["--target", &model.pull_target]) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .with_context(|| format!("running {} pull", cli.display()))?; + if !status.success() { + bail!("BaseRT model download exited with {status}"); + } + let rows = list(cli, false)?; + rows.into_iter() + .find(|row| { + row.get("id").and_then(Value::as_str) == Some(model.id.as_str()) + && row + .get("variant") + .and_then(Value::as_str) + .is_some_and(|variant| same_quantization(variant, &model.variant)) + }) + .and_then(|row| row.get("path").and_then(Value::as_str).map(PathBuf::from)) + .context("BaseRT completed the pull but did not list the installed model") +} + +pub(crate) fn record_download(root: &Path, model: &Path) -> Result<()> { + let sidecar_path = model + .parent() + .context("BaseRT model has no variant directory")? + .join("hub.json"); + let sidecar: Value = serde_json::from_slice( + &std::fs::read(&sidecar_path) + .with_context(|| format!("reading {}", sidecar_path.display()))?, + ) + .with_context(|| format!("parsing {}", sidecar_path.display()))?; + let repository = sidecar + .get("hf_repo") + .and_then(Value::as_str) + .context("BaseRT model provenance omitted its Hugging Face repository")?; + let artifact_sha256 = crate::adapters::file_sha256(model)?; + let expected_sha256 = sidecar.get("base_sha256").and_then(Value::as_str); + if expected_sha256.is_some_and(|expected| !expected.eq_ignore_ascii_case(&artifact_sha256)) { + bail!("the installed BaseRT model does not match its provenance SHA-256"); + } + let sidecar_revision = sidecar + .get("revision") + .and_then(Value::as_str) + .unwrap_or("main"); + if let Ok(file) = + crate::huggingface::find_file_by_sha256(repository, sidecar_revision, &artifact_sha256) + { + let canonical = sidecar + .get("source_repo") + .and_then(Value::as_str) + .or(file.canonical_repository.as_deref()); + return crate::model_identity::record_huggingface_download( + root, + repository, + &file.revision, + &file.path, + &artifact_sha256, + file.sha256.as_deref(), + canonical, + "basert_pull", + ); + } + let identity = crate::huggingface::repository_identity(repository).ok(); + let revision = identity + .as_ref() + .map(|identity| identity.revision.as_str()) + .or(Some(sidecar_revision)) + .unwrap_or("unknown"); + let canonical = sidecar + .get("source_repo") + .and_then(Value::as_str) + .or_else(|| { + identity + .as_ref() + .and_then(|identity| identity.canonical_repository.as_deref()) + }); + crate::model_identity::record_huggingface_download( + root, + repository, + revision, + model + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("model.base"), + &artifact_sha256, + expected_sha256, + canonical, + "basert_pull", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_catalog_variants_to_pull_targets() { + assert_eq!(target_for("default-q4").unwrap(), "base-q4"); + assert_eq!(target_for("cuda-q4mix").unwrap(), "base-q4"); + assert_eq!(target_for("default-q8").unwrap(), "base-q8"); + assert_eq!(target_for("bf16").unwrap(), "bf16"); + assert!(target_for("experimental").is_err()); + } + + #[test] + fn catalogue_collapses_variants_that_share_one_pull_target() { + let rows = vec![ + serde_json::json!({ + "id": "Qwen/Qwen3-4B", + "variant": "default-q4", + "arch": "qwen", + "quant": "base_q4", + "size_bytes": 100, + "installed": false + }), + serde_json::json!({ + "id": "Qwen/Qwen3-4B", + "variant": "metal-q4mix", + "arch": "qwen", + "quant": "base_q4", + "size_bytes": 101, + "installed": false + }), + serde_json::json!({ + "id": "Qwen/Qwen3-4B", + "variant": "default-q8", + "arch": "qwen", + "quant": "base_q8", + "size_bytes": 200, + "installed": false + }), + ]; + let models = normalize_available(rows); + assert_eq!(models.len(), 2); + assert_eq!(models[0].pull_target, "base-q4"); + assert_eq!(models[1].pull_target, "base-q8"); + } + + #[test] + fn catalogue_excludes_an_installed_target_and_unsupported_whisper_models() { + let rows = vec![ + serde_json::json!({ + "id": "basecompute/Qwen3-4B", + "variant": "default-q4", + "arch": "qwen", + "installed": true + }), + serde_json::json!({ + "id": "basecompute/Qwen3-4B", + "variant": "metal-q4mix", + "arch": "qwen", + "installed": false + }), + serde_json::json!({ + "id": "basecompute/whisper-base", + "variant": "default-q8", + "arch": "whisper", + "installed": false + }), + ]; + assert!(normalize_available(rows).is_empty()); + } +} diff --git a/crates/computearena-cli/src/benchmark.rs b/crates/computearena-cli/src/benchmark.rs index 2507f05..bcff9eb 100644 --- a/crates/computearena-cli/src/benchmark.rs +++ b/crates/computearena-cli/src/benchmark.rs @@ -558,11 +558,8 @@ pub(crate) fn run_benchmark( let public_bytes = public.to_bytes(); let key_id = sha256_hex(&public_bytes); let run_id = random_id(); - let mut model_metadata = result.model; - model_metadata["upstream_id"] = Value::Null; - model_metadata["upstream_id_source"] = json!("unresolved"); - model_metadata["identity_verification"] = json!("unverified"); - model_metadata["artifact_sha256"] = json!(model_sha256); + let model_metadata = + crate::model_identity::finalize(runtime, paths, model, result.model, &model_sha256); // Intentionally omit the user's account and local model path: a benchmark // can be created offline and attached to an authenticated account later. diff --git a/crates/computearena-cli/src/huggingface.rs b/crates/computearena-cli/src/huggingface.rs index 03afd2f..1757352 100644 --- a/crates/computearena-cli/src/huggingface.rs +++ b/crates/computearena-cli/src/huggingface.rs @@ -5,7 +5,10 @@ //! the environment (`HF_TOKEN`), which gated repositories need. use crate::config::{HTTP_CONNECT_TIMEOUT, HUGGINGFACE_API, HUGGINGFACE_HOST}; use anyhow::{bail, Context, Result}; +use percent_encoding::percent_decode_str; use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; use std::fs::File; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; @@ -21,9 +24,23 @@ pub(crate) struct HubModel { pub(crate) likes: u64, } +#[derive(Clone)] pub(crate) struct HubFile { pub(crate) path: String, pub(crate) size: u64, + pub(crate) revision: String, + pub(crate) sha256: Option, + pub(crate) canonical_repository: Option, +} + +pub(crate) struct RepositoryIdentity { + pub(crate) revision: String, + pub(crate) canonical_repository: Option, +} + +pub(crate) struct HubFileIdentity { + pub(crate) repository: String, + pub(crate) file: HubFile, } fn client(timeout: Duration) -> Result { @@ -93,17 +110,265 @@ pub(crate) fn search(query: &str) -> Result> { /// The GGUF files inside one repository, smallest first: quantizations are /// usually chosen by the size a device can hold. -pub(crate) fn gguf_files(repository: &str) -> Result> { - let url = format!("{HUGGINGFACE_API}/models/{repository}/tree/main?recursive=true"); - let body: Value = serde_json::from_str( +fn canonical_repository(info: &Value) -> Option { + fn add(repositories: &mut BTreeSet, value: &str) { + let value = value + .strip_prefix("quantized:") + .or_else(|| value.strip_prefix("finetune:")) + .or_else(|| value.strip_prefix("adapter:")) + .unwrap_or(value); + if value.split('/').filter(|part| !part.is_empty()).count() == 2 { + repositories.insert(value.to_string()); + } + } + let mut repositories = BTreeSet::new(); + match info.pointer("/cardData/base_model") { + Some(Value::String(value)) => add(&mut repositories, value), + Some(Value::Array(values)) => { + for value in values.iter().filter_map(Value::as_str) { + add(&mut repositories, value); + } + } + _ => {} + } + if repositories.is_empty() { + for tag in info + .get("tags") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter_map(|tag| tag.strip_prefix("base_model:")) + { + add(&mut repositories, tag); + } + } + (repositories.len() == 1).then(|| repositories.into_iter().next().unwrap()) +} + +fn repository_info(repository: &str) -> Result { + let url = format!("{HUGGINGFACE_API}/models/{repository}"); + serde_json::from_str( &request(&url, SEARCH_TIMEOUT)? .text() - .context("reading the repository listing")?, + .context("reading Hugging Face model metadata")?, ) - .context("parsing the repository listing")?; - let entries = body + .context("parsing Hugging Face model metadata") +} + +fn repository_info_at(repository: &str, revision: &str) -> Result { + let url = format!("{HUGGINGFACE_API}/models/{repository}/revision/{revision}"); + serde_json::from_str( + &request(&url, SEARCH_TIMEOUT)? + .text() + .context("reading Hugging Face model metadata")?, + ) + .context("parsing Hugging Face model metadata") +} + +pub(crate) fn repository_identity(repository: &str) -> Result { + let info = repository_info(repository)?; + Ok(RepositoryIdentity { + revision: info + .get("sha") + .and_then(Value::as_str) + .context("Hugging Face model metadata omitted its immutable revision")? + .to_string(), + canonical_repository: canonical_repository(&info), + }) +} + +fn repository_identity_at(repository: &str, revision: &str) -> Result { + let info = repository_info_at(repository, revision)?; + Ok(RepositoryIdentity { + revision: info + .get("sha") + .and_then(Value::as_str) + .context("Hugging Face model metadata omitted its immutable revision")? + .to_string(), + canonical_repository: canonical_repository(&info), + }) +} + +fn decode_path_segment(value: &str) -> Result { + percent_decode_str(value) + .decode_utf8() + .map(|value| value.into_owned()) + .context("Hugging Face URL contains invalid UTF-8") +} + +fn parse_file_url(value: &str) -> Result<(String, String, String)> { + let url = reqwest::Url::parse(value).context("parsing the Hugging Face file URL")?; + if url.scheme() != "https" || url.host_str() != Some("huggingface.co") { + bail!("expected an https://huggingface.co/... file URL"); + } + let parts: Vec<_> = url + .path_segments() + .context("Hugging Face file URL has no path")? + .map(decode_path_segment) + .collect::>()?; + if parts.len() < 5 || !matches!(parts[2].as_str(), "blob" | "resolve") { + bail!( + "expected a Hugging Face file URL such as https://huggingface.co/owner/model/blob/revision/path/to/model.gguf" + ); + } + Ok(( + format!("{}/{}", parts[0], parts[1]), + parts[3].clone(), + parts[4..].join("/"), + )) +} + +fn next_page(response: &reqwest::blocking::Response) -> Option { + response + .headers() + .get(reqwest::header::LINK)? + .to_str() + .ok()? + .split(',') + .find_map(|link| { + let (url, attributes) = link.trim().split_once(';')?; + if !attributes + .split(';') + .any(|attribute| attribute.trim() == "rel=\"next\"") + { + return None; + } + url.trim() + .strip_prefix('<')? + .strip_suffix('>') + .map(str::to_string) + }) +} + +fn repository_tree(repository: &str, revision: &str) -> Result> { + let mut url = Some(format!( + "{HUGGINGFACE_API}/models/{repository}/tree/{revision}?recursive=true&expand=true" + )); + let mut entries = Vec::new(); + let mut seen = BTreeSet::new(); + while let Some(page) = url.take() { + if !seen.insert(page.clone()) { + bail!("Hugging Face returned a cyclic pagination link"); + } + let response = request(&page, SEARCH_TIMEOUT)?; + url = next_page(&response); + if let Some(next) = &url { + let next = reqwest::Url::parse(next).context("parsing Hugging Face pagination URL")?; + if next.scheme() != "https" || next.host_str() != Some("huggingface.co") { + bail!("Hugging Face returned an unsafe pagination URL"); + } + } + let body: Value = + serde_json::from_str(&response.text().context("reading the repository listing")?) + .context("parsing the repository listing")?; + entries.extend( + body.as_array() + .context("Hugging Face returned an unexpected repository listing")? + .iter() + .cloned(), + ); + } + Ok(entries) +} + +/// Resolve a human-facing Hugging Face file URL to immutable repository and +/// file identity. `paths-info` exposes the content SHA-256 for LFS/Xet model +/// objects without downloading the model again. +pub(crate) fn file_identity(url: &str) -> Result { + let (repository, requested_revision, path) = parse_file_url(url)?; + let identity = repository_identity_at(&repository, &requested_revision)?; + let info_url = format!( + "{HUGGINGFACE_API}/models/{repository}/paths-info/{}", + identity.revision + ); + let mut builder = client(SEARCH_TIMEOUT)? + .post(&info_url) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(serde_json::to_vec(&serde_json::json!({"paths": [&path]}))?); + if let Some(token) = std::env::var_os("HF_TOKEN").filter(|token| !token.is_empty()) { + builder = builder.bearer_auth(token.to_string_lossy()); + } + let response = builder + .send() + .with_context(|| format!("contacting {info_url}"))?; + if !response.status().is_success() { + bail!("Hugging Face answered {} for {info_url}", response.status()); + } + let body: Value = serde_json::from_str( + &response + .text() + .context("reading Hugging Face file metadata")?, + ) + .context("parsing Hugging Face file metadata")?; + let entry = body .as_array() - .context("Hugging Face returned an unexpected repository listing")?; + .and_then(|entries| entries.first()) + .filter(|entry| entry.get("type").and_then(Value::as_str) == Some("file")) + .with_context(|| format!("{repository} has no file named {path} at that revision"))?; + let sha256 = entry + .pointer("/lfs/oid") + .and_then(Value::as_str) + .filter(|sha| sha.len() == 64 && sha.bytes().all(|byte| byte.is_ascii_hexdigit())) + .map(|sha| sha.to_ascii_lowercase()); + Ok(HubFileIdentity { + repository, + file: HubFile { + path, + size: entry + .get("size") + .and_then(Value::as_u64) + .unwrap_or_default(), + revision: identity.revision, + sha256, + canonical_repository: identity.canonical_repository, + }, + }) +} + +/// Find the single Hub file whose published LFS SHA-256 matches local bytes. +/// This recovers the actual artifact path when a runtime sidecar only records +/// a repository and a mutable ref such as `main`. +pub(crate) fn find_file_by_sha256( + repository: &str, + revision: &str, + sha256: &str, +) -> Result { + let identity = repository_identity_at(repository, revision)?; + let entries = repository_tree(repository, &identity.revision)?; + let mut matches = entries.iter().filter(|entry| { + entry + .pointer("/lfs/oid") + .and_then(Value::as_str) + .is_some_and(|oid| oid.eq_ignore_ascii_case(sha256)) + }); + let entry = matches.next().with_context(|| { + format!("{repository} has no file matching the local SHA-256 at {revision}") + })?; + if matches.next().is_some() { + bail!("{repository} contains multiple files with that SHA-256 at {revision}"); + } + Ok(HubFile { + path: entry + .get("path") + .and_then(Value::as_str) + .context("Hugging Face file metadata omitted its path")? + .to_string(), + size: entry + .get("size") + .and_then(Value::as_u64) + .unwrap_or_default(), + revision: identity.revision, + sha256: Some(sha256.to_ascii_lowercase()), + canonical_repository: identity.canonical_repository, + }) +} + +pub(crate) fn gguf_files(repository: &str) -> Result> { + let identity = repository_identity(repository)?; + let revision = identity.revision; + let canonical_repository = identity.canonical_repository; + let entries = repository_tree(repository, &revision)?; let mut files: Vec = entries .iter() .filter(|entry| entry.get("type").and_then(Value::as_str) == Some("file")) @@ -118,6 +383,15 @@ pub(crate) fn gguf_files(repository: &str) -> Result> { .get("size") .and_then(Value::as_u64) .unwrap_or_default(), + revision: revision.clone(), + sha256: entry + .pointer("/lfs/oid") + .and_then(Value::as_str) + .filter(|sha| { + sha.len() == 64 && sha.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + .map(|sha| sha.to_ascii_lowercase()), + canonical_repository: canonical_repository.clone(), }) }) .collect(); @@ -145,19 +419,43 @@ pub(crate) fn format_size(bytes: u64) -> String { /// Where a downloaded model lives: under the ComputeArena data directory, laid /// out like the repository it came from so two files never collide. -pub(crate) fn download_path(root: &Path, repository: &str, file: &str) -> PathBuf { +pub(crate) fn download_path(root: &Path, repository: &str, file: &str) -> Result { let mut path = root.join("models"); for segment in repository.split('/').chain(file.split('/')) { + if segment.is_empty() || matches!(segment, "." | "..") { + bail!("Hugging Face returned an unsafe model path"); + } path.push(segment); } - path + Ok(path) } /// Fetch one file, printing progress. Downloads to a temporary name first so an /// interrupted transfer never looks like a usable model. pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result { - let destination = download_path(root, repository, &file.path); + let destination = download_path(root, repository, &file.path)?; if destination.is_file() { + let sha256 = crate::adapters::file_sha256(&destination)?; + if file + .sha256 + .as_deref() + .is_some_and(|expected| expected != sha256) + { + bail!( + "the existing file at {} does not match Hugging Face's SHA-256", + destination.display() + ); + } + crate::model_identity::record_huggingface_download( + root, + repository, + &file.revision, + &file.path, + &sha256, + file.sha256.as_deref(), + file.canonical_repository.as_deref(), + "computearena_download", + )?; println!("Already downloaded: {}", destination.display()); return Ok(destination); } @@ -168,8 +466,8 @@ pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result< .with_context(|| format!("creating {}", directory.display()))?; let url = format!( - "{HUGGINGFACE_HOST}/{repository}/resolve/main/{}?download=true", - file.path + "{HUGGINGFACE_HOST}/{repository}/resolve/{}/{}?download=true", + file.revision, file.path ); println!("Downloading {} ({})", file.path, format_size(file.size)); let mut response = request(&url, DOWNLOAD_TIMEOUT)?; @@ -179,6 +477,7 @@ pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result< File::create(&partial).with_context(|| format!("creating {}", partial.display()))?; let mut buffer = vec![0_u8; 1024 * 1024]; let mut received: u64 = 0; + let mut digest = Sha256::new(); let mut reported = Instant::now(); loop { let count = response.read(&mut buffer).context("reading the download")?; @@ -188,6 +487,7 @@ pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result< output .write_all(&buffer[..count]) .context("writing the download")?; + digest.update(&buffer[..count]); received += count as u64; // One line every couple of seconds: enough to show life, few enough to // stay readable in a log pane. @@ -213,6 +513,19 @@ pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result< format_size(total) ); } + let sha256 = format!("{:x}", digest.finalize()); + if file + .sha256 + .as_deref() + .is_some_and(|expected| expected != sha256) + { + let _ = std::fs::remove_file(&partial); + bail!( + "Hugging Face SHA-256 mismatch for {}: expected {}, got {sha256}", + file.path, + file.sha256.as_deref().unwrap_or("unknown") + ); + } std::fs::rename(&partial, &destination).with_context(|| { format!( "moving {} into place at {}", @@ -220,6 +533,16 @@ pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result< destination.display() ) })?; + crate::model_identity::record_huggingface_download( + root, + repository, + &file.revision, + &file.path, + &sha256, + file.sha256.as_deref(), + file.canonical_repository.as_deref(), + "computearena_download", + )?; println!("Saved {}", destination.display()); Ok(destination) } @@ -252,12 +575,50 @@ mod tests { assert_eq!(format_size(3 * 1024 * 1024 * 1024), "3.0 GB"); } + #[test] + fn parses_huggingface_file_urls() { + assert_eq!( + parse_file_url( + "https://huggingface.co/bartowski/Qwen-GGUF/blob/main/models/Q4_K_M/model%20one.gguf?download=true" + ) + .unwrap(), + ( + "bartowski/Qwen-GGUF".to_string(), + "main".to_string(), + "models/Q4_K_M/model one.gguf".to_string() + ) + ); + assert!(parse_file_url("https://example.com/a/b/blob/main/model.gguf").is_err()); + } + #[test] fn downloads_mirror_the_repository_layout_under_the_data_directory() { - let path = download_path(Path::new("/data"), "TheBloke/Qwen-GGUF", "q4/model.gguf"); + let path = + download_path(Path::new("/data"), "TheBloke/Qwen-GGUF", "q4/model.gguf").unwrap(); assert_eq!( path, Path::new("/data/models/TheBloke/Qwen-GGUF/q4/model.gguf") ); + assert!(download_path(Path::new("/data"), "owner/model", "../model.gguf").is_err()); + } + + #[test] + fn reads_one_canonical_base_model_from_hub_metadata() { + let info = serde_json::json!({ + "cardData": {"base_model": "Qwen/Qwen3-4B"}, + "tags": ["base_model:quantized:Qwen/Qwen3-4B"] + }); + assert_eq!( + canonical_repository(&info).as_deref(), + Some("Qwen/Qwen3-4B") + ); + } + + #[test] + fn merged_models_do_not_claim_one_canonical_identity() { + let info = serde_json::json!({ + "cardData": {"base_model": ["one/model", "two/model"]} + }); + assert_eq!(canonical_repository(&info), None); } } diff --git a/crates/computearena-cli/src/main.rs b/crates/computearena-cli/src/main.rs index b1229cc..18cde61 100644 --- a/crates/computearena-cli/src/main.rs +++ b/crates/computearena-cli/src/main.rs @@ -2,10 +2,12 @@ mod adapters; mod api; use adapters::{BenchmarkRequest, Runtime}; mod auth; +mod basert_models; mod benchmark; mod conditioning; mod config; mod huggingface; +mod model_identity; mod models; mod protocol; mod recent_gguf; @@ -145,6 +147,13 @@ enum Action { Inspect { report: String }, /// Verify one report's Ed25519 signature. Verify { report: String }, + /// Bind a manually acquired model to an exact Hugging Face file by SHA-256. + Identify { + /// Local .base or .gguf model file. + model: PathBuf, + /// Full Hugging Face file URL, including blob/resolve revision and path. + huggingface_url: String, + }, /// Log in through a browser and connect this installation. Login, /// Revoke and remove the session for the selected API URL. @@ -397,6 +406,18 @@ fn execute( println!("Installation key: {key_id}"); Ok(()) } + Action::Identify { + model, + huggingface_url, + } => { + let ui = TerminalUi::detect(); + let started = start_activity(ui, "Matching the local file to Hugging Face…"); + let identity = + model_identity::identify_huggingface_file(&paths.root, &model, &huggingface_url)?; + finish_activity(ui, started, "Model identity saved"); + println!("{}", serde_json::to_string_pretty(&identity)?); + Ok(()) + } Action::Login => login(paths, api_url), Action::Logout => logout(paths, api_url), Action::Install { yes, archive } => { diff --git a/crates/computearena-cli/src/model_identity.rs b/crates/computearena-cli/src/model_identity.rs new file mode 100644 index 0000000..59b594b --- /dev/null +++ b/crates/computearena-cli/src/model_identity.rs @@ -0,0 +1,489 @@ +//! Runtime-neutral model identity and local acquisition provenance. +//! +//! A model class (the upstream Hugging Face repository), a converted artifact +//! repository, and the exact local bytes are separate identities. Reports keep +//! all three so the server can group equivalent model families without +//! pretending that unlike quantizations are the same artifact. + +use crate::adapters::Runtime; +use crate::reports::{atomic_write_json, Paths}; +use serde_json::{json, Value}; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +pub(crate) const IDENTITY_SCHEMA: &str = "computearena-model/1"; +const RECEIPT_SCHEMA: &str = "computearena-model-provenance/1"; + +fn receipts_dir(root: &Path) -> PathBuf { + root.join("model-provenance") +} + +fn receipt_path(root: &Path, sha256: &str) -> PathBuf { + receipts_dir(root).join(format!("{sha256}.json")) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn record_huggingface_download( + root: &Path, + repository: &str, + revision: &str, + file: &str, + artifact_sha256: &str, + expected_sha256: Option<&str>, + canonical_repository: Option<&str>, + method: &str, +) -> anyhow::Result<()> { + let verification = + if expected_sha256.is_some_and(|expected| expected.eq_ignore_ascii_case(artifact_sha256)) { + "sha256" + } else { + "immutable_revision" + }; + let receipt = json!({ + "schema": RECEIPT_SCHEMA, + "artifact": { + "provider": "huggingface", + "repo_id": repository, + "revision": revision, + "path": file, + "sha256": artifact_sha256 + }, + "canonical": canonical_repository.map(|repo_id| json!({ + "provider": "huggingface", + "repo_id": repo_id, + "revision": Value::Null, + "verification": "publisher_asserted" + })), + "provenance": { + "method": method, + "artifact_verification": verification, + "identity_source": if canonical_repository.is_some() { + "huggingface_base_model" + } else { + "unresolved" + } + } + }); + let path = receipt_path(root, artifact_sha256); + if path.is_file() { + let existing: Value = serde_json::from_slice(&fs::read(&path)?)?; + if existing.pointer("/artifact/sha256").and_then(Value::as_str) == Some(artifact_sha256) { + return Ok(()); + } + anyhow::bail!( + "model provenance for {artifact_sha256} conflicts with {}", + path.display() + ); + } + atomic_write_json(&path, &receipt) +} + +/// Bind manually acquired bytes to one exact Hugging Face file. The caller +/// has already compared the local SHA-256 with the Hub's LFS object ID, so the +/// receipt is safe to reuse even if the local file is later renamed. +pub(crate) fn identify_huggingface_file( + root: &Path, + model: &Path, + source_url: &str, +) -> anyhow::Result { + if !model.is_file() { + anyhow::bail!("model file does not exist: {}", model.display()); + } + let artifact_sha256 = crate::adapters::file_sha256(model)?; + let source = crate::huggingface::file_identity(source_url)?; + let expected = source.file.sha256.as_deref().ok_or_else(|| { + anyhow::anyhow!( + "Hugging Face did not publish a SHA-256 for {}; the local file cannot be verified", + source.file.path + ) + })?; + if !expected.eq_ignore_ascii_case(&artifact_sha256) { + anyhow::bail!( + "the local file does not match {}/{} at revision {}: expected {}, got {}", + source.repository, + source.file.path, + source.file.revision, + expected, + artifact_sha256 + ); + } + record_huggingface_download( + root, + &source.repository, + &source.file.revision, + &source.file.path, + &artifact_sha256, + Some(expected), + source.file.canonical_repository.as_deref(), + "manual_hash_match", + )?; + Ok(json!({ + "model": model, + "sha256": artifact_sha256, + "artifact_repository": source.repository, + "artifact_revision": source.file.revision, + "artifact_path": source.file.path, + "canonical_repository": source.file.canonical_repository, + "verification": "sha256" + })) +} + +fn read_receipt(paths: &Paths, artifact_sha256: &str) -> Option { + let bytes = fs::read(receipt_path(&paths.root, artifact_sha256)).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + (value.get("schema").and_then(Value::as_str) == Some(RECEIPT_SCHEMA) + && value.pointer("/artifact/sha256").and_then(Value::as_str) == Some(artifact_sha256)) + .then_some(value) +} + +/// Recover the repository, immutable snapshot, and file name from the standard +/// Hugging Face cache layout. The file itself is still hashed independently. +fn huggingface_cache_artifact(path: &Path, artifact_sha256: &str) -> Option { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir().ok()?.join(path) + }; + let components: Vec<_> = absolute.components().collect(); + let model_index = components.iter().position(|component| match component { + Component::Normal(value) => value + .to_str() + .is_some_and(|value| value.starts_with("models--")), + _ => false, + })?; + let encoded = match components.get(model_index)? { + Component::Normal(value) => value.to_str()?.strip_prefix("models--")?, + _ => return None, + }; + let (owner, model) = encoded.split_once("--")?; + let snapshots = match components.get(model_index + 1)? { + Component::Normal(value) => value.to_str()?, + _ => return None, + }; + if snapshots != "snapshots" { + return None; + } + let revision = match components.get(model_index + 2)? { + Component::Normal(value) => value.to_str()?, + _ => return None, + }; + if revision.len() < 7 || !revision.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let file = components[(model_index + 3)..] + .iter() + .fold(PathBuf::new(), |path, component| { + path.join(component.as_os_str()) + }); + let cache_blob_matches = fs::canonicalize(&absolute) + .ok() + .and_then(|resolved| resolved.file_name().map(|name| name.to_owned())) + .and_then(|name| name.to_str().map(str::to_string)) + .is_some_and(|name| name.eq_ignore_ascii_case(artifact_sha256)); + Some(json!({ + "provider": "huggingface", + "repo_id": format!("{owner}/{model}"), + "revision": revision, + "path": file.to_string_lossy(), + "sha256": artifact_sha256, + "cache_blob_sha256_verified": cache_blob_matches + })) +} + +fn huggingface_repo_id(url: &str) -> Option { + let value = url + .trim() + .strip_prefix("https://huggingface.co/")? + .trim_matches('/'); + let mut parts = value.split('/'); + let owner = parts.next()?; + let model = parts.next()?; + if owner.is_empty() || model.is_empty() { + return None; + } + Some(format!("{owner}/{model}")) +} + +fn base_sidecar(path: &Path, artifact_sha256: &str) -> Option { + let bytes = fs::read(path.parent()?.join("hub.json")).ok()?; + let sidecar: Value = serde_json::from_slice(&bytes).ok()?; + if sidecar + .get("base_sha256") + .and_then(Value::as_str) + .is_some_and(|sha| !sha.eq_ignore_ascii_case(artifact_sha256)) + { + return None; + } + let repository = sidecar.get("hf_repo").and_then(Value::as_str)?; + let revision = sidecar + .get("revision") + .and_then(Value::as_str) + .unwrap_or("unknown"); + let canonical = sidecar.get("source_repo").and_then(Value::as_str); + Some(json!({ + "artifact": { + "provider": "huggingface", + "repo_id": repository, + "revision": revision, + "path": path.file_name().and_then(|name| name.to_str()), + "sha256": artifact_sha256 + }, + "canonical": canonical.map(|repo_id| json!({ + "provider": "huggingface", + "repo_id": repo_id, + "revision": Value::Null, + "verification": "runtime_provenance" + })), + "provenance": { + "method": "basert_hub_sidecar", + "artifact_verification": if sidecar.get("base_sha256").is_some() { + "sha256" + } else { + "runtime_record" + }, + "identity_source": if canonical.is_some() { + "basert_source_repo" + } else { + "unresolved" + } + } + })) +} + +fn embedded_gguf_identity(model: &Value, artifact_sha256: &str) -> Option { + let artifact_repository = model + .get("repo_url") + .and_then(Value::as_str) + .and_then(huggingface_repo_id); + let canonical = model + .get("source_repo_url") + .and_then(Value::as_str) + .and_then(huggingface_repo_id) + .or_else(|| { + model + .get("base_model_repo_url") + .and_then(Value::as_str) + .and_then(huggingface_repo_id) + }); + if artifact_repository.is_none() && canonical.is_none() { + return None; + } + Some(json!({ + "artifact": { + "provider": artifact_repository.as_ref().map(|_| "huggingface"), + "repo_id": artifact_repository, + "revision": Value::Null, + "path": model.get("file_name"), + "sha256": artifact_sha256 + }, + "canonical": canonical.as_ref().map(|repo_id| json!({ + "provider": "huggingface", + "repo_id": repo_id, + "revision": Value::Null, + "verification": "embedded_metadata" + })), + "provenance": { + "method": "embedded_gguf_metadata", + "artifact_verification": "local_sha256", + "identity_source": if canonical.is_some() { + "gguf_source_metadata" + } else { + "unresolved" + } + } + })) +} + +fn fallback_evidence(runtime: Runtime, path: &Path, model: &Value, artifact_sha256: &str) -> Value { + if runtime == Runtime::Basert { + if let Some(evidence) = base_sidecar(path, artifact_sha256) { + return evidence; + } + } + if runtime == Runtime::LlamaCpp { + let cache = huggingface_cache_artifact(path, artifact_sha256); + let embedded = embedded_gguf_identity(model, artifact_sha256); + if let Some(artifact) = cache { + let canonical = embedded + .as_ref() + .and_then(|value| value.get("canonical")) + .cloned() + .unwrap_or(Value::Null); + let identity_source = if canonical.is_null() { + "unresolved" + } else { + "gguf_source_metadata" + }; + return json!({ + "artifact": artifact, + "canonical": canonical, + "provenance": { + "method": "huggingface_cache", + "artifact_verification": if artifact["cache_blob_sha256_verified"] == true { + "sha256" + } else { + "local_sha256" + }, + "identity_source": identity_source + } + }); + } + if let Some(evidence) = embedded { + return evidence; + } + } + json!({ + "artifact": { + "provider": Value::Null, + "repo_id": Value::Null, + "revision": Value::Null, + "path": Value::Null, + "sha256": artifact_sha256 + }, + "canonical": Value::Null, + "provenance": { + "method": "local_file", + "artifact_verification": "local_sha256", + "identity_source": "unresolved" + } + }) +} + +pub(crate) fn finalize( + runtime: Runtime, + paths: &Paths, + model_path: &Path, + mut model: Value, + artifact_sha256: &str, +) -> Value { + let evidence = read_receipt(paths, artifact_sha256) + .unwrap_or_else(|| fallback_evidence(runtime, model_path, &model, artifact_sha256)); + let artifact = evidence + .get("artifact") + .cloned() + .unwrap_or_else(|| json!({"sha256": artifact_sha256})); + let canonical = evidence.get("canonical").cloned().unwrap_or(Value::Null); + let provenance = evidence + .get("provenance") + .cloned() + .unwrap_or_else(|| json!({"method":"local_file"})); + let scheme = model + .get("quantization") + .and_then(Value::as_str) + .unwrap_or("unknown"); + let format = model + .get("format") + .and_then(Value::as_str) + .unwrap_or(match runtime { + Runtime::Basert => "base", + Runtime::LlamaCpp => "gguf", + }); + let mut artifact = artifact; + artifact["format"] = json!(format); + artifact["quantization"] = json!({ + "namespace": match runtime { + Runtime::Basert => "basert", + Runtime::LlamaCpp => "gguf", + }, + "scheme": scheme + }); + + let upstream_id = canonical.get("repo_id").cloned().unwrap_or(Value::Null); + let identity_source = provenance + .get("identity_source") + .cloned() + .unwrap_or_else(|| json!("unresolved")); + let identity_verification = canonical + .get("verification") + .cloned() + .unwrap_or_else(|| json!("unverified")); + model["identity_schema"] = json!(IDENTITY_SCHEMA); + model["canonical"] = canonical; + model["artifact"] = artifact; + model["provenance"] = provenance; + // Retain the released fields while the server learns the richer schema. + model["upstream_id"] = upstream_id; + model["upstream_id_source"] = identity_source; + model["identity_verification"] = identity_verification; + model["artifact_sha256"] = json!(artifact_sha256); + model +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recovers_huggingface_snapshot_identity() { + let path = Path::new( + "/cache/hub/models--bartowski--Qwen-GGUF/snapshots/0123456789abcdef/model.gguf", + ); + let artifact = huggingface_cache_artifact(path, "aa").unwrap(); + assert_eq!(artifact["repo_id"], "bartowski/Qwen-GGUF"); + assert_eq!(artifact["revision"], "0123456789abcdef"); + assert_eq!(artifact["path"], "model.gguf"); + } + + #[test] + fn extracts_huggingface_repository_from_urls() { + assert_eq!( + huggingface_repo_id("https://huggingface.co/Qwen/Qwen3-4B/tree/main"), + Some("Qwen/Qwen3-4B".to_string()) + ); + assert_eq!(huggingface_repo_id("https://example.com/a/b"), None); + } + + #[test] + fn download_receipt_populates_canonical_artifact_and_compatibility_fields() { + let temporary = tempfile::tempdir().unwrap(); + let paths = Paths::resolve(Some(temporary.path().to_path_buf())).unwrap(); + let sha256 = "a".repeat(64); + record_huggingface_download( + &paths.root, + "bartowski/Qwen3-4B-GGUF", + "0123456789abcdef", + "Qwen3-4B-Q4_K_M.gguf", + &sha256, + Some(&sha256), + Some("Qwen/Qwen3-4B"), + "computearena_download", + ) + .unwrap(); + let model = finalize( + Runtime::LlamaCpp, + &paths, + Path::new("/tmp/renamed.gguf"), + json!({"name": "Qwen", "format": "gguf", "quantization": "Q4_K_M"}), + &sha256, + ); + + assert_eq!(model["identity_schema"], IDENTITY_SCHEMA); + assert_eq!(model["canonical"]["repo_id"], "Qwen/Qwen3-4B"); + assert_eq!(model["artifact"]["repo_id"], "bartowski/Qwen3-4B-GGUF"); + assert_eq!(model["artifact"]["revision"], "0123456789abcdef"); + assert_eq!(model["artifact"]["quantization"]["namespace"], "gguf"); + assert_eq!(model["artifact"]["quantization"]["scheme"], "Q4_K_M"); + assert_eq!(model["upstream_id"], "Qwen/Qwen3-4B"); + assert_eq!(model["artifact_sha256"], sha256); + } + + #[test] + fn unknown_local_files_remain_unresolved_but_keep_exact_artifact_identity() { + let temporary = tempfile::tempdir().unwrap(); + let paths = Paths::resolve(Some(temporary.path().to_path_buf())).unwrap(); + let sha256 = "b".repeat(64); + let model = finalize( + Runtime::Basert, + &paths, + Path::new("/tmp/unknown.base"), + json!({"name": "unknown", "quantization": "base_q4"}), + &sha256, + ); + + assert!(model["canonical"].is_null()); + assert_eq!(model["artifact"]["format"], "base"); + assert_eq!(model["artifact"]["quantization"]["namespace"], "basert"); + assert_eq!(model["provenance"]["method"], "local_file"); + assert_eq!(model["artifact_sha256"], sha256); + } +} diff --git a/crates/computearena-cli/src/models.rs b/crates/computearena-cli/src/models.rs index 8274d96..0331910 100644 --- a/crates/computearena-cli/src/models.rs +++ b/crates/computearena-cli/src/models.rs @@ -1,4 +1,5 @@ use crate::config::{MODEL_ID_COLUMN_WIDTH, MODEL_QUANT_COLUMN_WIDTH, MODEL_VARIANT_COLUMN_WIDTH}; +use crate::reports::Paths; use crate::theme::selector_theme; use crate::ui::{finish_activity, prompt, start_activity, visible_rows, TerminalUi}; use anyhow::{bail, Context, Result}; @@ -30,6 +31,7 @@ pub(crate) fn inspect_model(path: &Path) -> Result { "name": fallback_name, "file_name": path.file_name().and_then(|name| name.to_str()).unwrap_or("unknown"), "size_bytes": file.len(), + "format": "base", "format_schema": required_header_u64(&header, "schema")?, "architecture": required_header_string(&header, "arch")?, "quantization": required_header_string(&header, "quant_scheme")?, @@ -111,7 +113,7 @@ pub(crate) fn display_name(path: &Path) -> String { } } -pub(crate) fn prompt_model_path() -> Result> { +pub(crate) fn prompt_model_path(paths: &Paths) -> Result> { let ui = TerminalUi::detect(); let started = start_activity(ui, "Scanning installed BaseRT model metadata…"); let installed = installed_models()?; @@ -122,14 +124,15 @@ pub(crate) fn prompt_model_path() -> Result> { ); if installed.is_empty() { print_model_acquisition_help(ui, true); - return model_path_from_input(prompt("Model path: ")?).map(Some); } - print_model_acquisition_help(ui, false); + if !installed.is_empty() { + print_model_acquisition_help(ui, false); + } if io::stdin().is_terminal() && io::stderr().is_terminal() { - prompt_model_path_interactive(&installed, ui) + prompt_model_path_interactive(&installed, paths, ui) } else { - prompt_model_path_numbered(&installed, ui) + prompt_model_path_numbered(&installed, paths, ui) } } @@ -146,24 +149,28 @@ fn print_model_acquisition_help(ui: TerminalUi, no_models_installed: bool) { println!( " {} {}", ui.neutral("Browse"), - ui.accent_bold(BASERT_REMOTE_MODELS_COMMAND) + ui.accent_bold("Browse and download in this model picker") ); println!( " {} {}", ui.neutral("Pull"), - ui.accent_bold(BASERT_PULL_MODEL_COMMAND) + ui.accent_bold(format!("or use {BASERT_PULL_MODEL_COMMAND}")) ); println!( "{}", - ui.muted("Run this step again after pulling to refresh the list.") + ui.muted(format!( + "The catalogue is the same one shown by `{BASERT_REMOTE_MODELS_COMMAND}`." + )) ); } fn prompt_model_path_interactive( installed: &[InstalledModel], + paths: &Paths, ui: TerminalUi, ) -> Result> { let mut choices = model_choice_labels(installed); + choices.push("Browse the BaseRT model catalogue…".to_string()); choices.push("Enter another model path…".to_string()); println!( "{}", @@ -184,6 +191,9 @@ fn prompt_model_path_interactive( println!("{} Model selection cancelled", ui.neutral("←")); return Ok(None); }; + if index == installed.len() { + return prompt_remote_model(paths, ui); + } let Some(model) = installed.get(index) else { return model_path_from_input(prompt("Model path: ")?).map(Some); }; @@ -193,12 +203,17 @@ fn prompt_model_path_interactive( fn prompt_model_path_numbered( installed: &[InstalledModel], + paths: &Paths, ui: TerminalUi, ) -> Result> { println!("Installed BaseRT models:"); for (index, label) in model_choice_labels(installed).iter().enumerate() { println!(" {} {label}", ui.brand_bold(format!("{}.", index + 1))); } + println!( + " {} Browse the BaseRT model catalogue", + ui.brand_bold("d.") + ); println!(" {} Enter another model path", ui.brand_bold("p.")); let input = prompt("Choose a model number or enter a path: ")?; if matches!(input.to_ascii_lowercase().as_str(), "q" | "quit" | "back") { @@ -215,9 +230,73 @@ fn prompt_model_path_numbered( if input.eq_ignore_ascii_case("p") { return model_path_from_input(prompt("Model path: ")?).map(Some); } + if input.eq_ignore_ascii_case("d") { + return prompt_remote_model(paths, ui); + } model_path_from_input(input).map(Some) } +fn prompt_remote_model(paths: &Paths, ui: TerminalUi) -> Result> { + let cli = crate::basert_models::locate_cli(None)?; + let started = start_activity(ui, "Loading the BaseRT model catalogue…"); + let models = crate::basert_models::available(None)?; + finish_activity( + ui, + started, + format!("Found {} downloadable model option(s)", models.len()), + ); + if models.is_empty() { + println!("Every compatible catalogue model is already installed."); + return Ok(None); + } + let labels: Vec<_> = models + .iter() + .map(|model| { + format!( + "{} {} {} {}", + model.id, + crate::basert_models::display_pull_target(&model.pull_target), + model.architecture, + model + .size_bytes + .map(crate::huggingface::format_size) + .unwrap_or_else(|| "size unavailable".to_string()) + ) + }) + .collect(); + let selected = if io::stdin().is_terminal() && io::stderr().is_terminal() { + FuzzySelect::with_theme(&selector_theme()) + .with_prompt("Download a BaseRT model") + .items(&labels) + .max_length(visible_rows(labels.len())) + .report(false) + .interact_opt() + .context("reading BaseRT model selection")? + } else { + for (index, label) in labels.iter().enumerate() { + println!(" {} {label}", ui.brand_bold(format!("{}.", index + 1))); + } + let input = prompt("Choose a model number (or q to go back): ")?; + if matches!(input.to_ascii_lowercase().as_str(), "q" | "quit" | "back") { + return Ok(None); + } + Some( + input + .parse::() + .ok() + .and_then(|index| index.checked_sub(1)) + .filter(|index| *index < models.len()) + .context("model selection is out of range")?, + ) + }; + let Some(index) = selected else { + return Ok(None); + }; + let path = crate::basert_models::pull(&cli, &models[index])?; + crate::basert_models::record_download(&paths.root, &path)?; + Ok(Some(path)) +} + pub(crate) fn model_choice_labels(installed: &[InstalledModel]) -> Vec { let id_width = installed .iter() diff --git a/crates/computearena-cli/src/tui/app.rs b/crates/computearena-cli/src/tui/app.rs index c129a90..afa0925 100644 --- a/crates/computearena-cli/src/tui/app.rs +++ b/crates/computearena-cli/src/tui/app.rs @@ -41,6 +41,12 @@ pub(crate) struct HubFileRow { pub(crate) file: crate::huggingface::HubFile, } +pub(crate) struct BaseRtModelRow { + pub(crate) label: String, + pub(crate) detail: String, + pub(crate) model: crate::basert_models::RemoteModel, +} + pub(crate) struct ReportRow { pub(crate) label: String, pub(crate) detail: String, @@ -91,6 +97,11 @@ pub(crate) enum Screen { rows: Vec, cursor: usize, }, + BaseRtModels { + rows: Vec, + filter: String, + cursor: usize, + }, Account { cursor: usize, }, @@ -145,6 +156,7 @@ enum Loaded { Reports(Vec, ReportMode), HubModels(Vec), HubFiles(String, Vec), + BaseRtModels(Vec), } struct Pending { @@ -176,7 +188,7 @@ pub(crate) struct App { pending: Option, /// Where the running download will land, so its plan can open when it /// finishes. - downloaded: Option, + downloaded: Arc>>, pub(crate) status: String, pub(crate) should_quit: bool, /// Where the benchmark job leaves the path of the report it saved, so @@ -205,7 +217,7 @@ impl App { screens: Vec::new(), job: None, pending: None, - downloaded: None, + downloaded: Arc::new(Mutex::new(None)), status: String::new(), should_quit: false, completed_report: Arc::new(Mutex::new(None)), @@ -460,6 +472,18 @@ impl App { cursor: 0, }); } + Ok(Loaded::BaseRtModels(rows)) => { + if rows.is_empty() { + self.status = + "Every compatible BaseRT catalogue model is installed".to_string(); + return; + } + self.screens.push(Screen::BaseRtModels { + rows, + filter: String::new(), + cursor: 0, + }); + } Ok(Loaded::Reports(rows, mode)) => { if rows.is_empty() { self.screens.push(Screen::Info { @@ -541,23 +565,64 @@ impl App { .next() .unwrap_or(&file.path) .to_string(); - self.downloaded = Some(crate::huggingface::download_path( - &root, - &repository, - &file.path, - )); + let downloaded = self.downloaded.clone(); + *downloaded.lock().unwrap_or_else(PoisonError::into_inner) = None; self.job = Some(Job::spawn( JobKind::Download, format!("Downloading {name} from {repository}"), move || { let path = crate::huggingface::download(&root, &repository, &file)?; crate::recent_gguf::remember(&paths, &path)?; + *downloaded.lock().unwrap_or_else(PoisonError::into_inner) = Some(path); Ok(format!("Downloaded {name}")) }, )); self.screens.push(Screen::Running); } + fn open_basert_models(&mut self) -> Result<()> { + let harness = self.executable.clone(); + self.screens.push(Screen::Loading { + message: "Loading the BaseRT model catalogue…".to_string(), + }); + self.pending = Some(Pending::spawn(move || { + let rows = crate::basert_models::available(harness.as_deref())? + .into_iter() + .map(|model| BaseRtModelRow { + label: model.id.clone(), + detail: format!( + "{} · {} · {}", + crate::basert_models::display_pull_target(&model.pull_target), + model.architecture, + model + .size_bytes + .map(crate::huggingface::format_size) + .unwrap_or_else(|| "size unavailable".to_string()) + ), + model, + }) + .collect(); + Ok(Loaded::BaseRtModels(rows)) + })); + Ok(()) + } + + fn start_basert_download(&mut self, model: crate::basert_models::RemoteModel) -> Result<()> { + let cli = crate::basert_models::locate_cli(self.executable.as_deref())?; + let root = self.paths.root.clone(); + let downloaded = self.downloaded.clone(); + *downloaded.lock().unwrap_or_else(PoisonError::into_inner) = None; + let title = format!("Downloading {} ({})", model.id, model.pull_target); + self.job = Some(Job::spawn(JobKind::Download, title, move || { + let path = crate::basert_models::pull(&cli, &model)?; + crate::basert_models::record_download(&root, &path)?; + *downloaded.lock().unwrap_or_else(PoisonError::into_inner) = Some(path); + Ok(format!("Downloaded {} ({})", model.id, model.pull_target)) + })); + self.screens.push(Screen::Running); + Ok(()) + } + fn start_verify(&mut self, report: PathBuf) { self.job = Some(Job::spawn( JobKind::Verify, @@ -691,11 +756,18 @@ impl App { JobKind::Download => { while matches!( self.screen(), - Screen::HubFiles { .. } | Screen::HubModels { .. } + Screen::HubFiles { .. } + | Screen::HubModels { .. } + | Screen::BaseRtModels { .. } ) { self.back(); } - if let (Some(Ok(_)), Some(path)) = (&saved, self.downloaded.take()) { + let path = self + .downloaded + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + if let (Some(Ok(_)), Some(path)) = (&saved, path) { self.open_plan(path)?; } } @@ -784,6 +856,11 @@ impl App { input.push(character); return Ok(()); } + Screen::BaseRtModels { filter, cursor, .. } => { + filter.push(character); + *cursor = 0; + return Ok(()); + } _ => {} } match character { @@ -810,6 +887,10 @@ impl App { Screen::HubSearch { input } => { input.pop(); } + Screen::BaseRtModels { filter, cursor, .. } => { + filter.pop(); + *cursor = 0; + } _ => {} } } @@ -852,6 +933,19 @@ impl App { .collect() } + fn visible_basert_models(rows: &[BaseRtModelRow], filter: &str) -> Vec { + let needle = filter.to_ascii_lowercase(); + rows.iter() + .enumerate() + .filter(|(_, row)| { + needle.is_empty() + || row.label.to_ascii_lowercase().contains(&needle) + || row.detail.to_ascii_lowercase().contains(&needle) + }) + .map(|(index, _)| index) + .collect() + } + fn list_length(&self) -> usize { match self.screen() { Screen::Runtime { .. } => 2, @@ -862,6 +956,9 @@ impl App { Screen::Reports { rows, .. } => rows.len(), Screen::HubModels { rows, .. } => rows.len(), Screen::HubFiles { rows, .. } => rows.len(), + Screen::BaseRtModels { rows, filter, .. } => { + Self::visible_basert_models(rows, filter).len() + } Screen::Account { .. } => 2, Screen::Preview { lines, .. } => lines.len(), Screen::PathEntry { .. } | Screen::Running | Screen::Info { .. } => 0, @@ -888,6 +985,7 @@ impl App { | Screen::Reports { cursor, .. } | Screen::HubModels { cursor, .. } | Screen::HubFiles { cursor, .. } + | Screen::BaseRtModels { cursor, .. } | Screen::Account { cursor } => cursor, Screen::Preview { scroll, .. } => scroll, _ => return, @@ -908,6 +1006,7 @@ impl App { | Screen::Reports { cursor, .. } | Screen::HubModels { cursor, .. } | Screen::HubFiles { cursor, .. } + | Screen::BaseRtModels { cursor, .. } | Screen::Account { cursor } => *cursor = position.min(length.saturating_sub(1)), Screen::Preview { scroll, .. } => *scroll = position.min(length.saturating_sub(1)), _ => {} @@ -1033,12 +1132,19 @@ impl App { cursor, } => { let repository = repository.clone(); - let file = crate::huggingface::HubFile { - path: rows[*cursor].file.path.clone(), - size: rows[*cursor].file.size, - }; + let file = rows[*cursor].file.clone(); self.start_download(repository, file); } + Screen::BaseRtModels { + rows, + filter, + cursor, + } => { + let visible = Self::visible_basert_models(rows, filter); + if let Some(index) = visible.get(*cursor) { + self.start_basert_download(rows[*index].model.clone())?; + } + } Screen::Models { rows, filter, @@ -1050,9 +1156,12 @@ impl App { }; match (rows[*index].path.clone(), rows[*index].source) { (Some(path), _) => self.open_plan(path)?, - (None, ModelSource::Hub) => self.screens.push(Screen::HubSearch { - input: String::new(), - }), + (None, ModelSource::Hub) => match self.runtime { + Runtime::Basert => self.open_basert_models()?, + Runtime::LlamaCpp => self.screens.push(Screen::HubSearch { + input: String::new(), + }), + }, (None, ModelSource::Local) => self.screens.push(Screen::PathEntry { input: String::new(), error: None, @@ -1223,16 +1332,15 @@ fn model_rows(runtime: Runtime, paths: &Paths) -> Result> { }) .collect(), }; - if runtime == Runtime::LlamaCpp { - // Most people have no GGUF on disk yet, so the Hub is offered before - // the path prompt rather than after it. - rows.push(ModelRow { - label: "Search Hugging Face for a GGUF…".to_string(), - detail: "Download a model to benchmark".to_string(), - path: None, - source: ModelSource::Hub, - }); - } + rows.push(ModelRow { + label: match runtime { + Runtime::Basert => "Browse the BaseRT model catalogue…".to_string(), + Runtime::LlamaCpp => "Search Hugging Face for a GGUF…".to_string(), + }, + detail: "Download a model to benchmark".to_string(), + path: None, + source: ModelSource::Hub, + }); rows.push(ModelRow { label: match runtime { Runtime::Basert => "Enter another model path…".to_string(), @@ -1285,7 +1393,7 @@ mod tests { screens: vec![Screen::Menu { cursor: 0 }], job: None, pending: None, - downloaded: None, + downloaded: Arc::new(Mutex::new(None)), status: String::new(), should_quit: false, completed_report: Arc::new(Mutex::new(None)), diff --git a/crates/computearena-cli/src/tui/draw.rs b/crates/computearena-cli/src/tui/draw.rs index 8e10e19..05836fb 100644 --- a/crates/computearena-cli/src/tui/draw.rs +++ b/crates/computearena-cli/src/tui/draw.rs @@ -1,8 +1,8 @@ //! Rendering. Every screen shares the same frame: a header naming the session, //! a body, and a key bar, so nothing moves between screens except the body. use super::app::{ - App, HubFileRow, HubModelRow, ModelRow, ReportMode, ReportRow, Screen, MENU_ITEMS, - SETUP_ACTIONS, + App, BaseRtModelRow, HubFileRow, HubModelRow, ModelRow, ReportMode, ReportRow, Screen, + MENU_ITEMS, SETUP_ACTIONS, }; use super::job::Job; use crate::benchmark::LOAD_WARNING; @@ -214,6 +214,7 @@ fn footer(frame: &mut Frame, area: Rect, app: &App) { Screen::HubSearch { .. } => "type a search · Enter search · Esc back", Screen::HubModels { .. } => "↑/↓ move · Enter list files · Esc back", Screen::HubFiles { .. } => "↑/↓ move · Enter download · Esc back", + Screen::BaseRtModels { .. } => "type to filter · ↑/↓ move · Enter download · Esc back", _ => "↑/↓ move · Enter select · Esc back · Ctrl+C quit", }; let status = if app.status.is_empty() { @@ -281,6 +282,11 @@ fn body(frame: &mut Frame, area: Rect, app: &mut App) { rows, cursor, } => hub_files_screen(frame, area, repository, rows, *cursor), + Screen::BaseRtModels { + rows, + filter, + cursor, + } => basert_models_screen(frame, area, rows, filter, *cursor), Screen::Account { cursor } => account_screen( frame, area, @@ -643,6 +649,33 @@ fn hub_files_screen( ); } +fn basert_models_screen( + frame: &mut Frame, + area: Rect, + rows: &[BaseRtModelRow], + filter: &str, + cursor: usize, +) { + let needle = filter.to_ascii_lowercase(); + let items = rows + .iter() + .enumerate() + .filter(|(_, row)| { + needle.is_empty() + || row.label.to_ascii_lowercase().contains(&needle) + || row.detail.to_ascii_lowercase().contains(&needle) + }) + .enumerate() + .map(|(visible, (_, row))| item(row.label.clone(), row.detail.clone(), visible == cursor)) + .collect(); + let title = if filter.is_empty() { + "BaseRT · available models".to_string() + } else { + format!("BaseRT · filter: {filter}") + }; + render_list(frame, area, &title, items, cursor); +} + fn account_screen( frame: &mut Frame, area: Rect, diff --git a/docs/model-identity.md b/docs/model-identity.md new file mode 100644 index 0000000..cbfca3e --- /dev/null +++ b/docs/model-identity.md @@ -0,0 +1,53 @@ +# Model identity and provenance + +ComputeArena needs to compare the same upstream model across runtimes without +pretending that converted files or quantizations are interchangeable. Every new +report therefore separates: + +- `model.canonical`: the upstream Hugging Face model class, such as + `Qwen/Qwen3-4B`; +- `model.artifact`: the repository, immutable revision, file path, format, + runtime-namespaced quantization, and SHA-256 of the exact bytes used; and +- `model.provenance`: how those identities were obtained and verified. + +The signed identity schema is `computearena-model/1`. The released compatibility +fields (`upstream_id`, `upstream_id_source`, `identity_verification`, and +`artifact_sha256`) remain populated so older servers can accept new reports. + +## Managed downloads + +For GGUF, ComputeArena resolves the repository's current ref to an immutable +commit, reads the publisher's `base_model` metadata, downloads that exact file, +hashes it while streaming, and compares it with the Hub's LFS object ID. A +content-addressed receipt is saved under `model-provenance/`. + +For BaseRT, ComputeArena delegates to `basert pull`. It then hashes the installed +`.base` file and records BaseRT's adjacent `hub.json` fields. When online, it +also resolves the artifact repository's Hugging Face metadata to recover the +canonical upstream model. Multiple internal catalogue variants that map to one +`basert pull --target` choice are shown once; BaseRT selects the compatible +artifact for the current backend. + +## Models acquired elsewhere + +Standard Hugging Face cache paths and embedded GGUF source URLs are retained as +evidence, but missing or ambiguous identity stays unresolved. A user can prove +that a copied or renamed file is one exact Hub object with: + +```sh +computearena identify /path/to/model.base \ + https://huggingface.co/owner/repository/blob//path/to/model.base +``` + +The command resolves the revision, fetches only file metadata, hashes the local +file, and saves a receipt only if the SHA-256 values match. A repository URL or +free-form model name is intentionally insufficient. + +## Trust boundary + +This proves which public artifact bytes were selected and preserves the +publisher's model lineage claim. It does not prove that a publisher labelled a +model correctly, that the runtime executed those bytes honestly, or that two +fine-tunes or merges with different lineage are equivalent. The server must +keep artifact and quantization dimensions when grouping by canonical model +class. diff --git a/docs/runtime-adapters.md b/docs/runtime-adapters.md index a731339..511ae8f 100644 --- a/docs/runtime-adapters.md +++ b/docs/runtime-adapters.md @@ -16,9 +16,13 @@ Both retain the computearena-benchmark/1 signed envelope, so old reports remain - Model selection for llama.cpp offers the Hugging Face Hub alongside recent files: search models filtered to GGUF, list a repository's .gguf files smallest first, download into the ComputeArena data directory, and go straight to that model's plan. `HF_TOKEN` is - forwarded when set, for gated repositories. -- computearena basert [run|list|inspect|verify|login|logout|submit|install] -- computearena llama-cpp [run|list|inspect|verify|login|logout|submit|install] + forwarded when set, for gated repositories. Downloads use immutable revisions and verify + published LFS SHA-256 values when available. +- Model selection for BaseRT offers BaseRT's public catalogue alongside installed files. + ComputeArena invokes the installed `basert list --remote` and `basert pull` commands so + BaseRT owns backend selection, split downloads, conversion, and `hub.json` provenance. +- computearena basert [run|list|inspect|verify|identify|login|logout|submit|install] +- computearena llama-cpp [run|list|inspect|verify|identify|login|logout|submit|install] - install downloads the runtime's prebuilt release after showing the plan; --archive unpacks a local bundle instead. Discovery prefers --runtime-path, then environment variables, then a ComputeArena-installed copy, then PATH, then the runtime's default @@ -44,7 +48,8 @@ of repetitions. The plan states this before execution. Automatic external teleme is collected over the whole process (see telemetry.md). Optional cooldown runs each workload in a fresh process using llama-bench-conditioned-pp-tg/1 (see benchmark-profiles.md). Native effective settings are preserved. -No equivalence between BaseRT and GGUF quantization names is assumed. +No equivalence between BaseRT and GGUF quantization names is assumed. Both adapters emit +the runtime-neutral `computearena-model/1` identity described in model-identity.md. The PP/TG counts alone do not establish equivalent timing semantics. llama.cpp records a distinct protocol ID and its exclusion of sampling/tokenization. From 1ee6a1c64258bf9d62cebd513d9a48ca629a8e9f Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 15:33:42 +1000 Subject: [PATCH 03/10] feat(models): verify artifact identity before submission --- crates/computearena-cli/src/huggingface.rs | 31 ++++++- crates/computearena-cli/src/model_identity.rs | 86 +++++++++++++++++++ crates/computearena-cli/src/submission.rs | 81 ++++++++++++++++- 3 files changed, 194 insertions(+), 4 deletions(-) diff --git a/crates/computearena-cli/src/huggingface.rs b/crates/computearena-cli/src/huggingface.rs index 1757352..5ff3926 100644 --- a/crates/computearena-cli/src/huggingface.rs +++ b/crates/computearena-cli/src/huggingface.rs @@ -277,7 +277,32 @@ fn repository_tree(repository: &str, revision: &str) -> Result> { /// objects without downloading the model again. pub(crate) fn file_identity(url: &str) -> Result { let (repository, requested_revision, path) = parse_file_url(url)?; - let identity = repository_identity_at(&repository, &requested_revision)?; + artifact_identity(&repository, &requested_revision, &path) +} + +/// Resolve an already identified repository/revision/path tuple without +/// searching the whole Hub. Submission preflight uses this bounded lookup to +/// compare the report's local SHA-256 with the publisher's LFS object ID. +pub(crate) fn artifact_identity( + repository: &str, + requested_revision: &str, + path: &str, +) -> Result { + if repository.split('/').count() != 2 + || repository + .split('/') + .any(|part| part.is_empty() || matches!(part, "." | "..")) + { + bail!("invalid Hugging Face repository ID in model identity"); + } + if path.starts_with('/') + || path + .split('/') + .any(|part| part.is_empty() || matches!(part, "." | "..")) + { + bail!("invalid Hugging Face artifact path in model identity"); + } + let identity = repository_identity_at(repository, requested_revision)?; let info_url = format!( "{HUGGINGFACE_API}/models/{repository}/paths-info/{}", identity.revision @@ -312,9 +337,9 @@ pub(crate) fn file_identity(url: &str) -> Result { .filter(|sha| sha.len() == 64 && sha.bytes().all(|byte| byte.is_ascii_hexdigit())) .map(|sha| sha.to_ascii_lowercase()); Ok(HubFileIdentity { - repository, + repository: repository.to_string(), file: HubFile { - path, + path: path.to_string(), size: entry .get("size") .and_then(Value::as_u64) diff --git a/crates/computearena-cli/src/model_identity.rs b/crates/computearena-cli/src/model_identity.rs index 59b594b..c7f15ef 100644 --- a/crates/computearena-cli/src/model_identity.rs +++ b/crates/computearena-cli/src/model_identity.rs @@ -14,6 +14,92 @@ use std::path::{Component, Path, PathBuf}; pub(crate) const IDENTITY_SCHEMA: &str = "computearena-model/1"; const RECEIPT_SCHEMA: &str = "computearena-model-provenance/1"; +#[derive(Clone, Debug)] +pub(crate) enum SubmissionModelVerification { + Verified(String), + Unresolved(String), + Unavailable(String), + Mismatch(String), +} + +/// Advisory client-side check used immediately before submission. The server +/// repeats this independently because a public CLI is not a trust boundary. +pub(crate) fn verify_submission_model(report: &Value) -> SubmissionModelVerification { + if report + .pointer("/model/identity_schema") + .and_then(Value::as_str) + != Some(IDENTITY_SCHEMA) + { + return SubmissionModelVerification::Unresolved( + "legacy report; no runtime-neutral model identity".to_string(), + ); + } + let artifact = match report.pointer("/model/artifact") { + Some(Value::Object(artifact)) => artifact, + _ => { + return SubmissionModelVerification::Unresolved( + "model artifact identity is incomplete".to_string(), + ) + } + }; + let sha256 = match artifact.get("sha256").and_then(Value::as_str) { + Some(value) if value.len() == 64 => value, + _ => { + return SubmissionModelVerification::Unresolved( + "model artifact SHA-256 is unavailable".to_string(), + ) + } + }; + let tuple = ( + artifact.get("provider").and_then(Value::as_str), + artifact.get("repo_id").and_then(Value::as_str), + artifact.get("revision").and_then(Value::as_str), + artifact.get("path").and_then(Value::as_str), + ); + let (Some("huggingface"), Some(repository), Some(revision), Some(path)) = tuple else { + return SubmissionModelVerification::Unresolved( + "exact Hugging Face repository, revision, and path are unavailable".to_string(), + ); + }; + + let identity = match crate::huggingface::artifact_identity(repository, revision, path) { + Ok(identity) => identity, + Err(error) => { + return SubmissionModelVerification::Unavailable(format!( + "could not contact Hugging Face: {error}" + )) + } + }; + let Some(published_sha256) = identity.file.sha256.as_deref() else { + return SubmissionModelVerification::Unresolved( + "Hugging Face does not publish a SHA-256 for this artifact".to_string(), + ); + }; + if !published_sha256.eq_ignore_ascii_case(sha256) { + return SubmissionModelVerification::Mismatch( + "artifact SHA-256 does not match the claimed Hugging Face file".to_string(), + ); + } + + let claimed_canonical = report + .pointer("/model/canonical/repo_id") + .and_then(Value::as_str); + if let (Some(claimed), Some(published)) = ( + claimed_canonical, + identity.file.canonical_repository.as_deref(), + ) { + if !claimed.eq_ignore_ascii_case(published) { + return SubmissionModelVerification::Mismatch( + "canonical model does not match the Hugging Face repository metadata".to_string(), + ); + } + } + SubmissionModelVerification::Verified(match identity.file.canonical_repository { + Some(canonical) => format!("artifact hash and model family verified as {canonical}"), + None => "artifact hash verified; model family remains unresolved".to_string(), + }) +} + fn receipts_dir(root: &Path) -> PathBuf { root.join("model-provenance") } diff --git a/crates/computearena-cli/src/submission.rs b/crates/computearena-cli/src/submission.rs index f2b9813..e4c952a 100644 --- a/crates/computearena-cli/src/submission.rs +++ b/crates/computearena-cli/src/submission.rs @@ -1,6 +1,7 @@ use crate::api::{client as api_client, error_message as api_error_message}; use crate::auth::load_api_session; use crate::config::SUBMISSION_HTTP_TIMEOUT; +use crate::model_identity::{verify_submission_model, SubmissionModelVerification}; use crate::reports::{ model_identity_for_report, report_summaries, resolve_report, short_id, verify_report, Paths, }; @@ -20,6 +21,7 @@ pub(crate) struct PreparedSubmission { pub(crate) path: PathBuf, value: Value, bytes: Vec, + model_verification: SubmissionModelVerification, } #[derive(Debug)] @@ -183,12 +185,18 @@ pub(crate) fn submit_reports( } let ui = TerminalUi::detect(); let checking_started = start_activity(ui, "Checking selected benchmarks…"); - let preflight = preflight_submissions(reports); + let mut preflight = preflight_submissions(reports); finish_activity( ui, checking_started, format!("Checked {} benchmark(s)", reports.len()), ); + if !preflight.ready.is_empty() { + let model_started = + start_activity(ui, "Verifying exact model artifacts with Hugging Face…"); + verify_submission_models(&mut preflight); + finish_activity(ui, model_started, "Model artifact check complete"); + } print_submission_preflight(ui, &preflight); if assume_yes && !skip_invalid && !preflight.invalid.is_empty() { @@ -298,6 +306,22 @@ pub(crate) fn submit_reports( println!("{}", ui.neutral(format!("Official downloads: {url}"))); } } + if let Some(verification) = response.get("model_verification") { + let status = verification + .get("status") + .and_then(Value::as_str) + .unwrap_or("unresolved"); + if let Some(message) = + verification.get("message").and_then(Value::as_str) + { + let marker = if status == "verified" { + ui.success("✓") + } else { + ui.warning("!") + }; + println!("{marker} Server model verification: {message}"); + } + } } outcomes.push(SubmissionOutcome { label, @@ -417,11 +441,29 @@ pub(crate) fn preflight_submissions(reports: &[PathBuf]) -> SubmissionPreflight path: path.clone(), value, bytes, + model_verification: SubmissionModelVerification::Unresolved("not checked".to_string()), }); } preflight } +fn verify_submission_models(preflight: &mut SubmissionPreflight) { + let candidates = std::mem::take(&mut preflight.ready); + for mut report in candidates { + let verification = verify_submission_model(&report.value); + if let SubmissionModelVerification::Mismatch(reason) = verification { + preflight.invalid.push(InvalidSubmission { + label: submission_label(&report.value, &report.path), + path: report.path, + reason: format!("Model provenance mismatch: {reason}"), + }); + } else { + report.model_verification = verification; + preflight.ready.push(report); + } + } +} + fn print_submission_preflight(ui: TerminalUi, preflight: &SubmissionPreflight) { println!(); println!("{}", ui.brand_bold("Submission check complete")); @@ -430,6 +472,43 @@ fn print_submission_preflight(ui: TerminalUi, preflight: &SubmissionPreflight) { "Ready to submit", ui.success(preflight.ready.len()) ); + + let mut verified = 0; + let mut unresolved = 0; + let mut unavailable = 0; + for report in &preflight.ready { + match report.model_verification { + SubmissionModelVerification::Verified(_) => verified += 1, + SubmissionModelVerification::Unresolved(_) => unresolved += 1, + SubmissionModelVerification::Unavailable(_) => unavailable += 1, + SubmissionModelVerification::Mismatch(_) => unreachable!("mismatches are invalid"), + } + } + println!("\n{}", ui.brand_bold("Model identity")); + println!(" {:<22} {}", "Verified artifacts", ui.success(verified)); + println!(" {:<22} {}", "Unresolved", ui.neutral(unresolved)); + println!(" {:<22} {}", "Check unavailable", ui.warning(unavailable)); + for report in &preflight.ready { + let detail = match &report.model_verification { + SubmissionModelVerification::Verified(message) => { + Some((ui.success("✓"), message.as_str())) + } + SubmissionModelVerification::Unresolved(message) => { + Some((ui.neutral("—"), message.as_str())) + } + SubmissionModelVerification::Unavailable(message) => { + Some((ui.warning("!"), message.as_str())) + } + SubmissionModelVerification::Mismatch(_) => None, + }; + if let Some((marker, message)) = detail { + println!( + " {marker} {}", + submission_label(&report.value, &report.path) + ); + println!(" {}", ui.neutral(message)); + } + } println!( " {:<22} {} {}", "Invalid reports", From a4ef1d93a28ee6f9769a5755d9dd0e194fa9c342 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 18:17:56 +1000 Subject: [PATCH 04/10] feat(models): explain report trust boundaries --- crates/computearena-cli/src/benchmark.rs | 3 ++ crates/computearena-cli/src/main.rs | 2 + crates/computearena-cli/src/model_identity.rs | 44 +++++++++++++++++++ crates/computearena-cli/src/reports.rs | 3 ++ crates/computearena-cli/src/submission.rs | 2 + crates/computearena-cli/src/tui/app.rs | 1 + 6 files changed, 55 insertions(+) diff --git a/crates/computearena-cli/src/benchmark.rs b/crates/computearena-cli/src/benchmark.rs index bcff9eb..8dbd368 100644 --- a/crates/computearena-cli/src/benchmark.rs +++ b/crates/computearena-cli/src/benchmark.rs @@ -560,6 +560,7 @@ pub(crate) fn run_benchmark( let run_id = random_id(); let model_metadata = crate::model_identity::finalize(runtime, paths, model, result.model, &model_sha256); + let model_identity_notice = crate::model_identity::report_identity_notice(&model_metadata); // Intentionally omit the user's account and local model path: a benchmark // can be created offline and attached to an authenticated account later. @@ -594,6 +595,8 @@ pub(crate) fn run_benchmark( println!("Saved signed benchmark: {}", path.display()); println!("Run ID: {run_id}"); println!("Report SHA-256: {digest}"); + println!("{}", ui.neutral(model_identity_notice)); + println!("{}", ui.muted(crate::reports::SIGNATURE_SCOPE_NOTICE)); if runtime == Runtime::LlamaCpp { if let Err(error) = crate::recent_gguf::remember(paths, model) { eprintln!( diff --git a/crates/computearena-cli/src/main.rs b/crates/computearena-cli/src/main.rs index 18cde61..82d01cc 100644 --- a/crates/computearena-cli/src/main.rs +++ b/crates/computearena-cli/src/main.rs @@ -404,6 +404,7 @@ fn execute( finish_activity(ui, started, "Signature is valid"); println!("Report: {}", path.display()); println!("Installation key: {key_id}"); + println!("{}", ui.neutral(reports::SIGNATURE_SCOPE_NOTICE)); Ok(()) } Action::Identify { @@ -602,6 +603,7 @@ fn interactive( ); println!(" Installation key: {}", short_id(&key)); println!(" File: {}", path.display()); + println!(" {}", ui.neutral(reports::SIGNATURE_SCOPE_NOTICE)); } Err(error) => { eprintln!("\n{} {error:#}", ui.error("✗ Verification failed:")); diff --git a/crates/computearena-cli/src/model_identity.rs b/crates/computearena-cli/src/model_identity.rs index c7f15ef..5b8cca5 100644 --- a/crates/computearena-cli/src/model_identity.rs +++ b/crates/computearena-cli/src/model_identity.rs @@ -495,6 +495,27 @@ pub(crate) fn finalize( model } +pub(crate) fn report_identity_notice(model: &Value) -> String { + let canonical = model.pointer("/canonical/repo_id").and_then(Value::as_str); + let exact_artifact = [ + "/artifact/repo_id", + "/artifact/revision", + "/artifact/path", + "/artifact/sha256", + ] + .into_iter() + .all(|pointer| model.pointer(pointer).and_then(Value::as_str).is_some()); + + match (exact_artifact, canonical) { + (true, Some(canonical)) => format!( + "Model identity recorded as {}; the server will independently verify the exact artifact when submitted.", + canonical + ), + (true, None) => "The exact model artifact was recorded for server verification, but its canonical model family is unresolved.".to_string(), + (false, _) => "Model identity is unresolved. The signed report remains submittable and will be labelled unverified; use computearena identify to bind manually acquired bytes to an exact Hugging Face file.".to_string(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -572,4 +593,27 @@ mod tests { assert_eq!(model["provenance"]["method"], "local_file"); assert_eq!(model["artifact_sha256"], sha256); } + + #[test] + fn report_notice_never_claims_local_execution_attestation() { + let unresolved = json!({ + "artifact": {"sha256": "aa"}, + "canonical": null + }); + assert!(report_identity_notice(&unresolved).contains("unresolved")); + assert!(report_identity_notice(&unresolved).contains("unverified")); + + let resolvable = json!({ + "artifact": { + "repo_id": "basecompute/Qwen3-4B", + "revision": "0123456789abcdef", + "path": "model.base", + "sha256": "aa" + }, + "canonical": {"repo_id": "Qwen/Qwen3-4B"} + }); + let notice = report_identity_notice(&resolvable); + assert!(notice.contains("server will independently verify")); + assert!(!notice.contains("execution verified")); + } } diff --git a/crates/computearena-cli/src/reports.rs b/crates/computearena-cli/src/reports.rs index aa7f39b..087195d 100644 --- a/crates/computearena-cli/src/reports.rs +++ b/crates/computearena-cli/src/reports.rs @@ -13,6 +13,9 @@ use std::fs::{self, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +pub(crate) const SIGNATURE_SCOPE_NOTICE: &str = + "A valid report signature detects changes made after signing. It does not attest that a modified client honestly executed the benchmark."; + #[derive(Clone, Debug)] pub(crate) struct Paths { pub(crate) root: PathBuf, diff --git a/crates/computearena-cli/src/submission.rs b/crates/computearena-cli/src/submission.rs index e4c952a..920f1a0 100644 --- a/crates/computearena-cli/src/submission.rs +++ b/crates/computearena-cli/src/submission.rs @@ -528,6 +528,8 @@ fn print_submission_preflight(ui: TerminalUi, preflight: &SubmissionPreflight) { println!(" {}", ui.muted(invalid.path.display())); } } + println!(); + println!("{}", ui.neutral(crate::reports::SIGNATURE_SCOPE_NOTICE)); } fn submission_file_label(path: &Path) -> String { diff --git a/crates/computearena-cli/src/tui/app.rs b/crates/computearena-cli/src/tui/app.rs index afa0925..933f3c0 100644 --- a/crates/computearena-cli/src/tui/app.rs +++ b/crates/computearena-cli/src/tui/app.rs @@ -643,6 +643,7 @@ impl App { .and_then(Value::as_str) .unwrap_or("unknown"); println!("Run ID: {run_id}"); + println!("{}", crate::reports::SIGNATURE_SCOPE_NOTICE); Ok("Signature is valid".to_string()) }, )); From cf63021d54f3360d33fd059fc1c9835b9d029b34 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 19:21:40 +1000 Subject: [PATCH 05/10] feat(cli): negotiate supported client versions --- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 8 +- crates/computearena-cli/Cargo.toml | 1 + crates/computearena-cli/src/api.rs | 68 +++++++- crates/computearena-cli/src/auth.rs | 17 +- crates/computearena-cli/src/config.rs | 3 + crates/computearena-cli/src/main.rs | 4 + crates/computearena-cli/src/submission.rs | 6 +- crates/computearena-cli/src/tui/app.rs | 16 +- crates/computearena-cli/src/updates.rs | 184 ++++++++++++++++++++++ 11 files changed, 290 insertions(+), 19 deletions(-) create mode 100644 crates/computearena-cli/src/updates.rs diff --git a/Cargo.lock b/Cargo.lock index 14145a2..2c477bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,6 +342,7 @@ dependencies = [ "rand_core 0.6.4", "ratatui", "reqwest", + "semver", "serde_json", "sha2", "sysinfo", diff --git a/Cargo.toml b/Cargo.toml index dfbf149..c169df0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ flate2 = "1" rand_core = { version = "0.6", features = ["getrandom"] } ratatui = "0.30" reqwest = { version = "0.13", default-features = false, features = ["blocking", "rustls"] } +semver = "1" serde_json = { version = "1", features = ["preserve_order", "float_roundtrip"] } sha2 = { version = "0.10", features = ["asm"] } tar = "0.4" diff --git a/README.md b/README.md index ca5fde8..16ea892 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,13 @@ Submitting the same report again succeeds rather than failing. The client talks to `https://computearena.ai/api/v1`. `--api-url` or `COMPUTEARENA_API_URL` point it at another deployment, such as a local -development server; sessions are kept per URL. +development server; sessions are kept per URL. Interactive sessions check the latest +stable GitHub release in the background, cache successful checks for 24 hours, +and show a quickstart link only when a newer semantic version exists. Network +failures never block offline benchmarking. API calls include the client version +in both the user agent and x-computearena-client-version header; a server retiring +an old client should return HTTP 426 with client_upgrade_required and may include +minimumClientVersion for an actionable upgrade message. ## Configuration diff --git a/crates/computearena-cli/Cargo.toml b/crates/computearena-cli/Cargo.toml index e9afe28..d0c406b 100644 --- a/crates/computearena-cli/Cargo.toml +++ b/crates/computearena-cli/Cargo.toml @@ -21,6 +21,7 @@ flate2.workspace = true rand_core.workspace = true ratatui.workspace = true reqwest.workspace = true +semver.workspace = true serde_json.workspace = true sha2.workspace = true tar.workspace = true diff --git a/crates/computearena-cli/src/api.rs b/crates/computearena-cli/src/api.rs index 6b11757..5f94c4a 100644 --- a/crates/computearena-cli/src/api.rs +++ b/crates/computearena-cli/src/api.rs @@ -1,14 +1,23 @@ -use crate::config::HTTP_CONNECT_TIMEOUT; +use crate::config::{COMPUTEARENA_INSTALL_SCRIPT, HTTP_CONNECT_TIMEOUT}; use anyhow::{Context, Result}; use reqwest::blocking::Client; +use reqwest::header::{HeaderMap, HeaderValue}; use serde_json::Value; use std::time::Duration; +pub(crate) const CLIENT_VERSION_HEADER: &str = "x-computearena-client-version"; + pub(crate) fn client(request_timeout: Duration) -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + CLIENT_VERSION_HEADER, + HeaderValue::from_static(env!("CARGO_PKG_VERSION")), + ); Client::builder() .connect_timeout(HTTP_CONNECT_TIMEOUT) .timeout(request_timeout) .user_agent(format!("computearena/{}", env!("CARGO_PKG_VERSION"))) + .default_headers(headers) .build() .context("building ComputeArena HTTP client") } @@ -28,3 +37,60 @@ pub(crate) fn error_code(body: &str) -> Option { .as_str() .map(str::to_owned) } + +/// Turn a structured API failure into an actionable message. The server can +/// retire an old client by returning HTTP 426 (or the matching error code) +/// without making older clients fail with an unexplained generic status. +pub(crate) fn server_error(status: reqwest::StatusCode, body: &str) -> String { + let message = + error_message(body).unwrap_or_else(|| format!("server returned HTTP {}", status.as_u16())); + let upgrade_required = status == reqwest::StatusCode::UPGRADE_REQUIRED + || matches!( + error_code(body).as_deref(), + Some("client_upgrade_required" | "unsupported_client_version") + ); + if !upgrade_required { + return message; + } + let value = serde_json::from_str::(body).ok(); + let minimum = value.as_ref().and_then(|value| { + [ + "/error/details/minimumClientVersion", + "/error/minimumClientVersion", + "/minimumClientVersion", + ] + .into_iter() + .find_map(|pointer| value.pointer(pointer).and_then(Value::as_str)) + }); + let requirement = minimum + .map(|version| format!(" The server requires {version} or newer.")) + .unwrap_or_default(); + format!( + "{message}\nInstalled ComputeArena CLI: {}.{requirement}\nUpdate with: {COMPUTEARENA_INSTALL_SCRIPT}", + env!("CARGO_PKG_VERSION") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upgrade_errors_include_the_installed_and_required_versions() { + let body = r#"{"error":{"code":"client_upgrade_required","message":"This client is no longer supported","details":{"minimumClientVersion":"0.2.0"}}}"#; + let message = server_error(reqwest::StatusCode::UPGRADE_REQUIRED, body); + assert!(message.contains("This client is no longer supported")); + assert!(message.contains(env!("CARGO_PKG_VERSION"))); + assert!(message.contains("0.2.0 or newer")); + assert!(message.contains("computearena.ai/install.sh")); + } + + #[test] + fn ordinary_api_errors_are_unchanged() { + let body = r#"{"error":{"code":"bad_request","message":"Invalid report"}}"#; + assert_eq!( + server_error(reqwest::StatusCode::BAD_REQUEST, body), + "Invalid report" + ); + } +} diff --git a/crates/computearena-cli/src/auth.rs b/crates/computearena-cli/src/auth.rs index 439e2f9..47baaae 100644 --- a/crates/computearena-cli/src/auth.rs +++ b/crates/computearena-cli/src/auth.rs @@ -1,5 +1,5 @@ use crate::api::{ - client as api_client, error_code as api_error_code, error_message as api_error_message, + client as api_client, error_code as api_error_code, server_error as api_server_error, }; use crate::config::{ AUTH_HTTP_TIMEOUT, DEFAULT_API_URL, DEFAULT_DEVICE_AUTH_EXPIRES_SECS, @@ -71,11 +71,7 @@ pub(crate) fn login(paths: &Paths, api_url: &str) -> Result<()> { let status = response.status(); let body = response.text().unwrap_or_default(); if !status.is_success() { - bail!( - "{}", - api_error_message(&body) - .unwrap_or_else(|| format!("server returned HTTP {}", status.as_u16())) - ); + bail!("{}", api_server_error(status, &body)); } let device: Value = serde_json::from_str(&body).context("parsing login response")?; let device_code = required_json_string(&device, "deviceCode")?; @@ -132,11 +128,7 @@ pub(crate) fn login(paths: &Paths, api_url: &str) -> Result<()> { if api_error_code(&body).as_deref() == Some("authorization_pending") { continue; } - bail!( - "{}", - api_error_message(&body) - .unwrap_or_else(|| format!("server returned HTTP {}", status.as_u16())) - ); + bail!("{}", api_server_error(status, &body)); } bail!("login code expired; run `computearena login` again") } @@ -161,8 +153,7 @@ fn validate_api_session(api_url: &str, session: &ApiSession) -> Result bool { | reqwest::StatusCode::FORBIDDEN | reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS + | reqwest::StatusCode::UPGRADE_REQUIRED ) || status.is_server_error() } diff --git a/crates/computearena-cli/src/tui/app.rs b/crates/computearena-cli/src/tui/app.rs index c129a90..9bcc1e3 100644 --- a/crates/computearena-cli/src/tui/app.rs +++ b/crates/computearena-cli/src/tui/app.rs @@ -185,6 +185,7 @@ pub(crate) struct App { /// A freshly saved report waiting for the person to sign in before it is /// offered for submission. pub(crate) pending_submission: Option, + update_check: Option, } impl App { @@ -195,6 +196,7 @@ impl App { harness_override: Option, api_url: String, ) -> Result { + let (update_notice, update_check) = crate::updates::start(&paths); let mut app = Self { paths, api_url, @@ -206,10 +208,13 @@ impl App { job: None, pending: None, downloaded: None, - status: String::new(), + status: update_notice + .map(|notice| notice.message()) + .unwrap_or_default(), should_quit: false, completed_report: Arc::new(Mutex::new(None)), pending_submission: None, + update_check, }; app.refresh_account(); // The same rule as the printed session: only ask which runtime to use @@ -305,6 +310,14 @@ impl App { pub(crate) fn tick(&mut self) -> bool { let mut changed = false; + let update = self.update_check.as_ref().and_then(|check| check.poll()); + if let Some(update) = update { + self.update_check = None; + if let Some(notice) = update { + self.status = notice.message(); + changed = true; + } + } if let Some(pending) = self.pending.as_ref() { match pending.receiver.try_recv() { Ok(loaded) => { @@ -1290,6 +1303,7 @@ mod tests { should_quit: false, completed_report: Arc::new(Mutex::new(None)), pending_submission: None, + update_check: None, } } diff --git a/crates/computearena-cli/src/updates.rs b/crates/computearena-cli/src/updates.rs new file mode 100644 index 0000000..487a723 --- /dev/null +++ b/crates/computearena-cli/src/updates.rs @@ -0,0 +1,184 @@ +//! A quiet, cached update hint for the interactive client. +//! +//! The GitHub request runs on a worker thread and failures are deliberately +//! ignored: running and retaining benchmarks must keep working offline. A +//! successful result is cached for a day to avoid unnecessary API traffic. + +use crate::config::COMPUTEARENA_QUICKSTART; +use crate::reports::Paths; +use semver::Version; +use serde_json::{json, Value}; +use std::fs; +use std::sync::mpsc::{self, Receiver, TryRecvError}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const LATEST_RELEASE_API: &str = + "https://api.github.com/repos/basecompute/computearena-cli/releases/latest"; +const UPDATE_CACHE_FILE: &str = "update-check.json"; +const UPDATE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60); +const UPDATE_HTTP_TIMEOUT: Duration = Duration::from_secs(3); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct UpdateNotice { + latest: Version, +} + +impl UpdateNotice { + pub(crate) fn message(&self) -> String { + format!( + "ComputeArena {} is available · update at {}", + self.latest, COMPUTEARENA_QUICKSTART + ) + } +} + +#[derive(Clone, Debug)] +struct CachedRelease { + checked_at: u64, + version: Version, + release_url: String, +} + +pub(crate) struct UpdateCheck { + receiver: Receiver>, +} + +impl UpdateCheck { + /// `None` means still running; `Some(None)` means the check completed with + /// no update (including an offline/network failure). + pub(crate) fn poll(&self) -> Option> { + match self.receiver.try_recv() { + Ok(notice) => Some(notice), + Err(TryRecvError::Empty) => None, + Err(TryRecvError::Disconnected) => Some(None), + } + } +} + +pub(crate) fn start(paths: &Paths) -> (Option, Option) { + let file = paths.root.join(UPDATE_CACHE_FILE); + let cached = read_cache(&file); + let initial = cached.as_ref().and_then(update_notice); + if cached.as_ref().is_some_and(cache_is_fresh) { + return (initial, None); + } + + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + let result = fetch_latest(); + if let Some(release) = result.as_ref() { + let _ = write_cache(&file, release); + } + let _ = sender.send(result.as_ref().and_then(update_notice)); + }); + (initial, Some(UpdateCheck { receiver })) +} + +fn current_version() -> Option { + Version::parse(env!("CARGO_PKG_VERSION")).ok() +} + +fn update_notice(release: &CachedRelease) -> Option { + (release.version > current_version()?).then(|| UpdateNotice { + latest: release.version.clone(), + }) +} + +fn cache_is_fresh(release: &CachedRelease) -> bool { + unix_seconds().saturating_sub(release.checked_at) < UPDATE_CACHE_TTL.as_secs() +} + +fn fetch_latest() -> Option { + let client = reqwest::blocking::Client::builder() + .connect_timeout(UPDATE_HTTP_TIMEOUT) + .timeout(UPDATE_HTTP_TIMEOUT) + .user_agent(concat!("computearena-cli/", env!("CARGO_PKG_VERSION"))) + .build() + .ok()?; + let response = client + .get(LATEST_RELEASE_API) + .header("Accept", "application/vnd.github+json") + .send() + .ok()? + .error_for_status() + .ok()?; + let body = response.text().ok()?; + let value: Value = serde_json::from_str(&body).ok()?; + let tag = value.get("tag_name")?.as_str()?.trim_start_matches('v'); + Some(CachedRelease { + checked_at: unix_seconds(), + version: Version::parse(tag).ok()?, + release_url: value + .get("html_url") + .and_then(Value::as_str) + .unwrap_or(COMPUTEARENA_QUICKSTART) + .to_string(), + }) +} + +fn read_cache(file: &std::path::Path) -> Option { + let value: Value = serde_json::from_slice(&fs::read(file).ok()?).ok()?; + Some(CachedRelease { + checked_at: value.get("checkedAtUnixSeconds")?.as_u64()?, + version: Version::parse(value.get("latestVersion")?.as_str()?).ok()?, + release_url: value.get("releaseUrl")?.as_str()?.to_string(), + }) +} + +fn write_cache(file: &std::path::Path, release: &CachedRelease) -> anyhow::Result<()> { + if let Some(parent) = file.parent() { + fs::create_dir_all(parent)?; + } + fs::write( + file, + serde_json::to_vec_pretty(&json!({ + "checkedAtUnixSeconds": release.checked_at, + "latestVersion": release.version.to_string(), + "releaseUrl": release.release_url, + }))?, + )?; + Ok(()) +} + +fn unix_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn release(version: &str) -> CachedRelease { + CachedRelease { + checked_at: unix_seconds(), + version: Version::parse(version).unwrap(), + release_url: format!("https://example.test/v{version}"), + } + } + + #[test] + fn only_newer_semantic_versions_create_a_notice() { + let current = current_version().unwrap(); + assert!(update_notice(&release(¤t.to_string())).is_none()); + let newer = Version::new(current.major, current.minor, current.patch + 1); + assert_eq!( + update_notice(&release(&newer.to_string())).unwrap().latest, + newer + ); + } + + #[test] + fn cache_round_trips() { + let directory = tempfile::tempdir().unwrap(); + let file = directory.path().join(UPDATE_CACHE_FILE); + let expected = release("9.8.7"); + write_cache(&file, &expected).unwrap(); + let actual = read_cache(&file).unwrap(); + assert_eq!(actual.version, expected.version); + assert_eq!(actual.release_url, expected.release_url); + assert!(cache_is_fresh(&actual)); + } +} From 89309e0cabb8eb5c0815b9d7fe845da35574a522 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 19:17:04 +1000 Subject: [PATCH 06/10] fix(basert): explain Linux x86-64 availability --- README.md | 4 +- crates/computearena-cli/src/main.rs | 4 ++ crates/computearena-cli/src/runtimes.rs | 49 ++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ca5fde8..b3824cb 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,9 @@ copy installed the same way. Neither touches shell profiles. `--archive` unpacks a bundle you already have instead of contacting GitHub. Prebuilt runtime bundles exist for macOS arm64 and Linux arm64 with CUDA (BaseRT) and for macOS and Linux CPU or Metal builds (llama.cpp); other platforms and GPU -builds of llama.cpp are installed by hand. +builds of llama.cpp are installed by hand. ComputeArena warns Linux x86-64 +users before entering BaseRT because no prebuilt BaseRT runtime is currently +published for that architecture; a compatible manually built harness still works. ### BaseRT diff --git a/crates/computearena-cli/src/main.rs b/crates/computearena-cli/src/main.rs index b1229cc..bb22696 100644 --- a/crates/computearena-cli/src/main.rs +++ b/crates/computearena-cli/src/main.rs @@ -328,6 +328,10 @@ fn execute( harness: Option, api_url: &str, ) -> Result<()> { + if runtime == Runtime::Basert && matches!(&command, Action::Run { .. } | Action::Install { .. }) + { + runtimes::print_platform_notice(TerminalUi::detect(), runtime); + } match command { Action::Run { model, diff --git a/crates/computearena-cli/src/runtimes.rs b/crates/computearena-cli/src/runtimes.rs index c1fdb49..ab3b347 100644 --- a/crates/computearena-cli/src/runtimes.rs +++ b/crates/computearena-cli/src/runtimes.rs @@ -29,6 +29,27 @@ const USER_AGENT: &str = concat!("computearena-cli/", env!("CARGO_PKG_VERSION")) const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(900); const RELEASE_LIST_SIZE: usize = 20; +fn platform_notice_for(runtime: Runtime, os: &str, arch: &str) -> Option { + (runtime == Runtime::Basert && os == "linux" && arch == "x86_64").then(|| { + "BaseRT does not currently publish a prebuilt Linux x86-64 runtime. ComputeArena itself works on this architecture: choose llama.cpp, or pass a compatible basert-benchmark-harness that you built yourself." + .to_string() + }) +} + +/// A warning for platforms on which the ComputeArena client is published but +/// the selected runtime is not. This is advisory: a manually built compatible +/// harness remains usable. +pub(crate) fn platform_notice(runtime: Runtime) -> Option { + platform_notice_for(runtime, std::env::consts::OS, std::env::consts::ARCH) +} + +pub(crate) fn print_platform_notice(ui: TerminalUi, runtime: Runtime) { + if let Some(notice) = platform_notice(runtime) { + println!(); + println!("{} {}", ui.warning("!"), ui.neutral(notice)); + } +} + // --- Discovery -------------------------------------------------------------- #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -146,19 +167,33 @@ fn not_found_message(runtime: Runtime) -> String { .iter() .map(|directory| compact_path(directory)), ); + let install_hint = if platform_notice(runtime).is_some() { + "ComputeArena cannot install a prebuilt BaseRT runtime on this architecture.".to_string() + } else { + format!( + "Or run `computearena {} install` to let ComputeArena download it.", + adapter.name() + ) + }; format!( - "{} was not found; ComputeArena looked for {} on {}.\n{}\nOr run `computearena {} install` to let ComputeArena download it.", + "{} was not found; ComputeArena looked for {} on {}.\n{}\n{}", adapter.display_name(), adapter.binary_name(), looked.join(" and in "), manual_instructions(runtime).join("\n"), - adapter.name() + install_hint ) } /// How to obtain the runtime without ComputeArena's help. pub(crate) fn manual_instructions(runtime: Runtime) -> Vec { match runtime { + Runtime::Basert if platform_notice(runtime).is_some() => vec![ + platform_notice(runtime).unwrap(), + "Choose llama.cpp for a supported prebuilt runtime on Linux x86-64.".to_string(), + "If you built BaseRT yourself, pass --runtime-path /path/to/basert-benchmark-harness." + .to_string(), + ], Runtime::Basert => vec![ format!("Install BaseRT with the official installer: {BASERT_INSTALL_SCRIPT}"), "Restart your terminal afterwards so basert-benchmark-harness is on PATH,".to_string(), @@ -1010,6 +1045,16 @@ pub(crate) fn ensure_runtime( mod tests { use super::*; + #[test] + fn linux_x86_64_explains_that_basert_has_no_prebuilt_runtime() { + let notice = platform_notice_for(Runtime::Basert, "linux", "x86_64").unwrap(); + assert!(notice.contains("does not currently publish")); + assert!(notice.contains("llama.cpp")); + assert!(notice.contains("built yourself")); + assert!(platform_notice_for(Runtime::LlamaCpp, "linux", "x86_64").is_none()); + assert!(platform_notice_for(Runtime::Basert, "linux", "aarch64").is_none()); + } + fn llama_releases() -> Vec { vec![ json!({"tag_name":"v0.4.0","draft":false,"assets":[{"name":"nightly-tag.txt","browser_download_url":"https://x/nightly-tag.txt","size":7}]}), From da6e84771f30183703977314784de056ca7b0719 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 19:50:13 +1000 Subject: [PATCH 07/10] test(runtimes): account for unsupported BaseRT platforms --- crates/computearena-cli/src/runtimes.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/computearena-cli/src/runtimes.rs b/crates/computearena-cli/src/runtimes.rs index ab3b347..69c9d09 100644 --- a/crates/computearena-cli/src/runtimes.rs +++ b/crates/computearena-cli/src/runtimes.rs @@ -1245,7 +1245,12 @@ mod tests { #[test] fn manual_instructions_name_the_official_channels() { let basert = manual_instructions(Runtime::Basert).join("\n"); - assert!(basert.contains(BASERT_INSTALL_SCRIPT)); + if let Some(notice) = platform_notice(Runtime::Basert) { + assert!(basert.contains(¬ice)); + assert!(!basert.contains(BASERT_INSTALL_SCRIPT)); + } else { + assert!(basert.contains(BASERT_INSTALL_SCRIPT)); + } assert!(basert.contains("--runtime-path")); let llama = manual_instructions(Runtime::LlamaCpp).join("\n"); assert!(llama.contains("brew install llama.cpp")); From 90526db1eedb5e33fda0c021aa38961f9aeab465 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 19:53:47 +1000 Subject: [PATCH 08/10] test(runtime): expect Linux x86 BaseRT guidance --- crates/computearena-cli/tests/runtime_setup.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/computearena-cli/tests/runtime_setup.rs b/crates/computearena-cli/tests/runtime_setup.rs index 8ac9d15..d4faee2 100644 --- a/crates/computearena-cli/tests/runtime_setup.rs +++ b/crates/computearena-cli/tests/runtime_setup.rs @@ -203,7 +203,13 @@ fn back_from_a_missing_runtime_returns_to_the_runtime_choice() { assert!(output.status.success(), "{}", text(&output)); let text = text(&output); assert!(text.contains("BaseRT was not found")); - assert!(text.contains("curl -LsSf https://basecompute.co/install.sh | sh")); + if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + assert!(text.contains("does not currently publish a prebuilt Linux x86-64 runtime")); + assert!(text.contains("--runtime-path /path/to/basert-benchmark-harness")); + assert!(!text.contains("https://basecompute.co/install.sh")); + } else { + assert!(text.contains("curl -LsSf https://basecompute.co/install.sh | sh")); + } assert!(text.contains("basert-home") || text.contains("~/.basert")); assert_eq!(text.matches("Choose a runtime").count(), 2); } From 80fce6f57188779f4c8c394de55672c53242155b Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 20:18:19 +1000 Subject: [PATCH 09/10] fix(models): stop inventing artifact paths and let the server settle grouping BaseRT's hub.json names the repository the bytes came from and usually a mutable ref, never the file, so every model pulled by basert itself was submitted with its local file name as the Hugging Face path and the server rejected it. The sidecar now yields a hash-only claim, the repository listing is searched for the file whose published SHA-256 matches (at pull time and, for models installed before receipts existed, the first time they are benchmarked), and receipts are replaced when a later lookup or `computearena identify` learns more about the same bytes. A repository's base_model link now counts only when it declares a quantization relation; finetunes, adapters, and merges are their own model, so an instruct model no longer folds into its pretraining base. A model family that differs from what the publisher says today is reported, not treated as a mismatch, because the server decides the grouping. Hub revisions and paths are percent-encoded per segment, and GGUF metadata no longer promotes general.base_model to the model family. The full-screen interface submits with invalid reports skipped instead of refusing the whole batch, and says how many were left out. --- crates/computearena-cli/src/basert_models.rs | 77 ++++---- crates/computearena-cli/src/benchmark.rs | 19 ++ crates/computearena-cli/src/huggingface.rs | 169 +++++++++++++---- crates/computearena-cli/src/main.rs | 2 +- crates/computearena-cli/src/model_identity.rs | 171 ++++++++++++++---- crates/computearena-cli/src/submission.rs | 38 +++- crates/computearena-cli/src/tui/app.rs | 10 +- docs/model-identity.md | 32 +++- 8 files changed, 394 insertions(+), 124 deletions(-) diff --git a/crates/computearena-cli/src/basert_models.rs b/crates/computearena-cli/src/basert_models.rs index d3df0e4..513c13a 100644 --- a/crates/computearena-cli/src/basert_models.rs +++ b/crates/computearena-cli/src/basert_models.rs @@ -170,7 +170,26 @@ pub(crate) fn pull(cli: &Path, model: &RemoteModel) -> Result { .context("BaseRT completed the pull but did not list the installed model") } +/// Record the provenance of a model `basert pull` just installed. The pull +/// itself is done; failing to match the file on the Hub only means the report +/// will identify the bytes by hash instead of by an exact file. pub(crate) fn record_download(root: &Path, model: &Path) -> Result<()> { + let artifact_sha256 = crate::adapters::file_sha256(model)?; + match resolve_artifact(root, model, &artifact_sha256) { + Ok(file) => println!("Matched the installed model to {file} on Hugging Face."), + Err(error) => println!( + "Could not match the installed model to a Hugging Face file ({error:#}); its reports will identify it by hash." + ), + } + Ok(()) +} + +/// Bind an installed BaseRT model to the exact Hub file whose published +/// SHA-256 matches it, and save a receipt. BaseRT's `hub.json` names the +/// repository and usually a mutable ref, never the file, so the file name is +/// recovered from the repository listing rather than guessed from the local +/// path. Returns the matched `repository/path`. +pub(crate) fn resolve_artifact(root: &Path, model: &Path, artifact_sha256: &str) -> Result { let sidecar_path = model .parent() .context("BaseRT model has no variant directory")? @@ -184,60 +203,34 @@ pub(crate) fn record_download(root: &Path, model: &Path) -> Result<()> { .get("hf_repo") .and_then(Value::as_str) .context("BaseRT model provenance omitted its Hugging Face repository")?; - let artifact_sha256 = crate::adapters::file_sha256(model)?; - let expected_sha256 = sidecar.get("base_sha256").and_then(Value::as_str); - if expected_sha256.is_some_and(|expected| !expected.eq_ignore_ascii_case(&artifact_sha256)) { - bail!("the installed BaseRT model does not match its provenance SHA-256"); + if let Some(expected) = sidecar.get("base_sha256").and_then(Value::as_str) { + if !expected.eq_ignore_ascii_case(artifact_sha256) { + bail!( + "the installed file does not match the SHA-256 recorded in {}", + sidecar_path.display() + ); + } } - let sidecar_revision = sidecar + let revision = sidecar .get("revision") .and_then(Value::as_str) .unwrap_or("main"); - if let Ok(file) = - crate::huggingface::find_file_by_sha256(repository, sidecar_revision, &artifact_sha256) - { - let canonical = sidecar - .get("source_repo") - .and_then(Value::as_str) - .or(file.canonical_repository.as_deref()); - return crate::model_identity::record_huggingface_download( - root, - repository, - &file.revision, - &file.path, - &artifact_sha256, - file.sha256.as_deref(), - canonical, - "basert_pull", - ); - } - let identity = crate::huggingface::repository_identity(repository).ok(); - let revision = identity - .as_ref() - .map(|identity| identity.revision.as_str()) - .or(Some(sidecar_revision)) - .unwrap_or("unknown"); + let file = crate::huggingface::find_file_by_sha256(repository, revision, artifact_sha256)?; let canonical = sidecar .get("source_repo") .and_then(Value::as_str) - .or_else(|| { - identity - .as_ref() - .and_then(|identity| identity.canonical_repository.as_deref()) - }); + .or(file.canonical_repository.as_deref()); crate::model_identity::record_huggingface_download( root, repository, - revision, - model - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("model.base"), - &artifact_sha256, - expected_sha256, + &file.revision, + &file.path, + artifact_sha256, + file.sha256.as_deref(), canonical, "basert_pull", - ) + )?; + Ok(format!("{repository}/{}", file.path)) } #[cfg(test)] diff --git a/crates/computearena-cli/src/benchmark.rs b/crates/computearena-cli/src/benchmark.rs index 8dbd368..c0ee364 100644 --- a/crates/computearena-cli/src/benchmark.rs +++ b/crates/computearena-cli/src/benchmark.rs @@ -525,6 +525,25 @@ pub(crate) fn run_benchmark( start_activity(ui, "Hashing the model artifact (outside benchmark timing)…"); let model_sha256 = file_sha256(model)?; finish_activity(ui, hashing_started, "Model artifact fingerprint ready"); + // A BaseRT model installed before ComputeArena tracked provenance has a + // hub.json but no receipt. Match it now, ahead of the timed section, so + // the report can name the exact published file. + if runtime == Runtime::Basert + && !crate::model_identity::has_receipt(&paths.root, &model_sha256) + && model + .parent() + .is_some_and(|directory| directory.join("hub.json").is_file()) + { + let matching_started = start_activity(ui, "Matching the model to its Hugging Face file…"); + match crate::basert_models::resolve_artifact(&paths.root, model, &model_sha256) { + Ok(file) => finish_activity(ui, matching_started, format!("Model matched to {file}")), + Err(error) => finish_activity( + ui, + matching_started, + format!("Model will be identified by hash ({error:#})"), + ), + } + } let benchmark_started = start_activity( ui, format!("Running the benchmark with {}…", harness.display()), diff --git a/crates/computearena-cli/src/huggingface.rs b/crates/computearena-cli/src/huggingface.rs index 5ff3926..a0c24d1 100644 --- a/crates/computearena-cli/src/huggingface.rs +++ b/crates/computearena-cli/src/huggingface.rs @@ -5,16 +5,37 @@ //! the environment (`HF_TOKEN`), which gated repositories need. use crate::config::{HTTP_CONNECT_TIMEOUT, HUGGINGFACE_API, HUGGINGFACE_HOST}; use anyhow::{bail, Context, Result}; -use percent_encoding::percent_decode_str; +use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC}; use serde_json::Value; use sha2::{Digest, Sha256}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; const SEARCH_TIMEOUT: Duration = Duration::from_secs(20); + +/// Characters that stay literal inside one URL path segment. +const PATH_SEGMENT: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'_') + .remove(b'.') + .remove(b'~'); + +/// Percent-encode one path segment, so a branch name such as `refs/pr/1` +/// stays a single segment and no value can steer a request to another path. +fn encode_segment(value: &str) -> String { + utf8_percent_encode(value, PATH_SEGMENT).to_string() +} + +fn encode_path(value: &str) -> String { + value + .split('/') + .map(encode_segment) + .collect::>() + .join("/") +} const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(60 * 60); const SEARCH_RESULTS: usize = 25; @@ -108,42 +129,95 @@ pub(crate) fn search(query: &str) -> Result> { .collect()) } -/// The GGUF files inside one repository, smallest first: quantizations are -/// usually chosen by the size a device can hold. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum BaseModelRelation { + Quantized, + Finetune, + Adapter, + Merge, + Unknown, +} + +fn relation_of(value: &str) -> Option { + match value { + "quantized" => Some(BaseModelRelation::Quantized), + "finetune" => Some(BaseModelRelation::Finetune), + "adapter" => Some(BaseModelRelation::Adapter), + "merge" => Some(BaseModelRelation::Merge), + _ => None, + } +} + +fn is_repository_id(value: &str) -> bool { + let mut parts = value.split('/'); + match (parts.next(), parts.next(), parts.next()) { + (Some(owner), Some(model), None) => [owner, model].iter().all(|part| { + !part.is_empty() + && part + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + }), + _ => false, + } +} + +/// The single upstream model a repository declares itself a quantization of. +/// +/// Only a `quantized` relation is followed. A finetune, adapter, or merge is a +/// different model, and an instruct model's card links to its pretraining +/// base exactly that way. The server applies the same rule together with its +/// own publisher trust list, so this is the client's best claim, not the +/// verdict. fn canonical_repository(info: &Value) -> Option { - fn add(repositories: &mut BTreeSet, value: &str) { - let value = value - .strip_prefix("quantized:") - .or_else(|| value.strip_prefix("finetune:")) - .or_else(|| value.strip_prefix("adapter:")) - .unwrap_or(value); - if value.split('/').filter(|part| !part.is_empty()).count() == 2 { - repositories.insert(value.to_string()); + fn add( + links: &mut BTreeMap, + declared: Option, + raw: &str, + ) { + let (relation, repository) = match raw.split_once(':') { + Some((prefix, rest)) if relation_of(prefix).is_some() => (relation_of(prefix), rest), + _ => (None, raw), + }; + if !is_repository_id(repository) { + return; + } + let relation = relation.or(declared).unwrap_or(BaseModelRelation::Unknown); + let entry = links + .entry(repository.to_ascii_lowercase()) + .or_insert((repository.to_string(), relation)); + if entry.1 == BaseModelRelation::Unknown { + entry.1 = relation; } } - let mut repositories = BTreeSet::new(); + let declared = info + .pointer("/cardData/base_model_relation") + .and_then(Value::as_str) + .and_then(relation_of); + let mut links = BTreeMap::new(); match info.pointer("/cardData/base_model") { - Some(Value::String(value)) => add(&mut repositories, value), + Some(Value::String(value)) => add(&mut links, declared, value), Some(Value::Array(values)) => { for value in values.iter().filter_map(Value::as_str) { - add(&mut repositories, value); + add(&mut links, declared, value); } } _ => {} } - if repositories.is_empty() { - for tag in info - .get("tags") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .filter_map(|tag| tag.strip_prefix("base_model:")) - { - add(&mut repositories, tag); - } + for tag in info + .get("tags") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter_map(|tag| tag.strip_prefix("base_model:")) + { + add(&mut links, declared, tag); } - (repositories.len() == 1).then(|| repositories.into_iter().next().unwrap()) + if links.len() != 1 { + return None; + } + let (repository, relation) = links.into_values().next()?; + (relation == BaseModelRelation::Quantized).then_some(repository) } fn repository_info(repository: &str) -> Result { @@ -157,7 +231,10 @@ fn repository_info(repository: &str) -> Result { } fn repository_info_at(repository: &str, revision: &str) -> Result { - let url = format!("{HUGGINGFACE_API}/models/{repository}/revision/{revision}"); + let url = format!( + "{HUGGINGFACE_API}/models/{repository}/revision/{}", + encode_segment(revision) + ); serde_json::from_str( &request(&url, SEARCH_TIMEOUT)? .text() @@ -243,7 +320,8 @@ fn next_page(response: &reqwest::blocking::Response) -> Option { fn repository_tree(repository: &str, revision: &str) -> Result> { let mut url = Some(format!( - "{HUGGINGFACE_API}/models/{repository}/tree/{revision}?recursive=true&expand=true" + "{HUGGINGFACE_API}/models/{repository}/tree/{}?recursive=true&expand=true", + encode_segment(revision) )); let mut entries = Vec::new(); let mut seen = BTreeSet::new(); @@ -305,7 +383,7 @@ pub(crate) fn artifact_identity( let identity = repository_identity_at(repository, requested_revision)?; let info_url = format!( "{HUGGINGFACE_API}/models/{repository}/paths-info/{}", - identity.revision + encode_segment(&identity.revision) ); let mut builder = client(SEARCH_TIMEOUT)? .post(&info_url) @@ -492,7 +570,8 @@ pub(crate) fn download(root: &Path, repository: &str, file: &HubFile) -> Result< let url = format!( "{HUGGINGFACE_HOST}/{repository}/resolve/{}/{}?download=true", - file.revision, file.path + encode_segment(&file.revision), + encode_path(&file.path) ); println!("Downloading {} ({})", file.path, format_size(file.size)); let mut response = request(&url, DOWNLOAD_TIMEOUT)?; @@ -646,4 +725,32 @@ mod tests { }); assert_eq!(canonical_repository(&info), None); } + + #[test] + fn only_quantizations_inherit_the_upstream_model() { + let instruct = serde_json::json!({ + "cardData": {"base_model": "google/gemma-3-1b-pt"}, + "tags": ["base_model:google/gemma-3-1b-pt", "base_model:finetune:google/gemma-3-1b-pt"] + }); + assert_eq!(canonical_repository(&instruct), None); + let declared = serde_json::json!({ + "cardData": {"base_model": "Qwen/Qwen3-4B", "base_model_relation": "quantized"} + }); + assert_eq!( + canonical_repository(&declared).as_deref(), + Some("Qwen/Qwen3-4B") + ); + let undeclared = serde_json::json!({"cardData": {"base_model": "Qwen/Qwen3-4B"}}); + assert_eq!(canonical_repository(&undeclared), None); + } + + #[test] + fn revisions_and_paths_are_encoded_per_segment() { + assert_eq!(encode_segment("refs/pr/1"), "refs%2Fpr%2F1"); + assert_eq!(encode_segment("../x"), "..%2Fx"); + assert_eq!( + encode_path("Q4_K_M/model one.gguf"), + "Q4_K_M/model%20one.gguf" + ); + } } diff --git a/crates/computearena-cli/src/main.rs b/crates/computearena-cli/src/main.rs index 82d01cc..1cb4b90 100644 --- a/crates/computearena-cli/src/main.rs +++ b/crates/computearena-cli/src/main.rs @@ -431,7 +431,7 @@ fn execute( skip_invalid, } => { let reports = select_reports_for_submission(paths, &reports, TerminalUi::detect())?; - submit_reports(paths, &reports, api_url, yes, skip_invalid) + submit_reports(paths, &reports, api_url, yes, skip_invalid).map(|_| ()) } } } diff --git a/crates/computearena-cli/src/model_identity.rs b/crates/computearena-cli/src/model_identity.rs index 5b8cca5..b7bee5d 100644 --- a/crates/computearena-cli/src/model_identity.rs +++ b/crates/computearena-cli/src/model_identity.rs @@ -6,9 +6,11 @@ //! pretending that unlike quantizations are the same artifact. use crate::adapters::Runtime; -use crate::reports::{atomic_write_json, Paths}; +use crate::reports::Paths; +use anyhow::Context; use serde_json::{json, Value}; use std::fs; +use std::io::Write; use std::path::{Component, Path, PathBuf}; pub(crate) const IDENTITY_SCHEMA: &str = "computearena-model/1"; @@ -58,7 +60,7 @@ pub(crate) fn verify_submission_model(report: &Value) -> SubmissionModelVerifica ); let (Some("huggingface"), Some(repository), Some(revision), Some(path)) = tuple else { return SubmissionModelVerification::Unresolved( - "exact Hugging Face repository, revision, and path are unavailable".to_string(), + "no exact Hugging Face file is recorded; the server will match the hash against artifacts it has verified".to_string(), ); }; @@ -81,25 +83,30 @@ pub(crate) fn verify_submission_model(report: &Value) -> SubmissionModelVerifica ); } + // The server decides the model family from what the publisher says + // now; a changed card is a grouping question, not a reason to refuse. let claimed_canonical = report .pointer("/model/canonical/repo_id") .and_then(Value::as_str); - if let (Some(claimed), Some(published)) = ( - claimed_canonical, - identity.file.canonical_repository.as_deref(), - ) { - if !claimed.eq_ignore_ascii_case(published) { - return SubmissionModelVerification::Mismatch( - "canonical model does not match the Hugging Face repository metadata".to_string(), - ); + let published = identity.file.canonical_repository.as_deref(); + SubmissionModelVerification::Verified(match (claimed_canonical, published) { + (Some(claimed), Some(published)) if !claimed.eq_ignore_ascii_case(published) => { + format!( + "artifact hash verified; the publisher now links it to {published} rather than {claimed}, and ComputeArena decides the grouping" + ) } - } - SubmissionModelVerification::Verified(match identity.file.canonical_repository { - Some(canonical) => format!("artifact hash and model family verified as {canonical}"), - None => "artifact hash verified; model family remains unresolved".to_string(), + (_, Some(published)) => format!("artifact hash and model family verified as {published}"), + (Some(claimed), None) => { + format!("artifact hash verified; ComputeArena decides whether it groups as {claimed}") + } + (None, None) => "artifact hash verified; model family remains unresolved".to_string(), }) } +pub(crate) fn has_receipt(root: &Path, artifact_sha256: &str) -> bool { + receipt_path(root, artifact_sha256).is_file() +} + fn receipts_dir(root: &Path) -> PathBuf { root.join("model-provenance") } @@ -153,15 +160,32 @@ pub(crate) fn record_huggingface_download( let path = receipt_path(root, artifact_sha256); if path.is_file() { let existing: Value = serde_json::from_slice(&fs::read(&path)?)?; - if existing.pointer("/artifact/sha256").and_then(Value::as_str) == Some(artifact_sha256) { + if existing.pointer("/artifact/sha256").and_then(Value::as_str) != Some(artifact_sha256) { + anyhow::bail!( + "model provenance for {artifact_sha256} conflicts with {}", + path.display() + ); + } + if existing == receipt { return Ok(()); } - anyhow::bail!( - "model provenance for {artifact_sha256} conflicts with {}", - path.display() - ); } - atomic_write_json(&path, &receipt) + write_receipt(&path, &receipt) +} + +/// A receipt describes one exact set of bytes, so a newer receipt for the +/// same SHA-256 replaces the older one: `computearena identify` and later +/// downloads can correct the repository, revision, or model family recorded. +fn write_receipt(path: &Path, receipt: &Value) -> anyhow::Result<()> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).with_context(|| format!("creating {}", parent.display()))?; + let mut temp = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("creating a receipt under {}", parent.display()))?; + serde_json::to_writer_pretty(&mut temp, receipt)?; + temp.write_all(b"\n")?; + temp.as_file().sync_all()?; + temp.persist(path).map_err(|error| error.error)?; + Ok(()) } /// Bind manually acquired bytes to one exact Hugging Face file. The caller @@ -290,6 +314,11 @@ fn huggingface_repo_id(url: &str) -> Option { Some(format!("{owner}/{model}")) } +/// BaseRT's `hub.json` names the repository the bytes came from and, usually, +/// a mutable ref, but never the file. Without a receipt from the listing +/// lookup the claim stays hash-only: the server matches the hash against the +/// artifacts it has already verified rather than being told a file name that +/// was never checked. fn base_sidecar(path: &Path, artifact_sha256: &str) -> Option { let bytes = fs::read(path.parent()?.join("hub.json")).ok()?; let sidecar: Value = serde_json::from_slice(&bytes).ok()?; @@ -301,17 +330,13 @@ fn base_sidecar(path: &Path, artifact_sha256: &str) -> Option { return None; } let repository = sidecar.get("hf_repo").and_then(Value::as_str)?; - let revision = sidecar - .get("revision") - .and_then(Value::as_str) - .unwrap_or("unknown"); let canonical = sidecar.get("source_repo").and_then(Value::as_str); Some(json!({ "artifact": { "provider": "huggingface", "repo_id": repository, - "revision": revision, - "path": path.file_name().and_then(|name| name.to_str()), + "revision": Value::Null, + "path": Value::Null, "sha256": artifact_sha256 }, "canonical": canonical.map(|repo_id| json!({ @@ -341,16 +366,13 @@ fn embedded_gguf_identity(model: &Value, artifact_sha256: &str) -> Option .get("repo_url") .and_then(Value::as_str) .and_then(huggingface_repo_id); + // `general.base_model` names what the source model was trained from (an + // instruct model's pretraining base, say), not the model itself, so only + // the source repository can stand for the model family. let canonical = model .get("source_repo_url") .and_then(Value::as_str) - .and_then(huggingface_repo_id) - .or_else(|| { - model - .get("base_model_repo_url") - .and_then(Value::as_str) - .and_then(huggingface_repo_id) - }); + .and_then(huggingface_repo_id); if artifact_repository.is_none() && canonical.is_none() { return None; } @@ -506,13 +528,18 @@ pub(crate) fn report_identity_notice(model: &Value) -> String { .into_iter() .all(|pointer| model.pointer(pointer).and_then(Value::as_str).is_some()); - match (exact_artifact, canonical) { - (true, Some(canonical)) => format!( + let repository = model.pointer("/artifact/repo_id").and_then(Value::as_str); + + match (exact_artifact, canonical, repository) { + (true, Some(canonical), _) => format!( "Model identity recorded as {}; the server will independently verify the exact artifact when submitted.", canonical ), - (true, None) => "The exact model artifact was recorded for server verification, but its canonical model family is unresolved.".to_string(), - (false, _) => "Model identity is unresolved. The signed report remains submittable and will be labelled unverified; use computearena identify to bind manually acquired bytes to an exact Hugging Face file.".to_string(), + (true, None, _) => "The exact model artifact was recorded for server verification, but its canonical model family is unresolved.".to_string(), + (false, _, Some(repository)) => format!( + "The model bytes were recorded from {repository} without an exact file; the server will match their hash against artifacts it has already verified." + ), + (false, _, None) => "Model identity is unresolved. The signed report remains submittable and will be labelled unverified; use computearena identify to bind manually acquired bytes to an exact Hugging Face file.".to_string(), } } @@ -582,7 +609,7 @@ mod tests { let model = finalize( Runtime::Basert, &paths, - Path::new("/tmp/unknown.base"), + &temporary.path().join("unknown.base"), json!({"name": "unknown", "quantization": "base_q4"}), &sha256, ); @@ -594,6 +621,74 @@ mod tests { assert_eq!(model["artifact_sha256"], sha256); } + #[test] + fn basert_sidecars_record_the_repository_but_never_invent_a_file() { + let temporary = tempfile::tempdir().unwrap(); + let paths = Paths::resolve(Some(temporary.path().join("data"))).unwrap(); + let sha256 = "c".repeat(64); + let variant = temporary.path().join("Qwen3-4B").join("default-q4"); + fs::create_dir_all(&variant).unwrap(); + fs::write( + variant.join("hub.json"), + serde_json::to_vec(&json!({ + "hf_repo": "basecompute/Qwen3-4B", + "source_repo": "Qwen/Qwen3-4B", + "revision": "main", + "base_sha256": sha256 + })) + .unwrap(), + ) + .unwrap(); + let model = finalize( + Runtime::Basert, + &paths, + &variant.join("model.base"), + json!({"name": "Qwen/Qwen3-4B", "quantization": "base_q4"}), + &sha256, + ); + + assert_eq!(model["artifact"]["repo_id"], "basecompute/Qwen3-4B"); + assert!(model["artifact"]["path"].is_null()); + assert!(model["artifact"]["revision"].is_null()); + assert_eq!(model["canonical"]["repo_id"], "Qwen/Qwen3-4B"); + assert_eq!(model["provenance"]["method"], "basert_hub_sidecar"); + assert!(report_identity_notice(&model).contains("without an exact file")); + } + + #[test] + fn receipts_are_refreshed_when_better_provenance_arrives() { + let temporary = tempfile::tempdir().unwrap(); + let paths = Paths::resolve(Some(temporary.path().to_path_buf())).unwrap(); + let sha256 = "d".repeat(64); + for canonical in [None, Some("Qwen/Qwen3-4B")] { + record_huggingface_download( + &paths.root, + "basecompute/Qwen3-4B", + "0123456789abcdef", + "Qwen3-4B-Q4.base", + &sha256, + Some(&sha256), + canonical, + "basert_pull", + ) + .unwrap(); + } + let receipt = read_receipt(&paths, &sha256).unwrap(); + assert_eq!(receipt["canonical"]["repo_id"], "Qwen/Qwen3-4B"); + assert!(has_receipt(&paths.root, &sha256)); + assert!(record_huggingface_download( + &paths.root, + "other/repo", + "abc", + "x.base", + &"e".repeat(64), + None, + None, + "basert_pull" + ) + .is_ok()); + } + #[test] fn report_notice_never_claims_local_execution_attestation() { let unresolved = json!({ diff --git a/crates/computearena-cli/src/submission.rs b/crates/computearena-cli/src/submission.rs index 920f1a0..a39b7af 100644 --- a/crates/computearena-cli/src/submission.rs +++ b/crates/computearena-cli/src/submission.rs @@ -51,6 +51,28 @@ struct SubmissionOutcome { kind: SubmissionOutcomeKind, detail: Option, } + +/// What a submission run did, for callers that summarise it in one line. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct SubmissionSummary { + pub(crate) submitted: usize, + pub(crate) duplicates: usize, + /// Reports left out because they failed the preflight checks. + pub(crate) skipped: usize, +} + +impl SubmissionSummary { + pub(crate) fn describe(&self) -> String { + let mut parts = vec![format!("Submitted {} benchmark(s)", self.submitted)]; + if self.duplicates > 0 { + parts.push(format!("{} already present", self.duplicates)); + } + if self.skipped > 0 { + parts.push(format!("{} skipped as invalid", self.skipped)); + } + parts.join("; ") + } +} pub(crate) fn select_reports_for_submission( paths: &Paths, requested: &[String], @@ -179,9 +201,9 @@ pub(crate) fn submit_reports( api_url: &str, assume_yes: bool, skip_invalid: bool, -) -> Result<()> { +) -> Result { if reports.is_empty() { - return Ok(()); + return Ok(SubmissionSummary::default()); } let ui = TerminalUi::detect(); let checking_started = start_activity(ui, "Checking selected benchmarks…"); @@ -243,13 +265,17 @@ pub(crate) fn submit_reports( "{} Nothing was uploaded.", ui.neutral("Submission cancelled.") ); - return Ok(()); + return Ok(SubmissionSummary { + skipped: preflight.invalid.len(), + ..SubmissionSummary::default() + }); } } let endpoint = format!("{api_url}/submissions"); let client = api_client(SUBMISSION_HTTP_TIMEOUT)?; let report_count = preflight.ready.len(); + let skipped = preflight.invalid.len(); let mut outcomes = Vec::with_capacity(report_count); let mut queue = preflight.ready.into_iter().enumerate(); while let Some((index, report)) = queue.next() { @@ -401,7 +427,11 @@ pub(crate) fn submit_reports( "{} Submission complete: {submitted} uploaded, {duplicates} already present.", ui.success("✓"), ); - Ok(()) + Ok(SubmissionSummary { + submitted, + duplicates, + skipped, + }) } pub(crate) fn preflight_submissions(reports: &[PathBuf]) -> SubmissionPreflight { diff --git a/crates/computearena-cli/src/tui/app.rs b/crates/computearena-cli/src/tui/app.rs index 933f3c0..f5bbfac 100644 --- a/crates/computearena-cli/src/tui/app.rs +++ b/crates/computearena-cli/src/tui/app.rs @@ -547,8 +547,14 @@ impl App { JobKind::Submit, format!("Submitting {count} benchmark(s)"), move || { - submit_reports(&paths, &reports, &api_url, true, false)?; - Ok(format!("Submitted {count} benchmark(s)")) + // A report that fails its checks is listed and left out; it + // must not stop the others from being uploaded. + let summary = submit_reports(&paths, &reports, &api_url, true, true)?; + Ok(if summary.skipped == 0 && summary.duplicates == 0 { + format!("Submitted {count} benchmark(s)") + } else { + summary.describe() + }) }, )); self.screens.push(Screen::Running); diff --git a/docs/model-identity.md b/docs/model-identity.md index cbfca3e..1526591 100644 --- a/docs/model-identity.md +++ b/docs/model-identity.md @@ -22,11 +22,17 @@ hashes it while streaming, and compares it with the Hub's LFS object ID. A content-addressed receipt is saved under `model-provenance/`. For BaseRT, ComputeArena delegates to `basert pull`. It then hashes the installed -`.base` file and records BaseRT's adjacent `hub.json` fields. When online, it -also resolves the artifact repository's Hugging Face metadata to recover the -canonical upstream model. Multiple internal catalogue variants that map to one -`basert pull --target` choice are shown once; BaseRT selects the compatible -artifact for the current backend. +`.base` file, reads BaseRT's adjacent `hub.json` (the artifact repository, the +source repository, and usually a mutable ref, but never a file name), and looks +the hash up in the repository's file listing to recover the exact published +file, saving a receipt when it finds one. A model installed before ComputeArena +tracked provenance is matched the same way the first time it is benchmarked. +When no file matches (offline, or a locally converted artifact that was never +published), the report carries the repository and the hash but no file name; +the server then matches the hash against artifacts it has already verified +instead of being told a path that was never checked. Multiple internal +catalogue variants that map to one `basert pull --target` choice are shown +once; BaseRT selects the compatible artifact for the current backend. ## Models acquired elsewhere @@ -41,7 +47,21 @@ computearena identify /path/to/model.base \ The command resolves the revision, fetches only file metadata, hashes the local file, and saves a receipt only if the SHA-256 values match. A repository URL or -free-form model name is intentionally insufficient. +free-form model name is intentionally insufficient. Receipts are keyed by +SHA-256 and replaced when a later download or `identify` learns more about the +same bytes, so a corrected repository or model family takes effect on the next +benchmark. + +## Model families + +A repository's card is followed to an upstream model only when it declares +itself a quantization of that model (the Hub's `base_model:quantized:` +relation). Finetunes, adapters, and merges are different models, so an +instruct model stays distinct from the pretraining base its card links to. The +server applies the same rule and additionally follows a link only from +publishers it trusts to quantize or convert models; its decision wins, and a +difference between the CLI's recorded family and the server's grouping never +blocks a submission. ## Trust boundary From f2ec60c74244e189c1bdaa76d3107650616d10b4 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 11 Sep 2026 20:34:24 +1000 Subject: [PATCH 10/10] docs: add v0.1.1 release notes --- docs/release-notes/0.1.1.md | 109 ++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/release-notes/0.1.1.md diff --git a/docs/release-notes/0.1.1.md b/docs/release-notes/0.1.1.md new file mode 100644 index 0000000..b1190dd --- /dev/null +++ b/docs/release-notes/0.1.1.md @@ -0,0 +1,109 @@ +# ComputeArena CLI 0.1.1 + +This release makes model identity verifiable. Reports now state exactly which +published model file was benchmarked, the client checks that claim against +Hugging Face before uploading, and computearena.ai verifies it again on its +own. It also adds update notices, server-driven upgrade guidance, and clearer +BaseRT support on Linux x86-64. + +## Install or upgrade + +On macOS or Linux: + +```sh +curl -LsSf https://computearena.ai/install.sh | sh +``` + +The same command upgrades an existing installation. Saved reports, the +installation signing key, login sessions, and installed runtimes are kept, and +the data directory gains a `model-provenance/` folder for the receipts +described below. Release archives and checksums are also available below for +manual installation. + +## Highlights + +- **Verifiable model identity.** Every new report carries a + `computearena-model/1` identity: the upstream Hugging Face model, the exact + artifact (repository, immutable revision, file path, format, and + runtime-specific quantization), the SHA-256 of the bytes that were + benchmarked, and how they were obtained. The fields from 0.1.0 remain + populated, and reports saved by earlier releases still submit. +- **Managed downloads leave a receipt.** GGUF downloads are pinned to an + immutable revision, hashed while streaming, and checked against the Hub's + published SHA-256. BaseRT models pulled through the picker are matched to + their exact published file by hash. Models installed before this release are + matched the same way the first time they are benchmarked; when no published + file matches, the report records the repository and the hash rather than a + guessed file name. +- **BaseRT catalogue in the model picker.** The BaseRT picker lists BaseRT's + public catalogue and runs `basert pull` with a backend-aware target, so + BaseRT keeps choosing the compatible artifact, downloading split files, and + converting. Existing `.base` and `.gguf` files can still be entered directly. +- **`computearena identify`.** Bind a file you downloaded or copied yourself to + one exact Hub file: `computearena identify ./model.gguf ` + succeeds only when the local SHA-256 matches the published object. A + repository URL or a model name is not enough, and a later `identify` can + correct an earlier receipt. +- **Checks before submission.** Before uploading, the client checks each + report's artifact against Hugging Face and lists it as verified, unresolved, + or check unavailable. A file whose hash contradicts the claimed published + file is set aside as invalid; in the full-screen interface the remaining + reports are still uploaded, and the summary says how many were left out. + When computearena.ai has verified an artifact, its outcome is shown after + the upload. +- **Model families.** A repository's `base_model` link counts only when the + repository declares itself a quantization of that model. Finetunes, + adapters, and merges stay their own model, so an instruct model is not folded + into its pretraining base. computearena.ai decides the final grouping, and a + difference between the client's record and that grouping never blocks a + submission. +- **Update notices and upgrade guidance.** Interactive sessions check for the + latest release in the background, keep the answer for a day, and mention a + newer version when one exists; the check never blocks offline use. API + requests carry the client version, and when the service retires a release + the client prints the installed version, the minimum required, and the + install command, then stops the submission batch. +- **BaseRT on Linux x86-64.** ComputeArena warns before entering BaseRT on + Linux x86-64, where no prebuilt BaseRT runtime is published, and points to + llama.cpp or a compatible harness you built yourself instead of an installer + that cannot help there. + +## Supported systems + +Prebuilt ComputeArena binaries are available for: + +- macOS on Apple Silicon; +- Linux x86_64 with glibc 2.31 or newer; and +- Linux arm64 with glibc 2.31 or newer. + +Windows and Intel Macs are not supported in this release. ComputeArena does not +bundle an inference runtime or model; install BaseRT or llama.cpp and obtain a +compatible model before benchmarking. Prebuilt BaseRT runtimes exist for macOS +on Apple Silicon and for Linux arm64 with CUDA; on Linux x86-64, use llama.cpp +or a BaseRT benchmark harness you built yourself. + +The macOS binary is not signed with an Apple Developer ID. The release archive +is checksum-verified and carries a Sigstore keyless signature from the release +workflow. Browser-downloaded copies may need their quarantine attribute removed +as described in the README. + +## Model identity and trust + +A verified artifact means the bytes that were benchmarked are a file published +on Hugging Face, and the model family follows what that file's publisher +declares. It does not prove that the publisher labelled the model correctly, +that the runtime executed those bytes honestly, or that two fine-tunes with +different lineage are equivalent. The client's checks are advisory; +computearena.ai repeats them independently, because a public client is not a +trust boundary. Files with no evidence, gated repositories the service cannot +read, and ambiguous merges stay unresolved rather than being grouped by file +name. Set `HF_TOKEN` in your environment to download from gated repositories. + +Accuracy benchmarks, MLX, vLLM, Windows, and Intel macOS support are not part +of v0.1.1. + +For usage and verification details, see the +[README](https://github.com/basecompute/computearena-cli#readme) and +[docs/model-identity.md](https://github.com/basecompute/computearena-cli/blob/main/docs/model-identity.md). +Questions and feedback are welcome in the +[ComputeArena Discord](https://discord.gg/CCT24GWhPG).