Rc 0.1.2

.github/scripts/smoke_test.sh

@@ -67,27 +67,50 @@ cli list >/dev/null || fail "list failed on an empty data directory"
 
 # 4. Offline benchmark round-trip against a stub llama-bench. The stub answers
 #    --help with the feature flags the adapter probes for and otherwise prints
-#    canned rows for the requested pp/tg sweep, exactly as tests/runtime_flow.rs
-#    does. The "model" is a GGUF header: magic, version 3, zero counts.
+#    one row per requested workload. Decode starts with one seed token; the
+#    headline PP512/TG128 pair must run before the remaining PP128 sweep.
+#    The "model" is a GGUF header: magic, version 3, zero counts.
 MODEL="$WORK/Qwen3-4B.gguf"
 { printf 'GGUF\003'; head -c 19 /dev/zero; } > "$MODEL"
-ROWS="$(printf '[{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"%s","model_size":24,"model_n_params":4000000000,"n_prompt":128,"n_gen":0,"n_depth":0,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]},{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"%s","model_size":24,"model_n_params":4000000000,"n_prompt":512,"n_gen":0,"n_depth":0,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]},{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"%s","model_size":24,"model_n_params":4000000000,"n_prompt":0,"n_gen":128,"n_depth":0,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]}]' "$MODEL" "$MODEL" "$MODEL")"
 STUB="$WORK/llama-bench"
-{
-  printf '#!/bin/sh\n'
-  printf 'if [ "$1" = --help ]; then\n'
-  printf "  printf '%%s\\\\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json'\n"
-  printf 'else\n'
-  printf "  cat <<'JSON'\n%s\nJSON\n" "$ROWS"
-  printf 'fi\n'
-} > "$STUB"
+cat > "$STUB" <<'SH'
+#!/bin/sh
+set -eu
+if [ "${1:-}" = --help ]; then
+  printf '%s\n' '--n-prompt --n-gen --n-depth --repetitions --no-warmup json'
+  exit 0
+fi
+pp= tg= depth= reps= format= model=
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -m) model="$2"; shift 2 ;;
+    -p) pp="$2"; shift 2 ;;
+    -n) tg="$2"; shift 2 ;;
+    -d) depth="$2"; shift 2 ;;
+    -r) reps="$2"; shift 2 ;;
+    -o) format="$2"; shift 2 ;;
+    --no-warmup) shift ;;
+    *) printf 'Unexpected fixture argument: %s\n' "$1" >&2; exit 2 ;;
+  esac
+done
+[ -f "$model" ] && [ "$reps" = 2 ] && [ "$format" = json ] || exit 2
+case "$pp:$tg:$depth" in
+  512:0:0|0:128:1|128:0:0) ;;
+  *) printf 'Unexpected fixture workload: %s\n' "$pp:$tg:$depth" >&2; exit 2 ;;
+esac
+printf '%s\n' "$pp:$tg:$depth" >> "$0.calls"
+printf '[{"build_commit":"abc123","build_number":123,"model_type":"Qwen3 Q4_K_M","model_filename":"Qwen3-4B.gguf","model_size":24,"model_n_params":4000000000,"n_prompt":%s,"n_gen":%s,"n_depth":%s,"n_gpu_layers":99,"gpu_info":"Apple M5 Pro","cpu_info":"Apple M5 Pro","backends":"Metal","samples_ns":[100000000,200000000]}]\n' "$pp" "$tg" "$depth"
+SH
 chmod 0755 "$STUB"
 
 REPORT="$WORK/report.json"
 if ! cli llama-cpp --runtime-path "$STUB" run "$MODEL" --pp 128,512 --reps 2 --yes --output "$REPORT" >"$WORK/run.log" 2>&1; then
   cat "$WORK/run.log" >&2
   fail "offline llama-cpp benchmark run failed"
 fi
+EXPECTED_CALLS="$(printf '%s\n' '512:0:0' '0:128:1' '128:0:0')"
+[ "$(cat "$STUB.calls")" = "$EXPECTED_CALLS" ] || fail "expected PP512, TG128 (depth 1), then PP128, exactly once each"
+say "headline-first workload order and decode seed depth verified"
 [ -f "$REPORT" ] || fail "run did not write $REPORT"
 grep -q '"name": *"llama-cpp"' "$REPORT" || fail "report does not record the llama-cpp runtime"
 grep -q "\"computearena_version\": *\"$EXPECTED\"" "$REPORT" || fail "report does not record computearena_version $EXPECTED"

.github/workflows/ci.yml

@@ -24,3 +24,14 @@ jobs:
       - run: cargo clippy --workspace --all-targets -- -D warnings
       - run: cargo test --workspace --all-targets
       - run: cargo build --workspace --release
+      - name: Smoke test the packaged binary (offline)
+        # Exercise the same fixture as staging/release on every PR, so protocol
+        # changes cannot pass unit tests while breaking the release smoke test.
+        shell: bash
+        run: |
+          PKGID="$(cargo pkgid --locked -p computearena-cli)"
+          VERSION="${PKGID##*#}"
+          VERSION="${VERSION##*@}"
+          SMOKE_DIR="$(mktemp -d)"
+          .github/scripts/package.sh target/release/computearena "computearena-smoke-${VERSION}" "$SMOKE_DIR"
+          sh .github/scripts/smoke_test.sh "$SMOKE_DIR/computearena-smoke-${VERSION}.tar.gz" "$VERSION"

Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/computearena-cli"]
 
 [workspace.package]
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
 license = "Apache-2.0"
 rust-version = "1.85"

README.md

@@ -146,6 +146,12 @@ computearena basert run model.base --pp 512,2048 --tg 128 --reps 5
 computearena llama-cpp run model.gguf --yes --output ./report.json
 ```
 
+Only a run with the full default sweep (PP128 to PP16384 and TG128) can be
+submitted, so every model and chip is comparable at every size. A custom `--pp`
+or `--tg` still produces a valid signed report for local use: the client says
+it will be local only before the run starts, shows such reports as `LOCAL ONLY`
+in its lists, and leaves them out of a submission with what is missing.
+
 Two profiles are offered before a run starts. Standard runs without an external
 cooldown wait. With the currently released BaseRT harness, thermally controlled
 (`--cooldown`) waits once before the complete harness run; llama.cpp waits before
@@ -155,12 +161,16 @@ and piped input must use `--yes`. Details are in
 [docs/benchmark-profiles.md](docs/benchmark-profiles.md).
 
 Telemetry is automatic for both runtimes and needs no flag, credential, or
-sudo. ComputeArena observes the single process it launches for current BaseRT
-and llama.cpp builds: resident memory, operating-system temperature sensors,
-power state, and NVIDIA or ROCm device snapshots where those vendor tools exist.
-A future BaseRT harness can advertise native same-run telemetry, which the CLI
-will use without a CLI release or version-string rule. Coverage and limitations
-are in [docs/telemetry.md](docs/telemetry.md).
+sudo. ComputeArena observes the processes it launches: resident memory,
+operating-system temperature sensors, power state, memory pressure and swap,
+plus NVIDIA or ROCm device snapshots where those vendor tools exist. Signed
+environment boundaries also record the OS/kernel, CPU layout, host memory, and
+available GPU configuration; static macOS display configuration is cached so
+`system_profiler` is not rerun at every boundary. A BaseRT harness can
+advertise native same-run telemetry, which the CLI uses without a CLI release
+or version-string rule. Runtime-native detail is preferred when it is more
+accurate; portable host boundaries remain available for cross-runtime analysis.
+Coverage and limitations are in [docs/telemetry.md](docs/telemetry.md).
 
 ## Runtimes
 
@@ -204,6 +214,21 @@ of `--runtime-path`. BaseRT 0.2.4 and newer can also start this client with
 BaseRT remains responsible for choosing a compatible backend artifact,
 downloading split files, conversion, and writing `hub.json` provenance.
 
+ComputeArena says when the BaseRT it found is worth updating, and never
+refuses to run an older one. A harness that does not advertise the
+headline-first protocol (BaseRT 0.2.4 and older) is named before the benchmark
+plan, with what its report will be signed as; this needs no network, because
+the harness describes itself. A newer BaseRT release is mentioned the same way.
+That lookup asks GitHub for BaseRT's latest release in the background, keeps
+the answer for 24 hours in `basert-update-check.json`, and never delays or
+fails a run; when it has not answered before a benchmark starts, the notice
+follows the run instead. `computearena basert install` installs the latest
+release where the official installer does, and the full-screen interface
+offers the same with `u` on its menu, asking for a second press before it
+replaces anything. A harness chosen with `--runtime-path` or
+`COMPUTEARENA_BASERT_HARNESS` is yours to update, and on platforms without a
+prebuilt BaseRT the notice points at the release to build from instead.
+
 ### llama.cpp
 
 The adapter asks for a GGUF file rather than scanning the disk, and lists the
@@ -214,10 +239,19 @@ reads its GGUF header only. The history lives in `recent-gguf.json` in the
 data directory, is never part of a report, and can be deleted to reset the
 list; a corrupt or unwritable history never blocks a benchmark.
 
-llama.cpp measurements carry their own protocol identifiers and record native
-warmup, zero context depth, and the exclusion of sampling and tokenization, so
-they are never presented as BaseRT numbers. The measurement contract is in
-[docs/runtime-adapters.md](docs/runtime-adapters.md).
+The standard headline is PP512 followed by TG128 **before** the remaining PP
+sweep. PP starts empty; TG starts with one untimed seed token. A capable BaseRT
+harness reserves 4K for the headline (`computearena-throughput/3`), without
+changing its existing warmup, repetitions, timing or telemetry. Older harnesses
+retain their existing invocation and recorded protocol.
+
+ComputeArena uses the user's unmodified llama-bench build. Its native capacity
+is retained: stock llama-bench has no independent 4K reservation option, and
+`-d 4096` would add real history instead. This difference, actual workload order,
+warmup and context requests are signed in the JSON. There is no claim of exact
+cross-runtime equivalence. Existing reports remain verifiable and submittable;
+protocol differences are not a new leaderboard filter.
+The measurement contract is in [docs/runtime-adapters.md](docs/runtime-adapters.md).
 
 ## Reports and signatures
 
@@ -300,7 +334,14 @@ refused.
 The server compares the signed runtime checksum with its catalogue of official
 builds. An unrecognized or custom build is still accepted and shown with
 download guidance; only a report whose signature does not verify is rejected.
-Submitting the same report again succeeds rather than failing.
+Submitting a report that is still published succeeds as an existing duplicate.
+If you deleted it on the website, re-submitting that saved report fails instead,
+including through **Select all**. The CLI labels it **Failed: previously deleted**
+and continues attempting the other valid reports. Successful uploads remain
+saved; the final summary counts the failures and the command exits nonzero.
+Local files are unchanged. You can rerun the same model with the same settings
+and submit the newly generated report as a separate benchmark. There is no need
+to choose a different configuration.
 
 The client talks to `https://computearena.ai/api/v1`. `--api-url` or
 `COMPUTEARENA_API_URL` point it at another deployment, such as a local
@@ -320,6 +361,7 @@ minimumClientVersion for an actionable upgrade message.
 | `COMPUTEARENA_API_URL` | API base URL, same as `--api-url` |
 | `COMPUTEARENA_BASERT_HARNESS` | Path to `basert-benchmark-harness`, same as `--runtime-path` for BaseRT |
 | `BASERT_INSTALL_DIR` | Where BaseRT is looked for and installed; `~/.basert` by default |
+| `COMPUTEARENA_BASERT_RELEASE_API` | Where BaseRT's latest release is looked up, for mirrors and tests; GitHub's API for `basecompute/baseRT` by default. Only a version number is read from the answer |
 | `BASERT_MODELS_DIR` | Where installed BaseRT models are listed from; BaseRT's own model cache by default |
 | `CUDA_VISIBLE_DEVICES` | Respected by the CUDA chip fallback; a mask leaves the chip unresolved |
 | `NO_COLOR` | Plain output |
@@ -334,12 +376,19 @@ reports, sessions, and the signing key.
 - Report envelope: `computearena-benchmark/1`
 - BaseRT harness output: `basert-benchmark-harness/1`; the older
   `basert-harness/1` is still accepted by the server
-- llama.cpp measurements: `computearena-measurements/1`, executed as
-  `llama-bench-independent-pp-tg/1` or, with cooldown,
-  `llama-bench-conditioned-pp-tg/1`
+- llama.cpp measurements: `computearena-measurements/1`
+- Comparable throughput semantics: `computearena-throughput/2`, with native
+  evidence retained as `basert-throughput-protocol/1` or
+  `llama-bench-json/1`. Older BaseRT results use
+  `computearena-throughput-legacy/1` and are marked non-comparable.
+- Headline-first BaseRT capacity: `computearena-throughput/3`, native evidence
+  `basert-throughput-protocol/2` / `basert-bench-capacity/1`. The historical
+  `comparable` metadata is not a guarantee of identical measured performance
+  and does not exclude older reports from the website.
 - Telemetry: `computearena-telemetry/1` for externally observed BaseRT and
   llama.cpp runs. A BaseRT harness advertising `features.same_run_telemetry`
   uses native `basert-telemetry/4` instead.
+- Host environment boundaries: `computearena-environment/1`
 - Signing: Ed25519 over `computearena-json-v1` canonical JSON
 
 ## Development