From 35b8e43af2ddef186cfa62e8bf8bfe9e08678c29 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 15:08:29 +0200 Subject: [PATCH 1/2] fix(lemonade): pin Lemonade 2026.44.1 so a ROCm 10.1 SDK gets a ROCm backend Lemonade 11.5.1 pins TheRock 7.13.0 and llama.cpp b9752, and downloads TheRock from a location that does not serve ROCm 10.x, so on a ROCm 10.1 SDK its llamacpp:rocm backend could not be paired with the SDK in use. Lemonade 2026.44.1 pins TheRock 10.1.0 and llama.cpp b11580, whose release ships ROCm 10.1 archives for Linux and Windows. The embeddable layout, binary names and the lemond/lemonade flags rocm-cli uses are unchanged; the new archive digests are recorded. Two adaptations the new release needs: - `lemonade backends` lists recipes rocm-cli did not know (llamacpp-hrx, ds4, thenoise) and adds Tier and Format columns. The parser matched recipe rows against a fixed list, so an unknown recipe's rows, and every indented row after them, were read as llamacpp backends. It now reads the recipe from the row layout, and reads Lemonade's update_required and update_available states as installable and installed instead of dropping the row. - The second alignment attempt took lemonade-sdk/llama.cpp's newest release, but recent releases are published with only qualification reports and no ROCm archives. It now lists the 30 newest releases and picks the most recently published one that ships the exact archive for the active ROCm version on this platform. The runtime-16 and runtime-17 scenarios assumed the active SDK never equals the packaged pin, which is no longer true on a ROCm 10.1 lane: runtime-16 now checks that the installed pin equals the active SDK by whichever route the host takes, and runtime-17 compares the pin with the one inside the verified archive. Signed-off-by: Roman Inflianskas --- MANIFEST.md | 13 +- .../e2e-report/src/consolidated/markdown.rs | 4 +- crates/rocm-dash-tui/src/skills.rs | 6 +- crates/rocm-deps/src/lib.rs | 4 +- docs/engine-plugins.md | 28 +- docs/testing.md | 21 + engines/lemonade/src/backend_alignment.rs | 417 ++++++++++++++---- engines/lemonade/src/direct_llama.rs | 6 +- engines/lemonade/src/install.rs | 90 ++-- engines/lemonade/src/llamacpp_backend.rs | 164 ++++++- runtime-deps.toml | 7 +- .../features/runtime_setup.feature | 36 +- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 215 +++++++-- 13 files changed, 773 insertions(+), 238 deletions(-) diff --git a/MANIFEST.md b/MANIFEST.md index 687b4493d..fc077ccef 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -703,11 +703,16 @@ Lemonade embeddable archive from the official Lemonade SDK GitHub releases at to the rocm-cli managed data directory and is reused for subsequent `rocm engines start lemonade` invocations. This is a self-contained binary distribution; no Python packages are installed for Lemonade by rocm-cli. -On Linux/WSL, `rocm engines install lemonade` may also query -`https://api.github.com/repos/lemonade-sdk/llama.cpp/releases/latest` (an -unauthenticated call) while aligning Lemonade's `llamacpp:rocm` backend to -the active ROCm SDK version; see +On Linux/WSL and Windows, `rocm engines install lemonade` may also list +`https://api.github.com/repos/lemonade-sdk/llama.cpp/releases` (the 30 newest +releases, one unauthenticated call) while aligning Lemonade's `llamacpp:rocm` +backend to the active ROCm SDK version; see [Lemonade backend alignment on engine install](docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install). +Lemonade itself then downloads the llama.cpp backend it installs from the +`lemonade-sdk/llama.cpp` GitHub releases, and the ROCm runtime that backend +runs on from AMD's ROCm package repository (as Python wheels into a virtual +environment of Lemonade's own, or as a TheRock tarball); those downloads are +Lemonade's, not rocm-cli's. ### Engine-Specific Python Dependencies diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index 798a04914..44a142ace 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -718,7 +718,7 @@ mod tests { let platform = r#"{ "platform_slug": "mi300x", "capability": {"effective_serve_engine": "vllm"}, - "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"11.5.1"}, + "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"2026.44.1"}, "expectations": [ {"id":"serve-x","effective_engine":"vllm","expected":"pass"} ] @@ -730,7 +730,7 @@ mod tests { "Ubuntu 24.04.3 LTS", "ROCm 7.13.0", "vLLM 0.23.0+rocm723", - "lemonade 11.5.1", + "lemonade 2026.44.1", ] { assert!(md.contains(token), "matrix cell missing {token:?}:\n{md}"); } diff --git a/crates/rocm-dash-tui/src/skills.rs b/crates/rocm-dash-tui/src/skills.rs index 44349d0c6..b9839a386 100644 --- a/crates/rocm-dash-tui/src/skills.rs +++ b/crates/rocm-dash-tui/src/skills.rs @@ -271,11 +271,11 @@ pub fn lemonade_embeddable_fallback_version() -> String { /// and locate the server binary. Pure data; no I/O. #[derive(Debug, Clone, PartialEq, Eq)] pub struct EmbeddableArtifact { - /// Version without a leading `v`, e.g. `11.5.1`. + /// Version without a leading `v`, e.g. `2026.44.1`. pub version: String, /// The archive's `browser_download_url`. pub url: String, - /// The archive file name, e.g. `lemonade-embeddable-11.5.1-ubuntu-x64.tar.gz`. + /// The archive file name, e.g. `lemonade-embeddable-2026.44.1-ubuntu-x64.tar.gz`. pub archive_name: String, /// The unpacked server executable name (`lemond` / `lemond.exe`). pub server_bin: String, @@ -303,7 +303,7 @@ fn server_bin_for(os: &str) -> &'static str { } } -/// Strip a leading `v` from a release tag (`v11.5.1` → `11.5.1`). +/// Strip a leading `v` from a release tag (`v2026.44.1` → `2026.44.1`). fn strip_v(tag: &str) -> &str { rocm_deps::strip_v(tag) } diff --git a/crates/rocm-deps/src/lib.rs b/crates/rocm-deps/src/lib.rs index e8044d760..4f29672b8 100644 --- a/crates/rocm-deps/src/lib.rs +++ b/crates/rocm-deps/src/lib.rs @@ -17,14 +17,14 @@ include!(concat!(env!("OUT_DIR"), "/pins.rs")); /// GitHub repository publishing the Lemonade embeddable release archives. pub const LEMONADE_GITHUB_REPO: &str = "lemonade-sdk/lemonade"; -/// Strip a leading `v` from a release tag (`v11.5.1` -> `11.5.1`). +/// Strip a leading `v` from a release tag (`v2026.44.1` -> `2026.44.1`). #[must_use] pub fn strip_v(tag: &str) -> &str { tag.strip_prefix('v').unwrap_or(tag) } /// Name of the Lemonade embeddable archive for a host token and extension, -/// e.g. `lemonade-embeddable-11.5.1-ubuntu-x64.tar.gz`. +/// e.g. `lemonade-embeddable-2026.44.1-ubuntu-x64.tar.gz`. #[must_use] pub fn lemonade_archive_name(version: &str, os_arch: &str, extension: &str) -> String { format!( diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 5e39aca23..6f85284c7 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -54,13 +54,20 @@ The rewrite tries two builds in order. The first keeps Lemonade's own pinned llama.cpp release and just repoints it at the active ROCm version — a deliberate, reproducible pin. If that attempt fails to resolve against the installed backend's shared libraries — for example because that release never -shipped an asset for the active ROCm version — the second queries GitHub's -`releases/latest` for `lemonade-sdk/llama.cpp` (an unauthenticated -`api.github.com` call, subject to GitHub's public rate limit) and installs -whatever build is newest at that moment. That second build is a moving -target, not a pin: which llama.cpp commit actually gets installed depends on -when the install ran, and the version installed is not recorded anywhere -`rocm examine`/`rocm version` report. An attempt that cannot be verified +shipped an asset for the active ROCm version — the second lists the 30 newest +`lemonade-sdk/llama.cpp` releases on GitHub (an unauthenticated +`api.github.com` call, subject to GitHub's public rate limit) and installs the +most recently published one that ships the llama.cpp archive for the active +ROCm version's `major.minor` on this platform +(`llama--bin-ubuntu-rocm--x64.tar.gz` on Linux, +`llama--bin-win-rocm--x64.zip` on Windows). The newest +release alone is not used, because a release can be published without its +ROCm archives. The second attempt is skipped when none of the 30 ships that +archive, when the listing cannot be fetched, and when the build it finds is the +one Lemonade already pins (the first attempt tried that build). That second build is a moving target, not a pin: which llama.cpp +commit actually gets installed depends on when the install ran, and the +version installed is not recorded anywhere `rocm examine`/`rocm version` +report. An attempt that cannot be verified against the installed backend's shared libraries reverts to whichever version was pinned before the attempt — the packaged pin on a fresh install or after `--reinstall`, but whatever alignment last wrote otherwise — rather than @@ -132,8 +139,11 @@ The versions of the third-party runtimes rocm-cli downloads are pinned in `runtime-deps.toml` at the repository root — one `[runtime.]` table per runtime. That file is the only place a runtime version is written down: archive names, download URLs, and the dashboard's offline fallback are all -derived from it, so a bump is a one-line edit and the tree cannot end up -holding two different versions of the same runtime. +derived from it, so the tree cannot end up holding two different versions of +the same runtime. A Lemonade bump also records the new release's archive +digests (`EMBEDDABLE_LINUX_SHA256` and `EMBEDDABLE_WINDOWS_SHA256` in +[install.rs](../engines/lemonade/src/install.rs)): the downloaded archive is +checked against them, and a digest cannot be derived from the version. `rocm engines list` shows the exact plugin directories for the current host. The same output is available in the TUI with `/engine`. diff --git a/docs/testing.md b/docs/testing.md index 237801b00..a1bd763e9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1195,6 +1195,27 @@ lemonade` against the shared tree) reaches the restore only on a runner whose shared tree still holds a stale pin; on a healthy runner the install succeeds and the restore never runs, so a green lane does not show that it works. +Lemonade backend selection and alignment focused tests (reading the +`llamacpp` rows out of `lemonade backends` when other recipes have indented +rows of their own, and the second alignment attempt choosing the newest +`lemonade-sdk/llama.cpp` release that ships the archive for the active ROCm +version rather than the newest release, and reading Lemonade's +`update_required`/`update_available` backend states): + +```bash +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::parse +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::another_recipes_backend +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::update_states +cargo test -p rocm-engine-lemonade backend_alignment::tests::tier2 +cargo test -p rocm-engine-lemonade backend_alignment::tests::align_falls_through_to_tier2 +cargo test -p rocm-engine-lemonade backend_alignment::tests::llamacpp_asset_name +``` + +They run on fixtures: the `lemonade backends` table Lemonade 2026.44.1 prints, +and a release listing shaped like the real one. The live paths (a real +`lemond`, the GitHub listing, a real backend install) run only on the gated +Lemonade GPU lanes. + Engine recipe adapter contract focused tests: ```bash diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 3899d32f9..39ec297c2 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -95,7 +95,7 @@ pub(crate) fn prepare_llamacpp_backend_for_active_rocm( disabled, try_llamacpp_backend_alignment, install_best_llamacpp_backend, - latest_llamacpp_rocm_stable_tag, + newest_llamacpp_rocm_stable_tag_for, ) }, ) @@ -341,7 +341,7 @@ fn install_pinned_backend_or_restore_packaged_pins( /// The Tier 1 / Tier 2 / revert state machine, isolated from the config and active-SDK /// lookups above so it can be exercised in tests against a temp `backend_versions.json` -/// with the install/align/latest-tag steps injected, instead of spawning a real +/// with the install/align/tag-lookup steps injected, instead of spawning a real /// `lemond` and reaching GitHub. /// /// `disabled` is the resolved value of [`lemonade_backend_alignment_disabled`], passed @@ -365,7 +365,7 @@ fn align_llamacpp_backend_to_version( disabled: bool, mut align: impl FnMut(&mut LemonadeInstallManifest, &str, bool, &str) -> bool, mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, - mut latest_tag: impl FnMut() -> Result, + mut newest_tag_with_asset: impl FnMut(&str) -> Result, ) -> Result> { if disabled { fallback_install(manifest, false)?; @@ -393,31 +393,41 @@ fn align_llamacpp_backend_to_version( // matching ROCm-version asset — Lemonade only started publishing per-ROCm- // version builds partway through its release history, and the embeddable's // pin (frozen at whatever build was current when this Lemonade version was - // cut) does not track that. Point at the current newest build instead, still - // paired with the active ROCm version, forcing a real reinstall attempt - // regardless of whatever Tier 1 left on disk. + // cut) does not track that. Point at the newest build that actually ships an + // archive for the active ROCm version instead, still paired with that version, + // forcing a real reinstall attempt regardless of whatever Tier 1 left on disk. let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); - let latest_tag_applied = match latest_tag() { - Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { - match write_backend_versions_llamacpp_tag(backend_versions_path, &latest_tag) { + let found_tag_applied = match newest_tag_with_asset(target_version) { + Ok(tag) if pinned_tag.as_deref() != Some(tag.as_str()) => { + match write_backend_versions_llamacpp_tag(backend_versions_path, &tag) { Ok(()) => true, Err(error) => { eprintln!( - "Warning: could not pin Lemonade's backend to llama.cpp {latest_tag}: \ + "Warning: could not pin Lemonade's backend to llama.cpp {tag}: \ {error:#}" ); false } } } - // The pinned tag already is the latest — Tier 1 already tried it. + // The pinned tag already is that build — Tier 1 already tried it. Ok(_) => false, Err(error) => { - eprintln!("Warning: could not determine Lemonade's latest llama.cpp build: {error:#}"); + eprintln!( + "Warning: could not look up a llama.cpp build that ships an archive for ROCm \ + {target_version}: {error:#}" + ); false } }; - if latest_tag_applied && align(manifest, target_version, true, "the latest llama.cpp build") { + if found_tag_applied + && align( + manifest, + target_version, + true, + "the newest llama.cpp build for this ROCm version", + ) + { return Ok(Some(target_version.to_owned())); } @@ -440,7 +450,7 @@ fn align_llamacpp_backend_to_version( ({pinned_version}): {error:#}" ); } - if latest_tag_applied { + if found_tag_applied { // Best-effort, matching the therock.version restore above: a failure here // must not skip the fallback reinstall below. Restore the original tag when // there was one; otherwise Tier 2 pinned a key that did not exist before, so @@ -549,19 +559,85 @@ const fn llamacpp_backend_alignment_succeeded( /// (and, at the time of writing, is) older than that. const LLAMACPP_ROCM_STABLE_REPO: &str = "lemonade-sdk/llama.cpp"; -/// The newest published release tag for [`LLAMACPP_ROCM_STABLE_REPO`] (e.g. -/// `b10952`), queried directly rather than assumed — the whole reason Tier 2 -/// exists is that the embeddable's own pinned tag cannot be trusted to have a -/// matching-ROCm-version asset. -fn latest_llamacpp_rocm_stable_tag() -> Result { - let url = format!("https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases/latest"); - let timeout = Duration::from_secs(15); +/// How many of [`LLAMACPP_ROCM_STABLE_REPO`]'s newest releases Tier 2 searches +/// for one that ships the archive it needs. The repository publishes a build +/// about twice a day, so this covers roughly the last two weeks; one listing page +/// of this size is about 2 MB. +const LLAMACPP_RELEASES_SEARCHED: usize = 30; + +/// The `llamacpp:rocm-stable` archive Lemonade downloads for build `tag` paired +/// with ROCm `rocm_version` — the name Lemonade's own installer builds, from the +/// ROCm version's `major.minor` (`10.1.0` -> `rocm-10.1`). +fn llamacpp_rocm_stable_asset_name(tag: &str, rocm_version: &str, windows: bool) -> String { + let major_minor = rocm_version + .splitn(3, '.') + .take(2) + .collect::>() + .join("."); + if windows { + format!("llama-{tag}-bin-win-rocm-{major_minor}-x64.zip") + } else { + format!("llama-{tag}-bin-ubuntu-rocm-{major_minor}-x64.tar.gz") + } +} + +/// From a GitHub release listing (`GET /repos/{repo}/releases`), the tag of the +/// most recently published release that actually ships the `llamacpp:rocm-stable` +/// archive for `rocm_version` on this platform. +/// +/// The newest release is not good enough on its own: a release can be published +/// before, or without, its ROCm archives (`b12013` through `b12062` carry only +/// qualification reports), and a build can skip one ROCm version or one platform. +/// Drafts, prereleases and assets still uploading are skipped. +fn newest_llamacpp_tag_with_asset( + releases: &Value, + rocm_version: &str, + windows: bool, +) -> Option { + releases + .as_array()? + .iter() + .filter(|release| { + !release + .get("draft") + .and_then(Value::as_bool) + .unwrap_or(false) + && !release + .get("prerelease") + .and_then(Value::as_bool) + .unwrap_or(false) + }) + .filter_map(|release| { + let tag = release.get("tag_name")?.as_str()?; + let wanted = llamacpp_rocm_stable_asset_name(tag, rocm_version, windows); + let ships_asset = release.get("assets")?.as_array()?.iter().any(|asset| { + asset.get("name").and_then(Value::as_str) == Some(wanted.as_str()) + && asset + .get("state") + .and_then(Value::as_str) + .is_none_or(|state| state == "uploaded") + }); + let published_at = release.get("published_at")?.as_str()?; + ships_asset.then_some((published_at, tag)) + }) + .max_by(|left, right| left.0.cmp(right.0)) + .map(|(_, tag)| tag.to_owned()) +} + +/// The newest [`LLAMACPP_ROCM_STABLE_REPO`] build that ships an archive for +/// `rocm_version` on this platform (e.g. `b12003`), queried directly rather than +/// assumed — the whole reason Tier 2 exists is that the embeddable's own pinned +/// tag cannot be trusted to have a matching-ROCm-version asset. +fn newest_llamacpp_rocm_stable_tag_for(rocm_version: &str) -> Result { + let url = format!( + "https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases?per_page={LLAMACPP_RELEASES_SEARCHED}" + ); // `timeout_connect` takes precedence over `timeout` and defaults to 30s, so without // it a host that blackholes rather than refuses would stall well past the intended - // ceiling. + // ceiling. The overall timeout also covers reading the listing. let agent = ureq::AgentBuilder::new() - .timeout_connect(timeout) - .timeout(timeout) + .timeout_connect(Duration::from_secs(15)) + .timeout(Duration::from_secs(30)) .build(); let response = agent .get(&url) @@ -571,10 +647,14 @@ fn latest_llamacpp_rocm_stable_tag() -> Result { let body: Value = response .into_json() .with_context(|| format!("failed to parse response from {url}"))?; - body.get("tag_name") - .and_then(Value::as_str) - .map(str::to_owned) - .with_context(|| format!("{url} response is missing 'tag_name'")) + let windows = runtime_is_windows(); + newest_llamacpp_tag_with_asset(&body, rocm_version, windows).with_context(|| { + format!( + "none of the {LLAMACPP_RELEASES_SEARCHED} newest {LLAMACPP_ROCM_STABLE_REPO} \ + releases ships {}", + llamacpp_rocm_stable_asset_name("", rocm_version, windows) + ) + }) } /// Read `resources/backend_versions.json`'s pinned `llamacpp.rocm-stable` build tag. @@ -1013,8 +1093,8 @@ mod tests { #[test] fn plain_semver_accepts_release_versions_only() { - assert!(looks_like_plain_semver("10.0.0")); - assert!(looks_like_plain_semver("7.13.0")); + assert!(looks_like_plain_semver("7.14.0")); + assert!(looks_like_plain_semver("10.1.0")); assert!(!looks_like_plain_semver("7.14.0a20260601")); assert!(!looks_like_plain_semver("10.0")); assert!(!looks_like_plain_semver("10.0.0.1")); @@ -1028,8 +1108,8 @@ mod tests { fs::write( &path, serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752" }, - "therock": { "version": "7.13.0", "architectures": ["gfx1151"] }, + "llamacpp": { "rocm-stable": "b11580" }, + "therock": { "version": "10.1.0", "architectures": ["gfx1151"] }, })) .unwrap(), ) @@ -1037,18 +1117,18 @@ mod tests { assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) + Some("10.1.0".to_owned()) ); - write_backend_versions_therock_version(&path, "10.0.0").unwrap(); + write_backend_versions_therock_version(&path, "7.14.0").unwrap(); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); // Every other key, including sibling keys under `therock`, is preserved. let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); assert_eq!(value["therock"]["architectures"][0], "gfx1151"); fs::remove_dir_all(&dir).ok(); @@ -1065,7 +1145,7 @@ mod tests { .unwrap(); assert_eq!(read_backend_versions_therock_version(&path), None); - assert!(write_backend_versions_therock_version(&path, "10.0.0").is_err()); + assert!(write_backend_versions_therock_version(&path, "7.14.0").is_err()); fs::remove_dir_all(&dir).ok(); } @@ -1077,8 +1157,8 @@ mod tests { fs::write( &path, serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752", "vulkan": "b9747" }, - "therock": { "version": "7.13.0" }, + "llamacpp": { "rocm-stable": "b11580", "vulkan": "b11393" }, + "therock": { "version": "10.1.0" }, })) .unwrap(), ) @@ -1086,20 +1166,20 @@ mod tests { assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()) + Some("b11580".to_owned()) ); - write_backend_versions_llamacpp_tag(&path, "b10952").unwrap(); + write_backend_versions_llamacpp_tag(&path, "b12003").unwrap(); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()) + Some("b12003".to_owned()) ); // Sibling keys, including the unrelated `llamacpp.vulkan` tag and the // whole `therock` object, are preserved. let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["vulkan"], "b9747"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["llamacpp"]["vulkan"], "b11393"); + assert_eq!(value["therock"]["version"], "10.1.0"); fs::remove_dir_all(&dir).ok(); } @@ -1115,7 +1195,7 @@ mod tests { .unwrap(); assert_eq!(read_backend_versions_llamacpp_tag(&path), None); - assert!(write_backend_versions_llamacpp_tag(&path, "b10952").is_err()); + assert!(write_backend_versions_llamacpp_tag(&path, "b12003").is_err()); fs::remove_dir_all(&dir).ok(); } @@ -1287,7 +1367,7 @@ mod tests { #[test] fn align_honors_the_disabled_flag_without_touching_the_pin_or_the_injected_steps() { // This pins `align`'s own half of the opt-out contract in isolation: the pin - // and llama.cpp tag stay byte-identical and `align`/`latest_tag` are never + // and llama.cpp tag stay byte-identical and `align`/`newest_tag_with_asset` are never // called. The real production gate -- whether this function is even reached // with `disabled == true` in the first place -- lives one level up, in // `prepare_llamacpp_backend_for_active_rocm_impl`; see @@ -1295,14 +1375,14 @@ mod tests { // below for that half. let dir = scratch_dir("align-disabled"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", true, |_manifest, _target, _force_reinstall, _label| { panic!("disabled means no alignment attempt of any kind") @@ -1314,18 +1394,18 @@ mod tests { ); Ok(()) }, - || panic!("disabled means tier 2's tag lookup must not run either"), + |_target| panic!("disabled means tier 2's tag lookup must not run either"), ); assert_eq!(result.unwrap(), None); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), + Some("10.1.0".to_owned()), "the packaged pin must survive untouched when alignment is disabled" ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), + Some("b11580".to_owned()), "the packaged llama.cpp tag must survive untouched when alignment is disabled" ); fs::remove_dir_all(&dir).ok(); @@ -1411,8 +1491,8 @@ mod tests { &mut manifest, false, false, - || Ok(Some("10.0.0".to_owned())), - |_path| Some("7.13.0".to_owned()), + || Ok(Some("7.14.0".to_owned())), + |_path| Some("10.1.0".to_owned()), |_manifest| Err(anyhow!("lemond failed to start")), |_manifest, force_reinstall| { assert!( @@ -1894,14 +1974,14 @@ mod tests { fn align_tier1_success_keeps_target_version_pinned() { let dir = scratch_dir("align-tier1-success"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, force_reinstall, _label| { // A stale backend installed against the OLD pin must not be able to @@ -1911,30 +1991,176 @@ mod tests { true }, |_manifest, _force_reinstall| panic!("tier 1 succeeded; no fallback install"), - || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), + |_target| panic!("tier 1 succeeded; tier 2's tag lookup must not run"), ); - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(result.unwrap(), Some("7.14.0".to_owned())); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); fs::remove_dir_all(&dir).ok(); } + /// One `GET /repos/lemonade-sdk/llama.cpp/releases` entry, trimmed to the + /// fields Tier 2 reads. + fn llamacpp_release(tag: &str, published_at: &str, assets: &[&str]) -> Value { + json!({ + "tag_name": tag, + "draft": false, + "prerelease": false, + "published_at": published_at, + "assets": assets + .iter() + .map(|name| json!({ "name": name, "state": "uploaded" })) + .collect::>(), + }) + } + + /// Synthetic, modelled on the real listing in early November 2026, where + /// b12062 carries only qualification reports and b11997 no ROCm 7.14 + /// archive. The real b12003 ships both platforms; here it ships Linux + /// archives only, so Linux and Windows pick different builds. + fn llamacpp_releases_fixture() -> Value { + json!([ + llamacpp_release( + "b12062", + "2026-11-03T03:41:03Z", + &[ + "qualification-llama-b12062-bin-ubuntu-rocm-10.1-x64.json", + "qualification-llama-b12062-bin-ubuntu-rocm-7.14-x64.json", + ], + ), + llamacpp_release( + "b12003", + "2026-10-30T03:47:50Z", + &[ + "llama-b12003-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b12003-bin-ubuntu-rocm-7.14-x64.tar.gz", + ], + ), + llamacpp_release( + "b11997", + "2026-10-29T16:11:35Z", + &[ + "llama-b11997-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11997-bin-win-rocm-10.1-x64.zip", + "qualification-llama-b11997-bin-ubuntu-rocm-7.14-x64.json", + ], + ), + llamacpp_release( + "b11972", + "2026-10-29T00:43:52Z", + &[ + "llama-b11972-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11972-bin-ubuntu-rocm-7.14-x64.tar.gz", + "llama-b11972-bin-win-rocm-10.1-x64.zip", + "llama-b11972-bin-win-rocm-7.14-x64.zip", + ], + ), + ]) + } + + #[test] + fn llamacpp_asset_name_matches_what_lemonade_downloads() { + assert_eq!( + llamacpp_rocm_stable_asset_name("b11580", "10.1.0", false), + "llama-b11580-bin-ubuntu-rocm-10.1-x64.tar.gz" + ); + assert_eq!( + llamacpp_rocm_stable_asset_name("b11580", "7.14.0", true), + "llama-b11580-bin-win-rocm-7.14-x64.zip" + ); + } + + #[test] + fn tier2_skips_a_newer_release_that_does_not_ship_the_archive() { + let releases = llamacpp_releases_fixture(); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", false).as_deref(), + Some("b12003"), + "the newest release ships only qualification reports" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", true).as_deref(), + Some("b11997"), + "b12003 ships no Windows archive" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.14.0", false).as_deref(), + Some("b12003") + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.14.0", true).as_deref(), + Some("b11972"), + "b12003 ships no Windows archive and b11997 no ROCm 7.14 archive" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.13.0", false), + None, + "no release ships ROCm 7.13" + ); + } + + #[test] + fn tier2_picks_by_publication_date_and_skips_unfinished_releases() { + let mut draft = llamacpp_release( + "b12100", + "2026-11-04T00:00:00Z", + &["llama-b12100-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + draft["draft"] = json!(true); + let mut prerelease = llamacpp_release( + "b12101", + "2026-11-04T01:00:00Z", + &["llama-b12101-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + prerelease["prerelease"] = json!(true); + let mut uploading = llamacpp_release( + "b12102", + "2026-11-04T02:00:00Z", + &["llama-b12102-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + uploading["assets"][0]["state"] = json!("open"); + // Listed out of date order: the listing's order is not trusted. + let releases = json!([ + llamacpp_release( + "b11900", + "2026-10-26T14:08:13Z", + &["llama-b11900-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ), + draft, + prerelease, + uploading, + llamacpp_release( + "b11945", + "2026-10-28T00:47:33Z", + &["llama-b11945-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ), + ]); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", false).as_deref(), + Some("b11945") + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&json!({}), "10.1.0", false), + None + ); + } + #[test] fn align_falls_through_to_tier2_and_succeeds() { let dir = scratch_dir("align-tier2-success"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut align_calls = 0; let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, force_reinstall, _label| { // Both tiers force a reinstall now, so the mock can no longer use @@ -1948,18 +2174,24 @@ mod tests { align_calls > 1 }, |_manifest, _force_reinstall| panic!("tier 2 succeeded; no fallback install"), - || Ok("b10952".to_owned()), + |target| { + assert_eq!( + target, "7.14.0", + "tier 2 looks for a build that ships the active ROCm version's archive" + ); + Ok("b12003".to_owned()) + }, ); - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(result.unwrap(), Some("7.14.0".to_owned())); assert_eq!(align_calls, 2, "both tiers were attempted"); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()), + Some("b12003".to_owned()), "tier 2 pins the newer llama.cpp tag" ); fs::remove_dir_all(&dir).ok(); @@ -1973,31 +2205,31 @@ mod tests { // value rather than staying on Tier 2's newer pin. let dir = scratch_dir("align-revert-restores-tag"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); Ok(()) }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert_eq!(result.unwrap(), None); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) + Some("10.1.0".to_owned()) ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), + Some("b11580".to_owned()), "the llama.cpp tag must be restored to its original value, not left on Tier 2's pin" ); fs::remove_dir_all(&dir).ok(); @@ -2015,7 +2247,7 @@ mod tests { &path, serde_json::to_vec_pretty(&json!({ "llamacpp": {}, - "therock": { "version": "7.13.0" }, + "therock": { "version": "10.1.0" }, })) .unwrap(), ) @@ -2025,15 +2257,15 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); Ok(()) }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert_eq!(result.unwrap(), None); @@ -2046,18 +2278,18 @@ mod tests { } #[test] - fn align_skips_tier2_reinstall_when_pinned_tag_is_already_latest() { + fn align_skips_tier2_reinstall_when_the_pinned_tag_is_the_one_found() { let dir = scratch_dir("align-tier2-tag-unchanged"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut align_calls = 0; let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| { align_calls += 1; @@ -2070,15 +2302,16 @@ mod tests { ); Ok(()) }, - // Already the latest tag: Tier 2's own reinstall must not be attempted. - || Ok("b9752".to_owned()), + // The pinned tag is the build Tier 2 found, which Tier 1 already tried: + // Tier 2's own reinstall must not be attempted. + |_target| Ok("b11580".to_owned()), ); assert_eq!(result.unwrap(), None); assert_eq!(align_calls, 1, "only tier 1 was attempted"); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), + Some("10.1.0".to_owned()), "reverted to the original pinned version" ); fs::remove_dir_all(&dir).ok(); @@ -2091,7 +2324,7 @@ mod tests { // path, and every sibling warning in this function is best-effort. let dir = scratch_dir("align-revert-write-fails"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut fallback_called = false; @@ -2100,8 +2333,8 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| { fs::write( @@ -2116,7 +2349,7 @@ mod tests { assert!(force_reinstall); Ok(()) }, - || bail!("network unavailable"), + |_target| bail!("network unavailable"), ); assert_eq!( @@ -2137,7 +2370,7 @@ mod tests { // the pre-existing backend must come back rather than staying deleted. let dir = scratch_dir("align-revert-fallback-install-fails"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let llamacpp = dir.join("bin").join("llamacpp"); let server = platform_binary_name("llama-server"); @@ -2148,15 +2381,15 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); bail!("network unavailable") }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert!( diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs index 0c3ba14e2..6bea51505 100644 --- a/engines/lemonade/src/direct_llama.rs +++ b/engines/lemonade/src/direct_llama.rs @@ -480,7 +480,7 @@ const DIRECT_LLAMA_SERVER_BACKENDS: [&str; 4] = ["rocm-stable", "rocm-nightly", /// /// Lemonade's backend installer extracts some llama.cpp releases straight into /// `/`, but others land one level deeper under a build-numbered directory -/// (e.g. `rocm-stable/llama-b9752/llama-server`), so both layouts are checked. +/// (e.g. `rocm-stable/llama-b11580/llama-server`), so both layouts are checked. pub(crate) fn find_llama_server_binary(manifest: &LemonadeInstallManifest) -> Option { let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); let binary = platform_binary_name("llama-server"); @@ -828,11 +828,11 @@ mod tests { #[test] fn find_llama_server_binary_finds_build_numbered_nesting() { // Some Lemonade llama.cpp releases extract one level deeper than - // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. + // `/llama-server`, e.g. `rocm-stable/llama-b11580/llama-server`. let dir = scratch_dir("find-server-nested"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let nested = llamacpp.join("rocm-stable").join("llama-b9752"); + let nested = llamacpp.join("rocm-stable").join("llama-b11580"); fs::create_dir_all(&nested).unwrap(); let server = nested.join(platform_binary_name("llama-server")); fs::write(&server, b"x").unwrap(); diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index e0fa931d1..cce15c26c 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -37,9 +37,9 @@ const MAX_EMBEDDABLE_SEARCH_DEPTH: usize = 4; /// release has its own digest, so bumping the pin means recording the new pair /// here as well. const EMBEDDABLE_WINDOWS_SHA256: &str = - "50a133bbc35c4f3f8971eafef2c9fe56c4bbbfb0f1032bf8728324d8d8c8a0e1"; + "7ff6c5d6ab612c33690411a7e53d6d0dd62eb515a073375c35015b0778a5a344"; const EMBEDDABLE_LINUX_SHA256: &str = - "bdfd3c3e5d6eda5101c8a32f36e6dd9236ceb9ab2eb66734c32628e6e86e18ac"; + "16f156a7f44a32b20936e76019b24dbccc7b82e60a3acea12e8614da8bca8e90"; /// Embeddable asset token and archive extension for the host this adapter runs /// on. Only `windows-x64` and `ubuntu-x64` are wired up here; the release also @@ -731,6 +731,23 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + #[test] + fn pinned_archive_digests_are_two_distinct_sha256_hex_strings() { + // A digest is pasted by hand on every bump and only a live download + // checks it, so a dropped character or a value pasted twice would + // otherwise pass every offline test. + for digest in [EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256] { + assert_eq!(digest.len(), 64, "{digest}"); + assert!( + digest + .chars() + .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)), + "{digest}" + ); + } + assert_ne!(EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256); + } + #[test] fn a_version_change_forces_re_extraction_over_the_existing_runtime() { let pin = rocm_deps::LEMONADE_VERSION; @@ -756,21 +773,16 @@ mod tests { assert_eq!(installed_runtime_version(runtime_dir), None); - record_runtime_version(runtime_dir, "7.13.0").unwrap(); + record_runtime_version(runtime_dir, "1.2.3").unwrap(); let installed = installed_runtime_version(runtime_dir); - assert_eq!(installed.as_deref(), Some("7.13.0")); + assert_eq!(installed.as_deref(), Some("1.2.3")); assert!(!needs_extraction( false, installed.as_deref(), - "7.13.0", - true - )); - assert!(needs_extraction( - false, - installed.as_deref(), - "7.14.0", + "1.2.3", true )); + assert!(needs_extraction(false, installed.as_deref(), "1.3.0", true)); } #[test] @@ -792,7 +804,7 @@ mod tests { data_dir: env_root.path().join("data"), // no manifest ever written here cache_dir: env_root.path().join("cache"), }; - let version = "7.13.0"; + let version = "1.2.3"; let root = lemonade_root(&paths, Some(env_root.path())); let runtime_dir = runtime_dir_in(&root); fs::create_dir_all(&runtime_dir).unwrap(); @@ -840,7 +852,7 @@ mod tests { data_dir: env_root.path().join("data"), cache_dir: env_root.path().join("cache"), }; - let version = "7.13.0"; + let version = "1.2.3"; // A real, well-formed archive (unlike the stub above): this test // exercises the extraction branch, so `extract_archive` must succeed. @@ -956,13 +968,13 @@ mod tests { fs::create_dir_all(&backend).unwrap(); fs::write(backend.join("llama-server"), b"installed backend").unwrap(); - let packaged = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#; + let packaged = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#; let source = dir.join("source-archive"); write_embeddable_with_pins(&source, packaged); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1027,12 +1039,20 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Lemonade version of the restore tests' manifests: synthetic, so a bump + /// of the real pin does not touch these tests. + const TEST_LEMONADE_VERSION: &str = "1.2.3"; + /// Puts `source` where `prepare_embeddable_with` leaves the verified - /// archive of the `11.5.1` test manifests: `/downloads/`. + /// archive of the [`TEST_LEMONADE_VERSION`] test manifests: `/downloads/`. fn cache_archive(root: &Path, source: &Path) { let downloads = root.join("downloads"); fs::create_dir_all(&downloads).unwrap(); - fs::copy(source, downloads.join(embeddable_archive_name("11.5.1"))).unwrap(); + fs::copy( + source, + downloads.join(embeddable_archive_name(TEST_LEMONADE_VERSION)), + ) + .unwrap(); } fn assert_no_pin_temp_files(dir: &Path) { @@ -1049,13 +1069,13 @@ mod tests { } /// Runs the restore against a fake embeddable archive that ships - /// `{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}`, + /// `{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}`, /// with `current` as the pin file already on disk. Returns what the restore /// returned and the pin file's content afterwards. fn restore_over(tag: &str, current: &str) -> (Option, String) { restore_over_packaged( tag, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, current, ) } @@ -1077,7 +1097,7 @@ mod tests { let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1107,8 +1127,8 @@ mod tests { let (backup, after) = restore_over("restore-keeps-user-edits", current); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["therock"]["version"], "10.1.0"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); assert_eq!(value["llamacpp"]["vulkan"], "b1111", "{after}"); assert_eq!(value["custom"]["x"], 1, "{after}"); assert!(backup.is_some()); @@ -1116,7 +1136,7 @@ mod tests { #[test] fn restore_is_a_no_op_when_only_other_entries_differ_from_the_packaged_file() { - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752","vulkan":"b1111"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580","vulkan":"b1111"}}"#; let (backup, after) = restore_over("restore-noop-other-keys", current); assert!(backup.is_none(), "the alignment pins already match"); @@ -1127,26 +1147,26 @@ mod tests { fn restore_removes_a_pin_the_packaged_file_does_not_have() { // A packaged file without `llamacpp.rocm-stable` must not leave the // kept alignment pin behind; the user's other entries still survive. - let packaged = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"vulkan":"b9747"}}"#; - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b11783","vulkan":"b1111"}}"#; + let packaged = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"vulkan":"b11393"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11783","vulkan":"b1111"}}"#; let (backup, after) = restore_over_packaged("restore-removes-pin", packaged, current); assert!(backup.is_some(), "the kept rocm-stable pin is drift"); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); assert!(value["llamacpp"].get("rocm-stable").is_none(), "{after}"); assert_eq!(value["llamacpp"]["vulkan"], "b1111", "{after}"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); } #[test] fn restore_happens_when_only_the_rocm_stable_pin_drifted() { - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b11783"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11783"}}"#; let (backup, after) = restore_over("restore-only-rocm-stable", current); assert!(backup.is_some(), "a drifted llama.cpp tag alone is drift"); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); + assert_eq!(value["therock"]["version"], "10.1.0"); } #[test] @@ -1155,7 +1175,7 @@ mod tests { assert!(backup.is_some()); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); } /// A runtime tree whose pin file holds a stale kept pin, and the manifest @@ -1173,7 +1193,7 @@ mod tests { .unwrap(); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1223,7 +1243,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); cache_archive(&root, &source); @@ -1254,7 +1274,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); cache_archive(&root, &source); @@ -1278,7 +1298,7 @@ mod tests { ); let value: serde_json::Value = serde_json::from_str(&fs::read_to_string(&pins).unwrap()).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); assert_eq!( value["custom"]["x"], 1, "the concurrent writer's other entries survive" @@ -1300,7 +1320,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); cache_archive(&root, &source); diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 5674afe21..4545fd173 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -140,19 +140,44 @@ pub(crate) fn ensure_best_llamacpp_backend( /// Parse `lemonade backends` table output into `(backend, status)` pairs for the /// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. +/// +/// Lemonade also reports two update states, read here as the state they act +/// like: `update_available` (an installed build that still runs, with a newer +/// release out) as `installed`, and `update_required` (an installed build +/// Lemonade will not run until it is reinstalled, e.g. after the pin in +/// `backend_versions.json` moved) as `installable`, so it is reinstalled rather +/// than reused or dropped. The other state Lemonade 2026.44.1 defines, +/// `action_required` (used by other recipes), leaves the row out. +/// +/// A row that starts in the first column names its recipe; an indented row +/// continues the recipe above it with another backend. The recipe is read from +/// the layout rather than matched against a list of known recipe names: each +/// Lemonade release adds recipes (`llamacpp-hrx`, `ds4`, `thenoise` in +/// 2026.44), and a row for a recipe missing from such a list used to be read as +/// one more `llamacpp` backend, along with every indented row below it. fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { - const RECIPES: [&str; 8] = [ - "flm", - "kokoro", - "llamacpp", - "ryzenai-llm", - "sd-cpp", - "vllm", - "whispercpp", - "embeddings", - ]; - const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; - let mut current_recipe = ""; + fn is_state(token: &str) -> bool { + token.starts_with("update_available") + || matches!( + token, + "installed" | "installable" | "update_required" | "unsupported" | "action_required" + ) + } + fn usable_status(token: &str) -> Option<&'static str> { + // Lemonade pads the status column to 16 characters with no separator + // after it, so the one 16-character state runs straight into the + // message (`update_availableNewer upstream release ...`). + if token.starts_with("update_available") { + return Some("installed"); + } + match token { + "installed" => Some("installed"), + "installable" | "update_required" => Some("installable"), + "unsupported" => Some("unsupported"), + _ => None, + } + } + let mut current_recipe = String::new(); let mut result = Vec::new(); for line in output.lines() { let tokens: Vec<&str> = line.split_whitespace().collect(); @@ -162,25 +187,26 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { if first == "Recipe" || first.starts_with("---") { continue; } - // A row either starts with a recipe name (recipe + its first backend) or, - // for grouped recipes, with a backend name continuing the previous recipe. - let (backend, rest) = if RECIPES.contains(&first) { - current_recipe = match RECIPES.iter().find(|r| **r == first) { - Some(r) => r, - None => current_recipe, - }; + let continues_recipe = line.starts_with(char::is_whitespace); + let (backend, rest) = if continues_recipe { + (first, &tokens[1..]) + } else { + first.clone_into(&mut current_recipe); match tokens.get(1) { Some(backend) => (*backend, &tokens[2..]), None => continue, } - } else { - (first, &tokens[1..]) }; if current_recipe != LLAMACPP_RECIPE { continue; } - if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { - result.push((backend.to_owned(), (*status).to_owned())); + // The first state token is the Status column; a state word later in + // the row is message text. + let Some(state) = rest.iter().find(|token| is_state(token)) else { + continue; + }; + if let Some(status) = usable_status(state) { + result.push((backend.to_owned(), status.to_owned())); } } result @@ -370,6 +396,98 @@ vllm rocm unsupported Requires Linux ); } + /// `lemonade backends` as Lemonade 2026.44.1 prints it on a Linux host with + /// an AMD GPU: a `Tier` and a `Format` column, and recipes (`ds4`, + /// `llamacpp-hrx`, `thenoise`) no earlier release listed. + const LEMONADE_2026_44_BACKENDS_TABLE: &str = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +acestep rocm experimental native installable Backend is supported but not installed. lemonade backends install acestep:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install acestep:vulkan +ds4 rocm guest native installable Backend is supported but not installed. lemonade backends install ds4:rocm +flm npu core native installable Backend is supported but not installed. lemonade backends install flm:npu +kokoro cpu core native installable Backend is supported but not installed. lemonade backends install kokoro:cpu +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install llamacpp:rocm + vulkan core native installable Backend is supported but not installed. lemonade backends install llamacpp:vulkan +llamacpp-hrx hrx experimental native installable Backend is supported but not installed. lemonade backends install llamacpp-hrx:hrx +moonshine cpu core python installable Backend is supported but not installed. lemonade backends install moonshine:cpu +onnxruntime cpu experimental native installable Backend is supported but not installed. lemonade backends install onnxruntime:cpu +openmoss vulkan experimental native installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +sd-cpp cpu core native installable Backend is supported but not installed. lemonade backends install sd-cpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install sd-cpp:rocm + vulkan core native installable Backend is supported but not installed. lemonade backends install sd-cpp:vulkan +thenoise rocm core python installable Backend is supported but not installed. lemonade backends install thenoise:rocm +thinksound rocm experimental native installable Backend is supported but not installed. lemonade backends install thinksound:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install thinksound:vulkan +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +"; + + #[test] + fn parses_only_llamacpp_rows_from_the_2026_44_table() { + let backends = parse_llamacpp_backend_statuses(LEMONADE_2026_44_BACKENDS_TABLE); + assert_eq!( + backends, + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ] + ); + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)) + ); + } + + #[test] + fn another_recipes_backend_is_never_read_as_a_llamacpp_backend() { + // Plain `lemonade backends` (no `--all`) leaves unsupported backends + // out, so on a host with no GPU backend for llama.cpp its only row is + // `cpu`. The indented `vulkan` row belongs to `openmoss`; read as a + // `llamacpp` backend, it would select a GPU backend Lemonade does not + // offer for llama.cpp here. + let output = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu +openmoss rocm experimental native installable Backend is supported but not installed. lemonade backends install openmoss:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!(backends, vec![("cpu".to_owned(), "installable".to_owned())]); + assert_eq!(select_best_llamacpp_backend(&backends), None); + } + + #[test] + fn update_states_read_as_the_state_they_act_like() { + // `update_required`: the installed build will not run until it is + // reinstalled, so it must be installed again, not reused. An + // `update_available` build still runs and is reused; Lemonade prints + // that 16-character state with no space before the message. An + // `action_required` row is left out, whatever words its message holds. + let output = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +llamacpp rocm core native update_required Backend update is required before use. lemonade backends install llamacpp:rocm + vulkan core native update_availableNewer upstream release available: b11400 lemonade backends install llamacpp:vulkan + system core native action_required Use the installed driver instead. - +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!( + backends, + vec![ + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ] + ); + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)), + "an update_required rocm backend is reinstalled, not dropped for vulkan" + ); + } + #[test] fn selects_vulkan_when_rocm_unsupported() { let backends = vec![ diff --git a/runtime-deps.toml b/runtime-deps.toml index 44885660b..2cddff748 100644 --- a/runtime-deps.toml +++ b/runtime-deps.toml @@ -3,8 +3,9 @@ # This file is the SINGLE SOURCE OF TRUTH for those pins. Nothing else in the # tree may spell a runtime version literal: archive names, download URLs, and # offline fallbacks are all derived from the values below by the `rocm-deps` -# crate, whose build script turns each entry into a Rust constant. Editing a -# version here is the whole bump. +# crate, whose build script turns each entry into a Rust constant. A bump also +# records the new archives' digests where the adapter verifies them; for +# Lemonade, that is `EMBEDDABLE_*_SHA256` in engines/lemonade/src/install.rs. # # Schema: one `[runtime.]` table per managed runtime, with one or more # string fields. Every field becomes a `pub const _: &str` in @@ -15,4 +16,4 @@ # Lemonade embeddable release, published as # `lemonade-embeddable--.{tar.gz|zip}` under the # `v` tag of the lemonade-sdk/lemonade GitHub releases. -version = "11.5.1" +version = "2026.44.1" diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 2ebd4fe96..ad3b2a2b1 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -311,33 +311,33 @@ Feature: Runtime configuration And the execution section says the consent came from the user's --yes # Lemonade's llama.cpp backend re-pins itself to match the ROCm SDK rocm-cli - # actually installed (Tier 1: point the pinned build at it; Tier 2: fall back - # to the newest build if the pin is too old to have shipped a matching - # ROCm-version asset; revert to the packaged default if neither verifies). - # The unit tests exercise that state machine directly against injected - # install/align steps, but nothing else asserts that `rocm engines install - # lemonade` actually surfaces the outcome to a real user -- this is the one - # part of that path with no other e2e coverage. + # actually installed (Tier 1: point the pinned build at it; Tier 2: the newest + # build that ships an archive for that ROCm version; revert to the packaged + # default if neither verifies). The unit tests exercise that state machine + # directly against injected install/align steps, but nothing else asserts that + # `rocm engines install lemonade` ends with a ROCm backend paired with the SDK + # on a real GPU -- this is the one part of that path with no other e2e + # coverage. # - # `--reinstall` re-extracts the packaged embeddable, resetting - # `backend_versions.json` to its pinned defaults, so this fires - # deterministically even against a shared runtime tree where an earlier - # scenario already left Lemonade's backend aligned (a plain install would - # find nothing left to do and print no alignment line at all). Verified - # against real hardware (Strix Halo, gfx1151): a fresh SDK's version does not - # match Lemonade's packaged pin, Tier 1's install 404s (the pinned build - # predates a matching ROCm-version asset), and Tier 2's newest build - # succeeds -- producing exactly the line this scenario asserts. + # Which route a lane takes depends on its SDK. An SDK older than Lemonade's + # packaged pin (ROCm 7.14 against 2026.44.1's 10.1.0) is aligned, and the CLI + # says so; an SDK that already is the packaged pin (ROCm 10.1) needs no + # alignment and gets none; a nightly SDK is never aligned and keeps the + # packaged pin. The Then accepts each route and fails a revert: + # it reads the pin the installed runtime tree holds and compares it with the + # active runtime's version. `--reinstall` re-extracts the packaged embeddable, + # resetting `backend_versions.json` to its pinned defaults, so this does not + # depend on what an earlier scenario left in a shared runtime tree. # # `@requires-engine:lemonade` because vLLM shares the SDK's own runtime # environment and has no llama.cpp backend to align; `@nightly` for the same # reason as the vLLM torch-alignment scenarios above -- a real managed SDK # and a real backend download, not something to repeat on every PR. @id:runtime-lemonade-backend-alignment-reported @requires-gpu @requires-os:windows @requires-engine:lemonade @nightly - Scenario: runtime-16 - Reinstalling Lemonade reports whether its ROCm backend was aligned + Scenario: runtime-16 - Reinstalling Lemonade pairs its ROCm backend with the active SDK Given a managed runtime is active When the user reinstalls the lemonade engine - Then the CLI reports that Lemonade's ROCm backend was aligned to the active SDK + Then Lemonade's ROCm backend is paired with the active SDK # `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` is the exit for a hand-edited # `backend_versions.json` -- the alignment runs on every Lemonade install, so diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index c7e6197d4..b68edf462 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -516,22 +516,104 @@ async fn user_reinstalls_lemonade_engine(world: &mut E2eWorld) { world.cli_stderr = Some(stderr); } -/// Verified against real hardware (Strix Halo, gfx1151): a fresh managed SDK -/// install's version does not match Lemonade's packaged pin, Tier 1's install -/// 404s (the pinned build predates a ROCm-7.14 asset), and Tier 2's newest -/// build succeeds -- producing exactly this line. This is the one part of the -/// alignment path with no other e2e coverage: the unit tests exercise the -/// Tier 1/Tier 2/revert state machine directly (against injected install/align -/// steps), but nothing else asserts that `rocm engines install lemonade` -/// actually surfaces the outcome to the user. -#[then("the CLI reports that Lemonade's ROCm backend was aligned to the active SDK")] -async fn assert_lemonade_backend_alignment_reported(world: &mut E2eWorld) { +/// The pin Lemonade installed against ends up equal to the active SDK, by +/// whichever route this host takes: +/// +/// - the active SDK differs from Lemonade's packaged `therock.version`, and the +/// install aligns the pin to it and says so ("Aligned Lemonade's ROCm llama.cpp +/// backend ..."), as on an SDK older than the pin (ROCm 7.14 against +/// Lemonade 2026.44.1's 10.1.0, whose pinned llama.cpp build also ships a 7.14 +/// archive); +/// - the active SDK already is the packaged pin (a ROCm 10.1 SDK), so there is +/// nothing to align: no alignment line, no alignment attempt, and the ROCm +/// backend installs as packaged. +/// +/// Either way the pin in the installed `backend_versions.json` is read back and +/// compared with the active runtime's own version from `rocm runtimes list`, and +/// the selected backend must be `rocm`: a failed alignment reverts the pin to the +/// packaged value and fails the comparison, and a message without the matching +/// pin fails it too. The unit tests drive the Tier 1/Tier 2/revert state machine +/// against injected steps; this is the only check that `rocm engines install +/// lemonade` reaches a ROCm backend paired with the SDK on real hardware. +#[then("Lemonade's ROCm backend is paired with the active SDK")] +async fn assert_lemonade_backend_paired_with_active_sdk(world: &mut E2eWorld) { let output = world.cli_output.as_deref().expect("no install output"); + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); assert!( output - .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"), - "expected the install to report the ROCm backend alignment outcome:\n{output}" + .contains("Selected the best supported llama.cpp backend for this GPU: llamacpp:rocm"), + "expected Lemonade's ROCm llama.cpp backend to be selected:\n{output}" ); + let pinned_version = installed_therock_pin(output); + let active_version = active_runtime_version(world); + if !is_plain_release_version(&active_version) { + // A dated (nightly) SDK is left alone by design: no alignment is + // attempted and the packaged pin stays. + assert_eq!( + pinned_version, + packaged_therock_pin(output), + "a nightly SDK ({active_version}) must leave Lemonade's packaged pin in place" + ); + assert!( + !output.contains("Aligned Lemonade's ROCm") + && !stderr.contains("Aligned Lemonade's ROCm"), + "a nightly SDK ({active_version}) was aligned to:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + return; + } + assert_eq!( + pinned_version, active_version, + "the installed backend_versions.json does not pin the active ROCm SDK:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + let reported_aligned = output + .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"); + if !reported_aligned { + // Only right when the packaged pin already was the active SDK: then no + // alignment is attempted at all. + assert!( + !stderr.contains("Aligned Lemonade's ROCm backend to") + && !stderr.contains("could not align Lemonade's ROCm backend to"), + "an alignment ran but the install did not report it:\n{stderr}" + ); + } +} + +/// Whether `version` is a plain `major.minor.patch` release, the only kind +/// Lemonade's backend alignment matches against (a nightly such as +/// `7.14.0a20260601` is left alone). +fn is_plain_release_version(version: &str) -> bool { + let parts: Vec<&str> = version.split('.').collect(); + parts.len() == 3 + && parts + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) +} + +/// `therock.version` in the `resources/backend_versions.json` of the runtime +/// tree the install reports (`env_path:`). +fn installed_therock_pin(output: &str) -> String { + let env_path = output + .lines() + .find_map(|line| line.trim().strip_prefix("env_path: ")) + .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); + let backend_versions_path = + std::path::Path::new(env_path).join("resources/backend_versions.json"); + let contents = std::fs::read_to_string(&backend_versions_path).unwrap_or_else(|error| { + panic!( + "failed to read {}: {error}", + backend_versions_path.display() + ) + }); + let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap_or_else(|error| { + panic!( + "failed to parse {}: {error}", + backend_versions_path.display() + ) + }); + parsed["therock"]["version"] + .as_str() + .unwrap_or_else(|| panic!("no therock.version in {}", backend_versions_path.display())) + .to_owned() } /// The CLI names the variable whenever it is set, so a user can tell it was @@ -562,9 +644,9 @@ async fn assert_lemonade_backend_alignment_opted_out(world: &mut E2eWorld) { /// (marked with `*`; see `ACTIVE_RUNTIME_MARKER` in `apps/rocm/src/main.rs`). /// /// Read independently of Lemonade's own reporting, on purpose: this is the -/// comparator [`assert_packaged_pin_survives`] uses to prove the pin was not -/// rewritten to match it, so it must not come from anything alignment itself -/// could produce (there is no `ROCm SDK:` line in `rocm version` on this +/// comparator [`assert_lemonade_backend_paired_with_active_sdk`] uses to prove +/// the pin was aligned to it, so it must not come from anything alignment +/// itself could produce (there is no `ROCm SDK:` line in `rocm version` on this /// branch -- that surface is a different, later change). fn active_runtime_version(world: &E2eWorld) -> String { let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]); @@ -582,12 +664,12 @@ fn active_runtime_version(world: &E2eWorld) -> String { /// The packaged pin was not rewritten. /// /// Reads `resources/backend_versions.json` inside the runtime tree the install -/// itself reports (`env_path:`) and compares it against the active runtime's -/// own version (from `rocm runtimes list`, not from anything Lemonade's -/// alignment reports), rather than inferring the outcome from an absent log -/// line alone. Both checks matter: the revert path can fail to restore the -/// pin (best-effort, matching its sibling warnings) while still printing no -/// "Aligned ..." line, so absence of that line alone cannot distinguish a +/// itself reports (`env_path:`) and compares it with the copy inside the +/// Lemonade archive the install verified, rather than inferring the outcome +/// from an absent log line alone. The absent-line checks matter too: the +/// revert path can fail to restore the pin (best-effort, matching its sibling +/// warnings) while still printing no "Aligned ..." line, so absence of that +/// line alone cannot distinguish a /// failed-restore from an honestly untouched pin -- hence also checking for /// the revert path's own "could not align ... reverting" warning, which does /// fire whenever a restore was attempted. `--reinstall` (the When's own @@ -610,34 +692,79 @@ async fn assert_packaged_pin_survives(world: &mut E2eWorld) { "an alignment attempt ran (and failed) instead of never starting:\n{stderr}" ); + let pinned_version = installed_therock_pin(output); + let packaged_version = packaged_therock_pin(output); + // Where the active SDK already is the packaged pin (a ROCm 10.1 SDK against + // Lemonade 2026.44.1) this cannot tell an untouched pin from one rewritten to + // the same value; the absent alignment and revert lines above are then the + // whole check. + assert_eq!( + pinned_version, packaged_version, + "the pin differs from the one the Lemonade archive ships even though the user opted out" + ); +} + +/// `therock.version` as the Lemonade embeddable archive the install verified +/// ships it, read from the cached archive beside the runtime tree +/// (`/downloads/` next to `/runtime`) with the system `tar`, which +/// reads both the Linux `.tar.gz` and, as bsdtar on Windows, the `.zip`. The +/// archive is the one named by the install's `env_id` +/// (`lemonade-embeddable-`): the downloads directory is never pruned, +/// so a runner that installed an earlier pin still holds that archive too. +fn packaged_therock_pin(output: &str) -> String { let env_path = output .lines() .find_map(|line| line.trim().strip_prefix("env_path: ")) .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); - let backend_versions_path = - std::path::Path::new(env_path).join("resources/backend_versions.json"); - let contents = std::fs::read_to_string(&backend_versions_path).unwrap_or_else(|error| { - panic!( - "failed to read {}: {error}", - backend_versions_path.display() - ) - }); - let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap_or_else(|error| { - panic!( - "failed to parse {}: {error}", - backend_versions_path.display() - ) - }); - let pinned_version = parsed["therock"]["version"] + let env_id = output + .lines() + .find_map(|line| line.trim().strip_prefix("env_id: ")) + .unwrap_or_else(|| panic!("no env_id in install output:\n{output}")); + let archive_prefix = format!("{env_id}-"); + let downloads = std::path::Path::new(env_path) + .parent() + .unwrap_or_else(|| panic!("env_path {env_path} has no parent")) + .join("downloads"); + let archive = std::fs::read_dir(&downloads) + .unwrap_or_else(|error| panic!("failed to list {}: {error}", downloads.display())) + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.starts_with(&archive_prefix) + && (name.ends_with(".tar.gz") || name.ends_with(".zip")) + }) + }) + .unwrap_or_else(|| panic!("no cached {env_id} archive in {}", downloads.display())); + let name = archive + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + let stem = name + .strip_suffix(".tar.gz") + .or_else(|| name.strip_suffix(".zip")) + .unwrap_or(name); + let member = format!("{stem}/resources/backend_versions.json"); + let extracted = std::process::Command::new("tar") + .arg("-xOf") + .arg(&archive) + .arg(&member) + .output() + .unwrap_or_else(|error| panic!("failed to run tar on {}: {error}", archive.display())); + assert!( + extracted.status.success(), + "tar could not read {member} from {}: {}", + archive.display(), + String::from_utf8_lossy(&extracted.stderr) + ); + let parsed: serde_json::Value = serde_json::from_slice(&extracted.stdout) + .unwrap_or_else(|error| panic!("failed to parse {member}: {error}")); + parsed["therock"]["version"] .as_str() - .unwrap_or_else(|| panic!("no therock.version in {}", backend_versions_path.display())); - - let active_version = active_runtime_version(world); - assert_ne!( - pinned_version, active_version, - "the packaged pin was rewritten to the active ROCm SDK version even though the user \ - opted out" - ); + .unwrap_or_else(|| panic!("no therock.version in {member}")) + .to_owned() } /// The engine inventory reports a usable engine runtime. From 806c9f0659c423c395d71f05a4aac38e6e6d1130 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 17:32:06 +0200 Subject: [PATCH 2/2] test(e2e): compare the Lemonade archive extension ignoring case The runtime-17 step that finds the cached Lemonade archive compared the `.zip` extension with `ends_with`, which clippy's pedantic `case_sensitive_file_extension_comparisons` rejects. The e2e target is excluded from `--all-targets`, so only the Windows job's `cargo clippy -p e2e-cucumber --test e2e` compiles it and caught it. Derive the archive's stem once, comparing either extension ignoring ASCII case, and reuse it for the member path. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index b68edf462..3094ba89f 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -725,27 +725,25 @@ fn packaged_therock_pin(output: &str) -> String { .parent() .unwrap_or_else(|| panic!("env_path {env_path} has no parent")) .join("downloads"); - let archive = std::fs::read_dir(&downloads) + // The archive's top-level directory is its file name without the + // extension. rocm-cli names the archive itself (`lemonade_archive_name`), + // always in lower case; the extension is compared ignoring ASCII case, as + // clippy's `case_sensitive_file_extension_comparisons` asks. + let archive_stem = |path: &std::path::Path| -> Option { + let name = path.file_name()?.to_str()?; + [".tar.gz", ".zip"].iter().find_map(|extension| { + let split = name.len().checked_sub(extension.len())?; + let (stem, tail) = (name.get(..split)?, name.get(split..)?); + (tail.eq_ignore_ascii_case(extension) && stem.starts_with(&archive_prefix)) + .then(|| stem.to_owned()) + }) + }; + let (archive, stem) = std::fs::read_dir(&downloads) .unwrap_or_else(|error| panic!("failed to list {}: {error}", downloads.display())) .filter_map(Result::ok) .map(|entry| entry.path()) - .find(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| { - name.starts_with(&archive_prefix) - && (name.ends_with(".tar.gz") || name.ends_with(".zip")) - }) - }) + .find_map(|path| archive_stem(&path).map(|stem| (path, stem))) .unwrap_or_else(|| panic!("no cached {env_id} archive in {}", downloads.display())); - let name = archive - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - let stem = name - .strip_suffix(".tar.gz") - .or_else(|| name.strip_suffix(".zip")) - .unwrap_or(name); let member = format!("{stem}/resources/backend_versions.json"); let extracted = std::process::Command::new("tar") .arg("-xOf")