diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 63be208..1cc3fc9 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -66,7 +66,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "base-arch" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-format", @@ -77,7 +77,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-format", @@ -91,7 +91,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-arch", @@ -114,7 +114,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "bitflags", @@ -130,7 +130,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-format", @@ -145,7 +145,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-format", @@ -157,7 +157,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-format", @@ -171,7 +171,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index ea2f8e1..9bee6df 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.1.6" +version = "0.1.7" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/README.md b/base-convert/README.md index fa7699e..56ea13d 100644 --- a/base-convert/README.md +++ b/base-convert/README.md @@ -47,14 +47,22 @@ basert list # installed models (table; --json for JSON) basert list --remote # also show catalog models not yet installed ``` +Downloads stage under `/.src/hf` — not the global HuggingFace +cache — and are moved into place on success, so a pulled model exists +exactly once on disk. Interrupted pulls leave their partial download in +staging and resume on retry. Convert-on-pull deletes the downloaded +safetensors sources after a successful conversion; set +`BASERT_KEEP_HF_SOURCES=1` to keep them (faster re-pulls of another quant +of the same repo, at the cost of a second multi-GB copy). + Gated/private repos use the standard HuggingFace token chain (`$HF_TOKEN` / `~/.cache/huggingface/token`). Convert-on-pull in the public build uses only generic profiles; tuned quantization is delivered through pre-converted catalog artifacts. -## Runtime tools +## Runtime commands -`basert ` forwards to the matching runtime binary (`basert-`): +Run a model with the BaseRT runtime: ``` basert serve [--model …] # OpenAI-compatible server (--model repeats for multi-model) diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 42540a7..7a22859 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -9,7 +9,7 @@ use crate::{AwqMode, ConvertArgs, ListArgs, PullArgs, TargetScheme}; use anyhow::{bail, Context, Result}; use base_hub::cache::{self, HubSidecar}; -use base_hub::fetch::{Fetcher, HfFetcher}; +use base_hub::fetch::{self, Fetcher, HfFetcher}; use base_hub::registry::{MergedRegistry, ModelEntry, ModelRef, Registry, SourceKind}; use std::path::{Path, PathBuf}; use std::process::Command; @@ -18,11 +18,13 @@ use std::process::Command; // per-arch profiles are intentionally NOT bundled — tuned quality reaches // users through pre-converted catalog artifacts, not by exposing the recipe. const PROFILE_Q4: &str = include_str!("../../../profiles/default-q4.json"); +const PROFILE_Q6: &str = include_str!("../../../profiles/default-q6.json"); const PROFILE_Q8: &str = include_str!("../../../profiles/default-q8.json"); fn bundled_profile_for_target(target: TargetScheme) -> Option<(&'static str, &'static str)> { match target { TargetScheme::BaseQ4 => Some(("default-q4", PROFILE_Q4)), + TargetScheme::BaseQ6 => Some(("default-q6", PROFILE_Q6)), TargetScheme::BaseQ8 => Some(("default-q8", PROFILE_Q8)), _ => None, } @@ -314,9 +316,7 @@ fn extract_variant_flag(rest: &[String]) -> Result<(Option, Vec) pub fn dispatch_external(argv: Vec) -> Result<()> { use std::os::unix::process::CommandExt; - let (cmd, rest) = argv - .split_first() - .context("no subcommand given to launcher")?; + let (cmd, rest) = argv.split_first().context("no command given")?; // `--variant ` is a launcher-level model selector; strip it before // forwarding (the runtime binary doesn't know it) and apply it during // hub-id resolution. @@ -347,13 +347,22 @@ pub fn dispatch_external(argv: Vec) -> Result<()> { return Err(err).with_context(|| format!("launching {}", target.display())); } } - bail!( - "unknown command `basert {cmd}` — no `basert-{cmd}` runtime tool found \ - next to this binary or on PATH.\n\ - Install the BaseRT runtime alongside the CLI, or run `basert --help`." - ) + if RUNTIME_COMMANDS.contains(&cmd.as_str()) { + bail!( + "`basert {cmd}` needs the BaseRT runtime, which wasn't found in this \ + installation.\n\ + Reinstall the full BaseRT package, or run `basert --help`." + ) + } + bail!("unknown command `basert {cmd}`.\nRun `basert --help` to see available commands.") } +/// Commands served by the BaseRT runtime rather than this binary. Used only +/// to shape the not-found error above; dispatch itself is name-driven, so +/// commands absent from this list (or from `basert --help`) still dispatch. +const RUNTIME_COMMANDS: [&str; 6] = + ["serve", "chat", "complete", "bench", "transcribe", "profile"]; + pub fn cmd_pull(args: PullArgs) -> Result<()> { let reg = MergedRegistry::load()?; let root = reg.root.clone(); @@ -381,7 +390,7 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { if quant_tag(variant) == want { pull_catalog(&root, &r) } else { - let fetcher = HfFetcher::new()?; + let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; let base_files = list_base_files(&fetcher, hf_repo, revision)?; pull_base_direct(&root, &args, id, hf_repo, revision, &fetcher, &base_files) } @@ -390,7 +399,7 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { // (e.g. the basecompute org), download the matching one directly — no // local conversion. Otherwise treat it as source safetensors. ModelRef::HuggingFace { id, repo, revision } => { - let fetcher = HfFetcher::new()?; + let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; let base_files = list_base_files(&fetcher, repo, revision)?; if base_files.is_empty() { pull_and_convert(&root, &args, id, repo, revision) @@ -442,18 +451,23 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { eprintln!("basert pull v{}", env!("CARGO_PKG_VERSION")); eprintln!(" catalog: {hf_repo}/{file}@{revision} (pre-converted)"); - let fetcher = HfFetcher::new()?; + let fetcher = HfFetcher::new(cache::hf_staging_dir(root))?; let src = fetcher.get_file(hf_repo, revision, file)?; let vdir = cache::variant_dir(root, id, variant)?; std::fs::create_dir_all(&vdir)?; let out = cache::base_artifact_path(&vdir); - std::fs::copy(&src, &out).with_context(|| format!("installing into {}", out.display()))?; + // Moves the staged download into place (same filesystem), so the pulled + // artifact exists exactly once on disk. + fetch::install_file(&fetcher, hf_repo, &src, &out)?; let got_sha = crate::compute_sha256_streaming(&out)?; if let Some(expected) = sha256 { if !got_sha.eq_ignore_ascii_case(expected) { std::fs::remove_file(&out).ok(); + // The staged bytes are equally bad — no resume value in them, so + // clear them and let a retry start from scratch. + fetch::cleanup_staging(&fetcher, hf_repo); bail!("sha256 mismatch for {id}: expected {expected}, got {got_sha}"); } eprintln!(" sha256: verified"); @@ -470,6 +484,7 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { None, Some(got_sha), )?; + fetch::cleanup_staging(&fetcher, hf_repo); eprintln!("installed {id} [{variant}] → {}", out.display()); Ok(()) } @@ -508,10 +523,13 @@ fn pull_base_direct( let out = cache::base_artifact_path(&vdir); let src = fetcher.get_file(repo, revision, file)?; - std::fs::copy(&src, &out).with_context(|| format!("installing into {}", out.display()))?; + // Moves the staged download into place (same filesystem), so the pulled + // artifact exists exactly once on disk. + fetch::install_file(fetcher, repo, &src, &out)?; let sha = crate::compute_sha256_streaming(&out).ok(); write_sidecar_for(&vdir, id, "huggingface", repo, None, revision, &variant, None, sha)?; + fetch::cleanup_staging(fetcher, repo); eprintln!("installed {id} [{variant}] → {}", out.display()); Ok(()) } @@ -536,7 +554,7 @@ fn pull_and_convert( let out = cache::base_artifact_path(&vdir); std::fs::create_dir_all(&vdir)?; - let fetcher = HfFetcher::new()?; + let fetcher = HfFetcher::new(cache::hf_staging_dir(root))?; let snapshot = download_source(repo, revision, &fetcher)?; let conv = ConvertArgs { @@ -565,10 +583,37 @@ fn pull_and_convert( Some(&variant), sha, )?; + // The staged source snapshot is now a redundant multi-GB copy of a model + // we keep in converted form (provenance lives in hub.json; a re-convert + // can re-fetch). Drop it unless the user opted to keep sources — keeping + // them makes a later pull of another quant of the same repo skip the + // re-download. + if keep_hf_sources() { + eprintln!( + " keeping HF source snapshot under {} (BASERT_KEEP_HF_SOURCES)", + cache::hf_staging_dir(root).display() + ); + } else { + fetch::cleanup_staging(&fetcher, repo); + } eprintln!("installed {id} [{variant}] → {}", out.display()); Ok(()) } +/// Whether convert-on-pull should keep the downloaded HF source snapshot +/// after a successful conversion. Off by default: the converted `.base` is +/// the artifact users asked for, and the safetensors sources would otherwise +/// linger as a second multi-GB copy. `BASERT_KEEP_HF_SOURCES=1` (anything but +/// `0`/`false`/empty) trades that disk for faster re-pulls of other quants. +fn keep_hf_sources() -> bool { + std::env::var("BASERT_KEEP_HF_SOURCES") + .map(|v| { + let v = v.trim(); + !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false") + }) + .unwrap_or(false) +} + /// Returns `(variant, profile_path, tempfile_guard)`. The guard keeps a /// bundled-profile tempfile alive until conversion finishes. fn choose_profile( @@ -835,6 +880,105 @@ mod tests { assert_eq!(std::fs::read(&out).unwrap(), b"q8-bytes"); // A provenance sidecar was written. assert!(root.join("basecompute/m/default-q8/hub.json").exists()); + // MockFetcher owns no staging: its fixtures are copied, never moved. + assert!(repo_dir.join("m-Q8.base").exists(), "fixture must survive"); + } + + /// Fetcher owning an hf-hub-style staging tree (blob + snapshot symlink), + /// mirroring what `HfFetcher` leaves on disk mid-pull. + struct StagedFetcher { + staging: PathBuf, + } + + impl StagedFetcher { + fn repo_dir(&self, repo: &str) -> PathBuf { + self.staging.join(format!("models--{}", repo.replace('/', "--"))) + } + + fn stage(&self, repo: &str, revision: &str, filename: &str, bytes: &[u8]) { + let rdir = self.repo_dir(repo); + let blobs = rdir.join("blobs"); + let snap = rdir.join("snapshots").join(revision); + std::fs::create_dir_all(&blobs).unwrap(); + std::fs::create_dir_all(&snap).unwrap(); + let blob = blobs.join(format!("etag-{filename}")); + std::fs::write(&blob, bytes).unwrap(); + std::os::unix::fs::symlink(&blob, snap.join(filename)).unwrap(); + } + } + + impl Fetcher for StagedFetcher { + fn get_file(&self, repo: &str, revision: &str, filename: &str) -> anyhow::Result { + let p = self.repo_dir(repo).join("snapshots").join(revision).join(filename); + anyhow::ensure!(p.exists(), "not staged: {}", p.display()); + Ok(p) + } + + fn list_files(&self, repo: &str, revision: &str) -> anyhow::Result> { + let dir = self.repo_dir(repo).join("snapshots").join(revision); + let mut out = Vec::new(); + for e in std::fs::read_dir(dir)? { + out.push(e?.file_name().to_string_lossy().into_owned()); + } + Ok(out) + } + + fn staging_dir(&self, repo: &str) -> Option { + Some(self.repo_dir(repo)) + } + } + + #[test] + fn pull_base_direct_from_staging_leaves_exactly_one_copy() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("cache"); + let fetcher = StagedFetcher { + staging: root.join(".src").join("hf"), + }; + fetcher.stage("basecompute/m", "main", "m-Q4.base", b"q4-bytes"); + let base_files = list_base_files(&fetcher, "basecompute/m", "main").unwrap(); + + let args = PullArgs { + id: "basecompute/m".into(), + profile: None, + target: TargetScheme::BaseQ4, + revision: "main".into(), + force: false, + dry_run: false, + }; + pull_base_direct(&root, &args, "basecompute/m", "basecompute/m", "main", &fetcher, &base_files) + .unwrap(); + + let out = root.join("basecompute/m/default-q4/model.base"); + assert_eq!(std::fs::read(&out).unwrap(), b"q4-bytes"); + // The staged download was consumed: no second copy anywhere. + assert!( + !fetcher.repo_dir("basecompute/m").exists(), + "staging tree must be cleaned after a successful install" + ); + } + + #[test] + fn keep_hf_sources_reads_env() { + // Single test for all cases: mutates shared process env (see the + // matching pattern in base-hub's fetch.rs tests). + let prev = std::env::var("BASERT_KEEP_HF_SOURCES").ok(); + + std::env::remove_var("BASERT_KEEP_HF_SOURCES"); + assert!(!keep_hf_sources(), "default must be: clean up sources"); + for off in ["", "0", "false", "FALSE", " 0 "] { + std::env::set_var("BASERT_KEEP_HF_SOURCES", off); + assert!(!keep_hf_sources(), "{off:?} should not keep sources"); + } + for on in ["1", "true", "yes"] { + std::env::set_var("BASERT_KEEP_HF_SOURCES", on); + assert!(keep_hf_sources(), "{on:?} should keep sources"); + } + + match prev { + Some(v) => std::env::set_var("BASERT_KEEP_HF_SOURCES", v), + None => std::env::remove_var("BASERT_KEEP_HF_SOURCES"), + } } #[test] @@ -1014,7 +1158,7 @@ mod tests { } #[test] - fn choose_profile_uses_bundled_for_q4_and_q8() { + fn choose_profile_uses_bundled_for_q4_q6_and_q8() { let mk = |t| PullArgs { id: "x/y".into(), profile: None, @@ -1027,6 +1171,10 @@ mod tests { assert_eq!(variant, "default-q4"); assert!(path.is_some() && guard.is_some()); + let (variant, path, guard) = choose_profile(&mk(TargetScheme::BaseQ6)).unwrap(); + assert_eq!(variant, "default-q6"); + assert!(path.is_some() && guard.is_some()); + let (variant, _, _) = choose_profile(&mk(TargetScheme::BaseQ8)).unwrap(); assert_eq!(variant, "default-q8"); diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index 024f348..3fe9c8c 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -4,29 +4,45 @@ use std::path::PathBuf; mod hub; -/// Listed in `basert --help` so the dispatched runtime tools are discoverable -/// (clap's external-subcommand catch-all is otherwise invisible in help). -const RUNTIME_TOOLS_HELP: &str = "\ -Runtime tools (forwarded to the matching `basert-` binary): - serve Start the OpenAI-compatible HTTP server - chat Interactive chat - complete One-shot text completion - bench Throughput benchmark - profile Profile prefill/decode timing - transcribe Audio transcription - -Run `basert --help` for a tool's own options. - -Model variant (serve/chat/complete/bench/profile) — pick which quant build to run: +/// Hand-written top-level help: the runtime commands are dispatched via +/// clap's external-subcommand catch-all (invisible to clap's auto listing), +/// so the whole command list is laid out here — runtime commands first, +/// grouped, with the built-in subcommands folded in. +const HELP_TEMPLATE: &str = "\ +{about-with-newline} +{usage-heading} {usage} + +Run models: + serve Start an OpenAI-compatible HTTP server + chat Chat with a model interactively + complete Generate a one-shot completion + bench Measure throughput + +Manage models: + pull Download a model from the BaseRT catalog or Hugging Face + list List installed models (`--remote` adds the catalog) + +Convert & author: + convert Convert a GGUF / safetensors model to `.base` + inspect Summarize a `.base` file (header, tensors, metadata) + sign Sign a `.base` file + verify Verify a signed `.base` file + keygen Generate a signing keypair + +Options: + -h, --help Print help + -V, --version Print version + +Model variants — pick which quant build to run: : e.g. basert serve basecompute/Gemma-4-E2B-it:default-q8 --variant same, as a flag (applies to ids without an inline ':variant') -Run `basert list` to see installed variants (default-q4, default-q8, …)."; +Run `basert list` to see installed variants (default-q4, default-q8, …). -/// The `basert` CLI: the model hub (pull/list from HuggingFace), the offline -/// GGUF / MLX-safetensors / HF-safetensors → `.base` converter, and a -/// launcher for the runtime tools (`basert serve`, `basert chat`, …). +Run `basert --help` for a command's options."; + +/// The BaseRT CLI: run, download, and convert models. #[derive(Parser, Debug)] -#[command(name = "basert", version, about, after_help = RUNTIME_TOOLS_HELP)] +#[command(name = "basert", version, about, help_template = HELP_TEMPLATE)] struct Args { #[command(subcommand)] cmd: Cmd, @@ -49,8 +65,8 @@ enum Cmd { Pull(PullArgs), /// List models in the local hub cache (and, with `--remote`, the catalog). List(ListArgs), - /// Runtime tools — `serve`, `chat`, `complete`, `bench`, … — forwarded to - /// the matching engine binary (`basert-`). + /// Runtime commands — `serve`, `chat`, `complete`, `bench`, … — handled + /// by the BaseRT runtime (dispatched in `hub::dispatch_external`). #[command(external_subcommand)] External(Vec), } diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index 6f08e7b..fe0b8fa 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -1,6 +1,6 @@ { "schema": 1, - "updated": "2026-07-01", + "updated": "2026-07-20", "models": [ { "id": "basecompute/Qwen3-0.6B", @@ -325,6 +325,106 @@ "quant": "default-q4", "size": 17182920704, "sha256": "7b294daeff8ba6e097f6cebe9f56c7b71f32476765a3f614eb256ef9d8c6ffce" + }, + { + "id": "basecompute/Qwen3.5-2B", + "hf_repo": "basecompute/Qwen3.5-2B", + "source_repo": "Qwen/Qwen3.5-2B", + "arch": "qwen35", + "file": "Qwen3.5-2B-Q4.base", + "quant": "default-q4", + "size": 1266810880, + "sha256": "1c95f41730d5713fcb886a71d2481a29ff1a6e2fc9cea8a4d9ddbe29082fa262" + }, + { + "id": "basecompute/Qwen3.5-2B", + "hf_repo": "basecompute/Qwen3.5-2B", + "source_repo": "Qwen/Qwen3.5-2B", + "arch": "qwen35", + "file": "Qwen3.5-2B-Q8.base", + "quant": "default-q8", + "size": 2301460480, + "sha256": "d4cf325278e873b3e0aefc18c5eff0a863f12fee5a82f0ac2f2e5ebce4067f3c" + }, + { + "id": "basecompute/Qwen3.5-2B-Base", + "hf_repo": "basecompute/Qwen3.5-2B-Base", + "source_repo": "Qwen/Qwen3.5-2B-Base", + "arch": "qwen35", + "file": "Qwen3.5-2B-Base-Q4.base", + "quant": "default-q4", + "size": 1266810880, + "sha256": "ef10005ccd4c6bd3e2d9fbcac31b7f4491eff2639584bdc8006e67226f63d986" + }, + { + "id": "basecompute/Qwen3.5-2B-Base", + "hf_repo": "basecompute/Qwen3.5-2B-Base", + "source_repo": "Qwen/Qwen3.5-2B-Base", + "arch": "qwen35", + "file": "Qwen3.5-2B-Base-Q8.base", + "quant": "default-q8", + "size": 2301460480, + "sha256": "f546b5074d92f104453a782a44143f3a44ed538569befdc7e2ff22675c719a30" + }, + { + "id": "basecompute/Qwen3.5-35B-A3B", + "hf_repo": "basecompute/Qwen3.5-35B-A3B", + "source_repo": "Qwen/Qwen3.5-35B-A3B", + "arch": "qwen35moe", + "file": "Qwen3.5-35B-A3B-Q4.base", + "quant": "default-q4", + "size": 19776225280, + "sha256": "e62635e523e26c7a31df2d0d26cec485630b3b2172a9fd710b06e90c45174ff5" + }, + { + "id": "basecompute/Qwen3.5-35B-A3B", + "hf_repo": "basecompute/Qwen3.5-35B-A3B", + "source_repo": "Qwen/Qwen3.5-35B-A3B", + "arch": "qwen35moe", + "file": "Qwen3.5-35B-A3B-Q8.base", + "quant": "default-q8", + "size": 36228218880, + "sha256": "6f8878ff2332451efbf2258887bad0535a97aeb657cf0b7ce8492219881184ff" + }, + { + "id": "basecompute/Qwen3.6-27B", + "hf_repo": "basecompute/Qwen3.6-27B", + "source_repo": "Qwen/Qwen3.6-27B", + "arch": "qwen35", + "file": "Qwen3.6-27B-Q4.base", + "quant": "default-q4", + "size": 15447605248, + "sha256": "f24b31c552994f82e18f2a3b7a7352ff0c41cf070351e941816365bd02bca79c" + }, + { + "id": "basecompute/Qwen3.6-27B", + "hf_repo": "basecompute/Qwen3.6-27B", + "source_repo": "Qwen/Qwen3.6-27B", + "arch": "qwen35", + "file": "Qwen3.6-27B-Q8.base", + "quant": "default-q8", + "size": 28256698368, + "sha256": "f65023ea2a5e1cc2a6091cc926a4156935e4ada58133d4bc84bc6d759f738039" + }, + { + "id": "basecompute/Qwen3.6-35B-A3B", + "hf_repo": "basecompute/Qwen3.6-35B-A3B", + "source_repo": "Qwen/Qwen3.6-35B-A3B", + "arch": "qwen35moe", + "file": "Qwen3.6-35B-A3B-Q4.base", + "quant": "default-q4", + "size": 19776225280, + "sha256": "6819d2152fe025af5b1eb12d02e862c5277258a016e6b901c6c91791ba24c48b" + }, + { + "id": "basecompute/Qwen3.6-35B-A3B", + "hf_repo": "basecompute/Qwen3.6-35B-A3B", + "source_repo": "Qwen/Qwen3.6-35B-A3B", + "arch": "qwen35moe", + "file": "Qwen3.6-35B-A3B-Q8.base", + "quant": "default-q8", + "size": 36228218880, + "sha256": "11d093a918e2a0b917430547dafcdcab2fe1c2d689ec86b5343382f72d2b2c3b" } ] } diff --git a/base-convert/crates/base-hub/src/cache.rs b/base-convert/crates/base-hub/src/cache.rs index ba9b448..43e2544 100644 --- a/base-convert/crates/base-hub/src/cache.rs +++ b/base-convert/crates/base-hub/src/cache.rs @@ -4,7 +4,7 @@ //! $BASERT_MODELS_DIR/ (default ~/.cache/baseRT/models) //! ///model.base ← artifact the runtime loads //! ///hub.json ← provenance sidecar -//! .src//// ← raw HF snapshot staging +//! .src/hf/models----/ ← hf-hub download staging //! ``` //! //! The fixed `model.base` filename makes server discovery a trivial @@ -19,7 +19,7 @@ use std::path::{Component, Path, PathBuf}; pub const ARTIFACT_NAME: &str = "model.base"; /// Provenance sidecar filename inside every variant directory. pub const SIDECAR_NAME: &str = "hub.json"; -/// Sub-tree holding raw HF source snapshots prior to conversion. +/// Sub-tree holding in-flight HF downloads prior to installation. pub const SRC_STAGING: &str = ".src"; /// Resolve the models root: `$BASERT_MODELS_DIR` or `~/.cache/baseRT/models`. @@ -82,14 +82,15 @@ pub fn base_artifact_path(variant_dir: &Path) -> PathBuf { variant_dir.join(ARTIFACT_NAME) } -/// Staging dir for a raw HF source snapshot: -/// `/.src///`. Kept apart from the canonical tree -/// so an interrupted download never pollutes it and `list` never trips over -/// loose safetensors. -pub fn src_staging_dir(root: &Path, repo: &str, revision: &str) -> Result { - let rel = id_to_relpath(repo)?; - let rev = sanitize_variant(revision)?; - Ok(root.join(SRC_STAGING).join(rel).join(rev)) +/// Root for hf-hub downloads: `/.src/hf`. Downloads land here (in +/// hf-hub's own `models----/{blobs,snapshots,refs}` layout) instead +/// of the user's global HuggingFace cache, so a pulled artifact is never +/// duplicated across two caches and a finished install is a same-filesystem +/// rename, not a copy. Kept apart from the canonical tree so an interrupted +/// download never pollutes it and `list` never trips over loose safetensors; +/// partial downloads left behind by a failed pull are resumed on retry. +pub fn hf_staging_dir(root: &Path) -> PathBuf { + root.join(SRC_STAGING).join("hf") } /// A single path component derived from free-form text (variant / revision). @@ -221,10 +222,12 @@ mod tests { } #[test] - fn src_staging_is_namespaced() { + fn hf_staging_is_under_dot_src() { let root = Path::new("/models"); - let d = src_staging_dir(root, "meta-llama/Llama-3.2-1B", "main").unwrap(); - assert_eq!(d, Path::new("/models/.src/meta-llama/Llama-3.2-1B/main")); + let d = hf_staging_dir(root); + assert_eq!(d, Path::new("/models/.src/hf")); + // Must stay inside the `.src` sub-tree that `list` skips. + assert!(d.starts_with(root.join(SRC_STAGING))); } #[test] diff --git a/base-convert/crates/base-hub/src/fetch.rs b/base-convert/crates/base-hub/src/fetch.rs index 51fdedd..e5bdebc 100644 --- a/base-convert/crates/base-hub/src/fetch.rs +++ b/base-convert/crates/base-hub/src/fetch.rs @@ -6,7 +6,7 @@ //! CI with no HuggingFace access. use anyhow::{Context, Result}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; /// Download retries hf-hub performs on transient failures (peer disconnects, /// truncated chunks — routine on multi-GB model pulls). hf-hub's own default is @@ -31,20 +31,42 @@ pub trait Fetcher { /// List the filenames available in `repo` at `revision`. fn list_files(&self, repo: &str, revision: &str) -> Result>; + + /// The staging directory this fetcher owns for `repo` — every byte it + /// downloaded for that repo lives under it, and nothing else does. `None` + /// when the fetcher serves files it does not own (fixtures, a shared + /// cache): those must be copied on install and never deleted. + fn staging_dir(&self, repo: &str) -> Option { + let _ = repo; + None + } } /// Real fetcher backed by hf-hub's synchronous API. Reads the HF token from /// `$HF_TOKEN` / `$HUGGING_FACE_HUB_TOKEN`, falling back to the cached login /// token (`~/.cache/huggingface/token`). +/// +/// Downloads land in a private staging directory (normally +/// `/.src/hf` — see [`crate::cache::hf_staging_dir`]), NOT the +/// user's global HuggingFace cache: multi-GB `.base` artifacts would otherwise +/// persist there as a second copy after installation. Keeping staging on the +/// same filesystem as the models root also lets installs move (rename) the +/// downloaded bytes instead of copying them. pub struct HfFetcher { api: hf_hub::api::sync::Api, + staging_root: PathBuf, } impl HfFetcher { - pub fn new() -> Result { + pub fn new(staging_root: impl Into) -> Result { + let staging_root = staging_root.into(); + // `ApiBuilder::new()` snapshots the login token from the default HF + // cache location before `with_cache_dir` re-points downloads at our + // staging dir, so `~/.cache/huggingface/token` keeps working. let mut builder = hf_hub::api::sync::ApiBuilder::new() .with_progress(true) - .with_retries(resolve_max_retries()); + .with_retries(resolve_max_retries()) + .with_cache_dir(staging_root.clone()); if let Some(tok) = std::env::var("HF_TOKEN") .ok() .or_else(|| std::env::var("HUGGING_FACE_HUB_TOKEN").ok()) @@ -55,7 +77,7 @@ impl HfFetcher { let api = builder .build() .context("initializing HuggingFace API client")?; - Ok(Self { api }) + Ok(Self { api, staging_root }) } fn repo(&self, repo: &str, revision: &str) -> hf_hub::api::sync::ApiRepo { @@ -81,6 +103,53 @@ impl Fetcher for HfFetcher { .with_context(|| format!("fetching repo info for {repo}@{revision}"))?; Ok(info.siblings.into_iter().map(|s| s.rfilename).collect()) } + + fn staging_dir(&self, repo: &str) -> Option { + // hf-hub keeps everything for a repo under `models----`. + let folder = hf_hub::Repo::model(repo.to_string()).folder_name(); + Some(self.staging_root.join(folder)) + } +} + +/// Install a file returned by [`Fetcher::get_file`] at `dst`, leaving at most +/// one surviving copy of the bytes. +/// +/// When `src` sits inside the fetcher's own staging tree for `repo`, the +/// underlying blob is *moved* (symlinks resolved first — hf-hub's snapshot +/// paths are pointers into `blobs/`), so no duplicate ever exists; a rename +/// that fails (e.g. across filesystems) degrades to a copy, and the source is +/// then reclaimed by [`cleanup_staging`]. Files the fetcher does not own +/// (fixtures, shared caches) are copied and left untouched. +pub fn install_file(fetcher: &dyn Fetcher, repo: &str, src: &Path, dst: &Path) -> Result<()> { + let owned = fetcher + .staging_dir(repo) + .is_some_and(|dir| src.starts_with(&dir)); + if owned { + // Resolve the snapshot symlink to the actual blob before renaming; + // renaming the symlink itself would strand the payload in staging. + let real = std::fs::canonicalize(src) + .with_context(|| format!("resolving {}", src.display()))?; + if std::fs::rename(&real, dst).is_ok() { + return Ok(()); + } + // Rename can fail across filesystems; fall through to a copy (the + // staged source is removed later by `cleanup_staging`). + } + std::fs::copy(src, dst) + .with_context(|| format!("installing {} into {}", src.display(), dst.display()))?; + Ok(()) +} + +/// Delete everything the fetcher staged for `repo`. Call only once the +/// installed artifact is in place (or known-bad): partial downloads left +/// behind by a failed pull are exactly what makes resume-on-retry work, so +/// failures should skip this. A no-op for fetchers that own no staging. +pub fn cleanup_staging(fetcher: &dyn Fetcher, repo: &str) { + if let Some(dir) = fetcher.staging_dir(repo) { + if dir.exists() { + let _ = std::fs::remove_dir_all(&dir); + } + } } /// Test fetcher that serves files from a local fixture directory laid out as @@ -133,6 +202,112 @@ impl Fetcher for MockFetcher { mod tests { use super::*; + /// Fetcher that owns an hf-hub-style staging tree: + /// `/models----/blobs/` with + /// `snapshots//` symlinks pointing at the blobs — the layout + /// `HfFetcher` produces. + struct StagedFetcher { + staging: PathBuf, + } + + impl StagedFetcher { + fn repo_dir(&self, repo: &str) -> PathBuf { + self.staging.join(format!("models--{}", repo.replace('/', "--"))) + } + + /// Materialize a staged download of `filename` with `bytes`. + fn stage(&self, repo: &str, revision: &str, filename: &str, bytes: &[u8]) -> PathBuf { + let rdir = self.repo_dir(repo); + let blobs = rdir.join("blobs"); + let snap = rdir.join("snapshots").join(revision); + std::fs::create_dir_all(&blobs).unwrap(); + std::fs::create_dir_all(&snap).unwrap(); + let blob = blobs.join(format!("etag-{filename}")); + std::fs::write(&blob, bytes).unwrap(); + let pointer = snap.join(filename); + std::os::unix::fs::symlink(&blob, &pointer).unwrap(); + pointer + } + } + + impl Fetcher for StagedFetcher { + fn get_file(&self, repo: &str, revision: &str, filename: &str) -> Result { + let p = self + .repo_dir(repo) + .join("snapshots") + .join(revision) + .join(filename); + anyhow::ensure!(p.exists(), "not staged: {}", p.display()); + Ok(p) + } + + fn list_files(&self, repo: &str, revision: &str) -> Result> { + let dir = self.repo_dir(repo).join("snapshots").join(revision); + let mut out = Vec::new(); + for e in std::fs::read_dir(dir)? { + out.push(e?.file_name().to_string_lossy().into_owned()); + } + Ok(out) + } + + fn staging_dir(&self, repo: &str) -> Option { + Some(self.repo_dir(repo)) + } + } + + #[test] + fn install_moves_owned_blob_then_cleanup_leaves_one_copy() { + let tmp = tempfile::tempdir().unwrap(); + let f = StagedFetcher { staging: tmp.path().join("staging") }; + let src = f.stage("org/m", "main", "m-Q4.base", b"payload"); + let dst = tmp.path().join("model.base"); + + install_file(&f, "org/m", &src, &dst).unwrap(); + assert_eq!(std::fs::read(&dst).unwrap(), b"payload"); + // The blob was moved, not copied: the staged payload is gone (only a + // dangling pointer symlink may remain until cleanup). + let blob = f.repo_dir("org/m").join("blobs").join("etag-m-Q4.base"); + assert!(!blob.exists(), "blob must be moved out of staging"); + + cleanup_staging(&f, "org/m"); + assert!(!f.repo_dir("org/m").exists(), "staging tree must be removed"); + // Exactly one copy survives. + assert_eq!(std::fs::read(&dst).unwrap(), b"payload"); + } + + #[test] + fn install_overwrites_existing_artifact() { + // `--force` re-pulls install over an existing model.base. + let tmp = tempfile::tempdir().unwrap(); + let f = StagedFetcher { staging: tmp.path().join("staging") }; + let src = f.stage("org/m", "main", "m-Q4.base", b"new-bytes"); + let dst = tmp.path().join("model.base"); + std::fs::write(&dst, b"old-bytes").unwrap(); + + install_file(&f, "org/m", &src, &dst).unwrap(); + assert_eq!(std::fs::read(&dst).unwrap(), b"new-bytes"); + } + + #[test] + fn install_copies_unowned_sources_and_preserves_them() { + // MockFetcher owns no staging: fixtures must survive installation and + // cleanup must be a no-op. + let tmp = tempfile::tempdir().unwrap(); + let repo_dir = tmp.path().join("org").join("m"); + std::fs::create_dir_all(&repo_dir).unwrap(); + let fixture = repo_dir.join("m.base"); + std::fs::write(&fixture, b"fixture-bytes").unwrap(); + let f = MockFetcher::new(tmp.path()); + + let src = f.get_file("org/m", "main", "m.base").unwrap(); + let dst = tmp.path().join("model.base"); + install_file(&f, "org/m", &src, &dst).unwrap(); + cleanup_staging(&f, "org/m"); + + assert_eq!(std::fs::read(&dst).unwrap(), b"fixture-bytes"); + assert!(fixture.exists(), "unowned source must not be deleted"); + } + // All assertions live in one test: they mutate the shared process env, so // splitting them into separate `#[test]` fns would race under Rust's // parallel test runner. Sequential mutation within a single fn is safe. diff --git a/base-convert/crates/base-hub/src/registry.rs b/base-convert/crates/base-hub/src/registry.rs index 188f659..6d2f145 100644 --- a/base-convert/crates/base-hub/src/registry.rs +++ b/base-convert/crates/base-hub/src/registry.rs @@ -423,7 +423,7 @@ mod tests { std::fs::create_dir_all(&vdir).unwrap(); std::fs::write(cache::base_artifact_path(&vdir), b"junk-not-a-header").unwrap(); // A staging artifact under .src must be ignored. - let sdir = cache::src_staging_dir(root, "meta/Foo", "main").unwrap(); + let sdir = cache::hf_staging_dir(root).join("models--meta--Foo/snapshots/main"); std::fs::create_dir_all(&sdir).unwrap(); std::fs::write(cache::base_artifact_path(&sdir), b"junk").unwrap(); diff --git a/base-convert/crates/base-readers/src/mlx.rs b/base-convert/crates/base-readers/src/mlx.rs index 701946e..14f0fd1 100644 --- a/base-convert/crates/base-readers/src/mlx.rs +++ b/base-convert/crates/base-readers/src/mlx.rs @@ -6,14 +6,18 @@ //! - `config.json` has a `"quantization": {"bits": 4, "group_size": 64}` //! key declaring the scheme. //! - A quantized weight `foo.weight` is stored as a `U32` tensor with -//! shape `[out_features, in_features / (32 / bits)]` — packed nibbles -//! (or bytes for 8-bit). +//! shape `[out_features, in_features * bits / 32]`: each row is a +//! contiguous little-endian bitstream of `bits`-wide codes. For bits +//! that divide 32 (2/4/8) every code sits at a fixed slot inside one +//! u32; for 3/5/6-bit codes cross byte and word boundaries, so the +//! stream must be decoded at bit granularity (MLX supports +//! bits ∈ {2, 3, 4, 5, 6, 8} for affine quant). //! - Two additional tensors accompany it: `foo.scales` and `foo.biases`, //! both `F16`, shape `[out_features, in_features / group_size]`. //! - Dequant: `x[i, j] = q[i, j] * scale[i, j / group_size] //! + bias[i, j / group_size]` -//! where `q[i, j]` is extracted from `packed[i, j / (32/bits)]` at -//! nibble position `j % (32/bits)` (low-nibble first). +//! where `q[i, j]` is the `bits`-wide field at bit offset `j * bits` +//! of row `i`'s bitstream. //! //! Some MLX models carry per-tensor AWQ overrides in //! `config.quantization_config.{tensor_name}`; those are read verbatim @@ -135,7 +139,13 @@ impl MlxDir { let q = self.quant_for_tensor(name); let bits = q.bits as usize; let group_size = q.group_size as usize; - let vals_per_u32 = 32 / bits; + if !matches!(bits, 2 | 3 | 4 | 5 | 6 | 8) { + bail!( + "MLX packed tensor {:?}: unsupported bits={} (MLX affine quant packs 2/3/4/5/6/8)", + name, + bits + ); + } // Batch dims are everything except the last; last-2 dims are // [out_features, packed_in]. For 2-D tensors batch is empty. @@ -143,7 +153,15 @@ impl MlxDir { let packed_in = packed_in[0] as usize; let (batch_dims_split, out_dim_slice) = batch_dims.split_at(batch_dims.len() - 1); let out_features = out_dim_slice[0] as usize; - let in_features = packed_in * vals_per_u32; + if packed_in * 32 % bits != 0 { + bail!( + "MLX packed tensor {:?}: packed width {} u32 does not hold a whole number of {}-bit codes", + name, + packed_in, + bits + ); + } + let in_features = packed_in * 32 / bits; let batch: usize = batch_dims_split.iter().product::() as usize; let batch = batch.max(1); @@ -185,7 +203,7 @@ impl MlxDir { let s_base = b * slice_scales_len; let o_base = b * slice_out_len; for i in 0..out_features { - let row_p = p_base + i * packed_in; + let row_bytes = (p_base + i * packed_in) * 4; let row_s = s_base + i * groups_per_row; for gj in 0..groups_per_row { let scale = read_half(scales_bytes, row_s + gj, scales_dtype); @@ -197,14 +215,19 @@ impl MlxDir { }; for lj in 0..group_size { let j = gj * group_size + lj; - let u32_idx = row_p + j / vals_per_u32; - let u = u32::from_le_bytes( - packed_bytes[u32_idx * 4..u32_idx * 4 + 4] - .try_into() - .unwrap(), - ); - let slot = j % vals_per_u32; - let q = (u >> (bits * slot)) & mask; + // Code j occupies bits [j*bits, (j+1)*bits) of the + // row's little-endian bitstream. With bits ≤ 8 a + // code spans at most two bytes, and the second + // byte is only touched when the code actually + // crosses into it — never past the row's end. + let bit = j * bits; + let byte0 = row_bytes + bit / 8; + let sh = bit % 8; + let mut word = packed_bytes[byte0] as u32; + if sh + bits > 8 { + word |= (packed_bytes[byte0 + 1] as u32) << 8; + } + let q = (word >> sh) & mask; out[o_base + i * in_features + j] = (q as f32) * scale + bias; } @@ -219,8 +242,8 @@ impl MlxDir { /// Resolves bits per-tensor — Gemma 4 26B-A4B 4-bit checkpoints /// override `mlp.{gate,up,down}_proj` and `router.proj` to 8-bit, so /// using the global bits here unpacks 8-bit data as 4-bit and the - /// resulting `last_dim *= 8` (instead of *= 4) doubles the logical - /// in_features the runtime expects. + /// resulting `last_dim * 32 / 4` (instead of `* 32 / 8`) doubles the + /// logical in_features the runtime expects. pub fn unpacked_shape(&self, name: &str) -> Option> { let info = self.hf.tensor_info(name)?; quant_sibling(name, "scales") @@ -229,10 +252,15 @@ impl MlxDir { return None; } let q = self.quant_for_tensor(name); - let vals_per_u32 = 32 / q.bits as u64; + let bits = q.bits as u64; let mut shape = info.shape.clone(); let last = shape.len() - 1; - shape[last] *= vals_per_u32; + // packed_in = in_features * bits / 32, exactly — a non-exact + // inverse means the tensor isn't MLX-packed with these settings. + if bits == 0 || (shape[last] * 32) % bits != 0 { + return None; + } + shape[last] = shape[last] * 32 / bits; Some(shape) } diff --git a/base-convert/crates/base-readers/tests/fixtures/mlx_dequant_fixtures.rs b/base-convert/crates/base-readers/tests/fixtures/mlx_dequant_fixtures.rs new file mode 100644 index 0000000..5436105 --- /dev/null +++ b/base-convert/crates/base-readers/tests/fixtures/mlx_dequant_fixtures.rs @@ -0,0 +1,35 @@ +// bits=3 shape=[3, 64] gs=32 packed=[3, 6] +const B3_PACKED: &[u32] = &[3031659763, 3367226694, 2713120138, 2932428627, 2640755402, 2978790221, 1917221183, 421437135, 583792307, 2731521361, 1460384870, 1055875710, 1555709293, 963485360, 3586469554, 1684745262, 656513879, 2231176022]; +const B3_SCALES: &[u16] = &[47177, 14011, 13670, 14057, 47572, 47170]; // f16 bits +const B3_BIASES: &[u16] = &[16457, 48827, 48832, 48873, 16852, 16450]; // f16 bits +const B3_EXPECTED: &[f32] = &[5.35644531e-01, -1.07128906e+00, 5.35644531e-01, 1.07128906e+00, -1.60742188e+00, -1.07128906e+00, 0.00000000e+00, -5.35644531e-01, 0.00000000e+00, -1.07128906e+00, 1.07128906e+00, 5.35644531e-01, 0.00000000e+00, 1.07128906e+00, 5.35644531e-01, -1.07128906e+00, 5.35644531e-01, -1.07128906e+00, 1.07128906e+00, 0.00000000e+00, 0.00000000e+00, -5.35644531e-01, 1.07128906e+00, 0.00000000e+00, -5.35644531e-01, -5.35644531e-01, 5.35644531e-01, 5.35644531e-01, 5.35644531e-01, 5.35644531e-01, 2.14257812e+00, -5.35644531e-01, -4.20654297e-01, -8.41308594e-01, 4.20654297e-01, 1.26171875e+00, 0.00000000e+00, -8.41308594e-01, -8.41308594e-01, 8.41308594e-01, 8.41308594e-01, 4.20654297e-01, -8.41308594e-01, 4.20654297e-01, 0.00000000e+00, 4.20654297e-01, 8.41308594e-01, 4.20654297e-01, 8.41308594e-01, 0.00000000e+00, 4.20654297e-01, 8.41308594e-01, -1.26171875e+00, -4.20654297e-01, -4.20654297e-01, -8.41308594e-01, -4.20654297e-01, 1.26171875e+00, -8.41308594e-01, 8.41308594e-01, -1.68261719e+00, -4.20654297e-01, 0.00000000e+00, 4.20654297e-01, 6.74316406e-01, 6.74316406e-01, -3.37890625e-01, -3.37890625e-01, 6.74316406e-01, -3.37890625e-01, -1.34960938e+00, -1.01269531e+00, -1.01269531e+00, 3.36914062e-01, -4.88281250e-04, 6.74316406e-01, -3.37890625e-01, -4.88281250e-04, 6.74316406e-01, -3.37890625e-01, 3.36914062e-01, -6.75292969e-01, -3.37890625e-01, -3.37890625e-01, -1.34960938e+00, 3.36914062e-01, -3.37890625e-01, -4.88281250e-04, 3.36914062e-01, 3.36914062e-01, 6.74316406e-01, -4.88281250e-04, -3.37890625e-01, -4.88281250e-04, -1.68750000e+00, -1.34960938e+00, -1.29589844e+00, -8.63769531e-01, 4.31884766e-01, -8.63769531e-01, -4.31884766e-01, 1.29589844e+00, -4.31884766e-01, 8.63769531e-01, -8.63769531e-01, 0.00000000e+00, -8.63769531e-01, -4.31884766e-01, 8.63769531e-01, -1.72753906e+00, 4.31884766e-01, 4.31884766e-01, -4.31884766e-01, -1.29589844e+00, 0.00000000e+00, -4.31884766e-01, 4.31884766e-01, 0.00000000e+00, 1.29589844e+00, -4.31884766e-01, -8.63769531e-01, 0.00000000e+00, 4.31884766e-01, 1.29589844e+00, 8.63769531e-01, 4.31884766e-01, 1.29589844e+00, -1.29589844e+00, -7.28515625e-01, -7.28515625e-01, -7.28515625e-01, -1.45703125e+00, 7.28515625e-01, 0.00000000e+00, -1.45703125e+00, -7.28515625e-01, 0.00000000e+00, 7.28515625e-01, 2.18554688e+00, 2.91406250e+00, 7.28515625e-01, -7.28515625e-01, -2.18554688e+00, 0.00000000e+00, -7.28515625e-01, -7.28515625e-01, -7.28515625e-01, 0.00000000e+00, 7.28515625e-01, 0.00000000e+00, 0.00000000e+00, -7.28515625e-01, -1.45703125e+00, -7.28515625e-01, 0.00000000e+00, 1.45703125e+00, 0.00000000e+00, 7.28515625e-01, -7.28515625e-01, -1.45703125e+00, -1.06445312e+00, -5.32226562e-01, 2.12890625e+00, -1.06445312e+00, 1.06445312e+00, -1.06445312e+00, 1.06445312e+00, 5.32226562e-01, 0.00000000e+00, 0.00000000e+00, -5.32226562e-01, 5.32226562e-01, -5.32226562e-01, -1.06445312e+00, -1.06445312e+00, 0.00000000e+00, 1.59667969e+00, 0.00000000e+00, 0.00000000e+00, 5.32226562e-01, 1.06445312e+00, 0.00000000e+00, -5.32226562e-01, 1.06445312e+00, 5.32226562e-01, 1.59667969e+00, 0.00000000e+00, -1.06445312e+00, -1.59667969e+00, 1.59667969e+00, 1.59667969e+00, 0.00000000e+00]; + +// bits=4 shape=[3, 64] gs=32 packed=[3, 8] +const B4_PACKED: &[u32] = &[2306626807, 1167116794, 3631824261, 1252550907, 3198955913, 3165633368, 1020456604, 2059700944, 2811877782, 2072540998, 1942712840, 2276222198, 2819532952, 2354029450, 3187402841, 1551469759, 2210315387, 1212591460, 2559618198, 3117590046, 2608297050, 3141059961, 106248279, 3460866809]; +const B4_SCALES: &[u16] = &[13131, 46378, 46128, 13456, 13461, 46248]; // f16 bits +const B4_BIASES: &[u16] = &[49178, 16847, 16432, 49442, 49301, 16552]; // f16 bits +const B4_EXPECTED: &[f32] = &[-4.55566406e-01, 1.36816406e+00, -1.13867188e+00, -9.11132812e-01, 6.84082031e-01, -4.55566406e-01, 3.66210938e-04, -2.27539062e-01, 2.28271484e-01, 1.36816406e+00, 3.66210938e-04, 6.84082031e-01, -2.05078125e+00, 3.66210938e-04, -9.11132812e-01, -1.13867188e+00, -9.11132812e-01, -2.27539062e-01, 9.12109375e-01, -1.36718750e+00, 3.66210938e-04, -4.55566406e-01, -2.27539062e-01, 9.12109375e-01, 4.56054688e-01, 1.36816406e+00, -2.27539062e-01, -6.83593750e-01, -2.27539062e-01, 2.28271484e-01, 2.28271484e-01, -1.13867188e+00, -4.88281250e-04, 3.22265625e-01, 2.58203125e+00, 1.93554688e+00, -9.68750000e-01, -3.23242188e-01, -1.61425781e+00, -6.45996094e-01, 3.22265625e-01, 1.29101562e+00, -6.45996094e-01, -6.45996094e-01, -1.93750000e+00, -3.23242188e-01, -9.68750000e-01, -6.45996094e-01, -9.68750000e-01, -4.88281250e-04, -1.61425781e+00, -1.61425781e+00, 2.25781250e+00, -1.29101562e+00, -9.68750000e-01, 1.93554688e+00, 2.90429688e+00, -1.29101562e+00, -3.23242188e-01, 3.22265625e-01, 1.61328125e+00, -9.68750000e-01, -3.23242188e-01, 6.45019531e-01, 5.23437500e-01, -2.61718750e-01, -2.61718750e-01, -1.30859375e+00, -2.61718750e-01, -2.61718750e-01, 2.61718750e-01, -5.23437500e-01, 5.23437500e-01, 1.04687500e+00, 2.61718750e-01, 2.61718750e-01, 0.00000000e+00, 0.00000000e+00, -7.85156250e-01, 2.61718750e-01, 0.00000000e+00, 2.09375000e+00, 1.57031250e+00, 2.61718750e-01, -7.85156250e-01, -1.04687500e+00, 1.30859375e+00, 2.61718750e-01, 5.23437500e-01, -1.83203125e+00, 1.04687500e+00, 5.23437500e-01, -1.04687500e+00, -5.23437500e-01, 2.61718750e-01, 0.00000000e+00, -2.85156250e-01, 0.00000000e+00, -2.85156250e-01, 2.85156250e-01, 1.42578125e+00, -2.56640625e+00, -2.85156250e-01, 2.85156250e-01, 2.85156250e-01, -2.85156250e-01, -1.71093750e+00, 2.85156250e-01, 1.71093750e+00, -1.42578125e+00, 8.55468750e-01, -2.85156250e-01, 0.00000000e+00, -1.14062500e+00, -2.85156250e-01, 1.42578125e+00, 5.70312500e-01, 1.71093750e+00, 1.14062500e+00, 5.70312500e-01, 1.71093750e+00, 5.70312500e-01, 8.55468750e-01, -2.85156250e-01, 0.00000000e+00, -5.70312500e-01, 8.55468750e-01, -1.14062500e+00, 8.59375000e-01, -2.86376953e-01, 1.14550781e+00, 8.59375000e-01, 1.71875000e+00, 8.59375000e-01, -1.43164062e+00, 0.00000000e+00, -1.14550781e+00, -5.72753906e-01, 1.43164062e+00, 5.72753906e-01, -5.72753906e-01, -1.14550781e+00, 0.00000000e+00, -1.14550781e+00, -5.72753906e-01, 2.86376953e-01, 1.14550781e+00, 5.72753906e-01, -2.29101562e+00, 2.86376953e-01, 0.00000000e+00, 2.86376953e-01, 1.71875000e+00, -2.00390625e+00, -5.72753906e-01, 5.72753906e-01, -1.71875000e+00, 1.43164062e+00, 2.86376953e-01, 8.59375000e-01, -5.82031250e-01, 8.73046875e-01, 1.16406250e+00, 2.91015625e-01, 2.91015625e-01, 2.91015625e-01, -8.73046875e-01, -2.91015625e-01, -2.91015625e-01, 2.91015625e-01, 8.73046875e-01, -1.16406250e+00, 0.00000000e+00, 1.45507812e+00, -8.73046875e-01, -8.73046875e-01, 2.91015625e-01, 8.73046875e-01, 0.00000000e+00, 1.45507812e+00, 8.73046875e-01, 8.73046875e-01, 5.82031250e-01, 2.32812500e+00, -2.91015625e-01, -2.03710938e+00, 1.74609375e+00, -5.82031250e-01, 0.00000000e+00, 1.16406250e+00, -1.74609375e+00, -1.16406250e+00]; + +// bits=5 shape=[3, 64] gs=32 packed=[3, 10] +const B5_PACKED: &[u32] = &[330198503, 3967521472, 4281698099, 1140697545, 3443327009, 3541974938, 1339926554, 3759120870, 1856147125, 1850138190, 1179109834, 4173261444, 1318649580, 329510562, 74492342, 1629113138, 4192119963, 434615272, 3225644436, 2049501693, 1534779955, 3639700470, 3218382687, 1857661685, 2367182339, 3637363316, 2597064555, 3388200823, 231003820, 4231334398]; +const B5_SCALES: &[u16] = &[11903, 12115, 45284, 12693, 12354, 44860]; // f16 bits +const B5_BIASES: &[u16] = &[49323, 48979, 16690, 49646, 49354, 16402]; // f16 bits +const B5_EXPECTED: &[f32] = &[-1.62304688e+00, -8.11523438e-01, 4.06494141e-01, 5.07812500e-01, 3.05175781e-01, -1.42089844e+00, -2.33398438e+00, 1.02050781e-01, -5.06835938e-01, 5.07812500e-01, 7.10937500e-01, 1.02050781e-01, 7.10937500e-01, 2.03613281e-01, 5.07812500e-01, -7.09960938e-01, -2.02392578e-01, 2.03613281e-01, 8.12500000e-01, -4.05517578e-01, 5.07812500e-01, -3.03955078e-01, -1.00952148e-01, 8.12500000e-01, -2.02929688e+00, -1.31933594e+00, -1.52148438e+00, -2.33398438e+00, -7.09960938e-01, 7.10937500e-01, -3.03955078e-01, 2.03613281e-01, 1.14453125e+00, 1.37304688e+00, -1.14440918e-01, 1.37304688e+00, 1.14440918e-01, -8.01269531e-01, -5.72265625e-01, -1.48730469e+00, -9.15527344e-01, -3.43261719e-01, 8.01269531e-01, 1.71679688e+00, -1.37304688e+00, 3.43261719e-01, -1.03027344e+00, 4.57763672e-01, -1.14440918e-01, -1.83105469e+00, 9.15527344e-01, -5.72265625e-01, -5.72265625e-01, -1.03027344e+00, -6.86523438e-01, 4.57763672e-01, -2.28881836e-01, 3.43261719e-01, 3.43261719e-01, 1.37304688e+00, -3.43261719e-01, -1.48730469e+00, 1.03027344e+00, -3.43261719e-01, 1.06933594e+00, 4.58007812e-01, -1.53320312e-01, 3.05175781e-01, 1.98632812e+00, 2.13867188e+00, -4.88281250e-04, 1.52343750e-01, 1.06933594e+00, -9.17480469e-01, 3.05175781e-01, -4.88281250e-04, 3.05175781e-01, -7.64648438e-01, -1.52929688e+00, -2.14062500e+00, -1.07031250e+00, -4.58984375e-01, -3.06152344e-01, 1.98632812e+00, 1.06933594e+00, -9.17480469e-01, 3.05175781e-01, -4.58984375e-01, -3.06152344e-01, 1.52343750e-01, 6.10839844e-01, -3.06152344e-01, 1.06933594e+00, -1.07031250e+00, -4.88281250e-04, 2.59765625e+00, 1.75048828e-01, 1.39648438e+00, 1.75048828e-01, 5.23925781e-01, 6.10351562e-04, -1.73828125e-01, -6.97265625e-01, 3.49609375e-01, -8.71582031e-01, 6.98242188e-01, 1.04687500e+00, 3.49609375e-01, -3.48144531e-01, 5.23925781e-01, -3.48144531e-01, 8.73046875e-01, -1.74414062e+00, -3.48144531e-01, -1.91796875e+00, -1.56933594e+00, 1.39648438e+00, 8.73046875e-01, -6.97265625e-01, -1.56933594e+00, -2.96484375e+00, -5.22460938e-01, 2.44335938e+00, 3.49609375e-01, -5.22460938e-01, 5.23925781e-01, -1.56933594e+00, -3.48144531e-01, 1.33544922e-01, -1.32568359e-01, 7.98828125e-01, 3.99658203e-01, 6.66015625e-01, -6.65039062e-01, 9.31640625e-01, 1.59765625e+00, -9.30664062e-01, -9.30664062e-01, 1.33105469e+00, -1.32568359e-01, 1.46386719e+00, -3.98681641e-01, 1.46386719e+00, 1.33544922e-01, 2.66601562e-01, 1.59765625e+00, -3.98681641e-01, -9.30664062e-01, -3.98681641e-01, 3.99658203e-01, -1.59570312e+00, 6.66015625e-01, -5.31738281e-01, 1.19824219e+00, -2.39453125e+00, -7.97851562e-01, -1.72949219e+00, -7.97851562e-01, 2.66601562e-01, -1.32568359e-01, -2.25585938e-01, -1.12548828e-01, 2.26562500e-01, -1.01660156e+00, 6.78710938e-01, 6.78710938e-01, 3.39599609e-01, 5.65429688e-01, 3.39599609e-01, 2.03515625e+00, -1.12548828e-01, -3.38623047e-01, -7.91015625e-01, -1.01660156e+00, -1.24316406e+00, -9.03808594e-01, -1.12548828e-01, 3.39599609e-01, 4.88281250e-04, -7.91015625e-01, 9.04785156e-01, 7.91992188e-01, -1.12548828e-01, -6.77734375e-01, 5.65429688e-01, 2.26562500e-01, -1.46875000e+00, -1.12548828e-01, 2.26562500e-01, -9.03808594e-01, 2.26562500e-01, -1.46875000e+00]; + +// bits=6 shape=[3, 64] gs=32 packed=[3, 12] +const B6_PACKED: &[u32] = &[2157833525, 531385828, 2568906073, 1534132988, 1185490523, 2732235111, 2994132597, 2270675622, 2715297338, 2906544602, 1311047887, 1704698514, 1587018575, 3349491384, 2662677127, 3430893911, 3542251896, 2665487528, 2154650794, 4128606646, 827958940, 2475542950, 3414142655, 2263404942, 1656583607, 697682540, 1443646538, 2536771172, 3570650815, 2797137822, 2952829804, 3152156972, 1981095079, 2479603666, 3400769195, 1713497770]; +const B6_SCALES: &[u16] = &[11401, 44313, 11440, 44228, 11183, 11356]; // f16 bits +const B6_BIASES: &[u16] = &[49362, 16665, 49441, 16580, 49296, 49279]; // f16 bits +const B6_EXPECTED: &[f32] = &[1.34570312e+00, 1.27441406e+00, -2.84179688e-01, 3.53515625e-01, -2.41015625e+00, -1.13476562e+00, -2.84179688e-01, -1.06347656e+00, 7.08007812e-01, 1.98339844e+00, -1.20507812e+00, -8.51074219e-01, 2.11669922e-01, 1.62890625e+00, -1.20507812e+00, 2.82714844e-01, 1.84179688e+00, -1.63085938e+00, -1.27636719e+00, -4.26025391e-01, -4.96826172e-01, 7.78808594e-01, 2.11669922e-01, -1.77246094e+00, 4.95117188e-01, -5.67871094e-01, 1.27441406e+00, -6.38671875e-01, -6.38671875e-01, 5.65917969e-01, 7.78808594e-01, 4.24316406e-01, -1.67285156e+00, 5.57617188e-01, -1.03515625e+00, 2.39013672e-01, -1.43359375e+00, 4.78027344e-01, -7.96386719e-01, -1.27441406e+00, 7.16796875e-01, 2.39013672e-01, -6.37207031e-01, 1.43359375e+00, -4.78027344e-01, 0.00000000e+00, 2.39013672e-01, -6.37207031e-01, 4.78027344e-01, -5.57617188e-01, -3.98193359e-01, 1.35449219e+00, -1.03515625e+00, -2.38867188e+00, 1.59277344e+00, 2.54882812e+00, -3.98193359e-01, -1.91210938e+00, -3.18603516e-01, -3.18603516e-01, -1.59301758e-01, -1.11523438e+00, 5.57617188e-01, 5.57617188e-01, -1.46582031e+00, 7.31445312e-01, 2.05078125e+00, 1.45507812e-01, -3.67187500e-01, -1.47460938e-01, 5.84960938e-01, -1.75878906e+00, 1.45507812e-01, -3.67187500e-01, 1.83007812e+00, -1.47460938e-01, 1.68359375e+00, -1.09960938e+00, 5.84960938e-01, 2.91992188e-01, -8.79882812e-01, 1.45507812e-01, 1.24414062e+00, -2.93945312e-01, -1.68554688e+00, -9.76562500e-04, -8.79882812e-01, -3.67187500e-01, -7.42187500e-02, -1.68554688e+00, -1.61230469e+00, 5.11718750e-01, -5.13671875e-01, -2.56445312e+00, 8.04687500e-01, 2.91992188e-01, -7.44628906e-01, -1.33984375e+00, 8.19335938e-01, 3.72314453e-01, 2.38281250e+00, 4.46777344e-01, 3.72314453e-01, -1.48925781e-01, 8.19335938e-01, 5.95703125e-01, 1.26562500e+00, -5.21484375e-01, -4.46777344e-01, -4.46777344e-01, 8.19335938e-01, 1.48925781e+00, -4.46777344e-01, -4.46777344e-01, 2.97851562e-01, -2.23388672e-01, 9.67773438e-01, -2.23437500e+00, -8.19335938e-01, -8.19335938e-01, -2.30859375e+00, -9.67773438e-01, -8.93554688e-01, -2.23388672e-01, 1.11718750e+00, -2.23388672e-01, -1.04296875e+00, -7.44628906e-02, 1.02050781e+00, -9.60449219e-01, -9.00390625e-01, 5.40039062e-01, -2.40234375e-01, 6.60156250e-01, -1.83105469e-04, 7.20214844e-01, -1.02050781e+00, -1.83105469e-04, -2.40234375e-01, -1.20117188e+00, -1.56054688e+00, 6.60156250e-01, -3.60351562e-01, -1.02050781e+00, -1.20239258e-01, 1.14062500e+00, -2.28125000e+00, -1.50097656e+00, -9.00390625e-01, 1.44042969e+00, 3.00048828e-01, 7.80273438e-01, -1.14062500e+00, -1.14062500e+00, 4.19921875e-01, 5.98449707e-02, 5.40039062e-01, -1.80297852e-01, 3.00048828e-01, 1.79931641e-01, 7.49023438e-01, 8.17382812e-01, -1.63476562e+00, -2.24804688e+00, 1.02148438e+00, 1.15820312e+00, -1.02148438e+00, -1.83984375e+00, 6.78710938e-02, 9.53125000e-01, 1.77050781e+00, 5.44921875e-01, -3.40820312e-01, -8.85742188e-01, -2.44140625e-04, -2.72705078e-01, -1.02148438e+00, 2.04296875e+00, 1.77050781e+00, 1.15820312e+00, -9.54101562e-01, 8.85253906e-01, 6.12792969e-01, 4.08447266e-01, 1.22558594e+00, 6.12792969e-01, 7.49023438e-01, 6.12792969e-01, 3.40332031e-01, -1.77148438e+00, 6.78710938e-02, -5.44921875e-01]; + +// bits=8 shape=[3, 64] gs=32 packed=[3, 16] +const B8_PACKED: &[u32] = &[520423643, 1920933998, 3715917006, 1073776260, 2390584959, 3652553677, 2315215964, 2100770105, 3098959982, 1452925935, 2123738766, 3027993520, 1555148937, 559675524, 2594415409, 4290314474, 3401822678, 2940426158, 2739379263, 1946193047, 3065016001, 2643500145, 2113833357, 2594301647, 2387599995, 1723705218, 623488731, 2829294505, 1516169913, 2382347872, 2222011904, 4053118350, 3899047345, 529497708, 3616800941, 1807141504, 977644897, 4264387368, 2860367258, 1787825296, 2979963284, 3098273232, 3301415532, 3534996647, 531468203, 2991784111, 3649006771, 2963210121]; +const B8_SCALES: &[u16] = &[41777, 9280, 42279, 9097, 41967, 9705]; // f16 bits +const B8_BIASES: &[u16] = &[16177, 49496, 16988, 49409, 16415, 49996]; // f16 bits +const B8_EXPECTED: &[f32] = &[-1.27832031e+00, 1.68554688e+00, 1.72753906e+00, 1.36230469e+00, 2.52929688e-01, 1.34863281e+00, 1.40457153e-02, 1.96655273e-01, -1.09570312e+00, 3.93310547e-01, 5.61828613e-02, -1.30664062e+00, -5.61828613e-02, -8.42895508e-02, 1.79785156e+00, 8.98925781e-01, 1.40457153e-02, 2.52929688e-01, 4.21447754e-02, -1.96655273e-01, -1.08105469e+00, -1.54541016e-01, -7.44628906e-01, -1.25000000e+00, 5.05859375e-01, 3.93310547e-01, -1.78417969e+00, -1.26464844e-01, 9.97070312e-01, 1.05371094e+00, 1.02539062e+00, 4.21447754e-02, -8.45703125e-01, -1.07812500e+00, 3.49609375e-01, 3.82812500e-01, 1.29589844e+00, 1.09667969e+00, -1.31835938e-01, -1.24414062e+00, -3.14453125e-01, 2.16796875e-01, -1.98242188e-01, -5.80078125e-01, 2.50000000e-01, -4.97070312e-01, -6.29882812e-01, 3.16406250e-01, -3.97460938e-01, 2.50000000e-01, 2.66601562e-01, -1.14453125e+00, -4.80468750e-01, 1.44531250e+00, -1.16113281e+00, -2.12500000e+00, -1.85839844e+00, 3.41796875e-02, 3.41796875e-02, -1.15234375e-01, 1.21289062e+00, -2.67187500e+00, 3.99414062e-01, 1.56152344e+00, -1.12695312e+00, -3.82568359e-01, -7.45117188e-01, -8.85742188e-01, -3.22265625e-01, 1.42871094e+00, 1.83105469e+00, -3.42529297e-01, 1.91210938e+00, 3.99780273e-02, 1.75097656e+00, -1.00891113e-01, 1.40625000e-01, 3.62060547e-01, 3.17968750e+00, 8.45214844e-01, -7.04589844e-01, 9.65820312e-01, -3.62548828e-01, -4.83398438e-01, 9.05273438e-01, 3.99780273e-02, 2.81494141e-01, 1.98516846e-02, 3.41796875e-01, 4.22363281e-01, -1.93261719e+00, 6.64062500e-01, -9.86328125e-01, -1.44921875e+00, -6.06689453e-02, 8.02001953e-02, -6.91894531e-01, 8.24218750e-01, -1.33886719e+00, -4.12109375e-01, -5.88867188e-01, -4.41894531e-02, 2.79541016e-01, -1.00097656e+00, 7.21191406e-01, 5.88378906e-02, -1.89843750e+00, -1.95703125e+00, -1.47628784e-02, -2.20825195e-01, -1.03088379e-01, -2.94799805e-02, 2.20703125e-01, 9.41894531e-01, -1.11816406e+00, -1.17773438e+00, -1.08886719e+00, 2.94189453e-01, 1.25097656e+00, -4.26757812e-01, -2.50195312e+00, -1.70703125e+00, -8.38867188e-01, -5.59082031e-01, -4.12109375e-01, 4.40979004e-02, -3.09082031e-01, 1.04492188e+00, -6.82128906e-01, -8.68164062e-01, 4.79980469e-01, -1.53417969e+00, 3.86962891e-01, 1.08154297e-01, -1.55273438e-01, 1.58007812e+00, -6.20117188e-01, 2.06054688e+00, -2.32788086e-01, -1.27050781e+00, 7.71484375e-02, -1.06933594e+00, -7.59765625e-01, 4.02587891e-01, 5.57617188e-01, -6.20117188e-01, 9.91210938e-01, 1.16210938e+00, 1.44042969e+00, 5.26367188e-01, 1.36328125e+00, -1.87500000e+00, -3.25683594e-01, -8.68164062e-01, 1.23657227e-01, -5.73730469e-01, -1.70776367e-01, 1.87500000e+00, -1.70776367e-01, 4.17968750e-01, -2.31689453e-01, 6.84814453e-02, -7.62939453e-04, 4.37988281e-01, 1.15332031e+00, 1.63867188e+00, 2.99316406e-01, 5.99609375e-01, -1.15527344e+00, -9.31396484e-02, 9.45800781e-01, 8.76464844e-01, 2.07031250e-01, 8.76464844e-01, 4.84130859e-01, 1.20019531e+00, 2.99316406e-01, -3.47167969e-01, 3.45458984e-01, -2.93359375e+00, 3.91601562e-01, -3.64843750e+00, -1.73242188e+00, 4.60937500e-01, 4.84130859e-01, -1.15527344e+00, -7.16308594e-01, 1.36132812e+00, -4.85595703e-01, 2.23828125e+00, -7.62939453e-04, 4.14794922e-01]; + +// bits=6 shape=[2, 3, 64] gs=32 packed=[2, 3, 12] +const B6BATCH_PACKED: &[u32] = &[3784001344, 1741602398, 3611630730, 2539637258, 3302573503, 217753191, 3547214400, 2611104247, 2706825558, 3927672467, 2040582991, 1503058778, 3783780797, 2259187988, 1634358406, 1803650350, 653159051, 926949523, 3934939302, 2054251252, 3803861899, 362242027, 715477760, 4224427678, 2441804308, 3586138887, 1565870620, 114291962, 167485476, 3718941314, 562137387, 1625328632, 3793639809, 1066547557, 3152310352, 3464311930, 1618835932, 963628019, 604696018, 1659304293, 3081426459, 1655009645, 714990626, 3139420008, 1398117103, 1518761745, 3653197206, 1074159730, 1675956231, 186042046, 1237020015, 1683921233, 4114151147, 500790643, 4027645924, 143832822, 2770849448, 2505251042, 3601340485, 2510335110, 650158645, 2084541754, 3263324286, 1717659408, 2166537938, 3243770069, 674355182, 1534023576, 3994428138, 1507063527, 119484920, 2985816224]; +const B6BATCH_SCALES: &[u16] = &[43640, 43765, 44222, 44158, 43506, 11296, 44266, 11518, 11264, 43711, 11470, 44208]; // f16 bits +const B6BATCH_BIASES: &[u16] = &[15992, 16173, 16612, 16798, 15953, 49316, 16618, 49486, 49280, 16117, 49550, 16748]; // f16 bits +const B6BATCH_EXPECTED: &[f32] = &[1.61718750e+00, 1.51611328e-01, -1.01074219e+00, -1.01074219e-01, -5.05371094e-02, -1.36425781e+00, -2.52685547e-01, -7.57812500e-01, 9.09667969e-01, 5.05371094e-02, -3.03222656e-01, -1.01074219e-01, 2.02148438e-01, 6.06445312e-01, -1.01074219e+00, -1.06152344e+00, 1.11132812e+00, -4.04296875e-01, -1.41503906e+00, 4.54833984e-01, 4.54833984e-01, -1.51562500e+00, 2.52685547e-01, 7.57812500e-01, 3.53759766e-01, 6.57226562e-01, -1.41503906e+00, 3.53759766e-01, -3.03222656e-01, -5.05371094e-01, 8.59375000e-01, 1.46582031e+00, 1.79394531e+00, 1.30468750e+00, -1.08398438e-01, 3.26416016e-01, 7.61230469e-01, 1.09008789e-01, 1.09008789e-01, 8.15429688e-01, -5.40161133e-02, -7.06054688e-01, -4.34570312e-01, 6.52343750e-01, 3.35693359e-04, 3.26416016e-01, 6.52343750e-01, -3.80126953e-01, 7.61230469e-01, 3.80859375e-01, -1.25000000e+00, 1.46777344e+00, -4.88769531e-01, -1.62988281e+00, -1.03222656e+00, -1.03222656e+00, 5.46875000e-02, -2.71484375e-01, -3.25683594e-01, 5.98144531e-01, 3.26416016e-01, 3.26416016e-01, 4.35058594e-01, 5.98144531e-01, -2.07421875e+00, -3.70361328e-01, -2.14843750e+00, 1.22070312e-04, 1.22070312e-04, 1.03710938e+00, 1.18554688e+00, 2.22412109e-01, -5.18554688e-01, 5.18554688e-01, -5.18554688e-01, 1.22070312e-04, 9.63378906e-01, -5.92773438e-01, 8.15429688e-01, 6.66992188e-01, -9.62890625e-01, 9.63378906e-01, 6.66992188e-01, 7.42187500e-02, -7.40722656e-01, -8.89160156e-01, -5.18554688e-01, 5.18554688e-01, -9.62890625e-01, 4.44824219e-01, -1.25976562e+00, -2.22167969e-01, 7.42187500e-02, 2.44531250e+00, -1.40820312e+00, 1.48242188e+00, 1.41357422e-01, 4.22119141e-01, 1.41357422e-01, 4.22119141e-01, -1.39404297e-01, 1.47460938e+00, -4.90234375e-01, 1.12402344e+00, -6.30859375e-01, -6.92138672e-02, -1.05175781e+00, 4.22119141e-01, 1.47460938e+00, -6.92138672e-02, -2.09594727e-01, -1.12207031e+00, -2.09594727e-01, -1.61328125e+00, -9.11621094e-01, 2.11547852e-01, 1.33496094e+00, 2.80859375e+00, -5.60546875e-01, 1.40429688e+00, 2.11547852e-01, -1.39404297e-01, 4.22119141e-01, 7.11669922e-02, 1.41357422e-01, -4.20166016e-01, -1.40234375e+00, -1.54296875e+00, 6.50390625e-01, -2.78808594e-01, -6.04003906e-01, -1.22070312e-04, 7.89550781e-01, 1.85668945e-01, -6.50390625e-01, 1.20703125e+00, 1.57910156e+00, 5.10742188e-01, 9.75097656e-01, 1.25390625e+00, 1.11425781e+00, 6.03515625e-01, 6.03515625e-01, 5.10742188e-01, -1.11523438e+00, 6.96777344e-01, -1.34667969e+00, -7.89550781e-01, 1.30078125e+00, 8.35937500e-01, 1.48632812e+00, -2.78808594e-01, -1.16113281e+00, -2.32421875e-01, 9.27734375e-02, 9.27734375e-02, 1.11425781e+00, -3.71582031e-01, 3.71582031e-01, -9.75585938e-01, 4.51171875e-01, 0.00000000e+00, -7.73437500e-01, -2.57812500e-01, -1.93359375e-01, -2.57812500e-01, 1.74023438e+00, -1.28906250e-01, -2.57812500e-01, -2.12695312e+00, -9.02343750e-01, -2.57812500e-01, -7.08984375e-01, 1.35351562e+00, -1.93359375e-01, 1.28906250e+00, 6.44531250e-02, 6.44531250e-02, -6.44531250e-02, 0.00000000e+00, 1.74023438e+00, -2.32031250e+00, -1.99804688e+00, -5.15625000e-01, 0.00000000e+00, 7.08984375e-01, 4.51171875e-01, -3.86718750e-01, 1.03125000e+00, 1.03125000e+00, 1.93359375e-01, 9.66796875e-01, 3.07128906e-01, -5.37597656e-01, 6.90917969e-01, 7.67822266e-02, 0.00000000e+00, 1.45898438e+00, -2.38085938e+00, -1.38183594e+00, -1.15136719e+00, -3.83789062e-01, -2.30346680e-01, -1.53515625e+00, -9.98046875e-01, -8.44726562e-01, 2.45703125e+00, 1.76562500e+00, -3.83789062e-01, -3.83789062e-01, -1.15136719e+00, -1.91992188e+00, -1.53564453e-01, -9.98046875e-01, -7.67822266e-02, -1.61230469e+00, -7.67578125e-01, 1.53564453e-01, 3.83789062e-01, 3.83789062e-01, -1.30566406e+00, 8.44726562e-01, -7.67578125e-01, 6.14257812e-01, -2.44140625e-04, -1.40429688e+00, -3.12255859e-01, 3.89648438e-01, 6.24023438e-01, -1.56250000e-01, 1.55957031e+00, 1.01367188e+00, -2.34252930e-01, 7.79785156e-01, 1.95019531e+00, 1.95019531e+00, -1.24804688e+00, -9.36523438e-01, 1.48144531e+00, -1.09179688e+00, -1.32617188e+00, -4.68261719e-01, 3.89648438e-01, -7.82470703e-02, -6.24023438e-01, -7.02148438e-01, -7.02148438e-01, -8.58398438e-01, 2.26171875e+00, 3.11767578e-01, 8.57910156e-01, -4.68261719e-01, -1.56250000e-01, -7.02148438e-01, -2.65234375e+00, -1.40429688e+00, -1.81250000e+00, -2.25000000e+00, -1.18750000e+00, 1.31250000e+00, -6.25000000e-02, 1.31250000e+00, 4.37500000e-01, 8.12500000e-01, -8.75000000e-01, 5.00000000e-01, 7.50000000e-01, -5.62500000e-01, 5.62500000e-01, 5.62500000e-01, -5.62500000e-01, -1.12500000e+00, -1.18750000e+00, 6.25000000e-02, 3.12500000e-01, -8.12500000e-01, 0.00000000e+00, 5.62500000e-01, 6.25000000e-01, 1.56250000e+00, 1.25000000e+00, -1.00000000e+00, 1.68750000e+00, -5.00000000e-01, 1.06250000e+00, 6.25000000e-02, -4.37500000e-01, -1.81250000e+00, -1.58081055e-01, -1.58105469e+00, 9.48730469e-01, 1.52832031e+00, -7.90527344e-01, 3.16162109e-01, -7.37792969e-01, -6.32324219e-01, 7.90527344e-01, -5.26733398e-02, 1.73925781e+00, -4.74365234e-01, 1.00097656e+00, 1.05468750e-01, 7.90527344e-01, -4.21630859e-01, -5.26733398e-02, -1.05346680e-01, -8.43261719e-01, 6.85058594e-01, 6.32324219e-01, 5.79589844e-01, 1.52832031e+00, 1.42285156e+00, -3.68896484e-01, 3.68896484e-01, -6.32324219e-01, 3.05175781e-05, -5.79589844e-01, 1.63378906e+00, 3.68896484e-01, -2.10815430e-01, 1.20117188e+00, -2.17578125e+00, -2.02734375e+00, 8.26171875e-01, 7.54394531e-02, 2.25585938e-01, -1.35058594e+00, 3.66210938e-04, 1.95214844e+00, 8.26171875e-01, 1.50512695e-01, -4.50195312e-01, -3.75000000e-01, -2.10156250e+00, 2.25585938e-01, 8.26171875e-01, -1.57617188e+00, -6.75292969e-01, -1.12597656e+00, -9.75585938e-01, 7.54394531e-02, -2.10156250e+00, 6.01074219e-01, 7.50976562e-01, -2.24853516e-01, -2.47656250e+00, -9.75585938e-01, 1.20117188e+00, -2.77734375e+00, -3.75000000e-01, -1.20117188e+00, 8.26171875e-01, -6.58203125e-01, 4.40429688e-01, 5.86914062e-01, 1.83203125e+00, -2.18750000e-01, 3.67187500e-01, -1.46386719e+00, 1.17285156e+00, -7.31445312e-01, -5.84960938e-01, 9.76562500e-04, -1.53710938e+00, 2.20703125e-01, 9.53125000e-01, 2.93945312e-01, -1.61035156e+00, -1.45507812e-01, 7.33398438e-01, -1.90332031e+00, -1.09765625e+00, 8.79882812e-01, 2.93945312e-01, 4.40429688e-01, 1.83203125e+00, 4.40429688e-01, 6.60156250e-01, 2.71093750e+00, -2.18750000e-01, -8.04687500e-01, 4.40429688e-01, 4.40429688e-01, -5.11718750e-01]; diff --git a/base-convert/crates/base-readers/tests/mlx_dequant.rs b/base-convert/crates/base-readers/tests/mlx_dequant.rs new file mode 100644 index 0000000..64880d1 --- /dev/null +++ b/base-convert/crates/base-readers/tests/mlx_dequant.rs @@ -0,0 +1,177 @@ +//! MLX packed-tensor dequant against ground truth generated with +//! `mlx.core.quantize` / `mlx.core.dequantize` (mlx 0.26, f16 weights, +//! group_size=32, seed 42). The fixture arrays in +//! `fixtures/mlx_dequant_fixtures.rs` are the verbatim packed words, +//! f16 scale/bias bit patterns, and mlx's own dequantized output. +//! +//! 3/5/6-bit are the interesting cases: their codes cross byte and u32 +//! boundaries (contiguous little-endian bitstream), which the old +//! `32 / bits` slot math silently mis-decoded (issue #20 on the public +//! repo: 6-bit checkpoints failed with a bogus group-size error). + +// The generated fixture literals carry full round-trip precision. +#![allow(clippy::excessive_precision)] + +use base_readers::mlx::MlxDir; +use std::io::Write; +use std::path::Path; + +include!("fixtures/mlx_dequant_fixtures.rs"); + +/// mlx dequantize emits f16; our reader recomputes in f32 from the same +/// f16 scales/biases, so results differ by at most one f16 rounding of +/// values drawn from N(0,1). A mis-decoded bitstream is off by O(1). +const TOL: f32 = 5e-3; + +fn write_safetensors(path: &Path, tensors: &[(&str, &str, &[u64], Vec)]) { + let mut header = serde_json::Map::new(); + let mut offset = 0u64; + for (name, dtype, shape, bytes) in tensors { + let end = offset + bytes.len() as u64; + header.insert( + name.to_string(), + serde_json::json!({ + "dtype": dtype, + "shape": shape, + "data_offsets": [offset, end], + }), + ); + offset = end; + } + let hdr = serde_json::to_vec(&serde_json::Value::Object(header)).unwrap(); + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(&(hdr.len() as u64).to_le_bytes()).unwrap(); + f.write_all(&hdr).unwrap(); + for (_, _, _, bytes) in tensors { + f.write_all(bytes).unwrap(); + } +} + +fn u32_bytes(v: &[u32]) -> Vec { + v.iter().flat_map(|x| x.to_le_bytes()).collect() +} + +fn u16_bytes(v: &[u16]) -> Vec { + v.iter().flat_map(|x| x.to_le_bytes()).collect() +} + +/// Build a one-tensor MLX checkpoint dir and return the opened reader. +fn mlx_dir( + dir: &Path, + bits: u32, + packed_shape: &[u64], + scales_shape: &[u64], + packed: &[u32], + scales: &[u16], + biases: &[u16], +) -> MlxDir { + let config = serde_json::json!({ + "model_type": "test", + "quantization": { "bits": bits, "group_size": 32 }, + }); + std::fs::write(dir.join("config.json"), serde_json::to_vec(&config).unwrap()).unwrap(); + write_safetensors( + &dir.join("model.safetensors"), + &[ + ("layer.weight", "U32", packed_shape, u32_bytes(packed)), + ("layer.scales", "F16", scales_shape, u16_bytes(scales)), + ("layer.biases", "F16", scales_shape, u16_bytes(biases)), + ], + ); + MlxDir::open(dir).unwrap() +} + +fn check_case( + bits: u32, + logical_shape: &[u64], + packed: &[u32], + scales: &[u16], + biases: &[u16], + expected: &[f32], +) { + let tmp = tempfile::tempdir().unwrap(); + let mut packed_shape = logical_shape.to_vec(); + let last = packed_shape.len() - 1; + packed_shape[last] = packed_shape[last] * bits as u64 / 32; + let mut scales_shape = logical_shape.to_vec(); + scales_shape[last] = logical_shape[last] / 32; + let mlx = mlx_dir( + tmp.path(), + bits, + &packed_shape, + &scales_shape, + packed, + scales, + biases, + ); + + assert_eq!( + mlx.unpacked_shape("layer.weight").unwrap(), + logical_shape, + "bits={bits}: unpacked_shape" + ); + + let got = mlx.tensor_to_f32("layer.weight").unwrap(); + assert_eq!(got.len(), expected.len(), "bits={bits}: element count"); + for (i, (g, e)) in got.iter().zip(expected).enumerate() { + assert!( + (g - e).abs() <= TOL, + "bits={bits}: element {i} mismatch: got {g}, mlx says {e}" + ); + } +} + +#[test] +fn dequant_matches_mlx_3bit() { + check_case(3, &[3, 64], B3_PACKED, B3_SCALES, B3_BIASES, B3_EXPECTED); +} + +#[test] +fn dequant_matches_mlx_4bit() { + check_case(4, &[3, 64], B4_PACKED, B4_SCALES, B4_BIASES, B4_EXPECTED); +} + +#[test] +fn dequant_matches_mlx_5bit() { + check_case(5, &[3, 64], B5_PACKED, B5_SCALES, B5_BIASES, B5_EXPECTED); +} + +#[test] +fn dequant_matches_mlx_6bit() { + check_case(6, &[3, 64], B6_PACKED, B6_SCALES, B6_BIASES, B6_EXPECTED); +} + +#[test] +fn dequant_matches_mlx_8bit() { + check_case(8, &[3, 64], B8_PACKED, B8_SCALES, B8_BIASES, B8_EXPECTED); +} + +/// Stacked-experts layout: batch dim ahead of [out, packed_in], as in +/// MLX-LM MoE checkpoints (switch_mlp stacks experts on dim 0). +#[test] +fn dequant_matches_mlx_6bit_batched() { + check_case( + 6, + &[2, 3, 64], + B6BATCH_PACKED, + B6BATCH_SCALES, + B6BATCH_BIASES, + B6BATCH_EXPECTED, + ); +} + +#[test] +fn unsupported_bits_is_a_clear_error() { + let tmp = tempfile::tempdir().unwrap(); + let mlx = mlx_dir( + tmp.path(), + 7, + &[3, 14], + &[3, 2], + &[0u32; 42], + &[0u16; 6], + &[0u16; 6], + ); + let err = mlx.tensor_to_f32("layer.weight").unwrap_err().to_string(); + assert!(err.contains("unsupported bits=7"), "got: {err}"); +} diff --git a/base-convert/profiles/default-q6.json b/base-convert/profiles/default-q6.json new file mode 100644 index 0000000..98e009b --- /dev/null +++ b/base-convert/profiles/default-q6.json @@ -0,0 +1,33 @@ +{ + "name": "default-q6", + "arch": "*", + "rules": [ + {"pattern": "model.embed_tokens.weight", "dtype": "f16"}, + {"pattern": "embed_tokens.weight", "dtype": "f16"}, + {"pattern": "**.input_layernorm.weight", "dtype": "f16"}, + {"pattern": "**.input_norm.weight", "dtype": "f16"}, + {"pattern": "**.post_attention_layernorm.weight", "dtype": "f16"}, + {"pattern": "**.post_attn_norm.weight", "dtype": "f16"}, + {"pattern": "**.post_attention_norm.weight", "dtype": "f16"}, + {"pattern": "**.post_ffw_norm.weight", "dtype": "f16"}, + {"pattern": "**.pre_ffw_norm.weight", "dtype": "f16"}, + {"pattern": "**.per_layer_post_norm.weight", "dtype": "f16"}, + {"pattern": "per_layer_proj_norm.weight", "dtype": "f16"}, + {"pattern": "**.q_norm.weight", "dtype": "f16"}, + {"pattern": "**.k_norm.weight", "dtype": "f16"}, + {"pattern": "**.attn_q_norm.weight", "dtype": "f16"}, + {"pattern": "**.attn_k_norm.weight", "dtype": "f16"}, + {"pattern": "**.layer_out_scale.weight", "dtype": "f16"}, + {"pattern": "**.ffn_norm.weight", "dtype": "f16"}, + {"pattern": "**.attn_norm.weight", "dtype": "f16"}, + {"pattern": "model.norm.weight", "dtype": "f16"}, + {"pattern": "final_norm.weight", "dtype": "f16"}, + {"pattern": "output_norm.weight", "dtype": "f16"}, + {"pattern": "**.{q,k,v,o}_proj.weight", "dtype": "base_q6", "scale_dtype": "bf16", "group_size": 64}, + {"pattern": "**.{gate,up,down}_proj.weight", "dtype": "base_q6", "scale_dtype": "bf16", "group_size": 64}, + {"pattern": "lm_head.weight", "dtype": "base_q6", "scale_dtype": "bf16", "group_size": 64}, + {"pattern": "output.weight", "dtype": "base_q6", "scale_dtype": "bf16", "group_size": 64}, + {"pattern": "**.weight", "dtype": "base_q6", "scale_dtype": "bf16", "group_size": 64}, + {"pattern": "**.bias", "dtype": "f16"} + ] +} diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 72b9cf7..6c112d8 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -180,6 +180,17 @@ const char *baseRT_decode_token(baseRT_model_t model, uint32_t token_id); /// string lives in a thread-local buffer that is overwritten on each call. const char *baseRT_decode_token_static(baseRT_model_t model, uint32_t token_id); +/// Length-preserving variant of `baseRT_decode_token_static` for callers +/// that need the token's EXACT raw bytes. Byte-level BPE / byte-fallback +/// tokens can decode to bytes containing 0x00, which the C-string variants +/// above silently truncate at. Writes up to `max_bytes` into `out` (no NUL +/// terminator appended) and returns the token's FULL byte length — if the +/// return value exceeds `max_bytes`, call again with a larger buffer. +/// Stateless; does not touch the incremental-decode state. Returns 0 for +/// tokens that decode to nothing (e.g. filtered special tokens), <0 on +/// invalid arguments. +int baseRT_decode_token_raw(baseRT_model_t model, uint32_t token_id, char *out, int max_bytes); + // === Generation === /// Callback for streaming token output. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 72b9cf7..6c112d8 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -180,6 +180,17 @@ const char *baseRT_decode_token(baseRT_model_t model, uint32_t token_id); /// string lives in a thread-local buffer that is overwritten on each call. const char *baseRT_decode_token_static(baseRT_model_t model, uint32_t token_id); +/// Length-preserving variant of `baseRT_decode_token_static` for callers +/// that need the token's EXACT raw bytes. Byte-level BPE / byte-fallback +/// tokens can decode to bytes containing 0x00, which the C-string variants +/// above silently truncate at. Writes up to `max_bytes` into `out` (no NUL +/// terminator appended) and returns the token's FULL byte length — if the +/// return value exceeds `max_bytes`, call again with a larger buffer. +/// Stateless; does not touch the incremental-decode state. Returns 0 for +/// tokens that decode to nothing (e.g. filtered special tokens), <0 on +/// invalid arguments. +int baseRT_decode_token_raw(baseRT_model_t model, uint32_t token_id, char *out, int max_bytes); + // === Generation === /// Callback for streaming token output.