From 6305ca7f87fcc09d3f868aec6d3b426c71e85d2e Mon Sep 17 00:00:00 2001 From: Prabod Rathnayaka Date: Fri, 17 Jul 2026 19:48:06 +1000 Subject: [PATCH 1/3] =?UTF-8?q?sync:=20engine=200.1.6=20=E2=80=94=20Qwen?= =?UTF-8?q?=203.5/3.6=20converter=20support,=20updated=20headers=20+=20bin?= =?UTF-8?q?dings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - base-convert: Qwen 3.5/3.6 hybrid arch (config extraction, GDN tensor precision rules, MoE MTP/fused-expert handling, mrope recording), catalog updates, CLI banner, version 0.1.6 - headers: new config fields + baseRT_set_verbose / baseRT_model_config_sizeof - bindings (python/rust/node/swift): BaseRTModelConfig mirrors updated to the current 1520-byte layout + load-time size guards Requires the 0.1.6 engine binaries: the size guard rejects older libbaseRT builds. --- base-convert/Cargo.lock | 16 +- base-convert/Cargo.toml | 2 +- base-convert/README.md | 4 +- base-convert/crates/base-arch/src/lib.rs | 121 ++- base-convert/crates/base-arch/src/qwen.rs | 351 ++++++++ base-convert/crates/base-convert/src/hub.rs | 121 ++- base-convert/crates/base-convert/src/main.rs | 803 +++++++++++++++++- base-convert/crates/base-hub/catalog.json | 10 + base-convert/crates/base-hub/src/fetch.rs | 2 +- bindings/node/src/index.ts | 39 + bindings/python/baseRT/__init__.py | 35 + bindings/python/tests/test_baseRT.py | 39 +- bindings/rust/baseRT-sys/src/lib.rs | 54 +- .../swift/Sources/CBaseRT/include/baseRT.h | 15 +- .../swift/Sources/CBaseRT/include/types.h | 41 + include/baseRT/baseRT.h | 15 +- include/baseRT/types.h | 41 + 17 files changed, 1658 insertions(+), 51 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 25adfd2..63be208 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -66,7 +66,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "base-arch" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-format", @@ -77,7 +77,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-format", @@ -91,7 +91,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-arch", @@ -114,7 +114,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "bitflags", @@ -130,7 +130,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-format", @@ -145,7 +145,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-format", @@ -157,7 +157,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-format", @@ -171,7 +171,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.1.0" +version = "0.1.6" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index dce0b6c..ea2f8e1 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.1.0" +version = "0.1.6" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/README.md b/base-convert/README.md index 6c341ea..fa7699e 100644 --- a/base-convert/README.md +++ b/base-convert/README.md @@ -57,8 +57,8 @@ through pre-converted catalog artifacts. `basert ` forwards to the matching runtime binary (`basert-`): ``` -basert serve --model [--model …] # OpenAI-compatible server -basert chat --model # interactive chat +basert serve [--model …] # OpenAI-compatible server (--model repeats for multi-model) +basert chat # interactive chat basert complete --prompt # one-shot completion basert bench # throughput benchmark ``` diff --git a/base-convert/crates/base-arch/src/lib.rs b/base-convert/crates/base-arch/src/lib.rs index 8e7522f..f705f59 100644 --- a/base-convert/crates/base-arch/src/lib.rs +++ b/base-convert/crates/base-arch/src/lib.rs @@ -61,9 +61,30 @@ pub trait HfMapper: Sync { pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMapper> { match model_type { - "llama" => Some(&llama::LlamaHfMapper), + // Mistral is Llama-shaped (RMSNorm, RoPE, SwiGLU, GQA), its HF tensor + // names are already canonical Llama names, and it reuses the Llama mapper + // (canonical_arch="llama" → llama model class). Validated end-to-end on + // Mistral-7B-Instruct-v0.3 and Ministral-8B-Instruct-2410. + // + // Phi-3 is NOT enabled. The converter side is ready (SplittingProvider + // splits its fused qkv_proj/gate_up_proj; weights convert; HD=96 + // attention is fine — regression-tested) and the generation_config eos + // merge below makes it stop cleanly. BUT Phi-3 chat output degenerates + // (raw completions are coherent; chat floods/repeats and is incoherent at + // both Q4 and Q8, temp 0 and 0.7) — a chat-path issue (same class as + // SmolLM2) that's not yet root-caused. Re-enable "phi3" once that's + // fixed. (Phi-3.5 additionally needs LongRoPE — engine is linear-only.) + "llama" | "mistral" => Some(&llama::LlamaHfMapper), "qwen2" | "qwen3" => Some(&qwen::QwenHfMapper), "qwen2_moe" | "qwen3_moe" => Some(&qwen::QwenMoeHfMapper), + // Qwen3.5 / 3.6: hybrid Gated-DeltaNet + full-attention decoder + // (reuses the Qwen3-Next design). The top-level HF model_type is + // `qwen3_5` (multimodal wrapper `Qwen3_5ForConditionalGeneration`) + // with the text tower under `text_config.model_type = qwen3_5_text`. + // Both resolve here so a text-only checkpoint (top-level + // `qwen3_5_text`) and the multimodal wrapper convert identically. + "qwen3_5" | "qwen3_5_text" | "qwen35" => Some(&qwen::Qwen35HfMapper), + "qwen3_5_moe" | "qwen3_5_moe_text" | "qwen35_moe" => Some(&qwen::Qwen35MoeHfMapper), "nomic_bert" | "nomic-bert" => Some(&bert::NomicBertHfMapper), "gemma" | "gemma2" | "gemma3" | "gemma3_text" => Some(&gemma::Gemma3HfMapper), // gemma3n is a distinct arch (AltUp/Laurel/per-layer-FFN); the @@ -81,10 +102,17 @@ pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMappe /// message) has a single source of truth for what convert-on-pull supports. pub const SUPPORTED_HF_MODEL_TYPES: &[&str] = &[ "llama", + "mistral", "qwen2", "qwen3", "qwen2_moe", "qwen3_moe", + "qwen3_5", + "qwen3_5_text", + "qwen35", + "qwen3_5_moe", + "qwen3_5_moe_text", + "qwen35_moe", "nomic_bert", "gemma", "gemma2", @@ -145,6 +173,10 @@ pub struct ArchConfig { /// MoE: 1 if router top-k weights are renormalized to sum to 1 /// (Qwen), 0 if left as-is (Gemma). pub norm_topk_prob: bool, + /// MoE: number of always-on shared experts running in parallel to the + /// routed ones (Qwen3.5/3.6-MoE: 1, width `intermediate_size`, plus a + /// per-token scalar sigmoid gate). 0 = no shared expert. + pub num_shared_experts: u32, /// Maximum positional embedding length. Pulled from /// `max_position_embeddings` (HF) or `context_length` (GGUF). pub max_position_embeddings: u32, @@ -193,6 +225,44 @@ pub struct ArchConfig { /// `rope_freqs.weight` divisor mask; HF stores it as /// `rope_parameters.full_attention.partial_rotary_factor`. pub global_rope_partial_factor: f32, + + // ── Qwen3.5 / 3.6 hybrid-linear-attention fields ───────────────── + // (all zero/empty for non-hybrid archs). Qwen3.5 interleaves + // Gated-DeltaNet linear-attention layers with periodic full + // (softmax) attention layers, reusing the Qwen3-Next decoder. + /// Per-layer attention kind, one entry per layer: + /// "linear_attention" (Gated DeltaNet) | "full_attention". + /// Empty = not a hybrid model. Length = num_hidden_layers. + pub layer_types: Vec, + /// Every Nth layer is full attention (the rest are linear). Mirror + /// of `full_attention_interval` in the HF config; 0 = not hybrid. + /// Redundant with `layer_types` but kept for a cheap runtime check. + pub full_attention_interval: u32, + /// Gated-DeltaNet: number of key ("k") heads. 0 = not hybrid. + pub linear_num_key_heads: u32, + /// Gated-DeltaNet: number of value ("v") heads. + pub linear_num_value_heads: u32, + /// Gated-DeltaNet: per-head key/query dimension. + pub linear_key_head_dim: u32, + /// Gated-DeltaNet: per-head value dimension. + pub linear_value_head_dim: u32, + /// Gated-DeltaNet: causal depthwise short-conv kernel width (e.g. 4). + pub linear_conv_kernel_dim: u32, + /// Full-attention layers apply an output (sigmoid) gate to the + /// attention output before o_proj (`attn_output_gate`). Qwen3.5=true. + pub attn_output_gate: bool, + /// Partial rotary factor for the full-attention layers: only the + /// first `factor * head_dim` dims are rotated. Qwen3.5 = 0.25. + /// 0 or 1 = full rotation. + pub partial_rotary_factor: f32, + /// Multimodal-RoPE section split (Qwen3.5: [11, 11, 10] over + /// temporal/height/width). Empty = plain 1-D RoPE. For text-only + /// inference all positions collapse so the runtime may treat this + /// as ordinary 1-D RoPE. + pub mrope_section: Vec, + /// mRoPE interleaves the section frequencies rather than + /// concatenating them (Qwen3.5 = true). + pub mrope_interleaved: bool, } impl ArchConfig { @@ -235,6 +305,12 @@ impl ArchConfig { m.insert("num_experts_per_tok".into(), json!(self.num_experts_per_tok)); m.insert("moe_intermediate_size".into(), json!(self.moe_intermediate_size)); m.insert("norm_topk_prob".into(), json!(self.norm_topk_prob)); + if self.num_shared_experts > 0 { + m.insert( + "num_shared_experts".into(), + json!(self.num_shared_experts), + ); + } } if self.max_position_embeddings > 0 { m.insert( @@ -295,6 +371,49 @@ impl ArchConfig { if self.rope_local_theta > 0.0 { m.insert("rope_local_theta".into(), json!(self.rope_local_theta)); } + // Qwen3.5 / 3.6 hybrid-linear-attention fields — only emit when + // set so non-hybrid archs' headers stay tidy. + if !self.layer_types.is_empty() { + m.insert("layer_types".into(), json!(self.layer_types)); + } + if self.full_attention_interval > 0 { + m.insert( + "full_attention_interval".into(), + json!(self.full_attention_interval), + ); + } + if self.linear_num_key_heads > 0 { + m.insert( + "linear_num_key_heads".into(), + json!(self.linear_num_key_heads), + ); + m.insert( + "linear_num_value_heads".into(), + json!(self.linear_num_value_heads), + ); + m.insert("linear_key_head_dim".into(), json!(self.linear_key_head_dim)); + m.insert( + "linear_value_head_dim".into(), + json!(self.linear_value_head_dim), + ); + m.insert( + "linear_conv_kernel_dim".into(), + json!(self.linear_conv_kernel_dim), + ); + } + if self.attn_output_gate { + m.insert("attn_output_gate".into(), json!(self.attn_output_gate)); + } + if self.partial_rotary_factor > 0.0 { + m.insert( + "partial_rotary_factor".into(), + json!(self.partial_rotary_factor), + ); + } + if !self.mrope_section.is_empty() { + m.insert("mrope_section".into(), json!(self.mrope_section)); + m.insert("mrope_interleaved".into(), json!(self.mrope_interleaved)); + } m } } diff --git a/base-convert/crates/base-arch/src/qwen.rs b/base-convert/crates/base-arch/src/qwen.rs index 2b0f3e6..7cafc7a 100644 --- a/base-convert/crates/base-arch/src/qwen.rs +++ b/base-convert/crates/base-arch/src/qwen.rs @@ -12,6 +12,160 @@ use std::collections::BTreeMap; pub struct QwenMapper; pub struct QwenHfMapper; pub struct QwenMoeHfMapper; +pub struct Qwen35HfMapper; +pub struct Qwen35MoeHfMapper; + +/// Qwen3.5 / 3.6 shared HF config extraction. +/// +/// Qwen3.5 (`Qwen3_5ForConditionalGeneration`, model_type `qwen3_5`) is a +/// natively-multimodal model whose language tower lives under a nested +/// `text_config` (model_type `qwen3_5_text`). It reuses the Qwen3-Next +/// hybrid decoder: most layers are Gated-DeltaNet linear-attention blocks, +/// with every `full_attention_interval`-th layer being a (gated) softmax +/// attention block. RoPE parameters (theta, partial-rotary factor, mRoPE +/// sections) live under `rope_parameters`, not at the top level. +/// +/// This reads the standard Llama-shaped fields from the text config, then +/// patches in the RoPE + hybrid-linear-attention fields the generic reader +/// doesn't know about. +fn qwen35_config_from_hf(c: &serde_json::Value) -> Result { + // A text-only checkpoint may hoist the text params to the top level; + // the multimodal wrapper nests them under `text_config`. Prefer the + // nested object when present. + let tc = c.get("text_config").unwrap_or(c); + let mut config = crate::llama::hf_generic_config(tc)?; + + let u32_v = + |v: &serde_json::Value, k: &str| v.get(k).and_then(|x| x.as_u64()).map(|n| n as u32); + let f32_v = + |v: &serde_json::Value, k: &str| v.get(k).and_then(|x| x.as_f64()).map(|f| f as f32); + let bool_v = |v: &serde_json::Value, k: &str| v.get(k).and_then(|x| x.as_bool()); + + // RoPE lives under `rope_parameters` on Qwen3.5 (not top-level + // `rope_theta`), so hf_generic_config's default (10000) is wrong — + // override it here along with the partial-rotary + mRoPE fields. + if let Some(rp) = tc.get("rope_parameters") { + if let Some(theta) = f32_v(rp, "rope_theta") { + config.rope_theta = theta; + } + if let Some(f) = f32_v(rp, "partial_rotary_factor") { + config.partial_rotary_factor = f; + } + if let Some(sec) = rp.get("mrope_section").and_then(|v| v.as_array()) { + config.mrope_section = sec + .iter() + .filter_map(|x| x.as_u64().map(|n| n as u32)) + .collect(); + } + config.mrope_interleaved = bool_v(rp, "mrope_interleaved").unwrap_or(false); + } + + // Hybrid layer schedule + Gated-DeltaNet ("linear attention") shapes. + if let Some(lt) = tc.get("layer_types").and_then(|v| v.as_array()) { + config.layer_types = lt + .iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect(); + } + config.full_attention_interval = u32_v(tc, "full_attention_interval").unwrap_or(0); + config.linear_num_key_heads = u32_v(tc, "linear_num_key_heads").unwrap_or(0); + config.linear_num_value_heads = u32_v(tc, "linear_num_value_heads").unwrap_or(0); + config.linear_key_head_dim = u32_v(tc, "linear_key_head_dim").unwrap_or(0); + config.linear_value_head_dim = u32_v(tc, "linear_value_head_dim").unwrap_or(0); + config.linear_conv_kernel_dim = u32_v(tc, "linear_conv_kernel_dim").unwrap_or(0); + config.attn_output_gate = bool_v(tc, "attn_output_gate").unwrap_or(false); + + Ok(config) +} + +/// Qwen3.5 / 3.6 RMSNorm gamma shift. +/// +/// Qwen3NextRMSNorm stores zero-centered gamma and applies `(1 + weight)` at +/// inference (same convention as Gemma 3). Bake the +1 in here so the runtime +/// uses the plain rmsnorm kernel. This covers EVERY norm — input_layernorm, +/// post_attention_layernorm, the attention q_norm/k_norm, and the final +/// `model.norm` — EXCEPT the Gated-DeltaNet output norm +/// (`linear_attn.norm.weight`), which is a Qwen3NextRMSNormGated and uses plain +/// `weight` with no offset. +fn qwen35_norm_shift(canonical: &str) -> f32 { + if canonical.ends_with("norm.weight") && !canonical.ends_with(".linear_attn.norm.weight") { + 1.0 + } else { + 0.0 + } +} + +impl crate::HfMapper for Qwen35HfMapper { + fn canonical_arch(&self) -> &'static str { + "qwen35" + } + fn config_from_hf(&self, c: &serde_json::Value) -> Result { + qwen35_config_from_hf(c) + } + fn norm_shift(&self, canonical: &str) -> f32 { + qwen35_norm_shift(canonical) + } +} + +impl crate::HfMapper for Qwen35MoeHfMapper { + fn canonical_arch(&self) -> &'static str { + "qwen35moe" + } + fn norm_shift(&self, canonical: &str) -> f32 { + qwen35_norm_shift(canonical) + } + fn config_from_hf(&self, c: &serde_json::Value) -> Result { + // The real Qwen3.5/3.6-MoE text_config carries NO `intermediate_size` + // (the HF config class deletes the attribute) — the dense-FFN slot is + // taken by the shared expert, whose width lives in + // `shared_expert_intermediate_size`. hf_generic_config hard-requires + // `intermediate_size`, so patch the shared-expert width in before + // delegating; the runtime then reads ffn_dim = shared-expert width + // for the `ffn_*_shexp` SwiGLU stream (routed experts use + // `moe_intermediate_size`). + let mut patched = c.clone(); + { + let tc_mut = if patched.get("text_config").is_some() { + patched.get_mut("text_config").unwrap() + } else { + &mut patched + }; + if tc_mut.get("intermediate_size").is_none() { + let shexp = tc_mut + .get("shared_expert_intermediate_size") + .and_then(|v| v.as_u64()); + if let (Some(sh), Some(obj)) = (shexp, tc_mut.as_object_mut()) { + obj.insert("intermediate_size".into(), serde_json::json!(sh)); + } + } + } + let mut config = qwen35_config_from_hf(&patched)?; + // MoE topology lives alongside the text params (nested under + // `text_config` for the multimodal wrapper). Keep `intermediate_size` + // as the dense/shared-FFN width and read the per-expert width into + // `moe_intermediate_size` — same distinction as Qwen3-MoE. + let tc = c.get("text_config").unwrap_or(c); + let u32_v = |k: &str| tc.get(k).and_then(|v| v.as_u64()).map(|n| n as u32); + let bool_v = |k: &str| tc.get(k).and_then(|v| v.as_bool()); + if let Some(n) = u32_v("num_experts") { + config.num_experts = n; + } + if let Some(n) = u32_v("num_experts_per_tok") { + config.num_experts_per_tok = n; + } + if let Some(n) = u32_v("moe_intermediate_size") { + config.moe_intermediate_size = n; + } + // Shared expert (Qwen3NextSparseMoeBlock): always present on the + // released 35B-A3B checkpoints. Signal it to the runtime so the + // qwen3_5 encoder emits the ffn_*_shexp stream + scalar gate. + if u32_v("shared_expert_intermediate_size").unwrap_or(0) > 0 { + config.num_shared_experts = 1; + } + config.norm_topk_prob = bool_v("norm_topk_prob").unwrap_or(true); + Ok(config) + } +} impl crate::HfMapper for QwenHfMapper { fn canonical_arch(&self) -> &'static str { @@ -298,4 +452,201 @@ mod tests { assert_eq!(hf.norm_topk_prob, g.norm_topk_prob); assert!(hf.norm_topk_prob); } + + /// Qwen3NextRMSNorm is zero-centered (`1 + weight`), so base-convert bakes + /// +1 into every norm EXCEPT the Gated-DeltaNet output norm + /// (`linear_attn.norm`, a Qwen3NextRMSNormGated that uses plain weight). + #[test] + fn qwen35_norm_shift_predicate() { + use super::qwen35_norm_shift; + // Regular norms shift by +1. + for s in [ + "layers.0.input_norm.weight", + "layers.3.post_attn_norm.weight", + "layers.3.self_attn.q_norm.weight", + "layers.3.self_attn.k_norm.weight", + "final_norm.weight", + ] { + assert_eq!(qwen35_norm_shift(s), 1.0, "should shift: {s}"); + } + // The gated GDN norm and everything non-norm do NOT shift. + for s in [ + "layers.0.linear_attn.norm.weight", + "layers.0.linear_attn.in_proj_qkv.weight", + "layers.0.mlp.gate_proj.weight", + "embed_tokens.weight", + ] { + assert_eq!(qwen35_norm_shift(s), 0.0, "should not shift: {s}"); + } + } + + /// Qwen3.5-35B-A3B / Qwen3.6-35B-A3B (identical config shape): the REAL + /// checkpoint's text_config has NO `intermediate_size` (the HF config + /// class deletes it — the dense-FFN slot is the shared expert), GQA GDN + /// (nv=32 ≠ nk=16), 256 experts top-8 with fused gate_up, a shared + /// expert of width `shared_expert_intermediate_size`, and an MTP head + /// (tensors skipped at map time). The mapper must not bail on the + /// missing `intermediate_size` and must surface every MoE + GDN field. + #[test] + fn qwen35_moe_35b_a3b_config_from_hf() { + let cfg = json!({ + "architectures": ["Qwen3_5MoeForConditionalGeneration"], + "model_type": "qwen3_5_moe", + "text_config": { + "model_type": "qwen3_5_moe_text", + "attn_output_gate": true, + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_size": 2048, + "layer_types": [ + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 8, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "shared_expert_intermediate_size": 512, + "vocab_size": 248320, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [11, 11, 10], + "rope_type": "default", + "rope_theta": 10000000, + "partial_rotary_factor": 0.25 + } + } + }); + let c = Qwen35MoeHfMapper.config_from_hf(&cfg).unwrap(); + + // intermediate_size backfilled from the shared-expert width (the + // runtime's ffn_dim = shared-expert SwiGLU width). + assert_eq!(c.intermediate_size, 512); + assert_eq!(c.num_experts, 256); + assert_eq!(c.num_experts_per_tok, 8); + assert_eq!(c.moe_intermediate_size, 512); + assert_eq!(c.num_shared_experts, 1); + assert!(c.norm_topk_prob); + + // GQA GDN: value heads ≠ key heads. + assert_eq!(c.linear_num_key_heads, 16); + assert_eq!(c.linear_num_value_heads, 32); + assert_eq!(c.rope_theta, 10_000_000.0); + assert_eq!(c.partial_rotary_factor, 0.25); + assert_eq!(c.num_attention_heads, 16); + assert_eq!(c.num_kv_heads, 2); + + // Round-trip: the runtime reads these back from the .base header. + let m = c.to_config_map(); + assert_eq!(m["num_experts"], json!(256)); + assert_eq!(m["num_shared_experts"], json!(1)); + assert_eq!(m["moe_intermediate_size"], json!(512)); + assert_eq!(m["intermediate_size"], json!(512)); + assert_eq!(m["linear_num_value_heads"], json!(32)); + } + + /// Qwen3.5-2B-Base: the multimodal wrapper nests the LM params under + /// `text_config`, RoPE under `rope_parameters`, and declares a hybrid + /// Gated-DeltaNet / full-attention layer schedule. The mapper must dig + /// into `text_config`, override rope_theta from `rope_parameters` + /// (top-level default 10000 is wrong — real value is 1e7), and capture + /// every linear-attention shape + the layer_types schedule. + #[test] + fn qwen35_2b_hybrid_config_from_hf() { + let cfg = json!({ + "architectures": ["Qwen3_5ForConditionalGeneration"], + "model_type": "qwen3_5", + "tie_word_embeddings": true, + "text_config": { + "model_type": "qwen3_5_text", + "attn_output_gate": true, + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_size": 2048, + "intermediate_size": 6144, + "layer_types": [ + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 16, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "num_attention_heads": 8, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "tie_word_embeddings": true, + "vocab_size": 248320, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [11, 11, 10], + "rope_type": "default", + "rope_theta": 10000000, + "partial_rotary_factor": 0.25 + } + } + }); + let c = Qwen35HfMapper.config_from_hf(&cfg).unwrap(); + + // Core Llama-shaped fields read from the nested text_config. + assert_eq!(c.hidden_size, 2048); + assert_eq!(c.num_hidden_layers, 24); + assert_eq!(c.num_attention_heads, 8); + assert_eq!(c.num_kv_heads, 2); + assert_eq!(c.head_dim, 256); + assert_eq!(c.intermediate_size, 6144); + assert_eq!(c.vocab_size, 248320); + assert_eq!(c.rms_norm_eps, 1e-6); + assert!(c.tie_word_embeddings); + assert_eq!(c.eos_token_id, 248044); + + // RoPE pulled from rope_parameters (NOT the 10000 default). + assert_eq!(c.rope_theta, 10_000_000.0); + assert_eq!(c.partial_rotary_factor, 0.25); + assert_eq!(c.mrope_section, vec![11, 11, 10]); + assert!(c.mrope_interleaved); + + // Hybrid schedule + Gated-DeltaNet shapes. + assert_eq!(c.full_attention_interval, 4); + assert_eq!(c.layer_types.len(), 24); + assert_eq!(c.layer_types[0], "linear_attention"); + assert_eq!(c.layer_types[3], "full_attention"); + assert_eq!(c.linear_num_key_heads, 16); + assert_eq!(c.linear_num_value_heads, 16); + assert_eq!(c.linear_key_head_dim, 128); + assert_eq!(c.linear_value_head_dim, 128); + assert_eq!(c.linear_conv_kernel_dim, 4); + assert!(c.attn_output_gate); + + // Every hybrid field must survive the round-trip into the .base + // header config map (the runtime reads them back from here). + let m = c.to_config_map(); + assert_eq!(m["rope_theta"], json!(10_000_000.0)); + assert_eq!(m["partial_rotary_factor"], json!(0.25)); + assert_eq!(m["full_attention_interval"], json!(4)); + assert_eq!(m["linear_num_key_heads"], json!(16)); + assert_eq!(m["linear_conv_kernel_dim"], json!(4)); + assert_eq!(m["attn_output_gate"], json!(true)); + assert_eq!(m["mrope_section"], json!([11, 11, 10])); + assert!(m.contains_key("layer_types")); + } } diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index fbcca24..42540a7 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -216,12 +216,30 @@ fn auto_pull_and_resolve(reg: &MergedRegistry, id: &str, want_variant: Option<&s .with_context(|| format!("fetched {pull_id} but could not locate the installed artifact")) } -/// Resolve a hub model reference (`org/model` or `org/model:variant`) to the -/// installed `.base` artifact path. When the model isn't installed yet it is -/// fetched on demand — preferring the pre-converted basecompute mirror, else -/// converting the source repo — so `basert chat`/`serve ` Just Works. -fn resolve_hub_model(token: &str) -> Result { - let (id, variant) = token.split_once(':').map_or((token, None), |(i, v)| (i, Some(v))); +/// Resolve a hub model reference to the installed `.base` artifact path. +/// +/// The variant (quant build) is chosen by an inline `org/model:default-q8` on +/// the token, or by the `--variant` launcher flag (`default_variant`) for +/// tokens without an inline `:variant` — the inline form wins. With neither, +/// the sole installed variant is used (and it's an error to resolve a model +/// with several installed without naming one). An uninstalled model is fetched +/// on demand — preferring the pre-converted basecompute mirror, else converting +/// the source repo — so `basert chat`/`serve ` Just Works. +fn resolve_hub_model(token: &str, default_variant: Option<&str>) -> Result { + let (id, inline) = token.split_once(':').map_or((token, None), |(i, v)| (i, Some(v))); + + // A trailing/empty `:` is a typo, not "default variant" — fail loudly so it + // doesn't silently resolve to q4. + if inline == Some("") { + bail!( + "empty variant after ':' in `{token}` — name a variant \ + (e.g. `{id}:default-q8`) or drop the colon. Run `basert list` to \ + see installed variants." + ); + } + + // Inline `:variant` wins over the `--variant` flag. + let variant = inline.or(default_variant); let reg = MergedRegistry::load()?; if let Some(v) = variant { @@ -239,12 +257,14 @@ fn resolve_hub_model(token: &str) -> Result { /// Rewrite every hub-id-shaped argument to the installed `.base` path so that /// `basert chat org/model` (positional) and `basert serve --model org/model` -/// (repeatable flag) both Just Work after `basert pull`. -fn resolve_model_args(rest: &[String]) -> Result> { +/// (repeatable flag) both Just Work after `basert pull`. `default_variant` +/// comes from the `--variant` launcher flag and applies to ids without an +/// inline `:variant`. +fn resolve_model_args(rest: &[String], default_variant: Option<&str>) -> Result> { rest.iter() .map(|arg| { if looks_like_hub_id(arg) { - Ok(resolve_hub_model(arg)?.to_string_lossy().into_owned()) + Ok(resolve_hub_model(arg, default_variant)?.to_string_lossy().into_owned()) } else { Ok(arg.clone()) } @@ -252,6 +272,41 @@ fn resolve_model_args(rest: &[String]) -> Result> { .collect() } +/// Pull a `--variant ` / `--variant=` selector out of the forwarded args, +/// returning the variant (if any) and the args with it removed — the runtime +/// binary never sees a flag it doesn't understand. The selector applies to +/// hub-id model args that don't already carry an inline `:variant`. +fn extract_variant_flag(rest: &[String]) -> Result<(Option, Vec)> { + let mut variant: Option = None; + let mut out: Vec = Vec::with_capacity(rest.len()); + let mut i = 0; + while i < rest.len() { + let a = &rest[i]; + if a == "--variant" { + let v = rest + .get(i + 1) + .context("`--variant` needs a value, e.g. `--variant default-q8`")?; + if v.is_empty() { + bail!("`--variant` value is empty"); + } + variant = Some(v.clone()); + i += 2; + continue; + } + if let Some(v) = a.strip_prefix("--variant=") { + if v.is_empty() { + bail!("`--variant=` value is empty"); + } + variant = Some(v.to_string()); + i += 1; + continue; + } + out.push(a.clone()); + i += 1; + } + Ok((variant, out)) +} + /// Forward `basert [args…]` to the matching runtime binary. Searches for /// `basert-` (release layout) then `baseRT_` (local dev build), /// looking next to this executable first and then on `PATH`. On success the @@ -262,7 +317,11 @@ pub fn dispatch_external(argv: Vec) -> Result<()> { let (cmd, rest) = argv .split_first() .context("no subcommand given to launcher")?; - let rest = resolve_model_args(rest)?; + // `--variant ` is a launcher-level model selector; strip it before + // forwarding (the runtime binary doesn't know it) and apply it during + // hub-id resolution. + let (variant_flag, rest) = extract_variant_flag(rest)?; + let rest = resolve_model_args(&rest, variant_flag.as_deref())?; let candidates = [format!("basert-{cmd}"), format!("baseRT_{cmd}")]; // Prefer a binary sitting next to `basert` (how the release ships); fall @@ -910,10 +969,50 @@ mod tests { "--max-tokens".to_string(), "32".to_string(), ]; - let out = resolve_model_args(&args).unwrap(); + let out = resolve_model_args(&args, None).unwrap(); assert_eq!(out, args); } + #[test] + fn resolve_hub_model_rejects_empty_variant() { + // A trailing `:` (empty variant) is a typo — error before any pull, + // rather than silently resolving to the q4 default. + let err = resolve_hub_model("basecompute/Gemma-4-E2B-it:", None) + .unwrap_err() + .to_string(); + assert!(err.contains("empty variant"), "{err}"); + } + + #[test] + fn extract_variant_flag_parses_all_forms() { + // `--variant ` + let (v, rest) = extract_variant_flag(&[ + "--model".into(), + "org/m".into(), + "--variant".into(), + "default-q8".into(), + "--port".into(), + "8080".into(), + ]) + .unwrap(); + assert_eq!(v.as_deref(), Some("default-q8")); + assert_eq!(rest, vec!["--model", "org/m", "--port", "8080"]); + + // `--variant=` + let (v, rest) = extract_variant_flag(&["--variant=q8".into(), "org/m".into()]).unwrap(); + assert_eq!(v.as_deref(), Some("q8")); + assert_eq!(rest, vec!["org/m"]); + + // Absent → None, args untouched. + let (v, rest) = extract_variant_flag(&["org/m".into()]).unwrap(); + assert!(v.is_none()); + assert_eq!(rest, vec!["org/m"]); + + // Missing / empty value → error. + assert!(extract_variant_flag(&["--variant".into()]).is_err()); + assert!(extract_variant_flag(&["--variant=".into()]).is_err()); + } + #[test] fn choose_profile_uses_bundled_for_q4_and_q8() { let mk = |t| PullArgs { diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index eb85db8..24b9c96 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -15,7 +15,12 @@ Runtime tools (forwarded to the matching `basert-` binary): profile Profile prefill/decode timing transcribe Audio transcription -Run `basert --help` for a tool's own options."; +Run `basert --help` for a tool's own options. + +Model variant (serve/chat/complete/bench/profile) — pick which quant build to run: + : e.g. basert serve basecompute/Gemma-4-E2B-it:default-q8 + --variant same, as a flag (applies to ids without an inline ':variant') +Run `basert list` to see installed variants (default-q4, default-q8, …)."; /// The `basert` CLI: the model hub (pull/list from HuggingFace), the offline /// GGUF / MLX-safetensors / HF-safetensors → `.base` converter, and a @@ -206,7 +211,60 @@ struct ListArgs { json: bool, } +/// BaseRT wordmark banner (mirrors the C++ CLIs — tools/basert_banner.h). +/// White→lime vertical gradient anchored on brand Lime #E8FFBD (the brand +/// tone alone is too close to white to read as terminal text). Printed only +/// on an interactive terminal; NO_COLOR drops the colors. +fn print_banner() { + use std::io::IsTerminal; + if !std::io::stdout().is_terminal() { + return; + } + let color = std::env::var_os("NO_COLOR").is_none(); + const ART: [&str; 8] = [ + " ███████████ ███████████ ███████████", + "░░███░░░░░███ ░░███░░░░░███ ░█░░░███░░░█", + " ░███ ░███ ██████ █████ ██████ ░███ ░███ ░ ░███ ░", + " ░██████████ ░░░░░███ ███░░ ███░░███ ░██████████ ░███", + " ░███░░░░░███ ███████ ░░█████ ░███████ ░███░░░░░███ ░███", + " ░███ ░███ ███░░███ ░░░░███░███░░░ ░███ ░███ ░███", + " ███████████ ░░████████ ██████ ░░██████ █████ █████ █████", + "░░░░░░░░░░░ ░░░░░░░░ ░░░░░░ ░░░░░░ ░░░░░ ░░░░░ ░░░░░", + ]; + const GRAD: [&str; 8] = [ + "\x1b[38;2;255;255;255m", + "\x1b[38;2;248;255;232m", + "\x1b[38;2;240;255;208m", + "\x1b[38;2;232;255;189m", + "\x1b[38;2;223;255;161m", + "\x1b[38;2;213;255;133m", + "\x1b[38;2;204;255;105m", + "\x1b[38;2;195;255;77m", + ]; + println!(); + for i in 0..8 { + if color { + println!(" \x1b[1m{}{}\x1b[0m", GRAD[i], ART[i]); + } else { + println!(" {}", ART[i]); + } + } + let (dim, lime, rst) = if color { + ("\x1b[2m", "\x1b[38;2;195;255;77m", "\x1b[0m") + } else { + ("", "", "") + }; + println!("\n {dim}{lime}https://basecompute.co{rst} {dim}·{rst} {dim}{lime}https://discord.gg/tB9YFTKZUV{rst}\n"); +} + fn main() -> Result<()> { + // Banner on the launcher's own help / no-args surface only — hub + // subcommands (convert/pull/list/...) stay quiet, and the runtime tools + // print their own banner (tools/basert_banner.h). + let raw_arg1 = std::env::args().nth(1); + if raw_arg1.is_none() || matches!(raw_arg1.as_deref(), Some("-h") | Some("--help") | Some("help")) { + print_banner(); + } let args = Args::parse(); match args.cmd { Cmd::Convert(a) => cmd_convert(a), @@ -706,6 +764,58 @@ fn convert_hf( config.intermediate_size, config.vocab_size ); + // Merge the generation stop set from generation_config.json. HF declares + // the authoritative end-of-generation ids there; config.json often carries + // only a single scalar eos. Phi-3 is the canonical case: config.json has + // eos_token_id=32000 (<|endoftext|>) but generation_config.json has + // [32000, 32001, 32007] where <|end|>(32007) ends every chat turn. Without + // the full set the runtime never stops on <|end|>, so the model floods / + // repeats past its turn (finish_reason=length forever). Keep config.json's + // primary eos first; append any extra ids from generation_config.json. + let mut config = config; + let gc_path = input.join("generation_config.json"); + if let Ok(text) = std::fs::read_to_string(&gc_path) { + if let Ok(gc) = serde_json::from_str::(&text) { + let mut gen_ids: Vec = Vec::new(); + match &gc["eos_token_id"] { + serde_json::Value::Number(n) => { + if let Some(u) = n.as_u64() { + gen_ids.push(u as u32); + } + } + serde_json::Value::Array(a) => { + for x in a { + if let Some(u) = x.as_u64() { + gen_ids.push(u as u32); + } + } + } + _ => {} + } + if !gen_ids.is_empty() { + let mut ids: Vec = Vec::new(); + if config.eos_token_id != 0 { + ids.push(config.eos_token_id); + } + for &e in &config.eos_token_ids { + if !ids.contains(&e) { + ids.push(e); + } + } + for &g in &gen_ids { + if g != 0 && !ids.contains(&g) { + ids.push(g); + } + } + if ids.len() > config.eos_token_ids.len() + 1 { + eprintln!(" eos: stop ids {:?} (merged generation_config.json)", ids); + } + config.eos_token_id = ids[0]; + config.eos_token_ids = ids[1..].to_vec(); + } + } + } + let provider = HfTensorProvider { hf: &hf }; let mmproj_cfg = mmproj_config_from_hf(&hf); convert_generic( @@ -920,6 +1030,216 @@ impl<'a> TensorProvider for MlxTensorProvider<'a> { } } +/// Wraps a provider to stack HF-mainline MoE *per-expert* tensors into the +/// fused 3D form the runtime requires. HF Qwen3-MoE (and Mixtral-style) ships +/// experts as `…layers.N.mlp.experts.{e}.{gate,up,down}_proj.weight` — one +/// matrix per expert. The runtime wants a single `layers.N.ffn_{…}_exps.weight` +/// of shape `[n_experts, out, in]`. We expose a *virtual* name with the expert +/// index dropped (`…layers.N.mlp.experts.{gate,up,down}_proj.weight`), which the +/// existing canonicalizer already remaps to `ffn_*_exps`; `to_f32` concatenates +/// the per-expert f32 in expert order so the unchanged 3D-expert quant path +/// handles it exactly like Gemma 4's already-fused experts. Pure pass-through +/// when no per-expert tensors are present (MLX / Gemma-4 sources). +struct StackingProvider<'a> { + inner: &'a dyn TensorProvider, + stacks: std::collections::BTreeMap>, // virtual name -> per-expert names (expert order) + rewritten: Vec, +} +impl<'a> StackingProvider<'a> { + fn build(inner: &'a dyn TensorProvider, source_names: &[String]) -> Self { + use std::collections::{BTreeMap, HashSet}; + const MARK: &str = ".mlp.experts."; + let mut groups: BTreeMap> = BTreeMap::new(); + let mut consumed: HashSet = HashSet::new(); + for n in source_names { + let Some(idx) = n.find(MARK) else { continue }; + let before = &n[..idx]; + let rest = &n[idx + MARK.len()..]; + let Some(dot) = rest.find('.') else { continue }; + let (e_str, tail) = rest.split_at(dot); + if e_str.is_empty() || !e_str.bytes().all(|b| b.is_ascii_digit()) { + continue; // already-fused (`experts.gate_proj.weight`) — leave to canon + } + if !matches!(tail, ".gate_proj.weight" | ".up_proj.weight" | ".down_proj.weight") { + continue; + } + let e: usize = e_str.parse().unwrap_or(usize::MAX); + let vname = format!("{before}{MARK}{tail_no_dot}", tail_no_dot = &tail[1..]); + groups.entry(vname).or_default().push((e, n.clone())); + consumed.insert(n.clone()); + } + let mut stacks: BTreeMap> = BTreeMap::new(); + for (v, mut es) in groups { + es.sort_by_key(|(e, _)| *e); + stacks.insert(v, es.into_iter().map(|(_, n)| n).collect()); + } + let mut rewritten: Vec = + source_names.iter().filter(|n| !consumed.contains(*n)).cloned().collect(); + rewritten.extend(stacks.keys().cloned()); + StackingProvider { inner, stacks, rewritten } + } + fn rewritten_names(&self) -> Vec { + self.rewritten.clone() + } +} +impl TensorProvider for StackingProvider<'_> { + fn source_shape(&self, name: &str) -> Result> { + match self.stacks.get(name) { + Some(parts) => { + let per = self.inner.source_shape(&parts[0])?; // [out, in] + let mut s = Vec::with_capacity(per.len() + 1); + s.push(parts.len() as u64); + s.extend_from_slice(&per); + Ok(s) + } + None => self.inner.source_shape(name), + } + } + fn to_f32(&self, name: &str) -> Result> { + match self.stacks.get(name) { + Some(parts) => { + let mut out: Vec = Vec::new(); + for p in parts { + out.extend(self.inner.to_f32(p)?); + } + Ok(out) + } + None => self.inner.to_f32(name), + } + } +} + +/// Inverse of [`StackingProvider`]: some HF checkpoints ship *fused* attention +/// and MLP projections that the canonical llama path expects to be separate. +/// Phi-3 stores `…self_attn.qkv_proj.weight` (output rows laid out as +/// `[q: n_heads·head_dim | k: n_kv·head_dim | v: n_kv·head_dim]`) and +/// `…mlp.gate_up_proj.weight` (`[gate: ffn | up: ffn]`, gate first). We expose +/// *virtual* separate names (`q_proj`/`k_proj`/`v_proj`, `gate_proj`/`up_proj`) +/// — which the canonicalizer already maps — and slice the fused matrix by +/// output rows in `to_f32`. Pure pass-through for sources that already ship +/// these separate (Llama, Qwen, Gemma, Mistral): no fused names → no splits. +#[derive(Clone)] +struct SplitSpec { + src: String, + row_off: u64, + row_cnt: u64, +} +struct SplittingProvider<'a> { + inner: &'a dyn TensorProvider, + splits: std::collections::BTreeMap, // virtual name -> slice of fused src + rewritten: Vec, +} +impl<'a> SplittingProvider<'a> { + fn build( + inner: &'a dyn TensorProvider, + source_names: &[String], + config: &base_arch::ArchConfig, + ) -> Result { + use std::collections::{BTreeMap, HashSet}; + let mut splits: BTreeMap = BTreeMap::new(); + let mut consumed: HashSet = HashSet::new(); + + let hd = config.head_dim as u64; + let nq = config.num_attention_heads as u64; + let nkv = config.num_kv_heads as u64; + let ffn = config.intermediate_size as u64; + + for n in source_names { + if let Some(before) = n.strip_suffix(".self_attn.qkv_proj.weight") { + if hd == 0 || nq == 0 || nkv == 0 { + bail!( + "fused {n}: head_dim/num_attention_heads/num_kv_heads must be set in \ + config to split qkv_proj (got hd={hd}, nq={nq}, nkv={nkv})" + ); + } + let (q, k, v) = (nq * hd, nkv * hd, nkv * hd); + let shape = inner.source_shape(n)?; // [out, in] + let rows = shape.first().copied().unwrap_or(0); + if rows != q + k + v { + bail!( + "fused qkv_proj {n} has {rows} output rows but config implies \ + n_heads·hd + 2·n_kv·hd = {} (nq={nq}, nkv={nkv}, hd={hd})", + q + k + v + ); + } + splits.insert( + format!("{before}.self_attn.q_proj.weight"), + SplitSpec { src: n.clone(), row_off: 0, row_cnt: q }, + ); + splits.insert( + format!("{before}.self_attn.k_proj.weight"), + SplitSpec { src: n.clone(), row_off: q, row_cnt: k }, + ); + splits.insert( + format!("{before}.self_attn.v_proj.weight"), + SplitSpec { src: n.clone(), row_off: q + k, row_cnt: v }, + ); + consumed.insert(n.clone()); + } else if let Some(before) = n.strip_suffix(".mlp.gate_up_proj.weight") { + if ffn == 0 { + bail!("fused {n}: intermediate_size must be set in config to split gate_up_proj"); + } + let shape = inner.source_shape(n)?; + let rows = shape.first().copied().unwrap_or(0); + if rows != 2 * ffn { + bail!( + "fused gate_up_proj {n} has {rows} output rows but config implies \ + 2·intermediate_size = {}", + 2 * ffn + ); + } + splits.insert( + format!("{before}.mlp.gate_proj.weight"), + SplitSpec { src: n.clone(), row_off: 0, row_cnt: ffn }, + ); + splits.insert( + format!("{before}.mlp.up_proj.weight"), + SplitSpec { src: n.clone(), row_off: ffn, row_cnt: ffn }, + ); + consumed.insert(n.clone()); + } + } + + let mut rewritten: Vec = + source_names.iter().filter(|n| !consumed.contains(*n)).cloned().collect(); + rewritten.extend(splits.keys().cloned()); + Ok(SplittingProvider { inner, splits, rewritten }) + } + fn rewritten_names(&self) -> Vec { + self.rewritten.clone() + } +} +impl TensorProvider for SplittingProvider<'_> { + fn source_shape(&self, name: &str) -> Result> { + match self.splits.get(name) { + Some(spec) => { + let full = self.inner.source_shape(&spec.src)?; // [out, in] + if full.len() != 2 { + bail!("fused tensor {} is not 2-D (shape {:?})", spec.src, full); + } + Ok(vec![spec.row_cnt, full[1]]) + } + None => self.inner.source_shape(name), + } + } + fn to_f32(&self, name: &str) -> Result> { + match self.splits.get(name) { + Some(spec) => { + let full = self.inner.source_shape(&spec.src)?; + if full.len() != 2 { + bail!("fused tensor {} is not 2-D (shape {:?})", spec.src, full); + } + let in_dim = full[1] as usize; + let data = self.inner.to_f32(&spec.src)?; + let start = spec.row_off as usize * in_dim; + let end = (spec.row_off + spec.row_cnt) as usize * in_dim; + Ok(data[start..end].to_vec()) + } + None => self.inner.to_f32(name), + } + } +} + #[allow(clippy::too_many_arguments)] fn convert_generic( input: &std::path::Path, @@ -952,6 +1272,34 @@ fn convert_generic( TargetScheme::Nvfp4 => QuantScheme::Nvfp4, }; + // Stack HF-mainline per-expert MoE tensors into fused 3D experts before + // mapping/quantizing (no-op for MLX / Gemma-4 sources that are already + // fused). Shadows `provider` + `source_names` for the rest of the function. + let stacking = StackingProvider::build(provider, &source_names); + if !stacking.stacks.is_empty() { + eprintln!( + " moe: stacked per-expert tensors into {} fused ffn_*_exps", + stacking.stacks.len() + ); + } + let source_names = stacking.rewritten_names(); + let provider: &dyn TensorProvider = &stacking; + + // Split HF fused attention/MLP projections (Phi-3: `qkv_proj`, + // `gate_up_proj`) into the separate q/k/v + gate/up tensors the llama path + // expects, slicing by output rows per the config dims. No-op for sources + // that already ship them separate (Llama, Qwen, Gemma, Mistral). Shadows + // `provider` + `source_names` again for the rest of the function. + let splitting = SplittingProvider::build(provider, &source_names, &config)?; + if !splitting.splits.is_empty() { + eprintln!( + " fused: split {} fused qkv_proj/gate_up_proj into q/k/v + gate/up", + splitting.splits.len() + ); + } + let source_names = splitting.rewritten_names(); + let provider: &dyn TensorProvider = &splitting; + // Map source names → canonical. Use the same llama-style map that // GGUF uses for blk.N.* tensors, plus a HF-style pass-through for // `model.layers.N.*` already-canonical names. Multimodal towers @@ -1118,12 +1466,15 @@ fn convert_generic( let is_ssm_a = canonical == "ssm.a_log" || canonical.ends_with(".ssm.a_log") - || src_name.ends_with(".ssm_a"); + || src_name.ends_with(".ssm_a") + || is_gdn_a_log(canonical); let is_ssm_sensitive = is_ssm_a || canonical.ends_with(".ssm.dt_bias") - || canonical.ends_with(".ssm.d"); - // See GGUF path above for rationale on the f16/GPU route. - let is_norm_like = shape.len() == 1; + || canonical.ends_with(".ssm.d") + || is_gdn_f32(canonical); + // See GGUF path above for rationale on the f16/GPU route. Qwen3.5 GDN + // conv1d + per-head gate projections also route here (f16 on GPU). + let is_norm_like = shape.len() == 1 || is_gdn_f16(canonical); let is_embedding = canonical == "embed_tokens.weight" || canonical == "lm_head.weight"; @@ -1562,11 +1913,49 @@ fn nomic_bert_hf_rename(name: &str) -> Option { Some(format!("layers.{layer}.{canonical}")) } +// ── Qwen3.5 / 3.6 Gated-DeltaNet tensor precision ──────────────────────── +// The GDN linear-attention block has params that the default profile's +// catch-all `**.weight` Q4 rule would wreck. These predicates (matched on +// canonical name) pin the precision-critical ones, mirroring how the GGUF +// SSM path protects Mamba's `ssm.*` scalars. + +/// GDN log-space state-decay `A_log` (A = -exp(A_log)). Same +/// NaN-after-~100-recurrent-steps sensitivity as the Mamba A-matrix, so it +/// must stay f32 in the CPU region (and carries the SSM_A_MATRIX flag). +fn is_gdn_a_log(canonical: &str) -> bool { + canonical.ends_with(".linear_attn.A_log") +} + +/// GDN recurrence scalars that must stay f32: `A_log` + the `dt` bias that +/// feeds the exp-gating. +fn is_gdn_f32(canonical: &str) -> bool { + is_gdn_a_log(canonical) || canonical.ends_with(".linear_attn.dt_bias") +} + +/// GDN short causal conv (`conv1d.weight`, [3*dim, 1, K]) and the tiny +/// per-head gate projections (`in_proj_a` = decay, `in_proj_b` = beta, each +/// [n_heads, dim]). Precision-critical but not recurrence-accumulated, so +/// f16-on-GPU is enough — the point is only to escape Q4 (the conv's 4-wide +/// last dim can't even form a sane quant group). +fn is_gdn_f16(canonical: &str) -> bool { + canonical.ends_with(".linear_attn.conv1d.weight") + || canonical.ends_with(".linear_attn.in_proj_a.weight") + || canonical.ends_with(".linear_attn.in_proj_b.weight") +} + /// Map an HF-style or MLX-style tensor name to canonical `.base` name. /// HF already uses the canonical `model.layers.N.*` convention, so we /// mostly strip the `model.` prefix. For anything not in HF convention, /// fall back to the llama-style GGUF mapper. fn to_canonical_name(name: &str, arch: &str) -> Option { + // Qwen3.5/3.6 checkpoints ship a Multi-Token-Prediction head (`mtp.*` — + // one extra decoder layer + fc/norms, its own experts on the MoE + // variants). It's a speculative-decoding auxiliary the runtime doesn't + // execute; keeping it would bloat the .base (256 extra expert stacks on + // 35B-A3B) and land unknown-named tensors in the main bundle. Drop it. + if arch.starts_with("qwen35") && (name == "mtp" || name.starts_with("mtp.")) { + return None; + } // Multimodal wrappers (Gemma-4) put the LLM under `language_model.model.*` // and the audio/vision towers under siblings. The towers ship in // the same .base bundle but live under `header.mmproj.tensors` so @@ -1585,6 +1974,13 @@ fn to_canonical_name(name: &str, arch: &str) -> Option { let mm_name = name.strip_prefix("model.").unwrap_or(name); if mm_name.starts_with("audio_tower.") || mm_name.starts_with("vision_tower.") + // Qwen3.5 / 3.6 wrap the vision tower under `model.visual.*` + // (Qwen-VL naming) rather than Gemma-4's `vision_tower.*`. Route + // it to the mmproj sub-bundle so a text-only runtime skips it and + // a future multimodal path can materialize it — the weights stay + // out of the main LM tensor set instead of being written as + // unknown pass-through tensors. + || mm_name.starts_with("visual.") || mm_name.starts_with("embed_audio") || mm_name.starts_with("embed_vision") || mm_name.starts_with("multi_modal_projector") @@ -1615,6 +2011,14 @@ fn to_canonical_name(name: &str, arch: &str) -> Option { .or_else(|| name.strip_prefix("model.")) .unwrap_or(name); + // MTP can also arrive wrapped (`model.mtp.*`, or `model.language_model. + // mtp.*` on VL-style checkpoints); the bare-prefix drop at the top of + // this function runs before wrapper stripping and misses those, which + // would canonicalize the dead MTP decoder/expert stacks into the bundle. + if arch.starts_with("qwen35") && (stripped == "mtp" || stripped.starts_with("mtp.")) { + return None; + } + // HF native names → canonical. if stripped != name { if stripped == "rotary_emb.inv_freq" { @@ -1690,6 +2094,15 @@ fn to_canonical_name(name: &str, arch: &str) -> Option { .replace(".experts.down_proj.weight", ".ffn_down_exps.weight") .replace(".experts.gate_proj.weight", ".ffn_gate_exps.weight") .replace(".experts.up_proj.weight", ".ffn_up_exps.weight") + // Bare (no `.weight`) forms with a `.mlp.` parent — Qwen3.5/3.6 + // MoE ships `layers.N.mlp.experts.gate_up_proj` (pre-stacked 3D). + // The `.mlp.` must be dropped here too, else the canonical comes + // out `layers.N.mlp.ffn_gate_up_exps.weight` and the runtime's + // alias table misses it (FATAL missing-tensor at load). + .replace(".mlp.experts.gate_up_proj", ".ffn_gate_up_exps.weight") + .replace(".mlp.experts.down_proj", ".ffn_down_exps.weight") + .replace(".mlp.experts.gate_proj", ".ffn_gate_exps.weight") + .replace(".mlp.experts.up_proj", ".ffn_up_exps.weight") .replace(".experts.gate_up_proj", ".ffn_gate_up_exps.weight") .replace(".experts.down_proj", ".ffn_down_exps.weight") .replace(".experts.gate_proj", ".ffn_gate_exps.weight") @@ -2585,6 +2998,138 @@ mod canonical_name_tests { } } + fn mmproj_canon(name: &str, arch: &str) -> Option { + match to_canonical_name(name, arch)? { + Canonical::Mmproj(s) => Some(s), + Canonical::Main(_) => None, + } + } + + // Qwen3.5 / 3.6 (`Qwen3_5ForConditionalGeneration`) wraps the LM under + // `model.language_model.*`. The Gated-DeltaNet ("linear_attn") tensors + // have no GGUF analogue, so they must pass through to_canonical_name + // verbatim (after prefix strip + the standard norm renames) — the + // runtime reads them under `layers.N.linear_attn.*`. The vision tower + // (`model.visual.*`, Qwen-VL naming) must route to the mmproj sub-bundle + // so a text-only bundle keeps it out of the main LM tensor set. + #[test] + fn qwen35_hybrid_tensor_canonicalization() { + // Gated-DeltaNet linear-attention block — verbatim passthrough. + for (src, want) in [ + ( + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight", + "layers.0.linear_attn.in_proj_qkv.weight", + ), + ( + "model.language_model.layers.0.linear_attn.in_proj_z.weight", + "layers.0.linear_attn.in_proj_z.weight", + ), + ( + "model.language_model.layers.0.linear_attn.in_proj_a.weight", + "layers.0.linear_attn.in_proj_a.weight", + ), + ( + "model.language_model.layers.0.linear_attn.in_proj_b.weight", + "layers.0.linear_attn.in_proj_b.weight", + ), + ( + "model.language_model.layers.0.linear_attn.conv1d.weight", + "layers.0.linear_attn.conv1d.weight", + ), + ( + "model.language_model.layers.0.linear_attn.norm.weight", + "layers.0.linear_attn.norm.weight", + ), + ( + "model.language_model.layers.0.linear_attn.out_proj.weight", + "layers.0.linear_attn.out_proj.weight", + ), + // 1-D params without a `.weight` suffix must survive too. + ( + "model.language_model.layers.0.linear_attn.A_log", + "layers.0.linear_attn.A_log", + ), + ( + "model.language_model.layers.0.linear_attn.dt_bias", + "layers.0.linear_attn.dt_bias", + ), + ] { + assert_eq!(main_canon(src, "qwen35"), Some(want.to_string()), "src={src}"); + } + + // Full-attention block: standard Qwen QK-norm + gate passthrough, + // and the two per-layer norms get the usual HF→canonical renames. + assert_eq!( + main_canon( + "model.language_model.layers.3.self_attn.q_norm.weight", + "qwen35", + ), + Some("layers.3.self_attn.q_norm.weight".to_string()), + ); + assert_eq!( + main_canon( + "model.language_model.layers.3.input_layernorm.weight", + "qwen35", + ), + Some("layers.3.input_norm.weight".to_string()), + ); + assert_eq!( + main_canon( + "model.language_model.layers.3.post_attention_layernorm.weight", + "qwen35", + ), + Some("layers.3.post_attn_norm.weight".to_string()), + ); + + // Top-level final norm + embedding. + assert_eq!( + main_canon("model.language_model.norm.weight", "qwen35"), + Some("final_norm.weight".to_string()), + ); + assert_eq!( + main_canon("model.language_model.embed_tokens.weight", "qwen35"), + Some("embed_tokens.weight".to_string()), + ); + + // Vision tower routes to mmproj (kept out of the main LM tensors). + assert_eq!( + mmproj_canon("model.visual.patch_embed.proj.weight", "qwen35"), + Some("visual.patch_embed.proj.weight".to_string()), + ); + assert!(main_canon("model.visual.patch_embed.proj.weight", "qwen35").is_none()); + } + + // Qwen3.5 GDN precision policy: the delta-rule recurrence scalars stay + // f32; the short conv + per-head gate projections escape Q4 to f16; the + // big projections (in_proj_qkv/z, out_proj) and MLP quantize normally. + // Verified end-to-end against a real Qwen3.5-2B conversion — under the + // default profile these three would otherwise be Q4-quantized, which + // corrupts the linear-attention recurrence. + #[test] + fn qwen35_gdn_tensor_precision() { + use super::{is_gdn_a_log, is_gdn_f16, is_gdn_f32}; + // f32 recurrence scalars. + assert!(is_gdn_a_log("layers.0.linear_attn.A_log")); + assert!(is_gdn_f32("layers.0.linear_attn.A_log")); + assert!(is_gdn_f32("layers.7.linear_attn.dt_bias")); + assert!(!is_gdn_a_log("layers.0.linear_attn.dt_bias")); + // f16 conv + gate projections. + assert!(is_gdn_f16("layers.0.linear_attn.conv1d.weight")); + assert!(is_gdn_f16("layers.0.linear_attn.in_proj_a.weight")); + assert!(is_gdn_f16("layers.0.linear_attn.in_proj_b.weight")); + // Big projections + everything else quantize normally (no override). + for n in [ + "layers.0.linear_attn.in_proj_qkv.weight", + "layers.0.linear_attn.in_proj_z.weight", + "layers.0.linear_attn.out_proj.weight", + "layers.0.linear_attn.norm.weight", + "layers.0.mlp.gate_proj.weight", + "layers.3.self_attn.q_proj.weight", + ] { + assert!(!is_gdn_f32(n) && !is_gdn_f16(n), "unexpected override for {n}"); + } + } + // MLX Qwen3-MoE: `model.layers.N.mlp.switch_mlp.X_proj.weight` // (per-expert stack) must canonicalize to `layers.N.ffn_X_exps.weight` // without doubling the `.weight` suffix. @@ -2643,7 +3188,67 @@ mod canonical_name_tests { ); } + // Qwen3.5/3.6-MoE ships the pre-stacked expert tensors bare under a + // `.mlp.` parent: `model.language_model.layers.N.mlp.experts.gate_up_proj`. + // The `.mlp.` prefix must be dropped along with the rename — pre-fix the + // canonical came out `layers.N.mlp.ffn_gate_up_exps.weight` and the + // runtime FATALed with missing-tensor at load. + #[test] + fn qwen35_moe_fused_bare_mlp_parent() { + assert_eq!( + main_canon( + "model.language_model.layers.7.mlp.experts.gate_up_proj", + "qwen35moe", + ), + Some("layers.7.ffn_gate_up_exps.weight".to_string()), + ); + assert_eq!( + main_canon( + "model.language_model.layers.7.mlp.experts.down_proj", + "qwen35moe", + ), + Some("layers.7.ffn_down_exps.weight".to_string()), + ); + } + // The shared-expert path (Qwen2-MoE / some Qwen3 variants) uses + // Qwen3.5/3.6 MTP (multi-token-prediction) head tensors must be DROPPED + // (`mtp.*` — an auxiliary decoder layer the runtime never executes; on + // 35B-A3B it carries 256 extra expert stacks that would bloat the .base). + #[test] + fn qwen35_mtp_tensors_dropped() { + for n in [ + "mtp.fc.weight", + "mtp.norm.weight", + "mtp.pre_fc_norm_embedding.weight", + "mtp.layers.0.mlp.experts.gate_up_proj", + "mtp.layers.0.self_attn.q_proj.weight", + ] { + assert!( + to_canonical_name(n, "qwen35moe").is_none(), + "should drop: {n}" + ); + assert!(to_canonical_name(n, "qwen35").is_none(), "should drop: {n}"); + } + // Wrapped variants (`model.mtp.*`, VL-style `model.language_model. + // mtp.*`) must be dropped too — the drop re-applies after wrapper + // prefix stripping. + for n in [ + "model.mtp.fc.weight", + "model.mtp.layers.0.mlp.experts.gate_up_proj", + "model.language_model.mtp.fc.weight", + "language_model.mtp.norm.weight", + ] { + assert!( + to_canonical_name(n, "qwen35moe").is_none(), + "should drop wrapped: {n}" + ); + } + // Non-qwen35 arches keep their behavior for names that merely start + // with "mtp" — the drop is arch-gated. + assert!(to_canonical_name("model.layers.0.mlp.gate_proj.weight", "qwen35moe").is_some()); + } + // `mlp.shared_expert.X_proj.weight`. The experts-rename rules must NOT // touch it — shared-expert tensors map through `mlp.shared_expert.*` // aliases on the runtime side. @@ -2752,3 +3357,191 @@ mod canonical_name_tests { assert_eq!(main_canon("0.auto_model.pooler.dense.weight", "nomic-bert"), None); } } + +#[cfg(test)] +mod stacking_tests { + use super::{StackingProvider, TensorProvider}; + use anyhow::Result; + + // Mock provider: each per-expert matrix is [out=4, in=2] = 8 elems, all + // equal to the expert index, so stacking order is checkable. + struct Mock; + impl TensorProvider for Mock { + fn source_shape(&self, _name: &str) -> Result> { + Ok(vec![4, 2]) + } + fn to_f32(&self, name: &str) -> Result> { + let e: f32 = name + .split(".mlp.experts.") + .nth(1) + .and_then(|s| s.split('.').next()) + .and_then(|s| s.parse().ok()) + .unwrap_or(-1.0); + Ok(vec![e; 8]) + } + } + + fn names() -> Vec { + // experts out of order incl. two-digit index to exercise numeric sort + let mut v = vec!["model.layers.0.self_attn.q_proj.weight".to_string()]; + for e in [0u32, 1, 10, 2] { + for p in ["gate", "up", "down"] { + v.push(format!("model.layers.0.mlp.experts.{e}.{p}_proj.weight")); + } + } + v + } + + #[test] + fn hf_per_expert_moe_stacks_into_fused_3d() { + let mock = Mock; + let sp = StackingProvider::build(&mock, &names()); + + // three fused virtual tensors (gate/up/down), per-expert names consumed + assert_eq!(sp.stacks.len(), 3); + let gate = "model.layers.0.mlp.experts.gate_proj.weight"; + assert!(sp.stacks.contains_key(gate)); + + // shape becomes [n_experts=4, out=4, in=2] + assert_eq!(sp.source_shape(gate).unwrap(), vec![4, 4, 2]); + + // stacked in NUMERIC expert order 0,1,2,10 (not lexical 0,1,10,2) + let d = sp.to_f32(gate).unwrap(); + assert_eq!(d.len(), 32); + assert_eq!(d[0], 0.0); // expert 0 + assert_eq!(d[8], 1.0); // expert 1 + assert_eq!(d[16], 2.0); // expert 2 + assert_eq!(d[24], 10.0); // expert 10 + + // rewritten names: per-expert dropped, attn kept, virtuals added + let rw = sp.rewritten_names(); + assert!(rw.iter().any(|n| n == "model.layers.0.self_attn.q_proj.weight")); + assert!(rw.iter().any(|n| n == gate)); + assert!(!rw.iter().any(|n| n.contains(".mlp.experts.0."))); + } + + #[test] + fn no_experts_is_passthrough() { + let mock = Mock; + let plain = vec![ + "model.layers.0.self_attn.q_proj.weight".to_string(), + "model.embed_tokens.weight".to_string(), + ]; + let sp = StackingProvider::build(&mock, &plain); + assert!(sp.stacks.is_empty()); + assert_eq!(sp.rewritten_names(), plain); + } +} + +#[cfg(test)] +mod splitting_tests { + use super::{SplittingProvider, TensorProvider}; + use anyhow::Result; + use base_arch::ArchConfig; + + // Mock: qkv_proj is [8, 2], gate_up_proj is [6, 2], anything else [4, 2]. + // Each element encodes its (row, col) as row*10 + col so slices are + // checkable by output row. + struct SplitMock; + impl TensorProvider for SplitMock { + fn source_shape(&self, name: &str) -> Result> { + if name.ends_with(".self_attn.qkv_proj.weight") { + Ok(vec![8, 2]) + } else if name.ends_with(".mlp.gate_up_proj.weight") { + Ok(vec![6, 2]) + } else { + Ok(vec![4, 2]) + } + } + fn to_f32(&self, name: &str) -> Result> { + let s = self.source_shape(name)?; + let (rows, cols) = (s[0] as usize, s[1] as usize); + let mut v = Vec::with_capacity(rows * cols); + for r in 0..rows { + for c in 0..cols { + v.push((r * 10 + c) as f32); + } + } + Ok(v) + } + } + + fn phi_config() -> ArchConfig { + // qkv rows = nq*hd + 2*nkv*hd = 2*2 + 2*(1*2) = 8; gate_up rows = 2*ffn = 6 + ArchConfig { + head_dim: 2, + num_attention_heads: 2, + num_kv_heads: 1, + intermediate_size: 3, + ..Default::default() + } + } + + #[test] + fn phi_fused_qkv_and_gate_up_split_by_row() { + let mock = SplitMock; + let names = vec![ + "model.layers.0.self_attn.qkv_proj.weight".to_string(), + "model.layers.0.mlp.gate_up_proj.weight".to_string(), + "model.layers.0.self_attn.o_proj.weight".to_string(), + ]; + let sp = SplittingProvider::build(&mock, &names, &phi_config()).unwrap(); + + // 5 virtual splits (q/k/v + gate/up); the two fused tensors consumed. + assert_eq!(sp.splits.len(), 5); + let q = "model.layers.0.self_attn.q_proj.weight"; + let k = "model.layers.0.self_attn.k_proj.weight"; + let v = "model.layers.0.self_attn.v_proj.weight"; + let g = "model.layers.0.mlp.gate_proj.weight"; + let u = "model.layers.0.mlp.up_proj.weight"; + + // sliced shapes: q=[nq*hd,in], k/v=[nkv*hd,in], gate/up=[ffn,in] + assert_eq!(sp.source_shape(q).unwrap(), vec![4, 2]); + assert_eq!(sp.source_shape(k).unwrap(), vec![2, 2]); + assert_eq!(sp.source_shape(v).unwrap(), vec![2, 2]); + assert_eq!(sp.source_shape(g).unwrap(), vec![3, 2]); + assert_eq!(sp.source_shape(u).unwrap(), vec![3, 2]); + + // qkv layout = [q rows 0..4 | k rows 4..6 | v rows 6..8] + assert_eq!(sp.to_f32(q).unwrap(), vec![0., 1., 10., 11., 20., 21., 30., 31.]); + assert_eq!(sp.to_f32(k).unwrap(), vec![40., 41., 50., 51.]); + assert_eq!(sp.to_f32(v).unwrap(), vec![60., 61., 70., 71.]); + // gate_up layout = [gate rows 0..3 | up rows 3..6], gate first + assert_eq!(sp.to_f32(g).unwrap(), vec![0., 1., 10., 11., 20., 21.]); + assert_eq!(sp.to_f32(u).unwrap(), vec![30., 31., 40., 41., 50., 51.]); + + // rewritten: fused dropped, o_proj passed through, virtuals present + let rw = sp.rewritten_names(); + assert!(rw.iter().any(|n| n == "model.layers.0.self_attn.o_proj.weight")); + assert!(rw.iter().any(|n| n == q)); + assert!(!rw.iter().any(|n| n.ends_with("qkv_proj.weight"))); + assert!(!rw.iter().any(|n| n.ends_with("gate_up_proj.weight"))); + } + + #[test] + fn no_fused_is_passthrough() { + let mock = SplitMock; + let plain = vec![ + "model.layers.0.self_attn.q_proj.weight".to_string(), + "model.embed_tokens.weight".to_string(), + ]; + let sp = SplittingProvider::build(&mock, &plain, &phi_config()).unwrap(); + assert!(sp.splits.is_empty()); + assert_eq!(sp.rewritten_names(), plain); + } + + #[test] + fn mismatched_config_is_rejected() { + let mock = SplitMock; // qkv_proj is [8, 2] + // nq=3 implies q+k+v = 3*2 + 2*(1*2) = 10 ≠ 8 rows → loud error, no garbage. + let bad = ArchConfig { + head_dim: 2, + num_attention_heads: 3, + num_kv_heads: 1, + intermediate_size: 3, + ..Default::default() + }; + let names = vec!["model.layers.0.self_attn.qkv_proj.weight".to_string()]; + assert!(SplittingProvider::build(&mock, &names, &bad).is_err()); + } +} diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index 8fb9e8c..6f08e7b 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -315,6 +315,16 @@ "quant": "default-q8", "size": 4155544576, "sha256": "930d78ec853410b68a433bfb8f608fe389566936899bf0a9dbc867618d80bce4" + }, + { + "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "file": "Qwen3-30B-A3B-Instruct-2507-Q4.base", + "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "arch": "qwen3_moe", + "quant": "default-q4", + "size": 17182920704, + "sha256": "7b294daeff8ba6e097f6cebe9f56c7b71f32476765a3f614eb256ef9d8c6ffce" } ] } diff --git a/base-convert/crates/base-hub/src/fetch.rs b/base-convert/crates/base-hub/src/fetch.rs index f2e284c..51fdedd 100644 --- a/base-convert/crates/base-hub/src/fetch.rs +++ b/base-convert/crates/base-hub/src/fetch.rs @@ -143,7 +143,7 @@ mod tests { // Unset -> the opted-in default (must be > 0, else the retry loop that // makes multi-GB pulls resilient stays disabled — the bug this fixes). std::env::remove_var("BASERT_HF_MAX_RETRIES"); - assert!(DEFAULT_HF_MAX_RETRIES > 0, "retries must be opted in by default"); + const { assert!(DEFAULT_HF_MAX_RETRIES > 0, "retries must be opted in by default") }; assert_eq!(resolve_max_retries(), DEFAULT_HF_MAX_RETRIES); // A valid override is honored. diff --git a/bindings/node/src/index.ts b/bindings/node/src/index.ts index 3162247..db384d1 100644 --- a/bindings/node/src/index.ts +++ b/bindings/node/src/index.ts @@ -106,6 +106,17 @@ const ModelConfigC = koffi.struct("BaseRTModelConfig", { swa_layers: koffi.array("uint8", 64), ffn_dims: koffi.array("uint32", 128), n_kv_heads_per_layer: koffi.array("uint32", 128), + // Qwen3.5 / 3.6 hybrid linear-attention (Gated DeltaNet) + attn_output_gate: "uint8", + _qwen35_pad: koffi.array("uint8", 3), + partial_rotary_factor: "float", + full_attention_interval: "uint32", + linear_attn_layers: koffi.array("uint8", 64), + gdn_num_k_heads: "uint32", + gdn_num_v_heads: "uint32", + gdn_key_head_dim: "uint32", + gdn_value_head_dim: "uint32", + gdn_conv_kernel: "uint32", // MoE n_experts: "uint32", n_experts_used: "uint32", @@ -130,6 +141,11 @@ const ModelConfigC = koffi.struct("BaseRTModelConfig", { image_token_id: "uint32", boi_token_id: "uint32", eoi_token_id: "uint32", + // Vision-tower family selector + Qwen3-VL-style extras + vision_arch: "uint32", + vision_spatial_merge: "uint32", + vision_temporal_patch: "uint32", + vision_out_dim: "uint32", // Audio tower audio_n_layers: "uint32", audio_dim: "uint32", @@ -150,6 +166,9 @@ const ModelConfigC = koffi.struct("BaseRTModelConfig", { audio_token_id: "uint32", boa_token_id: "uint32", eoa_token_id: "uint32", + mrope_section: koffi.array("uint32", 3), + mrope_interleaved: "uint8", + _mrope_pad: koffi.array("uint8", 3), }); const SamplingConfigC = koffi.struct("BaseRTSamplingConfig", { @@ -486,6 +505,26 @@ function lib(): BaseRTLib { "const char *baseRT_transcribe_pcm_stream(void *, float *, int, const char *, _Out_ BaseRTTranscribeStats *, SegmentCallback *, void *)" ), } as BaseRTLib; + + // ModelConfigC is mirrored by hand above; a size mismatch means the mirror + // drifted from include/baseRT/types.h and every field after the divergence + // point decodes as garbage. Fail at load, not at use. (Older libraries + // predate the symbol; skip the check there.) + try { + const configSizeof = k.func("size_t baseRT_model_config_sizeof()"); + const cSize = Number(configSizeof()); + const jsSize = koffi.sizeof(ModelConfigC); + if (cSize !== jsSize) { + throw new Error( + `BaseRTModelConfig layout drift: library says ${cSize} bytes, ` + + `Node mirror is ${jsSize} bytes. Update ModelConfigC in ` + + `bindings/node/src/index.ts to match include/baseRT/types.h.` + ); + } + } catch (e) { + if (e instanceof Error && e.message.includes("layout drift")) throw e; + // Symbol missing in an older library — no check possible. + } return _lib; } diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index a048ee4..3ade04d 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -127,6 +127,17 @@ class BaseRTModelConfig(ctypes.Structure): ("swa_layers", ctypes.c_uint8 * 64), ("ffn_dims", ctypes.c_uint32 * 128), ("n_kv_heads_per_layer", ctypes.c_uint32 * 128), + # Qwen3.5 / 3.6 hybrid linear-attention (Gated DeltaNet) + ("attn_output_gate", ctypes.c_uint8), + ("_qwen35_pad", ctypes.c_uint8 * 3), + ("partial_rotary_factor", ctypes.c_float), + ("full_attention_interval", ctypes.c_uint32), + ("linear_attn_layers", ctypes.c_uint8 * 64), + ("gdn_num_k_heads", ctypes.c_uint32), + ("gdn_num_v_heads", ctypes.c_uint32), + ("gdn_key_head_dim", ctypes.c_uint32), + ("gdn_value_head_dim", ctypes.c_uint32), + ("gdn_conv_kernel", ctypes.c_uint32), # Mixture-of-Experts (0 = dense) ("n_experts", ctypes.c_uint32), ("n_experts_used", ctypes.c_uint32), @@ -151,6 +162,11 @@ class BaseRTModelConfig(ctypes.Structure): ("image_token_id", ctypes.c_uint32), ("boi_token_id", ctypes.c_uint32), ("eoi_token_id", ctypes.c_uint32), + # Vision-tower family selector + Qwen3-VL-style extras + ("vision_arch", ctypes.c_uint32), + ("vision_spatial_merge", ctypes.c_uint32), + ("vision_temporal_patch", ctypes.c_uint32), + ("vision_out_dim", ctypes.c_uint32), # Audio tower ("audio_n_layers", ctypes.c_uint32), ("audio_dim", ctypes.c_uint32), @@ -171,6 +187,9 @@ class BaseRTModelConfig(ctypes.Structure): ("audio_token_id", ctypes.c_uint32), ("boa_token_id", ctypes.c_uint32), ("eoa_token_id", ctypes.c_uint32), + ("mrope_section", ctypes.c_uint32 * 3), + ("mrope_interleaved", ctypes.c_uint8), + ("_mrope_pad", ctypes.c_uint8 * 3), ] @@ -365,6 +384,22 @@ def _setup_signatures(lib: ctypes.CDLL) -> None: lib.baseRT_get_config.argtypes = [ctypes.c_void_p] lib.baseRT_get_config.restype = BaseRTModelConfig + # BaseRTModelConfig is mirrored by hand above; a size mismatch means the + # mirror drifted from include/baseRT/types.h and every field after the + # divergence point decodes as garbage. Fail at import, not at use. + # (Older libraries predate the symbol; skip the check there.) + if hasattr(lib, "baseRT_model_config_sizeof"): + lib.baseRT_model_config_sizeof.argtypes = [] + lib.baseRT_model_config_sizeof.restype = ctypes.c_size_t + c_size = lib.baseRT_model_config_sizeof() + py_size = ctypes.sizeof(BaseRTModelConfig) + if c_size != py_size: + raise RuntimeError( + f"BaseRTModelConfig layout drift: library says {c_size} bytes, " + f"Python mirror is {py_size} bytes. Update the _fields_ list in " + "bindings/python/baseRT/__init__.py to match include/baseRT/types.h." + ) + lib.baseRT_model_memory.argtypes = [ctypes.c_void_p] lib.baseRT_model_memory.restype = ctypes.c_size_t diff --git a/bindings/python/tests/test_baseRT.py b/bindings/python/tests/test_baseRT.py index 06e850d..de13a24 100644 --- a/bindings/python/tests/test_baseRT.py +++ b/bindings/python/tests/test_baseRT.py @@ -33,10 +33,14 @@ TranscribeStats, TensorInfo, BaseRTError, - _LIB_NAME, _find_library, + _lib_names, ) +# macOS shared-library name (the only shipping backend); tests below +# simulate the project layout with this filename. +_LIB_NAME = "libbaseRT.dylib" + # ------------------------------------------------------------------------- # ctypes struct sizes and field expectations @@ -48,18 +52,30 @@ class TestBaseRTModelConfig: def test_field_names(self): names = [f[0] for f in BaseRTModelConfig._fields_] - expected = [ + # Head of the struct — exact order matters for every field after it. + expected_head = [ "dim", "n_layers", "n_heads", "n_kv_heads", "head_dim", "q_dim", "kv_dim", "ffn_dim", "vocab_size", "max_seq_len", - "norm_eps", "rope_theta", "sliding_window_pattern", "rope_local_theta", - "architecture", + "norm_eps", "rope_theta", "sliding_window_pattern", "sliding_window", + "rope_local_theta", "architecture", "enc_n_layers", "enc_n_heads", "enc_dim", "enc_ffn_dim", "n_mels", "enc_max_seq_len", ] - assert names == expected + assert names[: len(expected_head)] == expected_head + # Block ordering — one sentinel per block, in types.h order. + block_order = [ + "n_embd_per_layer", # Gemma 4 + "attn_output_gate", # Qwen3.5 GDN + "n_experts", # MoE + "vision_n_layers", # vision tower + "vision_arch", # vision family selector + "audio_n_layers", # audio tower + ] + indices = [names.index(f) for f in block_order] + assert indices == sorted(indices) def test_field_count(self): - assert len(BaseRTModelConfig._fields_) == 21 + assert len(BaseRTModelConfig._fields_) == 87 def test_architecture_field_is_char_array(self): # architecture should be a fixed 32-byte char array @@ -459,7 +475,7 @@ def test_finds_library_in_build_dir(self, tmp_path): assert result == str(fake_lib.resolve()) def test_lib_name_constant(self): - assert _LIB_NAME == "libbaseRT.dylib" + assert _lib_names()[0] == _LIB_NAME # ------------------------------------------------------------------------- @@ -540,11 +556,10 @@ class TestStructSizes: """ def test_model_config_size(self): - size = ctypes.sizeof(BaseRTModelConfig) - # 14 uint32 + 2 float + 32 char + 6 uint32 = 20*4 + 2*4 + 32 = 120 min - assert size >= 112 - # Should not be absurdly large - assert size <= 256 + # Exact sizeof(BaseRTModelConfig) from include/baseRT/types.h; the + # library cross-check happens at import via baseRT_model_config_sizeof. + # Must match the Rust mirror test (bindings/rust/baseRT-sys). + assert ctypes.sizeof(BaseRTModelConfig) == 1520 def test_sampling_config_size(self): size = ctypes.sizeof(BaseRTSamplingConfig) diff --git a/bindings/rust/baseRT-sys/src/lib.rs b/bindings/rust/baseRT-sys/src/lib.rs index 9175f3d..58a54b1 100644 --- a/bindings/rust/baseRT-sys/src/lib.rs +++ b/bindings/rust/baseRT-sys/src/lib.rs @@ -58,6 +58,18 @@ pub struct BaseRTModelConfig { pub ffn_dims: [u32; 128], pub n_kv_heads_per_layer: [u32; 128], + // Qwen3.5 / 3.6 hybrid linear-attention (Gated DeltaNet) + pub attn_output_gate: u8, + pub _qwen35_pad: [u8; 3], + pub partial_rotary_factor: c_float, + pub full_attention_interval: u32, + pub linear_attn_layers: [u8; 64], + pub gdn_num_k_heads: u32, + pub gdn_num_v_heads: u32, + pub gdn_key_head_dim: u32, + pub gdn_value_head_dim: u32, + pub gdn_conv_kernel: u32, + // Mixture-of-Experts (0 = dense) pub n_experts: u32, pub n_experts_used: u32, @@ -84,6 +96,12 @@ pub struct BaseRTModelConfig { pub boi_token_id: u32, pub eoi_token_id: u32, + // Vision-tower family selector + Qwen3-VL-style extras + pub vision_arch: u32, + pub vision_spatial_merge: u32, + pub vision_temporal_patch: u32, + pub vision_out_dim: u32, + // Audio tower (all zero = none) pub audio_n_layers: u32, pub audio_dim: u32, @@ -104,6 +122,9 @@ pub struct BaseRTModelConfig { pub audio_token_id: u32, pub boa_token_id: u32, pub eoa_token_id: u32, + pub mrope_section: [u32; 3], + pub mrope_interleaved: u8, + pub _mrope_pad: [u8; 3], } /// Transcription result statistics. @@ -197,6 +218,7 @@ extern "C" { // === Model info === pub fn baseRT_get_config(model: baseRT_model_t) -> BaseRTModelConfig; + pub fn baseRT_model_config_sizeof() -> usize; pub fn baseRT_model_memory(model: baseRT_model_t) -> usize; pub fn baseRT_get_error() -> *const c_char; @@ -370,7 +392,8 @@ mod tests { #[test] fn model_config_size() { - // Full struct: decoder + encoder + Gemma4 + MoE + vision + audio. + // Full struct: decoder + encoder + Gemma4 + Qwen3.5-GDN + MoE + + // vision + audio. // Must be far larger than the old 112-byte truncation; returning a // 112-byte struct by value from baseRT_get_config corrupted memory. assert!( @@ -378,7 +401,20 @@ mod tests { "config struct unexpectedly small ({})", mem::size_of::() ); - assert_eq!(mem::size_of::(), 1392); + assert_eq!(mem::size_of::(), 1520); + } + + #[test] + fn model_config_size_matches_library() { + // The authoritative drift check: compare this hand-written mirror + // against sizeof(BaseRTModelConfig) as compiled into libbaseRT. A + // mismatch means every field after the divergence point decodes as + // garbage through baseRT_get_config. + assert_eq!( + unsafe { baseRT_model_config_sizeof() }, + mem::size_of::(), + "BaseRTModelConfig mirror drifted from include/baseRT/types.h" + ); } #[test] @@ -446,10 +482,16 @@ mod tests { // Spot-check the tail blocks to confirm the full layout. assert_eq!(&base.swa_layers as *const _ as usize - base_ptr, 144); assert_eq!(&base.ffn_dims as *const _ as usize - base_ptr, 208); - assert_eq!(&base.n_experts as *const _ as usize - base_ptr, 1232); - assert_eq!(&base.vision_n_layers as *const _ as usize - base_ptr, 1252); - assert_eq!(&base.audio_n_layers as *const _ as usize - base_ptr, 1312); - assert_eq!(&base.eoa_token_id as *const _ as usize - base_ptr, 1388); + assert_eq!(&base.attn_output_gate as *const _ as usize - base_ptr, 1232); + assert_eq!(&base.linear_attn_layers as *const _ as usize - base_ptr, 1244); + assert_eq!(&base.gdn_num_k_heads as *const _ as usize - base_ptr, 1308); + assert_eq!(&base.n_experts as *const _ as usize - base_ptr, 1328); + assert_eq!(&base.vision_n_layers as *const _ as usize - base_ptr, 1348); + assert_eq!(&base.vision_arch as *const _ as usize - base_ptr, 1408); + assert_eq!(&base.audio_n_layers as *const _ as usize - base_ptr, 1424); + assert_eq!(&base.eoa_token_id as *const _ as usize - base_ptr, 1500); + assert_eq!(&base.mrope_section as *const _ as usize - base_ptr, 1504); + assert_eq!(&base.mrope_interleaved as *const _ as usize - base_ptr, 1516); } #[test] diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index d8523af..72b9cf7 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -63,8 +63,8 @@ extern "C" { // === Versioning === #define BASERT_VERSION_MAJOR 0 -#define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 1 +#define BASERT_VERSION_MINOR 1 +#define BASERT_VERSION_PATCH 6 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. @@ -101,6 +101,11 @@ baseRT_model_t baseRT_load_model(const char *model_path, const char *kernel_libr /// baseRT_load_model. Other values are ignored. void baseRT_set_kv_bits(int bits); +/// Enable engine diagnostics (RoPE/tokenizer/GPU/architecture dumps, the +/// per-token dispatch-command count, "Warming up"). Off by default so end +/// users see only model output. Also honored via the BASERT_VERBOSE env var. +void baseRT_set_verbose(int on); + /// Toggle paged-KV mode for the next baseRT_load_model call. /// enable = 0 → contiguous KV cache (default; existing layout) /// enable = 1 → paged KV cache + block-table dispatch @@ -134,6 +139,12 @@ void baseRT_free_model(baseRT_model_t model); /// Get model configuration. BaseRTModelConfig baseRT_get_config(baseRT_model_t model); +/// sizeof(BaseRTModelConfig) as compiled into the library. Language +/// bindings that mirror the struct by hand (Python/Rust/Node) compare +/// this against their mirror's size at load time so layout drift fails +/// loudly instead of decoding garbage fields. +size_t baseRT_model_config_sizeof(void); + /// Get total GPU memory used by model (bytes). size_t baseRT_model_memory(baseRT_model_t model); diff --git a/bindings/swift/Sources/CBaseRT/include/types.h b/bindings/swift/Sources/CBaseRT/include/types.h index 83f9710..069968f 100644 --- a/bindings/swift/Sources/CBaseRT/include/types.h +++ b/bindings/swift/Sources/CBaseRT/include/types.h @@ -84,6 +84,27 @@ typedef struct { // E4B / E2B: all zeros (uniform cfg.n_kv_heads applies). uint32_t n_kv_heads_per_layer[128]; + // ── Qwen3.5 / 3.6 hybrid linear-attention (Gated DeltaNet) ─────── + // 0 / empty = not a hybrid model. Qwen3.5 interleaves Gated-DeltaNet + // linear-attention layers with periodic full (softmax) attention + // (reuses the Qwen3-Next decoder). full_attention_interval / the + // linear_attn_layers bitfield select which layers are which. + // Full-attention layers additionally use an output gate + partial RoPE. + uint8_t attn_output_gate; // 1 = full-attn q_proj is doubled [q|gate]; attn *= sigmoid(gate) pre-o_proj + uint8_t _qwen35_pad[3]; // align to 4 bytes + float partial_rotary_factor; // full-attn: only first factor*head_dim dims rotate (Qwen3.5 = 0.25). 0/1 = full + uint32_t full_attention_interval; // every Nth layer is full attention (rest are GDN). 0 = not hybrid + // Per-layer GDN mask: bit i = 1 means layer i is a Gated-DeltaNet + // linear-attention layer; bit 0 = full (softmax) attention. Authoritative + // copy of the config `layer_types` schedule. Up to 512 layers (64 bytes). + uint8_t linear_attn_layers[64]; + // Gated-DeltaNet shapes (0 = not hybrid). Consumed by the GDN kernel. + uint32_t gdn_num_k_heads; // linear_num_key_heads + uint32_t gdn_num_v_heads; // linear_num_value_heads + uint32_t gdn_key_head_dim; // linear_key_head_dim + uint32_t gdn_value_head_dim; // linear_value_head_dim + uint32_t gdn_conv_kernel; // linear_conv_kernel_dim (short causal conv width) + // Mixture-of-Experts (0 = dense model). // Gemma 4 26B-A4B: n_experts=128, n_experts_used=8, n_experts_shared=1 (via dense ffn.*), expert_gating=0 // (softmax), norm_topk_prob=0 Qwen3.6-35B-A3B (qwen35moe): n_experts=128, n_experts_used=8, n_experts_shared=0, @@ -114,6 +135,16 @@ typedef struct { uint32_t boi_token_id; // begin-of-image token (0 = none) uint32_t eoi_token_id; // end-of-image token (0 = none) + // ── Vision-tower family selector + Qwen3-VL-style extras ───────── + // vision_arch: 0 = SigLIP/Gemma-4 (pooled ViT + factorized pos-embed); + // 1 = Qwen3.5 / Qwen3-VL ViT (LayerNorm blocks, fused QKV, + // learned pos-embed bilinear interp, 2D M-RoPE, spatial + // merger instead of avg-pool). Populated from mmproj.arch. + uint32_t vision_arch; // 0 = siglip/gemma4, 1 = qwen3_5 / qwen3_vl + uint32_t vision_spatial_merge; // Qwen: spatial_merge_size (2 → 2x2 patch merge). 0 = n/a + uint32_t vision_temporal_patch; // Qwen: temporal_patch_size (frame is tiled this many times) + uint32_t vision_out_dim; // Qwen: merger out_hidden_size (== text dim). 0 = use cfg.dim + // Audio tower (Gemma 4 Conformer encoder). All zero = no audio tower. uint32_t audio_n_layers; // Conformer blocks (e.g. 12) uint32_t audio_dim; // hidden size (e.g. 1024) @@ -134,6 +165,16 @@ typedef struct { uint32_t audio_token_id; // placeholder token for audio features uint32_t boa_token_id; // begin-of-audio token (0 = none) uint32_t eoa_token_id; // end-of-audio token (0 = none) + + // ── Multimodal RoPE (Qwen3.5-VL interleaved M-RoPE) ────────────── + // Rotary-pair split over the [temporal, height, width] axes for the + // full-attention layers of VL models (Qwen3.5/3.6-VL: [11, 11, 10], + // summing to n_rot/2). All-zero = plain 1-D RoPE (text-only model or a + // .base converted before the converter recorded the key — the runtime + // falls back to [11, 11, 10] on the image path then). + uint32_t mrope_section[3]; + uint8_t mrope_interleaved; // 1 = interleaved axis map THWTHW…TT (Qwen3.5), 0 = sectioned/absent + uint8_t _mrope_pad[3]; // align to 4 bytes } BaseRTModelConfig; /// Transcription result statistics. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index d8523af..72b9cf7 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -63,8 +63,8 @@ extern "C" { // === Versioning === #define BASERT_VERSION_MAJOR 0 -#define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 1 +#define BASERT_VERSION_MINOR 1 +#define BASERT_VERSION_PATCH 6 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. @@ -101,6 +101,11 @@ baseRT_model_t baseRT_load_model(const char *model_path, const char *kernel_libr /// baseRT_load_model. Other values are ignored. void baseRT_set_kv_bits(int bits); +/// Enable engine diagnostics (RoPE/tokenizer/GPU/architecture dumps, the +/// per-token dispatch-command count, "Warming up"). Off by default so end +/// users see only model output. Also honored via the BASERT_VERBOSE env var. +void baseRT_set_verbose(int on); + /// Toggle paged-KV mode for the next baseRT_load_model call. /// enable = 0 → contiguous KV cache (default; existing layout) /// enable = 1 → paged KV cache + block-table dispatch @@ -134,6 +139,12 @@ void baseRT_free_model(baseRT_model_t model); /// Get model configuration. BaseRTModelConfig baseRT_get_config(baseRT_model_t model); +/// sizeof(BaseRTModelConfig) as compiled into the library. Language +/// bindings that mirror the struct by hand (Python/Rust/Node) compare +/// this against their mirror's size at load time so layout drift fails +/// loudly instead of decoding garbage fields. +size_t baseRT_model_config_sizeof(void); + /// Get total GPU memory used by model (bytes). size_t baseRT_model_memory(baseRT_model_t model); diff --git a/include/baseRT/types.h b/include/baseRT/types.h index 83f9710..069968f 100644 --- a/include/baseRT/types.h +++ b/include/baseRT/types.h @@ -84,6 +84,27 @@ typedef struct { // E4B / E2B: all zeros (uniform cfg.n_kv_heads applies). uint32_t n_kv_heads_per_layer[128]; + // ── Qwen3.5 / 3.6 hybrid linear-attention (Gated DeltaNet) ─────── + // 0 / empty = not a hybrid model. Qwen3.5 interleaves Gated-DeltaNet + // linear-attention layers with periodic full (softmax) attention + // (reuses the Qwen3-Next decoder). full_attention_interval / the + // linear_attn_layers bitfield select which layers are which. + // Full-attention layers additionally use an output gate + partial RoPE. + uint8_t attn_output_gate; // 1 = full-attn q_proj is doubled [q|gate]; attn *= sigmoid(gate) pre-o_proj + uint8_t _qwen35_pad[3]; // align to 4 bytes + float partial_rotary_factor; // full-attn: only first factor*head_dim dims rotate (Qwen3.5 = 0.25). 0/1 = full + uint32_t full_attention_interval; // every Nth layer is full attention (rest are GDN). 0 = not hybrid + // Per-layer GDN mask: bit i = 1 means layer i is a Gated-DeltaNet + // linear-attention layer; bit 0 = full (softmax) attention. Authoritative + // copy of the config `layer_types` schedule. Up to 512 layers (64 bytes). + uint8_t linear_attn_layers[64]; + // Gated-DeltaNet shapes (0 = not hybrid). Consumed by the GDN kernel. + uint32_t gdn_num_k_heads; // linear_num_key_heads + uint32_t gdn_num_v_heads; // linear_num_value_heads + uint32_t gdn_key_head_dim; // linear_key_head_dim + uint32_t gdn_value_head_dim; // linear_value_head_dim + uint32_t gdn_conv_kernel; // linear_conv_kernel_dim (short causal conv width) + // Mixture-of-Experts (0 = dense model). // Gemma 4 26B-A4B: n_experts=128, n_experts_used=8, n_experts_shared=1 (via dense ffn.*), expert_gating=0 // (softmax), norm_topk_prob=0 Qwen3.6-35B-A3B (qwen35moe): n_experts=128, n_experts_used=8, n_experts_shared=0, @@ -114,6 +135,16 @@ typedef struct { uint32_t boi_token_id; // begin-of-image token (0 = none) uint32_t eoi_token_id; // end-of-image token (0 = none) + // ── Vision-tower family selector + Qwen3-VL-style extras ───────── + // vision_arch: 0 = SigLIP/Gemma-4 (pooled ViT + factorized pos-embed); + // 1 = Qwen3.5 / Qwen3-VL ViT (LayerNorm blocks, fused QKV, + // learned pos-embed bilinear interp, 2D M-RoPE, spatial + // merger instead of avg-pool). Populated from mmproj.arch. + uint32_t vision_arch; // 0 = siglip/gemma4, 1 = qwen3_5 / qwen3_vl + uint32_t vision_spatial_merge; // Qwen: spatial_merge_size (2 → 2x2 patch merge). 0 = n/a + uint32_t vision_temporal_patch; // Qwen: temporal_patch_size (frame is tiled this many times) + uint32_t vision_out_dim; // Qwen: merger out_hidden_size (== text dim). 0 = use cfg.dim + // Audio tower (Gemma 4 Conformer encoder). All zero = no audio tower. uint32_t audio_n_layers; // Conformer blocks (e.g. 12) uint32_t audio_dim; // hidden size (e.g. 1024) @@ -134,6 +165,16 @@ typedef struct { uint32_t audio_token_id; // placeholder token for audio features uint32_t boa_token_id; // begin-of-audio token (0 = none) uint32_t eoa_token_id; // end-of-audio token (0 = none) + + // ── Multimodal RoPE (Qwen3.5-VL interleaved M-RoPE) ────────────── + // Rotary-pair split over the [temporal, height, width] axes for the + // full-attention layers of VL models (Qwen3.5/3.6-VL: [11, 11, 10], + // summing to n_rot/2). All-zero = plain 1-D RoPE (text-only model or a + // .base converted before the converter recorded the key — the runtime + // falls back to [11, 11, 10] on the image path then). + uint32_t mrope_section[3]; + uint8_t mrope_interleaved; // 1 = interleaved axis map THWTHW…TT (Qwen3.5), 0 = sectioned/absent + uint8_t _mrope_pad[3]; // align to 4 bytes } BaseRTModelConfig; /// Transcription result statistics. From a731f6212b4f327755ff3fdf1764355ddd141b1c Mon Sep 17 00:00:00 2001 From: Prabod Rathnayaka Date: Fri, 17 Jul 2026 19:49:24 +1000 Subject: [PATCH 2/3] ci: fix engine release asset pattern casing (baseRT->basert) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same casing fix as #15 applied to ci.yml: the Fetch engine step matched nothing since the rebrand, so the engine-linked ABI tests were silently skipped. With the pattern fixed the rust-sys ABI check runs against the latest engine release — it will fail on this PR until the 0.1.6 engine is published, which enforces the intended merge ordering. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abbb5ff..790aff0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,8 +58,8 @@ jobs: run: | mkdir -p build if gh release download --repo ${{ github.repository }} \ - --pattern 'baseRT-engine-macos-arm64*.tar.gz' -D build 2>/dev/null; then - tar -xzf build/baseRT-engine-macos-arm64*.tar.gz -C build + --pattern 'basert-engine-macos-arm64*.tar.gz' -D build 2>/dev/null; then + tar -xzf build/basert-engine-macos-arm64*.tar.gz -C build echo "have=1" >> "$GITHUB_OUTPUT" else echo "no engine release yet — skipping engine-linked tests" From 97588fad82d1f4728be6cdfd58a38c0b1931f043 Mon Sep 17 00:00:00 2001 From: Prabod Rathnayaka Date: Sat, 18 Jul 2026 17:14:47 +1000 Subject: [PATCH 3/3] =?UTF-8?q?sync:=20source=20from=20the=20v0.1.6=20tag?= =?UTF-8?q?=20=E2=80=94=20llama3=20rope-scaling=20config=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The initial sync was taken from the release branch before it merged; the tag additionally carries llama3 rope_scaling config fields (struct is 1540 bytes) and the converter's llama rope fixes. Mirrors verified against the published v0.1.6 engine binaries. --- base-convert/crates/base-arch/src/lib.rs | 44 ++++++++++ base-convert/crates/base-arch/src/llama.rs | 38 ++++++++- base-convert/crates/base-convert/src/main.rs | 84 +++++++++++++++++++ bindings/node/src/index.ts | 5 ++ bindings/python/baseRT/__init__.py | 5 ++ bindings/python/tests/test_baseRT.py | 2 +- bindings/rust/baseRT-sys/src/lib.rs | 10 ++- .../swift/Sources/CBaseRT/include/types.h | 14 ++++ include/baseRT/types.h | 14 ++++ 9 files changed, 212 insertions(+), 4 deletions(-) diff --git a/base-convert/crates/base-arch/src/lib.rs b/base-convert/crates/base-arch/src/lib.rs index f705f59..9e6cf90 100644 --- a/base-convert/crates/base-arch/src/lib.rs +++ b/base-convert/crates/base-arch/src/lib.rs @@ -57,6 +57,21 @@ pub trait HfMapper: Sync { fn norm_shift(&self, _canonical: &str) -> f32 { 0.0 } + + /// RoPE row-permutation head count for a canonical tensor at HF→.base + /// conversion time, or None when the tensor needs no permutation. + /// + /// HF llama-family checkpoints store `q_proj` / `k_proj` in the + /// transformers "split-half" rotary layout (rotate_half); the runtime + /// rope kernels — and GGUF sources — use Meta's original interleaved + /// pair layout. Mirrors `convert_hf_to_gguf.py::LlamaModel.permute`: + /// out_row[h*HD + 2j + k] = in_row[h*HD + k*HD/2 + j]. Skipping this + /// keeps attention internally consistent (Q and K scramble identically, + /// so relative-position structure survives) but assigns every dim-pair + /// the wrong trained frequency — retrieval collapses as context grows. + fn rope_permute_heads(&self, _canonical: &str, _cfg: &ArchConfig) -> Option { + None + } } pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMapper> { @@ -150,6 +165,14 @@ pub struct ArchConfig { pub vocab_size: u32, pub rope_theta: f32, pub rope_scale: f32, + /// HF `rope_scaling.rope_type` (empty = none). The runtime applies the + /// llama3 piecewise divisor formula only for "llama3" (or, for legacy + /// headers with no type, llama-arch + factor > 1); "linear" gets the + /// uniform divisor; anything else is skipped with a warning. + pub rope_scaling_type: String, + pub rope_low_freq_factor: f32, + pub rope_high_freq_factor: f32, + pub rope_original_max_pos: u32, pub rms_norm_eps: f32, pub tie_word_embeddings: bool, /// Per-layer FFN widths when the model declares heterogeneous FFN @@ -283,6 +306,27 @@ impl ArchConfig { m.insert("vocab_size".into(), json!(self.vocab_size)); m.insert("rope_theta".into(), json!(self.rope_theta)); m.insert("rope_scaling_factor".into(), json!(self.rope_scale)); + if !self.rope_scaling_type.is_empty() { + m.insert("rope_scaling_type".into(), json!(self.rope_scaling_type)); + } + if self.rope_low_freq_factor > 0.0 { + m.insert( + "rope_scaling_low_freq_factor".into(), + json!(self.rope_low_freq_factor), + ); + } + if self.rope_high_freq_factor > 0.0 { + m.insert( + "rope_scaling_high_freq_factor".into(), + json!(self.rope_high_freq_factor), + ); + } + if self.rope_original_max_pos > 0 { + m.insert( + "rope_scaling_original_max_position_embeddings".into(), + json!(self.rope_original_max_pos), + ); + } m.insert("rms_norm_eps".into(), json!(self.rms_norm_eps)); m.insert( "tie_word_embeddings".into(), diff --git a/base-convert/crates/base-arch/src/llama.rs b/base-convert/crates/base-arch/src/llama.rs index 243e1f8..c9f2db3 100644 --- a/base-convert/crates/base-arch/src/llama.rs +++ b/base-convert/crates/base-arch/src/llama.rs @@ -32,6 +32,16 @@ impl crate::HfMapper for LlamaHfMapper { fn config_from_hf(&self, c: &serde_json::Value) -> Result { hf_generic_config(c) } + fn rope_permute_heads(&self, canonical: &str, cfg: &crate::ArchConfig) -> Option { + // Weights only: llama/mistral attention carries no q/k bias. + if canonical.ends_with("self_attn.q_proj.weight") { + Some(cfg.num_attention_heads) + } else if canonical.ends_with("self_attn.k_proj.weight") { + Some(cfg.num_kv_heads) + } else { + None + } + } } pub(crate) fn hf_generic_config(c: &serde_json::Value) -> Result { @@ -70,12 +80,32 @@ pub(crate) fn hf_generic_config(c: &serde_json::Value) -> Result Result, mmproj_config: std::collections::BTreeMap, norm_shift: &dyn Fn(&str) -> f32, + rope_permute: &dyn Fn(&str) -> Option, ) -> Result<()> { use base_format::{ AlignmentConfig, BaseReader, BaseWriter, ComputeRegion, Header, HeaderFlags, LayerKind, @@ -1462,6 +1467,17 @@ fn convert_generic( } } + // Per-arch hook: normalize HF "split-half" rotary q/k row layout to + // the interleaved layout the runtime rope kernels (and GGUF sources) + // use. Mirrors `convert_hf_to_gguf.py::LlamaModel.permute`. + if shape.len() >= 2 { + if let Some(n_heads) = rope_permute(canonical) { + let rows = shape[0] as usize; + let cols = f32s.len() / rows; + f32s = rope_permute_rows(&f32s, rows, cols, n_heads); + } + } + let f32s_for = || -> &[f32] { &f32s }; let is_ssm_a = canonical == "ssm.a_log" @@ -3545,3 +3561,71 @@ mod splitting_tests { assert!(SplittingProvider::build(&mock, &names, &bad).is_err()); } } + +/// HF "split-half" rotary q/k row layout -> Meta/GGUF interleaved layout. +/// Per head of HD rows: out[h*HD + 2j + k] = in[h*HD + k*HD/2 + j]. +/// Mirrors `convert_hf_to_gguf.py::LlamaModel.permute`. Skipping this for +/// llama-family HF sources keeps attention internally consistent but assigns +/// every rotary dim-pair the wrong trained frequency — long-context +/// retrieval collapses while short prompts still look plausible. +fn rope_permute_rows(f32s: &[f32], rows: usize, cols: usize, n_heads: u32) -> Vec { + let hd = rows / n_heads as usize; + let half = hd / 2; + let mut out = vec![0f32; f32s.len()]; + for h in 0..n_heads as usize { + for j in 0..half { + for k in 0..2usize { + let dst = h * hd + 2 * j + k; + let src = h * hd + k * half + j; + out[dst * cols..(dst + 1) * cols].copy_from_slice(&f32s[src * cols..(src + 1) * cols]); + } + } + } + out +} + +#[cfg(test)] +mod rope_permute_tests { + use super::rope_permute_rows; + + /// 2 heads x HD=4, cols=1: rows tagged h.r — split-half [h0r0,h0r1,h0r2,h0r3] + /// where (r0,r1) are first-half dims and (r2,r3) second-half. Interleaved + /// output must pair them as [r0,r2,r1,r3] per head. + #[test] + fn split_half_to_interleaved() { + let input: Vec = vec![ + 0.0, 1.0, 2.0, 3.0, // head 0 + 10.0, 11.0, 12.0, 13.0, // head 1 + ]; + let out = rope_permute_rows(&input, 8, 1, 2); + assert_eq!(out, vec![0.0, 2.0, 1.0, 3.0, 10.0, 12.0, 11.0, 13.0]); + } + + /// Multi-column rows move as whole rows. + #[test] + fn rows_move_whole() { + let input: Vec = vec![ + 0.0, 0.5, // r0 + 1.0, 1.5, // r1 + 2.0, 2.5, // r2 + 3.0, 3.5, // r3 + ]; + let out = rope_permute_rows(&input, 4, 2, 1); + assert_eq!(out, vec![0.0, 0.5, 2.0, 2.5, 1.0, 1.5, 3.0, 3.5]); + } + + /// The permutation is an involution-inverse pair with the GGUF->HF + /// direction: applying it twice must NOT be identity (it is a genuine + /// reordering), but it must be a bijection — every input row appears + /// exactly once. + #[test] + fn is_bijection() { + let rows = 64usize; // one llama-3.2 head + let input: Vec = (0..rows).map(|i| i as f32).collect(); + let out = rope_permute_rows(&input, rows, 1, 1); + let mut seen = out.clone(); + seen.sort_by(|a, b| a.partial_cmp(b).unwrap()); + assert_eq!(seen, input); + assert_ne!(out, input); + } +} diff --git a/bindings/node/src/index.ts b/bindings/node/src/index.ts index db384d1..6c30ad5 100644 --- a/bindings/node/src/index.ts +++ b/bindings/node/src/index.ts @@ -169,6 +169,11 @@ const ModelConfigC = koffi.struct("BaseRTModelConfig", { mrope_section: koffi.array("uint32", 3), mrope_interleaved: "uint8", _mrope_pad: koffi.array("uint8", 3), + rope_scaling_factor: "float", + rope_low_freq_factor: "float", + rope_high_freq_factor: "float", + rope_orig_max_pos: "uint32", + rope_scaling_type: "uint32", }); const SamplingConfigC = koffi.struct("BaseRTSamplingConfig", { diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index 3ade04d..5a43eb4 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -190,6 +190,11 @@ class BaseRTModelConfig(ctypes.Structure): ("mrope_section", ctypes.c_uint32 * 3), ("mrope_interleaved", ctypes.c_uint8), ("_mrope_pad", ctypes.c_uint8 * 3), + ("rope_scaling_factor", ctypes.c_float), + ("rope_low_freq_factor", ctypes.c_float), + ("rope_high_freq_factor", ctypes.c_float), + ("rope_orig_max_pos", ctypes.c_uint32), + ("rope_scaling_type", ctypes.c_uint32), ] diff --git a/bindings/python/tests/test_baseRT.py b/bindings/python/tests/test_baseRT.py index de13a24..01370a0 100644 --- a/bindings/python/tests/test_baseRT.py +++ b/bindings/python/tests/test_baseRT.py @@ -559,7 +559,7 @@ def test_model_config_size(self): # Exact sizeof(BaseRTModelConfig) from include/baseRT/types.h; the # library cross-check happens at import via baseRT_model_config_sizeof. # Must match the Rust mirror test (bindings/rust/baseRT-sys). - assert ctypes.sizeof(BaseRTModelConfig) == 1520 + assert ctypes.sizeof(BaseRTModelConfig) == 1540 def test_sampling_config_size(self): size = ctypes.sizeof(BaseRTSamplingConfig) diff --git a/bindings/rust/baseRT-sys/src/lib.rs b/bindings/rust/baseRT-sys/src/lib.rs index 58a54b1..ed3b7c9 100644 --- a/bindings/rust/baseRT-sys/src/lib.rs +++ b/bindings/rust/baseRT-sys/src/lib.rs @@ -125,6 +125,11 @@ pub struct BaseRTModelConfig { pub mrope_section: [u32; 3], pub mrope_interleaved: u8, pub _mrope_pad: [u8; 3], + pub rope_scaling_factor: c_float, + pub rope_low_freq_factor: c_float, + pub rope_high_freq_factor: c_float, + pub rope_orig_max_pos: u32, + pub rope_scaling_type: u32, } /// Transcription result statistics. @@ -401,7 +406,7 @@ mod tests { "config struct unexpectedly small ({})", mem::size_of::() ); - assert_eq!(mem::size_of::(), 1520); + assert_eq!(mem::size_of::(), 1540); } #[test] @@ -492,6 +497,9 @@ mod tests { assert_eq!(&base.eoa_token_id as *const _ as usize - base_ptr, 1500); assert_eq!(&base.mrope_section as *const _ as usize - base_ptr, 1504); assert_eq!(&base.mrope_interleaved as *const _ as usize - base_ptr, 1516); + assert_eq!(&base.rope_scaling_factor as *const _ as usize - base_ptr, 1520); + assert_eq!(&base.rope_orig_max_pos as *const _ as usize - base_ptr, 1532); + assert_eq!(&base.rope_scaling_type as *const _ as usize - base_ptr, 1536); } #[test] diff --git a/bindings/swift/Sources/CBaseRT/include/types.h b/bindings/swift/Sources/CBaseRT/include/types.h index 069968f..d222ab1 100644 --- a/bindings/swift/Sources/CBaseRT/include/types.h +++ b/bindings/swift/Sources/CBaseRT/include/types.h @@ -175,6 +175,20 @@ typedef struct { uint32_t mrope_section[3]; uint8_t mrope_interleaved; // 1 = interleaved axis map THWTHW…TT (Qwen3.5), 0 = sectioned/absent uint8_t _mrope_pad[3]; // align to 4 bytes + // RoPE frequency scaling (HF `rope_scaling`, "llama3" type). factor <= 1 + // means none. The low/high freq factors and original max position + // parameterize the piecewise per-frequency rescale; when factor > 1 and + // the detail fields are 0 the runtime falls back to the Llama-3-family + // defaults (low 1.0, high 4.0, original 8192). + float rope_scaling_factor; // e.g. 32 on Llama-3.2 (1 or 0 = unscaled) + float rope_low_freq_factor; // wavelengths above orig_max/low rescale by 1/factor + float rope_high_freq_factor; // wavelengths below orig_max/high stay unscaled + uint32_t rope_orig_max_pos; // original_max_position_embeddings (e.g. 8192) + // rope_scaling type: 0 = none/legacy header (llama arch + factor > 1 + // defaults to llama3 — correct for every published llama-family .base), + // 1 = llama3 (piecewise), 2 = linear (uniform divisor), 3 = other / + // unsupported (skipped with a warning rather than mis-applied). + uint32_t rope_scaling_type; } BaseRTModelConfig; /// Transcription result statistics. diff --git a/include/baseRT/types.h b/include/baseRT/types.h index 069968f..d222ab1 100644 --- a/include/baseRT/types.h +++ b/include/baseRT/types.h @@ -175,6 +175,20 @@ typedef struct { uint32_t mrope_section[3]; uint8_t mrope_interleaved; // 1 = interleaved axis map THWTHW…TT (Qwen3.5), 0 = sectioned/absent uint8_t _mrope_pad[3]; // align to 4 bytes + // RoPE frequency scaling (HF `rope_scaling`, "llama3" type). factor <= 1 + // means none. The low/high freq factors and original max position + // parameterize the piecewise per-frequency rescale; when factor > 1 and + // the detail fields are 0 the runtime falls back to the Llama-3-family + // defaults (low 1.0, high 4.0, original 8192). + float rope_scaling_factor; // e.g. 32 on Llama-3.2 (1 or 0 = unscaled) + float rope_low_freq_factor; // wavelengths above orig_max/low rescale by 1/factor + float rope_high_freq_factor; // wavelengths below orig_max/high stay unscaled + uint32_t rope_orig_max_pos; // original_max_position_embeddings (e.g. 8192) + // rope_scaling type: 0 = none/legacy header (llama arch + factor > 1 + // defaults to llama3 — correct for every published llama-family .base), + // 1 = llama3 (piecewise), 2 = linear (uniform divisor), 3 = other / + // unsupported (skipped with a warning rather than mis-applied). + uint32_t rope_scaling_type; } BaseRTModelConfig; /// Transcription result statistics.