sync: engine 0.1.6 — Qwen 3.5/3.6 support, headers, binding mirrors

.github/workflows/ci.yml

@@ -58,8 +58,8 @@ jobs:
         run: |
           mkdir -p build
           if gh release download --repo ${{ github.repository }} \
-               --pattern 'baseRT-engine-macos-arm64*.tar.gz' -D build 2>/dev/null; then
-            tar -xzf build/baseRT-engine-macos-arm64*.tar.gz -C build
+               --pattern 'basert-engine-macos-arm64*.tar.gz' -D build 2>/dev/null; then
+            tar -xzf build/basert-engine-macos-arm64*.tar.gz -C build
             echo "have=1" >> "$GITHUB_OUTPUT"
           else
             echo "no engine release yet — skipping engine-linked tests"

base-convert/Cargo.toml

@@ -12,7 +12,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.1.0"
+version = "0.1.6"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/basecompute/baseRT"

base-convert/README.md

@@ -57,8 +57,8 @@ through pre-converted catalog artifacts.
 `basert <cmd>` forwards to the matching runtime binary (`basert-<cmd>`):
 
 ```
-basert serve    --model <path> [--model <path2> …]   # OpenAI-compatible server
-basert chat     --model <path>                       # interactive chat
+basert serve    <model> [--model <model2> …]         # OpenAI-compatible server (--model repeats for multi-model)
+basert chat     <model>                              # interactive chat
 basert complete <model> --prompt <text>              # one-shot completion
 basert bench    <model>                              # throughput benchmark
 ```

base-convert/crates/base-arch/src/lib.rs

@@ -57,13 +57,49 @@ pub trait HfMapper: Sync {
     fn norm_shift(&self, _canonical: &str) -> f32 {
         0.0
     }
+
+    /// RoPE row-permutation head count for a canonical tensor at HF→.base
+    /// conversion time, or None when the tensor needs no permutation.
+    ///
+    /// HF llama-family checkpoints store `q_proj` / `k_proj` in the
+    /// transformers "split-half" rotary layout (rotate_half); the runtime
+    /// rope kernels — and GGUF sources — use Meta's original interleaved
+    /// pair layout. Mirrors `convert_hf_to_gguf.py::LlamaModel.permute`:
+    /// out_row[h*HD + 2j + k] = in_row[h*HD + k*HD/2 + j]. Skipping this
+    /// keeps attention internally consistent (Q and K scramble identically,
+    /// so relative-position structure survives) but assigns every dim-pair
+    /// the wrong trained frequency — retrieval collapses as context grows.
+    fn rope_permute_heads(&self, _canonical: &str, _cfg: &ArchConfig) -> Option<u32> {
+        None
+    }
 }
 
 pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMapper> {
     match model_type {
-        "llama" => Some(&llama::LlamaHfMapper),
+        // Mistral is Llama-shaped (RMSNorm, RoPE, SwiGLU, GQA), its HF tensor
+        // names are already canonical Llama names, and it reuses the Llama mapper
+        // (canonical_arch="llama" → llama model class). Validated end-to-end on
+        // Mistral-7B-Instruct-v0.3 and Ministral-8B-Instruct-2410.
+        //
+        // Phi-3 is NOT enabled. The converter side is ready (SplittingProvider
+        // splits its fused qkv_proj/gate_up_proj; weights convert; HD=96
+        // attention is fine — regression-tested) and the generation_config eos
+        // merge below makes it stop cleanly. BUT Phi-3 chat output degenerates
+        // (raw completions are coherent; chat floods/repeats and is incoherent at
+        // both Q4 and Q8, temp 0 and 0.7) — a chat-path issue (same class as
+        // SmolLM2) that's not yet root-caused. Re-enable "phi3" once that's
+        // fixed. (Phi-3.5 additionally needs LongRoPE — engine is linear-only.)
+        "llama" | "mistral" => Some(&llama::LlamaHfMapper),
         "qwen2" | "qwen3" => Some(&qwen::QwenHfMapper),
         "qwen2_moe" | "qwen3_moe" => Some(&qwen::QwenMoeHfMapper),
+        // Qwen3.5 / 3.6: hybrid Gated-DeltaNet + full-attention decoder
+        // (reuses the Qwen3-Next design). The top-level HF model_type is
+        // `qwen3_5` (multimodal wrapper `Qwen3_5ForConditionalGeneration`)
+        // with the text tower under `text_config.model_type = qwen3_5_text`.
+        // Both resolve here so a text-only checkpoint (top-level
+        // `qwen3_5_text`) and the multimodal wrapper convert identically.
+        "qwen3_5" | "qwen3_5_text" | "qwen35" => Some(&qwen::Qwen35HfMapper),
+        "qwen3_5_moe" | "qwen3_5_moe_text" | "qwen35_moe" => Some(&qwen::Qwen35MoeHfMapper),
         "nomic_bert" | "nomic-bert" => Some(&bert::NomicBertHfMapper),
         "gemma" | "gemma2" | "gemma3" | "gemma3_text" => Some(&gemma::Gemma3HfMapper),
         // gemma3n is a distinct arch (AltUp/Laurel/per-layer-FFN); the
@@ -81,10 +117,17 @@ pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMappe
 /// message) has a single source of truth for what convert-on-pull supports.
 pub const SUPPORTED_HF_MODEL_TYPES: &[&str] = &[
     "llama",
+    "mistral",
     "qwen2",
     "qwen3",
     "qwen2_moe",
     "qwen3_moe",
+    "qwen3_5",
+    "qwen3_5_text",
+    "qwen35",
+    "qwen3_5_moe",
+    "qwen3_5_moe_text",
+    "qwen35_moe",
     "nomic_bert",
     "gemma",
     "gemma2",
@@ -122,6 +165,14 @@ pub struct ArchConfig {
     pub vocab_size: u32,
     pub rope_theta: f32,
     pub rope_scale: f32,
+    /// HF `rope_scaling.rope_type` (empty = none). The runtime applies the
+    /// llama3 piecewise divisor formula only for "llama3" (or, for legacy
+    /// headers with no type, llama-arch + factor > 1); "linear" gets the
+    /// uniform divisor; anything else is skipped with a warning.
+    pub rope_scaling_type: String,
+    pub rope_low_freq_factor: f32,
+    pub rope_high_freq_factor: f32,
+    pub rope_original_max_pos: u32,
     pub rms_norm_eps: f32,
     pub tie_word_embeddings: bool,
     /// Per-layer FFN widths when the model declares heterogeneous FFN
@@ -145,6 +196,10 @@ pub struct ArchConfig {
     /// MoE: 1 if router top-k weights are renormalized to sum to 1
     /// (Qwen), 0 if left as-is (Gemma).
     pub norm_topk_prob: bool,
+    /// MoE: number of always-on shared experts running in parallel to the
+    /// routed ones (Qwen3.5/3.6-MoE: 1, width `intermediate_size`, plus a
+    /// per-token scalar sigmoid gate). 0 = no shared expert.
+    pub num_shared_experts: u32,
     /// Maximum positional embedding length.  Pulled from
     /// `max_position_embeddings` (HF) or `context_length` (GGUF).
     pub max_position_embeddings: u32,
@@ -193,6 +248,44 @@ pub struct ArchConfig {
     /// `rope_freqs.weight` divisor mask; HF stores it as
     /// `rope_parameters.full_attention.partial_rotary_factor`.
     pub global_rope_partial_factor: f32,
+
+    // ── Qwen3.5 / 3.6 hybrid-linear-attention fields ─────────────────
+    // (all zero/empty for non-hybrid archs). Qwen3.5 interleaves
+    // Gated-DeltaNet linear-attention layers with periodic full
+    // (softmax) attention layers, reusing the Qwen3-Next decoder.
+    /// Per-layer attention kind, one entry per layer:
+    /// "linear_attention" (Gated DeltaNet) | "full_attention".
+    /// Empty = not a hybrid model. Length = num_hidden_layers.
+    pub layer_types: Vec<String>,
+    /// Every Nth layer is full attention (the rest are linear). Mirror
+    /// of `full_attention_interval` in the HF config; 0 = not hybrid.
+    /// Redundant with `layer_types` but kept for a cheap runtime check.
+    pub full_attention_interval: u32,
+    /// Gated-DeltaNet: number of key ("k") heads. 0 = not hybrid.
+    pub linear_num_key_heads: u32,
+    /// Gated-DeltaNet: number of value ("v") heads.
+    pub linear_num_value_heads: u32,
+    /// Gated-DeltaNet: per-head key/query dimension.
+    pub linear_key_head_dim: u32,
+    /// Gated-DeltaNet: per-head value dimension.
+    pub linear_value_head_dim: u32,
+    /// Gated-DeltaNet: causal depthwise short-conv kernel width (e.g. 4).
+    pub linear_conv_kernel_dim: u32,
+    /// Full-attention layers apply an output (sigmoid) gate to the
+    /// attention output before o_proj (`attn_output_gate`). Qwen3.5=true.
+    pub attn_output_gate: bool,
+    /// Partial rotary factor for the full-attention layers: only the
+    /// first `factor * head_dim` dims are rotated. Qwen3.5 = 0.25.
+    /// 0 or 1 = full rotation.
+    pub partial_rotary_factor: f32,
+    /// Multimodal-RoPE section split (Qwen3.5: [11, 11, 10] over
+    /// temporal/height/width). Empty = plain 1-D RoPE. For text-only
+    /// inference all positions collapse so the runtime may treat this
+    /// as ordinary 1-D RoPE.
+    pub mrope_section: Vec<u32>,
+    /// mRoPE interleaves the section frequencies rather than
+    /// concatenating them (Qwen3.5 = true).
+    pub mrope_interleaved: bool,
 }
 
 impl ArchConfig {
@@ -213,6 +306,27 @@ impl ArchConfig {
         m.insert("vocab_size".into(), json!(self.vocab_size));
         m.insert("rope_theta".into(), json!(self.rope_theta));
         m.insert("rope_scaling_factor".into(), json!(self.rope_scale));
+        if !self.rope_scaling_type.is_empty() {
+            m.insert("rope_scaling_type".into(), json!(self.rope_scaling_type));
+        }
+        if self.rope_low_freq_factor > 0.0 {
+            m.insert(
+                "rope_scaling_low_freq_factor".into(),
+                json!(self.rope_low_freq_factor),
+            );
+        }
+        if self.rope_high_freq_factor > 0.0 {
+            m.insert(
+                "rope_scaling_high_freq_factor".into(),
+                json!(self.rope_high_freq_factor),
+            );
+        }
+        if self.rope_original_max_pos > 0 {
+            m.insert(
+                "rope_scaling_original_max_position_embeddings".into(),
+                json!(self.rope_original_max_pos),
+            );
+        }
         m.insert("rms_norm_eps".into(), json!(self.rms_norm_eps));
         m.insert(
             "tie_word_embeddings".into(),
@@ -235,6 +349,12 @@ impl ArchConfig {
             m.insert("num_experts_per_tok".into(), json!(self.num_experts_per_tok));
             m.insert("moe_intermediate_size".into(), json!(self.moe_intermediate_size));
             m.insert("norm_topk_prob".into(), json!(self.norm_topk_prob));
+            if self.num_shared_experts > 0 {
+                m.insert(
+                    "num_shared_experts".into(),
+                    json!(self.num_shared_experts),
+                );
+            }
         }
         if self.max_position_embeddings > 0 {
             m.insert(
@@ -295,6 +415,49 @@ impl ArchConfig {
         if self.rope_local_theta > 0.0 {
             m.insert("rope_local_theta".into(), json!(self.rope_local_theta));
         }
+        // Qwen3.5 / 3.6 hybrid-linear-attention fields — only emit when
+        // set so non-hybrid archs' headers stay tidy.
+        if !self.layer_types.is_empty() {
+            m.insert("layer_types".into(), json!(self.layer_types));
+        }
+        if self.full_attention_interval > 0 {
+            m.insert(
+                "full_attention_interval".into(),
+                json!(self.full_attention_interval),
+            );
+        }
+        if self.linear_num_key_heads > 0 {
+            m.insert(
+                "linear_num_key_heads".into(),
+                json!(self.linear_num_key_heads),
+            );
+            m.insert(
+                "linear_num_value_heads".into(),
+                json!(self.linear_num_value_heads),
+            );
+            m.insert("linear_key_head_dim".into(), json!(self.linear_key_head_dim));
+            m.insert(
+                "linear_value_head_dim".into(),
+                json!(self.linear_value_head_dim),
+            );
+            m.insert(
+                "linear_conv_kernel_dim".into(),
+                json!(self.linear_conv_kernel_dim),
+            );
+        }
+        if self.attn_output_gate {
+            m.insert("attn_output_gate".into(), json!(self.attn_output_gate));
+        }
+        if self.partial_rotary_factor > 0.0 {
+            m.insert(
+                "partial_rotary_factor".into(),
+                json!(self.partial_rotary_factor),
+            );
+        }
+        if !self.mrope_section.is_empty() {
+            m.insert("mrope_section".into(), json!(self.mrope_section));
+            m.insert("mrope_interleaved".into(), json!(self.mrope_interleaved));
+        }
         m
     }
 }

base-convert/crates/base-arch/src/llama.rs

@@ -32,6 +32,16 @@ impl crate::HfMapper for LlamaHfMapper {
     fn config_from_hf(&self, c: &serde_json::Value) -> Result<crate::ArchConfig> {
         hf_generic_config(c)
     }
+    fn rope_permute_heads(&self, canonical: &str, cfg: &crate::ArchConfig) -> Option<u32> {
+        // Weights only: llama/mistral attention carries no q/k bias.
+        if canonical.ends_with("self_attn.q_proj.weight") {
+            Some(cfg.num_attention_heads)
+        } else if canonical.ends_with("self_attn.k_proj.weight") {
+            Some(cfg.num_kv_heads)
+        } else {
+            None
+        }
+    }
 }
 
 pub(crate) fn hf_generic_config(c: &serde_json::Value) -> Result<crate::ArchConfig> {
@@ -70,12 +80,32 @@ pub(crate) fn hf_generic_config(c: &serde_json::Value) -> Result<crate::ArchConf
     let vocab_size = u32_key("vocab_size")?;
     let head_dim = u32_key("head_dim").unwrap_or(hidden_size / num_attention_heads);
     let rope_theta = f32_key("rope_theta").unwrap_or(10_000.0);
-    let rope_scale = c
-        .get("rope_scaling")
+    let rope_scaling = c.get("rope_scaling");
+    let rope_scale = rope_scaling
         .and_then(|v| v.get("factor"))
         .and_then(|v| v.as_f64())
         .map(|f| f as f32)
         .unwrap_or(1.0);
+    // `rope_type` (current HF) with `type` (older configs) as fallback.
+    let rope_scaling_type = rope_scaling
+        .and_then(|v| v.get("rope_type").or_else(|| v.get("type")))
+        .and_then(|v| v.as_str())
+        .unwrap_or("")
+        .to_string();
+    let rs_f32 = |k: &str| {
+        rope_scaling
+            .and_then(|v| v.get(k))
+            .and_then(|v| v.as_f64())
+            .map(|f| f as f32)
+            .unwrap_or(0.0)
+    };
+    let rope_low_freq_factor = rs_f32("low_freq_factor");
+    let rope_high_freq_factor = rs_f32("high_freq_factor");
+    let rope_original_max_pos = rope_scaling
+        .and_then(|v| v.get("original_max_position_embeddings"))
+        .and_then(|v| v.as_u64())
+        .map(|n| n as u32)
+        .unwrap_or(0);
     let rms_norm_eps = f32_key("rms_norm_eps").unwrap_or(1e-6);
     let tie_word_embeddings = bool_key("tie_word_embeddings").unwrap_or(false);
 
@@ -112,6 +142,10 @@ pub(crate) fn hf_generic_config(c: &serde_json::Value) -> Result<crate::ArchConf
         vocab_size,
         rope_theta,
         rope_scale,
+        rope_scaling_type,
+        rope_low_freq_factor,
+        rope_high_freq_factor,
+        rope_original_max_pos,
         rms_norm_eps,
         tie_word_embeddings,
         max_position_embeddings,