Refactor JinaCLIP vision mmproj mapping to use tensor_mapping table

Author: liyang

convert_hf_to_gguf.py

@@ -1521,7 +1521,9 @@ class MmprojModel(ModelBase):
     preprocessor_config: dict[str, Any]
     global_config: dict[str, Any]
 
-    n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]
+    # Prefer explicit "layers"  (e.g. JinaCLIP),
+    # keep legacy keys for other models.
+    n_block_keys = ["layers", "n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]
 
     has_vision_encoder: bool = True # by default
     has_audio_encoder: bool = False
@@ -6804,6 +6806,11 @@ def __init__(self, *args, **kwargs):
         with open(config_path, encoding="utf-8") as f:
             self.vision_config = json.load(f)
 
+    def get_vision_config(self) -> dict[str, Any] | None:
+        # For JinaCLIPVisionModel, the top-level AutoConfig dict is already
+        # the vision-only configuration.
+        return self.global_config
+
     def set_vocab(self):
         # Vision encoder doesn't need vocabulary
         pass
@@ -6861,73 +6868,10 @@ def set_gguf_parameters(self):
     def _strip_vm_prefix(self, name: str) -> str:
         return name[len('vision_model.'):] if name.startswith('vision_model.') else name
 
-    def _map_block_tensor(self, layer: int, rest: str, data_torch: Tensor, name: str) -> list[tuple[str, Tensor]] | None:
-        parts = rest.split('.')
-        # layer norms
-        if rest.startswith('norm1.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ln1.{suffix}', data_torch)]
-        if rest.startswith('norm2.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ln2.{suffix}', data_torch)]
-        if rest.startswith('attn.inner_attn_ln.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.attn_ln.{suffix}', data_torch)]
-
-        if rest == 'attn.q_bias':
-            return [(f'v.blk.{layer}.attn_q.bias', data_torch)]
-        if rest == 'attn.v_bias':
-            return [(f'v.blk.{layer}.attn_v.bias', data_torch)]
-
-        if rest.startswith('attn.q_proj.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.attn_q.{suffix}', data_torch)]
-        if rest.startswith('attn.k_proj.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.attn_k.{suffix}', data_torch)]
-        if rest.startswith('attn.v_proj.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.attn_v.{suffix}', data_torch)]
-        if rest.startswith('attn.proj.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.attn_out.{suffix}', data_torch)]
-
-        # MLP
-        if rest.startswith('mlp.w1.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ffn_gate.{suffix}', data_torch)]
-        if rest.startswith('mlp.w2.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ffn_up.{suffix}', data_torch)]
-        if rest.startswith('mlp.w3.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ffn_down.{suffix}', data_torch)]
-        if rest.startswith('mlp.ffn_ln.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ffn_norm.{suffix}', data_torch)]
-        if rest.startswith('mlp.fc1.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ffn_up.{suffix}', data_torch)]
-        if rest.startswith('mlp.fc2.'):
-            suffix = parts[-1]
-            return [(f'v.blk.{layer}.ffn_down.{suffix}', data_torch)]
-        return None
-
     def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
-        """Prefer base table-driven mapping; keep Jina-specific targets if already mapped; fallback to legacy mapper."""
-        # Already a GGUF target name (e.g., "v.*" or "mm.*"): return as-is
         if name.startswith('v.') or name.startswith('mm.'):
             return name
-        # Try the base mapping first
-        try:
-            return super().map_tensor_name(name, try_suffixes=try_suffixes)
-        except Exception:
-            # Fallback to legacy Jina-specific mapper for any remaining edge keys
-            if hasattr(self, "_map_jinaclip_tensor_name"):
-                mapped = self._map_jinaclip_tensor_name(name)  # type: ignore[attr-defined]
-                if mapped:
-                    return mapped
-            return name
+        return super().map_tensor_name(name, try_suffixes=try_suffixes)
 
     def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
         yielded_any = False
@@ -6991,15 +6935,6 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
             base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_POST_NORM]
             return [(f'{base}.{suffix}', data_torch)]
 
-        if src_no_vm.startswith('blocks.'):
-            parts = src_no_vm.split('.')
-            if len(parts) >= 3 and parts[1].isdigit():
-                layer = int(parts[1])
-                rest = '.'.join(parts[2:])
-                mapped = self._map_block_tensor(layer, rest, data_torch, name)
-                if mapped is not None:
-                    return mapped
-
         try:
             return [(self.map_tensor_name(name), data_torch)]
         except Exception:

gguf-py/gguf/constants.py

@@ -634,9 +634,13 @@ class MODEL_TENSOR(IntEnum):
     V_ENC_ATTN_O         = auto()
     V_ENC_ATTN_O_NORM    = auto()
     V_ENC_POST_ATTN_NORM = auto()
+    V_ENC_ATTN_LN        = auto()
     V_ENC_FFN_UP         = auto()
     V_ENC_FFN_GATE       = auto()
     V_ENC_FFN_DOWN       = auto()
+    V_ENC_FFN_NORM       = auto()
+    V_ENC_ATTN_Q_BIAS    = auto()
+    V_ENC_ATTN_V_BIAS    = auto()
     V_LAYER_SCALE_1      = auto()
     V_LAYER_SCALE_2      = auto()
     V_PRE_NORM           = auto()
@@ -1002,9 +1006,13 @@ class MODEL_TENSOR(IntEnum):
     MODEL_TENSOR.V_ENC_ATTN_O:              "v.blk.{bid}.attn_out",
     MODEL_TENSOR.V_ENC_ATTN_O_NORM:         "v.blk.{bid}.attn_out_norm",
     MODEL_TENSOR.V_ENC_POST_ATTN_NORM:      "v.blk.{bid}.ln2",
+    MODEL_TENSOR.V_ENC_ATTN_LN:             "v.blk.{bid}.attn_ln",
     MODEL_TENSOR.V_ENC_FFN_UP:              "v.blk.{bid}.ffn_up",
     MODEL_TENSOR.V_ENC_FFN_GATE:            "v.blk.{bid}.ffn_gate",
     MODEL_TENSOR.V_ENC_FFN_DOWN:            "v.blk.{bid}.ffn_down",
+    MODEL_TENSOR.V_ENC_FFN_NORM:            "v.blk.{bid}.ffn_norm",
+    MODEL_TENSOR.V_ENC_ATTN_Q_BIAS:         "v.blk.{bid}.attn_q.bias",
+    MODEL_TENSOR.V_ENC_ATTN_V_BIAS:         "v.blk.{bid}.attn_v.bias",
     MODEL_TENSOR.V_LAYER_SCALE_1:           "v.blk.{bid}.ls1",
     MODEL_TENSOR.V_LAYER_SCALE_2:           "v.blk.{bid}.ls2",
     MODEL_TENSOR.V_PRE_NORM:                "v.pre_ln",
@@ -1080,9 +1088,13 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.V_ENC_ATTN_O,
         MODEL_TENSOR.V_ENC_ATTN_O_NORM,
         MODEL_TENSOR.V_ENC_POST_ATTN_NORM,
+        MODEL_TENSOR.V_ENC_ATTN_LN,
         MODEL_TENSOR.V_ENC_FFN_UP,
         MODEL_TENSOR.V_ENC_FFN_GATE,
         MODEL_TENSOR.V_ENC_FFN_DOWN,
+        MODEL_TENSOR.V_ENC_FFN_NORM,
+        MODEL_TENSOR.V_ENC_ATTN_Q_BIAS,
+        MODEL_TENSOR.V_ENC_ATTN_V_BIAS,
         MODEL_TENSOR.V_LAYER_SCALE_1,
         MODEL_TENSOR.V_LAYER_SCALE_2,
         MODEL_TENSOR.V_PRE_NORM,

gguf-py/gguf/tensor_mapping.py

@@ -1243,6 +1243,7 @@ class TensorNameMap:
             "vision_encoder.transformer.layers.{bid}.attention.wq", # pixtral
             "visual.blocks.{bid}.attn.q", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wq", # kimi-vl, generated
+            "blocks.{bid}.attn.q_proj", # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
@@ -1260,6 +1261,7 @@ class TensorNameMap:
             "vision_encoder.transformer.layers.{bid}.attention.wk", # pixtral
             "visual.blocks.{bid}.attn.k", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wk", # kimi-vl, generated
+            "blocks.{bid}.attn.k_proj", # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
@@ -1277,6 +1279,7 @@ class TensorNameMap:
             "vision_encoder.transformer.layers.{bid}.attention.wv", # pixtral
             "visual.blocks.{bid}.attn.v", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wv", # kimi-vl, generated
+            "blocks.{bid}.attn.v_proj", # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_INPUT_NORM: (
@@ -1291,6 +1294,7 @@ class TensorNameMap:
             "visual.blocks.{bid}.norm1", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm0", # kimi-vl (norm0/norm1)
             "model.vision.transformer.layers.{bid}.input_layernorm", # cogvlm
+            "blocks.{bid}.norm1", # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_O: (
@@ -1306,6 +1310,7 @@ class TensorNameMap:
             "visual.blocks.{bid}.attn.proj", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.wo", # kimi-vl
             "model.vision.transformer.layers.{bid}.attention.dense", # cogvlm
+            "blocks.{bid}.attn.proj", # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
@@ -1320,6 +1325,11 @@ class TensorNameMap:
             "visual.blocks.{bid}.norm2", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm1", # kimi-vl (norm0/norm1)
             "model.vision.transformer.layers.{bid}.post_attention_layernorm", # cogvlm
+            "blocks.{bid}.norm2", # JinaCLIP v2 vision
+        ),
+
+        MODEL_TENSOR.V_ENC_ATTN_LN: (
+            "blocks.{bid}.attn.inner_attn_ln",              # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_FFN_UP: (
@@ -1335,12 +1345,14 @@ class TensorNameMap:
             "visual.blocks.{bid}.mlp.linear_fc1", # qwen3vl
             "vision_tower.encoder.blocks.{bid}.mlp.fc0", # kimi-vl (fc0/fc1)
             "model.vision.transformer.layers.{bid}.mlp.fc1", # cogvlm
+            "blocks.{bid}.mlp.w2", # JinaCLIP v2 vision (up)
         ),
 
         MODEL_TENSOR.V_ENC_FFN_GATE: (
             "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
             "visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
+            "blocks.{bid}.mlp.w1", # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_ENC_FFN_DOWN: (
@@ -1356,6 +1368,11 @@ class TensorNameMap:
             "visual.blocks.{bid}.mlp.linear_fc2", # qwen3vl
             "vision_tower.encoder.blocks.{bid}.mlp.fc1", # kimi-vl (fc0/fc1)
             "model.vision.transformer.layers.{bid}.mlp.fc2", # cogvlm
+            "blocks.{bid}.mlp.w3", # JinaCLIP v2 vision (down)
+        ),
+
+        MODEL_TENSOR.V_ENC_FFN_NORM: (
+            "blocks.{bid}.mlp.ffn_ln",              # JinaCLIP v2 vision
         ),
 
         MODEL_TENSOR.V_LAYER_SCALE_1: (
@@ -1368,6 +1385,14 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1
         ),
 
+        MODEL_TENSOR.V_ENC_ATTN_Q_BIAS: (
+            "blocks.{bid}.attn.q_bias",              # JinaCLIP v2 vision
+        ),
+
+        MODEL_TENSOR.V_ENC_ATTN_V_BIAS: (
+            "blocks.{bid}.attn.v_bias",              # JinaCLIP v2 vision
+        ),
+
         MODEL_TENSOR.V_PRE_NORM: (
             "vision_tower.vision_model.pre_layrnorm",
             "vision_tower.ln_pre", # pixtral-hf