Feat: Support LoRA and FP8 Adapter of Minimax-H3 and FastH3

examples/minimax_h3/README.md

@@ -76,15 +76,54 @@ unless overridden.
 
 ```bash
 python examples/minimax_h3/minimax_h3_turbo_lora_h100.py \
-  --gpu-num 4 \
+  --gpu-num 1 \
   --model-root /hhb-data/aigc/model_zoo/MiniMaxAI_MiniMax-H3 \
   --lora-path /data/zuoxin/workspace/TeleFuser/work_dirs/models/lightx2v/Minimax-h3-Turbo/minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors \
+  --quantization fp8 \
+  --attn-impl SOL_ATTN \
+  --sol-fp8 \
+  --sol-dense-steps 1 \
+  --sol-dense-layers 0 \
+  --sol-tau 1.0 \
+  --sol-threshold-type exact \
   --duration 5 \
-  --output outputs/minimax_h3_turbo_lora.mp4
+  --output outputs/minimax_h3_turbo_lora_fp8_sol.mp4
 ```
 
 The example supports one, two, or four GPUs. Four GPUs use tensor plus Ulysses parallelism; LoRA is merged before
-parallel sharding.
+parallel sharding. Online AdaLN caching is enabled by default, matching the standard H100 FL2VA entrypoint.
+
+### FastH3 hybrid adapter
+
+The standard FL2VA entrypoint also accepts FastVideo's `fastvideo-lora-v2` hybrid adapters. These checkpoints mix
+ordinary low-rank `B @ A` updates with exact weight and bias deltas. TeleFuser applies both to the BF16 source weights
+before optional online FP8 Linear conversion:
+
+```bash
+python -m examples.minimax_h3.minimax_h3_fl2va_h100 \
+  --mode t2va \
+  --adapter-path /path/to/dense-datafree/adapter_model.safetensors \
+  --adapter-strength 1.0 \
+  --quantization fp8 \
+  --attn-impl SOL_ATTN \
+  --sol-fp8 \
+  --sol-dense-steps 1 \
+  --sol-dense-layers 0 \
+  --sol-tau 1.0 \
+  --sol-threshold-type exact \
+  --steps 5 \
+  --duration 5 \
+  --seed 1000 \
+  --prompt "integrated_multimodal_description: A red fox runs through fresh snow at dawn. overall_soundscape: Fast pawsteps in snow, winter wind, and distant birds." \
+  --output outputs/minimax_h3_fasth3_fp8_sol.mp4
+```
+
+The commands combine the merged adapter with FP8 Linear and FP8 Sol attention. The shown `1/0` dense
+step/layer profile is the fastest quality-valid single-H100 profile from the 5-second benchmark; adjust
+`--sol-dense-steps`, `--sol-dense-layers`, `--sol-tau`, and `--sol-threshold-type` for other workloads.
+The `vsa-*` FastH3 adapters contain trained attention compression-gate replacement tensors. TeleFuser does not
+implement that VSA-H3 backend and rejects those files instead of silently dropping the gates; use the dense adapter
+variant unless a VSA backend is available.
 
 ## Feature Cache
 

examples/minimax_h3/minimax_h3_fl2va_h100.py

@@ -40,6 +40,8 @@
     "feature_cache_n_derivatives": 1,
     "feature_cache_taylor_threshold": 2,
     "quantization": None,
+    "adapter_path": None,
+    "adapter_strength": 1.0,
 }
 
 
@@ -98,6 +100,8 @@ def get_pipeline(
     feature_cache_n_derivatives: int = PPL_CONFIG["feature_cache_n_derivatives"],
     feature_cache_taylor_threshold: int = PPL_CONFIG["feature_cache_taylor_threshold"],
     quantization: str | None = PPL_CONFIG["quantization"],
+    adapter_path: str | None = PPL_CONFIG["adapter_path"],
+    adapter_strength: float = PPL_CONFIG["adapter_strength"],
 ) -> MiniMaxH3Pipeline:
     """Load the FL2VA checkpoint partition for one, two, or four GPUs."""
     tp_degree = 2 if parallelism == 4 else 1
@@ -126,6 +130,8 @@ def get_pipeline(
             taylor_threshold=feature_cache_taylor_threshold,
         ),
         quantization=quantization,
+        lora_path=adapter_path,
+        lora_strength=adapter_strength,
     )
 
 
@@ -283,6 +289,14 @@ def _main(default_quantization: str | None = PPL_CONFIG["quantization"]) -> None
     parser.add_argument("--flow-shift", type=float, default=PPL_CONFIG["flow_shift"])
     parser.add_argument("--audio-flow-shift", type=float, default=PPL_CONFIG["audio_flow_shift"])
     parser.add_argument("--device", default=PPL_CONFIG["device"])
+    parser.add_argument(
+        "--adapter-path",
+        "--lora-path",
+        dest="adapter_path",
+        default=PPL_CONFIG["adapter_path"],
+        help="MiniMax H3 Turbo LoRA or dense FastVideo FastH3 hybrid adapter.",
+    )
+    parser.add_argument("--adapter-strength", type=float, default=PPL_CONFIG["adapter_strength"])
     parser.add_argument(
         "--quantization",
         choices=("fp8", "torchao-fp8", "tf-kernel-fp8", "bnb-nf4"),
@@ -358,6 +372,8 @@ def _main(default_quantization: str | None = PPL_CONFIG["quantization"]) -> None
         feature_cache_n_derivatives=args.feature_cache_n_derivatives,
         feature_cache_taylor_threshold=args.feature_cache_taylor_threshold,
         quantization=args.quantization,
+        adapter_path=args.adapter_path,
+        adapter_strength=args.adapter_strength,
     )
     try:
         result = run_with_file(

examples/minimax_h3/minimax_h3_turbo_lora_h100.py

@@ -34,7 +34,10 @@
     "aspect_ratio": "16:9",
     "device": "cuda:0",
     "enable_fsdp": False,
+    "online_adaln_cache": True,
     "attn_impl": AttnImplType.FLASH_ATTN_4,
+    "sol_fp8": False,
+    "quantization": None,
 }
 
 PIPELINE_MANIFEST = build_pipeline_manifest(
@@ -63,7 +66,16 @@ def get_pipeline(
     lora_strength: float = PPL_CONFIG["lora_strength"],
     num_inference_steps: int = PPL_CONFIG["num_inference_steps"],
     enable_fsdp: bool | None = PPL_CONFIG["enable_fsdp"],
+    online_adaln_cache: bool = PPL_CONFIG["online_adaln_cache"],
     attn_impl: AttnImplType | str = PPL_CONFIG["attn_impl"],
+    sol_fp8: bool = PPL_CONFIG["sol_fp8"],
+    sol_dense_steps: int = 2,
+    sol_dense_layers: int = 2,
+    sol_tau: float = 1.0,
+    sol_threshold_type: str = "exact",
+    sol_fp8_layer_start: int = 0,
+    sol_fp8_layer_end: int | None = None,
+    quantization: str | None = PPL_CONFIG["quantization"],
 ) -> MiniMaxH3Pipeline:
     """Load FL2VA and merge Turbo LoRA with the requested GPU parallelism."""
     if parallelism not in {1, 2, 4}:
@@ -78,7 +90,16 @@ def get_pipeline(
         tp_degree=tp_degree,
         text_encoder_tp_degree=parallelism,
         enable_fsdp=enable_fsdp,
+        online_adaln_cache=online_adaln_cache,
         attn_impl=attn_impl,
+        sol_fp8=sol_fp8,
+        sol_dense_steps=sol_dense_steps,
+        sol_dense_layers=sol_dense_layers,
+        sol_tau=sol_tau,
+        sol_threshold_type=sol_threshold_type,
+        sol_fp8_layer_start=sol_fp8_layer_start,
+        sol_fp8_layer_end=sol_fp8_layer_end,
+        quantization=quantization,
         lora_path=lora_path,
         lora_strength=lora_strength,
     )
@@ -145,9 +166,22 @@ def main() -> None:
     parser.add_argument("--device", default=PPL_CONFIG["device"])
     parser.add_argument(
         "--attn-impl",
-        choices=("FLASH_ATTN_4", "SAGE_ATTN_2_8_8_SM90"),
+        choices=("FLASH_ATTN_4", "SAGE_ATTN_2_8_8_SM90", "SOL_ATTN"),
         default=PPL_CONFIG["attn_impl"].name,
     )
+    parser.add_argument(
+        "--quantization",
+        choices=("fp8", "torchao-fp8", "tf-kernel-fp8", "bnb-nf4"),
+        default=PPL_CONFIG["quantization"],
+        help="Online DiT Linear quantization backend.",
+    )
+    parser.add_argument("--sol-fp8", action="store_true", help="Use FP8 Q/K/V in active Sol-Attn layers.")
+    parser.add_argument("--sol-dense-steps", type=int, default=2)
+    parser.add_argument("--sol-dense-layers", type=int, default=2)
+    parser.add_argument("--sol-tau", type=float, default=1.0)
+    parser.add_argument("--sol-threshold-type", choices=("exact", "diag"), default="exact")
+    parser.add_argument("--sol-fp8-layer-start", type=int, default=0)
+    parser.add_argument("--sol-fp8-layer-end", type=int)
     fsdp_group = parser.add_mutually_exclusive_group()
     fsdp_group.add_argument("--enable-fsdp", dest="enable_fsdp", action="store_true")
     fsdp_group.add_argument("--disable-fsdp", dest="enable_fsdp", action="store_false")
@@ -162,6 +196,14 @@ def main() -> None:
         num_inference_steps=args.steps,
         enable_fsdp=args.enable_fsdp,
         attn_impl=args.attn_impl,
+        sol_fp8=args.sol_fp8,
+        sol_dense_steps=args.sol_dense_steps,
+        sol_dense_layers=args.sol_dense_layers,
+        sol_tau=args.sol_tau,
+        sol_threshold_type=args.sol_threshold_type,
+        sol_fp8_layer_start=args.sol_fp8_layer_start,
+        sol_fp8_layer_end=args.sol_fp8_layer_end,
+        quantization=args.quantization,
     )
     try:
         run(

telefuser/models/minimax_h3_encoder.py

@@ -226,25 +226,36 @@ def encode_ids(
         host_image_grid = None if image_grid_thw is None else image_grid_thw.to(device="cpu", dtype=torch.long)
         host_video_grid = None if video_grid_thw is None else video_grid_thw.to(device="cpu", dtype=torch.long)
         position_ids = None
+        mm_token_type_ids = None
         if host_image_grid is not None or host_video_grid is not None:
+            mm_token_type_ids = torch.zeros_like(host_ids)
+            if host_image_grid is not None:
+                mm_token_type_ids[host_ids == self.model.config.image_token_id] = 1
+            if host_video_grid is not None:
+                mm_token_type_ids[host_ids == self.model.config.video_token_id] = 2
             position_ids, _ = self.model.get_rope_index(
-                host_ids,
-                host_image_grid,
-                host_video_grid,
+                input_ids=host_ids,
+                mm_token_type_ids=mm_token_type_ids,
+                image_grid_thw=host_image_grid,
+                video_grid_thw=host_video_grid,
                 attention_mask=torch.ones_like(host_ids),
             )
         call_kwargs: dict[str, Any] = {
             "input_ids": host_ids.to(self.device),
             "attention_mask": torch.ones_like(host_ids).to(self.device),
         }
         if position_ids is not None:
+            assert mm_token_type_ids is not None
             call_kwargs["position_ids"] = position_ids.to(self.device)
+            call_kwargs["mm_token_type_ids"] = mm_token_type_ids.to(self.device)
         if pixel_values is not None:
+            assert host_image_grid is not None
             call_kwargs["pixel_values"] = pixel_values.to(self.device, torch.bfloat16)
-            call_kwargs["image_grid_thw"] = host_image_grid
+            call_kwargs["image_grid_thw"] = host_image_grid.to(self.device)
         if pixel_values_videos is not None:
+            assert host_video_grid is not None
             call_kwargs["pixel_values_videos"] = pixel_values_videos.to(self.device, torch.bfloat16)
-            call_kwargs["video_grid_thw"] = host_video_grid
+            call_kwargs["video_grid_thw"] = host_video_grid.to(self.device)
         hidden = self(**call_kwargs)[0].to(torch.bfloat16)
         expected = (input_ids.numel(), self.hidden_dim)
         if tuple(hidden.shape) != expected:

telefuser/models/minimax_h3_lora.py

@@ -1,25 +1,57 @@
 # SPDX-License-Identifier: Apache-2.0
-"""MiniMax H3 Turbo LoRA mapping rules."""
+"""MiniMax H3 LoRA and FastVideo adapter mapping rules."""
 
 from __future__ import annotations
 
 from collections.abc import Iterable, Mapping
+from pathlib import Path
 
 import torch
+from safetensors import safe_open
 
 from telefuser.core.config import LoraConfig
 from telefuser.utils.logging import logger
 from telefuser.utils.lora_loader import LoRALoader, LoRATarget
 
 MINIMAX_H3_LORA_KEY_MAPPING_RULES = [
     (r"^(?:base_model\.model\.|model\.diffusion_model\.|diffusion_model\.|transformer\.|model\.)", ""),
+    (r"^proj_in\.", "video_patch_proj."),
+    (r"^audio_proj_in\.", "audio_patch_proj."),
+    (r"^context_embedder\.", "condition_proj."),
+    (r"^time_embedder\.linear_1\.", "time_embedder.proj_in."),
+    (r"^time_embedder\.linear_2\.", "time_embedder.proj_out."),
+    (r"^norm_out\.norm\.", "final_layer.norm."),
+    (r"^norm_out\.linear\.", "final_layer.adaln_proj.linear."),
+    (r"^proj_out\.", "final_layer.video_out."),
+    (r"^audio_proj_out\.", "final_layer.audio_out."),
     (r"^transformer_blocks\.", "blocks."),
     (r"^token_refiner\.refiner_blocks\.", "token_refiner.blocks."),
+    (r"\.attn\.norm_q\.", ".attn.q_norm."),
+    (r"\.attn\.norm_k\.", ".attn.k_norm."),
     (r"\.attn\.to_out\.0\.", ".attn.out_proj."),
     (r"\.ff\.net\.0\.proj\.", ".mlp.fc1."),
     (r"\.ff\.net\.2\.", ".mlp.fc2."),
 ]
 
+FASTVIDEO_LORA_FORMAT = "fastvideo-lora-v2"
+FASTVIDEO_REPLACEMENT_SUFFIX = ".set_weight"
+
+
+def _adapter_header(path: str | Path) -> tuple[Path, dict[str, str], tuple[str, ...]]:
+    adapter_path = Path(path).expanduser()
+    if adapter_path.is_dir():
+        files = sorted(adapter_path.glob("*.safetensors"))
+        if len(files) != 1:
+            raise ValueError(
+                f"MiniMax H3 adapter directory must contain exactly one safetensors file, got {len(files)}: "
+                f"{adapter_path}"
+            )
+        adapter_path = files[0]
+    if not adapter_path.is_file():
+        raise FileNotFoundError(f"MiniMax H3 adapter not found: {adapter_path}")
+    with safe_open(str(adapter_path), framework="pt", device="cpu") as source:
+        return adapter_path, source.metadata() or {}, tuple(source.keys())
+
 
 def minimax_h3_lora_target(
     model_key: str,
@@ -40,33 +72,50 @@ def minimax_h3_lora_target(
 
 
 class MiniMaxH3LoraAdapter:
-    """Configure the generic loader for the released H3 Turbo LoRA."""
+    """Merge released Turbo LoRAs and FastVideo hybrid MiniMax H3 adapters."""
 
     DEFAULT_ALPHA = 128.0
 
     @classmethod
     def apply(cls, model: torch.nn.Module, configs: Iterable[LoraConfig]) -> int:
         if getattr(model, "quant_type", None) is not None:
-            raise ValueError("MiniMax H3 Turbo LoRA requires original DiT weights during merging")
-        loader = LoRALoader(
-            MINIMAX_H3_LORA_KEY_MAPPING_RULES,
-            target_resolver=minimax_h3_lora_target,
-            strict=True,
-            default_alpha=cls.DEFAULT_ALPHA,
-            stream_safetensors=True,
-            merge_dtype=torch.float32,
-        )
+            raise ValueError("MiniMax H3 adapters require original DiT weights during merging")
         total = 0
         for config in configs:
-            applied = loader.apply_lora(model, config.path, strength=config.strength)
+            adapter_path, metadata, keys = _adapter_header(config.path)
+            replacement_keys = tuple(key for key in keys if key.endswith(FASTVIDEO_REPLACEMENT_SUFFIX))
+            if replacement_keys:
+                raise ValueError(
+                    "This MiniMax H3 adapter contains FastVideo VSA compression-gate .set_weight tensors, "
+                    "but TeleFuser does not implement the VSA-H3 gate backend. Use the dense FastH3 adapter "
+                    f"variant instead; first unsupported key: {replacement_keys[0]}"
+                )
+            fastvideo_format = metadata.get("format") == FASTVIDEO_LORA_FORMAT
+            loader = LoRALoader(
+                MINIMAX_H3_LORA_KEY_MAPPING_RULES,
+                target_resolver=minimax_h3_lora_target,
+                strict=True,
+                # FastVideo adapters define W = W_base + B @ A. Turbo LoRAs
+                # omit alpha metadata and use their released alpha=128 contract.
+                default_alpha=None if fastvideo_format else cls.DEFAULT_ALPHA,
+                stream_safetensors=True,
+                merge_dtype=torch.float32,
+            )
+            applied = loader.apply_lora(model, adapter_path, strength=config.strength)
             total += applied
             logger.info(
-                "Loaded MiniMax H3 Turbo LoRA: {} (strength={}, layers={})", config.path, config.strength, applied
+                "Loaded MiniMax H3 adapter: {} (format={}, strength={}, tensors={})",
+                config.path,
+                metadata.get("format", "turbo-lora"),
+                config.strength,
+                applied,
             )
         return total
 
 
 __all__ = [
+    "FASTVIDEO_LORA_FORMAT",
+    "FASTVIDEO_REPLACEMENT_SUFFIX",
     "MINIMAX_H3_LORA_KEY_MAPPING_RULES",
     "MiniMaxH3LoraAdapter",
     "minimax_h3_lora_target",