From db7edfe26b46d7d08825c3e11599915a95835821 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Wed, 2 Sep 2026 02:06:01 +0000 Subject: [PATCH 1/2] feat(minimax-h3): support hybrid LoRA adapters Add FastVideo fastvideo-lora-v2 mappings for low-rank updates and exact weight or bias deltas while preserving the released Turbo LoRA alpha contract. Merge adapters into source weights before optional online FP8 conversion, reject unsupported VSA replacement gates, and expose adapter options through the standard FL2VA example. Document BF16 and FP8 usage and cover hybrid merging, VSA rejection, CLI forwarding, and adapter-before-quantization ordering. --- examples/minimax_h3/README.md | 23 ++++++ examples/minimax_h3/minimax_h3_fl2va_h100.py | 16 ++++ telefuser/models/minimax_h3_lora.py | 75 +++++++++++++++---- tests/unit/models/test_minimax_h3_lora.py | 51 +++++++++++++ .../pipelines/minimax_h3/test_examples.py | 4 + .../pipelines/minimax_h3/test_parallelism.py | 31 ++++++++ 6 files changed, 187 insertions(+), 13 deletions(-) diff --git a/examples/minimax_h3/README.md b/examples/minimax_h3/README.md index abe001d8..01758de1 100644 --- a/examples/minimax_h3/README.md +++ b/examples/minimax_h3/README.md @@ -86,6 +86,29 @@ python examples/minimax_h3/minimax_h3_turbo_lora_h100.py \ The example supports one, two, or four GPUs. Four GPUs use tensor plus Ulysses parallelism; LoRA is merged before parallel sharding. +### FastH3 hybrid adapter + +The standard FL2VA entrypoint also accepts FastVideo's `fastvideo-lora-v2` hybrid adapters. These checkpoints mix +ordinary low-rank `B @ A` updates with exact weight and bias deltas. TeleFuser applies both to the BF16 source weights +before optional online FP8 Linear conversion: + +```bash +python -m examples.minimax_h3.minimax_h3_fl2va_h100 \ + --mode t2va \ + --adapter-path /path/to/dense-datafree/adapter_model.safetensors \ + --adapter-strength 1.0 \ + --steps 5 \ + --duration 5 \ + --seed 1000 \ + --prompt "integrated_multimodal_description: A red fox runs through fresh snow at dawn. overall_soundscape: Fast pawsteps in snow, winter wind, and distant birds." \ + --output outputs/minimax_h3_fasth3_bf16_fa4.mp4 +``` + +Add `--quantization fp8` to run the merged adapter with the existing FP8 Linear path while keeping FA4 attention. +The `vsa-*` FastH3 adapters contain trained attention compression-gate replacement tensors. TeleFuser does not +implement that VSA-H3 backend and rejects those files instead of silently dropping the gates; use the dense adapter +variant unless a VSA backend is available. + ## Feature Cache MiniMax H3 uses AdaTaylorCache around the complete joint audio-video DiT block stack. Calibrate once on one H100 diff --git a/examples/minimax_h3/minimax_h3_fl2va_h100.py b/examples/minimax_h3/minimax_h3_fl2va_h100.py index d05c5b56..905b25a3 100644 --- a/examples/minimax_h3/minimax_h3_fl2va_h100.py +++ b/examples/minimax_h3/minimax_h3_fl2va_h100.py @@ -40,6 +40,8 @@ "feature_cache_n_derivatives": 1, "feature_cache_taylor_threshold": 2, "quantization": None, + "adapter_path": None, + "adapter_strength": 1.0, } @@ -98,6 +100,8 @@ def get_pipeline( feature_cache_n_derivatives: int = PPL_CONFIG["feature_cache_n_derivatives"], feature_cache_taylor_threshold: int = PPL_CONFIG["feature_cache_taylor_threshold"], quantization: str | None = PPL_CONFIG["quantization"], + adapter_path: str | None = PPL_CONFIG["adapter_path"], + adapter_strength: float = PPL_CONFIG["adapter_strength"], ) -> MiniMaxH3Pipeline: """Load the FL2VA checkpoint partition for one, two, or four GPUs.""" tp_degree = 2 if parallelism == 4 else 1 @@ -126,6 +130,8 @@ def get_pipeline( taylor_threshold=feature_cache_taylor_threshold, ), quantization=quantization, + lora_path=adapter_path, + lora_strength=adapter_strength, ) @@ -283,6 +289,14 @@ def _main(default_quantization: str | None = PPL_CONFIG["quantization"]) -> None parser.add_argument("--flow-shift", type=float, default=PPL_CONFIG["flow_shift"]) parser.add_argument("--audio-flow-shift", type=float, default=PPL_CONFIG["audio_flow_shift"]) parser.add_argument("--device", default=PPL_CONFIG["device"]) + parser.add_argument( + "--adapter-path", + "--lora-path", + dest="adapter_path", + default=PPL_CONFIG["adapter_path"], + help="MiniMax H3 Turbo LoRA or dense FastVideo FastH3 hybrid adapter.", + ) + parser.add_argument("--adapter-strength", type=float, default=PPL_CONFIG["adapter_strength"]) parser.add_argument( "--quantization", choices=("fp8", "torchao-fp8", "tf-kernel-fp8", "bnb-nf4"), @@ -358,6 +372,8 @@ def _main(default_quantization: str | None = PPL_CONFIG["quantization"]) -> None feature_cache_n_derivatives=args.feature_cache_n_derivatives, feature_cache_taylor_threshold=args.feature_cache_taylor_threshold, quantization=args.quantization, + adapter_path=args.adapter_path, + adapter_strength=args.adapter_strength, ) try: result = run_with_file( diff --git a/telefuser/models/minimax_h3_lora.py b/telefuser/models/minimax_h3_lora.py index 2eadc75b..a55b7b6c 100644 --- a/telefuser/models/minimax_h3_lora.py +++ b/telefuser/models/minimax_h3_lora.py @@ -1,11 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 -"""MiniMax H3 Turbo LoRA mapping rules.""" +"""MiniMax H3 LoRA and FastVideo adapter mapping rules.""" from __future__ import annotations from collections.abc import Iterable, Mapping +from pathlib import Path import torch +from safetensors import safe_open from telefuser.core.config import LoraConfig from telefuser.utils.logging import logger @@ -13,13 +15,43 @@ MINIMAX_H3_LORA_KEY_MAPPING_RULES = [ (r"^(?:base_model\.model\.|model\.diffusion_model\.|diffusion_model\.|transformer\.|model\.)", ""), + (r"^proj_in\.", "video_patch_proj."), + (r"^audio_proj_in\.", "audio_patch_proj."), + (r"^context_embedder\.", "condition_proj."), + (r"^time_embedder\.linear_1\.", "time_embedder.proj_in."), + (r"^time_embedder\.linear_2\.", "time_embedder.proj_out."), + (r"^norm_out\.norm\.", "final_layer.norm."), + (r"^norm_out\.linear\.", "final_layer.adaln_proj.linear."), + (r"^proj_out\.", "final_layer.video_out."), + (r"^audio_proj_out\.", "final_layer.audio_out."), (r"^transformer_blocks\.", "blocks."), (r"^token_refiner\.refiner_blocks\.", "token_refiner.blocks."), + (r"\.attn\.norm_q\.", ".attn.q_norm."), + (r"\.attn\.norm_k\.", ".attn.k_norm."), (r"\.attn\.to_out\.0\.", ".attn.out_proj."), (r"\.ff\.net\.0\.proj\.", ".mlp.fc1."), (r"\.ff\.net\.2\.", ".mlp.fc2."), ] +FASTVIDEO_LORA_FORMAT = "fastvideo-lora-v2" +FASTVIDEO_REPLACEMENT_SUFFIX = ".set_weight" + + +def _adapter_header(path: str | Path) -> tuple[Path, dict[str, str], tuple[str, ...]]: + adapter_path = Path(path).expanduser() + if adapter_path.is_dir(): + files = sorted(adapter_path.glob("*.safetensors")) + if len(files) != 1: + raise ValueError( + f"MiniMax H3 adapter directory must contain exactly one safetensors file, got {len(files)}: " + f"{adapter_path}" + ) + adapter_path = files[0] + if not adapter_path.is_file(): + raise FileNotFoundError(f"MiniMax H3 adapter not found: {adapter_path}") + with safe_open(str(adapter_path), framework="pt", device="cpu") as source: + return adapter_path, source.metadata() or {}, tuple(source.keys()) + def minimax_h3_lora_target( model_key: str, @@ -40,33 +72,50 @@ def minimax_h3_lora_target( class MiniMaxH3LoraAdapter: - """Configure the generic loader for the released H3 Turbo LoRA.""" + """Merge released Turbo LoRAs and FastVideo hybrid MiniMax H3 adapters.""" DEFAULT_ALPHA = 128.0 @classmethod def apply(cls, model: torch.nn.Module, configs: Iterable[LoraConfig]) -> int: if getattr(model, "quant_type", None) is not None: - raise ValueError("MiniMax H3 Turbo LoRA requires original DiT weights during merging") - loader = LoRALoader( - MINIMAX_H3_LORA_KEY_MAPPING_RULES, - target_resolver=minimax_h3_lora_target, - strict=True, - default_alpha=cls.DEFAULT_ALPHA, - stream_safetensors=True, - merge_dtype=torch.float32, - ) + raise ValueError("MiniMax H3 adapters require original DiT weights during merging") total = 0 for config in configs: - applied = loader.apply_lora(model, config.path, strength=config.strength) + adapter_path, metadata, keys = _adapter_header(config.path) + replacement_keys = tuple(key for key in keys if key.endswith(FASTVIDEO_REPLACEMENT_SUFFIX)) + if replacement_keys: + raise ValueError( + "This MiniMax H3 adapter contains FastVideo VSA compression-gate .set_weight tensors, " + "but TeleFuser does not implement the VSA-H3 gate backend. Use the dense FastH3 adapter " + f"variant instead; first unsupported key: {replacement_keys[0]}" + ) + fastvideo_format = metadata.get("format") == FASTVIDEO_LORA_FORMAT + loader = LoRALoader( + MINIMAX_H3_LORA_KEY_MAPPING_RULES, + target_resolver=minimax_h3_lora_target, + strict=True, + # FastVideo adapters define W = W_base + B @ A. Turbo LoRAs + # omit alpha metadata and use their released alpha=128 contract. + default_alpha=None if fastvideo_format else cls.DEFAULT_ALPHA, + stream_safetensors=True, + merge_dtype=torch.float32, + ) + applied = loader.apply_lora(model, adapter_path, strength=config.strength) total += applied logger.info( - "Loaded MiniMax H3 Turbo LoRA: {} (strength={}, layers={})", config.path, config.strength, applied + "Loaded MiniMax H3 adapter: {} (format={}, strength={}, tensors={})", + config.path, + metadata.get("format", "turbo-lora"), + config.strength, + applied, ) return total __all__ = [ + "FASTVIDEO_LORA_FORMAT", + "FASTVIDEO_REPLACEMENT_SUFFIX", "MINIMAX_H3_LORA_KEY_MAPPING_RULES", "MiniMaxH3LoraAdapter", "minimax_h3_lora_target", diff --git a/tests/unit/models/test_minimax_h3_lora.py b/tests/unit/models/test_minimax_h3_lora.py index 003b969a..aee57c83 100644 --- a/tests/unit/models/test_minimax_h3_lora.py +++ b/tests/unit/models/test_minimax_h3_lora.py @@ -70,3 +70,54 @@ def test_h3_turbo_lora_rejects_missing_pairs(tmp_path) -> None: assert "incomplete LoRA pairs" in str(exc) else: raise AssertionError("incomplete H3 LoRA should fail") + + +def test_h3_fastvideo_hybrid_adapter_merges_low_rank_and_exact_deltas(tmp_path) -> None: + model = _small_model() + qkv_before = model.blocks[0].attn.qkv_proj.weight.detach().clone() + patch_before = model.video_patch_proj.weight.detach().clone() + condition_bias_before = model.condition_proj.bias.detach().clone() + adaln_bias_before = model.blocks[0].adaln_proj.linear.bias.detach().clone() + q_norm_before = model.blocks[0].attn.q_norm.weight.detach().clone() + weights = { + "transformer_blocks.0.attn.to_q.lora_A.weight": torch.ones(2, 8), + "transformer_blocks.0.attn.to_q.lora_B.weight": torch.ones(8, 2), + "proj_in.diff": torch.ones_like(patch_before), + "context_embedder.diff_b": torch.ones_like(condition_bias_before), + "transformer_blocks.0.adaln_proj.linear.diff_b": torch.ones_like(adaln_bias_before), + "transformer_blocks.0.attn.norm_q.diff": torch.ones_like(q_norm_before), + } + adapter_dir = tmp_path / "dense-datafree" + adapter_dir.mkdir() + save_file( + weights, + str(adapter_dir / "adapter_model.safetensors"), + metadata={"format": "fastvideo-lora-v2", "rank": "2"}, + ) + + assert MiniMaxH3LoraAdapter.apply(model, [LoraConfig(path=str(adapter_dir), strength=0.5)]) == 5 + + # FastVideo's hybrid format is W += B @ A, with no implicit alpha/rank scale. + torch.testing.assert_close(model.blocks[0].attn.qkv_proj.weight[:8], qkv_before[:8] + 1) + torch.testing.assert_close(model.blocks[0].attn.qkv_proj.weight[8:], qkv_before[8:]) + torch.testing.assert_close(model.video_patch_proj.weight, patch_before + 0.5) + torch.testing.assert_close(model.condition_proj.bias, condition_bias_before + 0.5) + torch.testing.assert_close(model.blocks[0].adaln_proj.linear.bias, adaln_bias_before + 0.5) + torch.testing.assert_close(model.blocks[0].attn.q_norm.weight, q_norm_before + 0.5) + + +def test_h3_fastvideo_vsa_adapter_rejects_missing_gate_backend(tmp_path) -> None: + path = tmp_path / "vsa.safetensors" + save_file( + {"transformer_blocks.0.attn.to_gate_compress.set_weight": torch.ones(8, 8)}, + str(path), + metadata={"format": "fastvideo-lora-v2"}, + ) + + try: + MiniMaxH3LoraAdapter.apply(_small_model(), [LoraConfig(path=str(path))]) + except ValueError as exc: + assert "VSA compression-gate" in str(exc) + assert "dense FastH3 adapter" in str(exc) + else: + raise AssertionError("VSA-only FastH3 adapter should fail without its gate backend") diff --git a/tests/unit/pipelines/minimax_h3/test_examples.py b/tests/unit/pipelines/minimax_h3/test_examples.py index c10acc5a..e1feefc0 100644 --- a/tests/unit/pipelines/minimax_h3/test_examples.py +++ b/tests/unit/pipelines/minimax_h3/test_examples.py @@ -119,6 +119,8 @@ def fake_loader(model_root: str, **kwargs: object) -> object: num_inference_steps=20, enable_fsdp=True, enable_feature_cache=True, + adapter_path="/models/fast-h3/adapter_model.safetensors", + adapter_strength=0.75, ) assert result is sentinel @@ -149,6 +151,8 @@ def fake_loader(model_root: str, **kwargs: object) -> object: taylor_threshold=2, ), "quantization": None, + "lora_path": "/models/fast-h3/adapter_model.safetensors", + "lora_strength": 0.75, }, ) ] diff --git a/tests/unit/pipelines/minimax_h3/test_parallelism.py b/tests/unit/pipelines/minimax_h3/test_parallelism.py index da9537e4..778be4d5 100644 --- a/tests/unit/pipelines/minimax_h3/test_parallelism.py +++ b/tests/unit/pipelines/minimax_h3/test_parallelism.py @@ -6,6 +6,7 @@ from telefuser.core.config import ( AttentionConfig, AttnImplType, + LoraConfig, ModelRuntimeConfig, OffloadConfig, ParallelConfig, @@ -154,6 +155,36 @@ def enable_quant(config: QuantConfig) -> None: empty_cache.assert_called_once_with() +def test_adapter_is_merged_before_online_fp8_quantization() -> None: + transformer = MagicMock() + transformer.quant_type = None + manager = MagicMock() + manager.fetch_module.return_value = transformer + runtime = ModelRuntimeConfig( + device_type="cuda", + lora_configs=[LoraConfig(path="adapter.safetensors")], + quant_config=QuantConfig(enabled=True, quant_type=QuantType.FP8), + ) + calls: list[str] = [] + + def enable_quant(config: QuantConfig) -> None: + calls.append("quantize") + transformer.quant_type = config.quant_type + + transformer.enable_quant.side_effect = enable_quant + with ( + patch( + "telefuser.pipelines.minimax_h3.denoising.MiniMaxH3LoraAdapter.apply", + side_effect=lambda *_: calls.append("merge_adapter"), + ), + patch("telefuser.pipelines.minimax_h3.denoising.current_platform.empty_cache"), + ): + stage = MiniMaxH3DenoisingStage(manager, runtime) + stage._ensure_online_quantized() + + assert calls == ["merge_adapter", "quantize"] + + def test_text_encoder_direct_handoff_keeps_token_tags_on_cpu() -> None: manager = MagicMock() manager.fetch_module.return_value = MagicMock() From b9c1ca733c0cc4825aca7d6c70531cecb1fa0ac7 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Wed, 2 Sep 2026 04:20:45 +0000 Subject: [PATCH 2/2] feat(minimax-h3): complete FP8 Sol adapter support --- examples/minimax_h3/README.md | 26 ++++++++--- .../minimax_h3/minimax_h3_turbo_lora_h100.py | 44 ++++++++++++++++++- telefuser/models/minimax_h3_encoder.py | 21 ++++++--- tests/unit/models/test_minimax_h3_encoder.py | 40 ++++++++++++++++- .../pipelines/minimax_h3/test_examples.py | 40 +++++++++++++++++ 5 files changed, 159 insertions(+), 12 deletions(-) diff --git a/examples/minimax_h3/README.md b/examples/minimax_h3/README.md index 01758de1..f6bc83c4 100644 --- a/examples/minimax_h3/README.md +++ b/examples/minimax_h3/README.md @@ -76,15 +76,22 @@ unless overridden. ```bash python examples/minimax_h3/minimax_h3_turbo_lora_h100.py \ - --gpu-num 4 \ + --gpu-num 1 \ --model-root /hhb-data/aigc/model_zoo/MiniMaxAI_MiniMax-H3 \ --lora-path /data/zuoxin/workspace/TeleFuser/work_dirs/models/lightx2v/Minimax-h3-Turbo/minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors \ + --quantization fp8 \ + --attn-impl SOL_ATTN \ + --sol-fp8 \ + --sol-dense-steps 1 \ + --sol-dense-layers 0 \ + --sol-tau 1.0 \ + --sol-threshold-type exact \ --duration 5 \ - --output outputs/minimax_h3_turbo_lora.mp4 + --output outputs/minimax_h3_turbo_lora_fp8_sol.mp4 ``` The example supports one, two, or four GPUs. Four GPUs use tensor plus Ulysses parallelism; LoRA is merged before -parallel sharding. +parallel sharding. Online AdaLN caching is enabled by default, matching the standard H100 FL2VA entrypoint. ### FastH3 hybrid adapter @@ -97,14 +104,23 @@ python -m examples.minimax_h3.minimax_h3_fl2va_h100 \ --mode t2va \ --adapter-path /path/to/dense-datafree/adapter_model.safetensors \ --adapter-strength 1.0 \ + --quantization fp8 \ + --attn-impl SOL_ATTN \ + --sol-fp8 \ + --sol-dense-steps 1 \ + --sol-dense-layers 0 \ + --sol-tau 1.0 \ + --sol-threshold-type exact \ --steps 5 \ --duration 5 \ --seed 1000 \ --prompt "integrated_multimodal_description: A red fox runs through fresh snow at dawn. overall_soundscape: Fast pawsteps in snow, winter wind, and distant birds." \ - --output outputs/minimax_h3_fasth3_bf16_fa4.mp4 + --output outputs/minimax_h3_fasth3_fp8_sol.mp4 ``` -Add `--quantization fp8` to run the merged adapter with the existing FP8 Linear path while keeping FA4 attention. +The commands combine the merged adapter with FP8 Linear and FP8 Sol attention. The shown `1/0` dense +step/layer profile is the fastest quality-valid single-H100 profile from the 5-second benchmark; adjust +`--sol-dense-steps`, `--sol-dense-layers`, `--sol-tau`, and `--sol-threshold-type` for other workloads. The `vsa-*` FastH3 adapters contain trained attention compression-gate replacement tensors. TeleFuser does not implement that VSA-H3 backend and rejects those files instead of silently dropping the gates; use the dense adapter variant unless a VSA backend is available. diff --git a/examples/minimax_h3/minimax_h3_turbo_lora_h100.py b/examples/minimax_h3/minimax_h3_turbo_lora_h100.py index 13647afc..d578c259 100644 --- a/examples/minimax_h3/minimax_h3_turbo_lora_h100.py +++ b/examples/minimax_h3/minimax_h3_turbo_lora_h100.py @@ -34,7 +34,10 @@ "aspect_ratio": "16:9", "device": "cuda:0", "enable_fsdp": False, + "online_adaln_cache": True, "attn_impl": AttnImplType.FLASH_ATTN_4, + "sol_fp8": False, + "quantization": None, } PIPELINE_MANIFEST = build_pipeline_manifest( @@ -63,7 +66,16 @@ def get_pipeline( lora_strength: float = PPL_CONFIG["lora_strength"], num_inference_steps: int = PPL_CONFIG["num_inference_steps"], enable_fsdp: bool | None = PPL_CONFIG["enable_fsdp"], + online_adaln_cache: bool = PPL_CONFIG["online_adaln_cache"], attn_impl: AttnImplType | str = PPL_CONFIG["attn_impl"], + sol_fp8: bool = PPL_CONFIG["sol_fp8"], + sol_dense_steps: int = 2, + sol_dense_layers: int = 2, + sol_tau: float = 1.0, + sol_threshold_type: str = "exact", + sol_fp8_layer_start: int = 0, + sol_fp8_layer_end: int | None = None, + quantization: str | None = PPL_CONFIG["quantization"], ) -> MiniMaxH3Pipeline: """Load FL2VA and merge Turbo LoRA with the requested GPU parallelism.""" if parallelism not in {1, 2, 4}: @@ -78,7 +90,16 @@ def get_pipeline( tp_degree=tp_degree, text_encoder_tp_degree=parallelism, enable_fsdp=enable_fsdp, + online_adaln_cache=online_adaln_cache, attn_impl=attn_impl, + sol_fp8=sol_fp8, + sol_dense_steps=sol_dense_steps, + sol_dense_layers=sol_dense_layers, + sol_tau=sol_tau, + sol_threshold_type=sol_threshold_type, + sol_fp8_layer_start=sol_fp8_layer_start, + sol_fp8_layer_end=sol_fp8_layer_end, + quantization=quantization, lora_path=lora_path, lora_strength=lora_strength, ) @@ -145,9 +166,22 @@ def main() -> None: parser.add_argument("--device", default=PPL_CONFIG["device"]) parser.add_argument( "--attn-impl", - choices=("FLASH_ATTN_4", "SAGE_ATTN_2_8_8_SM90"), + choices=("FLASH_ATTN_4", "SAGE_ATTN_2_8_8_SM90", "SOL_ATTN"), default=PPL_CONFIG["attn_impl"].name, ) + parser.add_argument( + "--quantization", + choices=("fp8", "torchao-fp8", "tf-kernel-fp8", "bnb-nf4"), + default=PPL_CONFIG["quantization"], + help="Online DiT Linear quantization backend.", + ) + parser.add_argument("--sol-fp8", action="store_true", help="Use FP8 Q/K/V in active Sol-Attn layers.") + parser.add_argument("--sol-dense-steps", type=int, default=2) + parser.add_argument("--sol-dense-layers", type=int, default=2) + parser.add_argument("--sol-tau", type=float, default=1.0) + parser.add_argument("--sol-threshold-type", choices=("exact", "diag"), default="exact") + parser.add_argument("--sol-fp8-layer-start", type=int, default=0) + parser.add_argument("--sol-fp8-layer-end", type=int) fsdp_group = parser.add_mutually_exclusive_group() fsdp_group.add_argument("--enable-fsdp", dest="enable_fsdp", action="store_true") fsdp_group.add_argument("--disable-fsdp", dest="enable_fsdp", action="store_false") @@ -162,6 +196,14 @@ def main() -> None: num_inference_steps=args.steps, enable_fsdp=args.enable_fsdp, attn_impl=args.attn_impl, + sol_fp8=args.sol_fp8, + sol_dense_steps=args.sol_dense_steps, + sol_dense_layers=args.sol_dense_layers, + sol_tau=args.sol_tau, + sol_threshold_type=args.sol_threshold_type, + sol_fp8_layer_start=args.sol_fp8_layer_start, + sol_fp8_layer_end=args.sol_fp8_layer_end, + quantization=args.quantization, ) try: run( diff --git a/telefuser/models/minimax_h3_encoder.py b/telefuser/models/minimax_h3_encoder.py index 3890d490..682483e5 100644 --- a/telefuser/models/minimax_h3_encoder.py +++ b/telefuser/models/minimax_h3_encoder.py @@ -226,11 +226,18 @@ def encode_ids( host_image_grid = None if image_grid_thw is None else image_grid_thw.to(device="cpu", dtype=torch.long) host_video_grid = None if video_grid_thw is None else video_grid_thw.to(device="cpu", dtype=torch.long) position_ids = None + mm_token_type_ids = None if host_image_grid is not None or host_video_grid is not None: + mm_token_type_ids = torch.zeros_like(host_ids) + if host_image_grid is not None: + mm_token_type_ids[host_ids == self.model.config.image_token_id] = 1 + if host_video_grid is not None: + mm_token_type_ids[host_ids == self.model.config.video_token_id] = 2 position_ids, _ = self.model.get_rope_index( - host_ids, - host_image_grid, - host_video_grid, + input_ids=host_ids, + mm_token_type_ids=mm_token_type_ids, + image_grid_thw=host_image_grid, + video_grid_thw=host_video_grid, attention_mask=torch.ones_like(host_ids), ) call_kwargs: dict[str, Any] = { @@ -238,13 +245,17 @@ def encode_ids( "attention_mask": torch.ones_like(host_ids).to(self.device), } if position_ids is not None: + assert mm_token_type_ids is not None call_kwargs["position_ids"] = position_ids.to(self.device) + call_kwargs["mm_token_type_ids"] = mm_token_type_ids.to(self.device) if pixel_values is not None: + assert host_image_grid is not None call_kwargs["pixel_values"] = pixel_values.to(self.device, torch.bfloat16) - call_kwargs["image_grid_thw"] = host_image_grid + call_kwargs["image_grid_thw"] = host_image_grid.to(self.device) if pixel_values_videos is not None: + assert host_video_grid is not None call_kwargs["pixel_values_videos"] = pixel_values_videos.to(self.device, torch.bfloat16) - call_kwargs["video_grid_thw"] = host_video_grid + call_kwargs["video_grid_thw"] = host_video_grid.to(self.device) hidden = self(**call_kwargs)[0].to(torch.bfloat16) expected = (input_ids.numel(), self.hidden_dim) if tuple(hidden.shape) != expected: diff --git a/tests/unit/models/test_minimax_h3_encoder.py b/tests/unit/models/test_minimax_h3_encoder.py index 8039d7ee..397e92e1 100644 --- a/tests/unit/models/test_minimax_h3_encoder.py +++ b/tests/unit/models/test_minimax_h3_encoder.py @@ -1,4 +1,5 @@ -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import Mock, patch import torch @@ -48,6 +49,43 @@ def test_encode_ids_enters_encoder_root_forward() -> None: assert set(call) == {"input_ids", "attention_mask"} +def test_encode_ids_passes_qwen3_vl_multimodal_token_types_by_keyword() -> None: + encoder = MiniMaxH3Encoder.__new__(MiniMaxH3Encoder) + torch.nn.Module.__init__(encoder) + encoder.register_parameter("anchor", torch.nn.Parameter(torch.zeros(()))) + encoder.hidden_dim = 4 + encoder.model = torch.nn.Module() + encoder.model.config = SimpleNamespace(image_token_id=151655, video_token_id=151656) + position_ids = torch.arange(12).reshape(3, 1, 4) + encoder.model.get_rope_index = Mock(return_value=(position_ids, torch.zeros(1))) + input_ids = torch.tensor([17, 151655, 151656, 18]) + image_grid = torch.tensor([[1, 2, 2]]) + video_grid = torch.tensor([[1, 2, 2]]) + expected = torch.arange(16, dtype=torch.bfloat16).reshape(1, 4, 4) + + with patch.object(encoder, "forward", return_value=expected) as forward: + actual = encoder.encode_ids( + input_ids, + pixel_values=torch.ones(4, 3), + image_grid_thw=image_grid, + pixel_values_videos=torch.ones(4, 3), + video_grid_thw=video_grid, + ) + + torch.testing.assert_close(actual, expected[0]) + rope_call = encoder.model.get_rope_index.call_args + assert not rope_call.args + assert torch.equal(rope_call.kwargs["input_ids"], input_ids.unsqueeze(0)) + assert torch.equal(rope_call.kwargs["mm_token_type_ids"], torch.tensor([[0, 1, 2, 0]])) + assert torch.equal(rope_call.kwargs["image_grid_thw"], image_grid) + assert torch.equal(rope_call.kwargs["video_grid_thw"], video_grid) + call = forward.call_args.kwargs + assert torch.equal(call["position_ids"], position_ids) + assert torch.equal(call["mm_token_type_ids"], torch.tensor([[0, 1, 2, 0]])) + assert torch.equal(call["image_grid_thw"], image_grid) + assert torch.equal(call["video_grid_thw"], video_grid) + + def test_encoder_tp_shards_fused_columns_by_logical_section() -> None: linear = torch.nn.Linear(4, 12, bias=True) original_weight = linear.weight.detach().clone() diff --git a/tests/unit/pipelines/minimax_h3/test_examples.py b/tests/unit/pipelines/minimax_h3/test_examples.py index e1feefc0..d2a7ed64 100644 --- a/tests/unit/pipelines/minimax_h3/test_examples.py +++ b/tests/unit/pipelines/minimax_h3/test_examples.py @@ -327,6 +327,46 @@ def __call__(self, **kwargs: object) -> object: assert calls[0]["conditions"] == [{"type": "image", "role": "keyframe", "uri": "input.png", "frame_index": 0}] +def test_turbo_example_forwards_fp8_sol_configuration(monkeypatch: pytest.MonkeyPatch) -> None: + calls = [] + sentinel = object() + + def fake_loader(*args: object, **kwargs: object) -> object: + calls.append((args, kwargs)) + return sentinel + + monkeypatch.setattr(turbo_example, "load_minimax_h3_pipeline", fake_loader) + result = turbo_example.get_pipeline( + 1, + "/models/h3", + lora_path="turbo.safetensors", + attn_impl="SOL_ATTN", + sol_fp8=True, + sol_dense_steps=2, + sol_dense_layers=1, + sol_tau=0.9, + sol_threshold_type="diag", + sol_fp8_layer_start=3, + sol_fp8_layer_end=40, + quantization="tf-kernel-fp8", + ) + + assert result is sentinel + assert calls[0][0] == ("/models/h3",) + options = calls[0][1] + assert options["lora_path"] == "turbo.safetensors" + assert options["online_adaln_cache"] is True + assert options["attn_impl"] == "SOL_ATTN" + assert options["sol_fp8"] is True + assert options["sol_dense_steps"] == 2 + assert options["sol_dense_layers"] == 1 + assert options["sol_tau"] == 0.9 + assert options["sol_threshold_type"] == "diag" + assert options["sol_fp8_layer_start"] == 3 + assert options["sol_fp8_layer_end"] == 40 + assert options["quantization"] == "tf-kernel-fp8" + + def test_turbo_run_with_file_accepts_service_image_alias(monkeypatch: pytest.MonkeyPatch) -> None: calls = []