[modular] Add LTX-2.5 DFR blocks

docs/source/en/api/pipelines/ltx2.md

@@ -1028,6 +1028,100 @@ encode_video(
 )
 ```
 
+#### Diffusion Fidelity Rendering (DFR)
+
+`LTX2DFRBlocks` trades wall-clock time for detail fidelity. It generates on a canvas padded to a whole number of keyframe segments and spends one extra latent frame of tokens per segment border on a **keyframe slot** — a single-pixel-frame latent the model fills in. Relaxing the effective temporal compression at those positions means the surrounding video is conditioned on genuinely new frames rather than interpolated ones. This needs a transformer whose config sets `use_keyframes_abs_pos_embedding`, which LTX-2.5 checkpoints ship.
+
+The recipe is two passes of the same blocks: a base pass at half resolution, then a detailing pass at full resolution seeded from it. Both the video latents and the keyframe slots are upsampled in between, and the spatial detailing IC-LoRA applies to the second pass only — switched on in the seam between the two calls, like the [stage 2 distilled LoRA](#stage-2-with-the-distilled-lora). The DFR schedules are distilled and run without guidance, so these blocks carry no guider and take no `guidance_scale`.
+
+```py
+import torch
+from diffusers import ComponentsManager
+from diffusers.modular_pipelines import LTX2DFRBlocks
+from diffusers.pipelines.ltx2 import LTX2LatentUpsamplePipeline
+from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
+from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
+from diffusers.utils import encode_video
+
+device, model_path = "cuda", "Lightricks/LTX-2.5-Diffusers"
+height, width, frame_rate = 704, 1216, 24.0
+prompt = "A cinematic shot of a red fox walking through a snowy forest at dawn."
+
+cm = ComponentsManager()
+pipe = LTX2DFRBlocks().init_pipeline(model_path, components_manager=cm)
+pipe.load_components(dtype=torch.bfloat16)
+
+# Load the detailing IC-LoRA *before* offload is enabled. `load_lora_adapter` restores offload
+# afterwards through `DiffusionPipeline.enable_model_cpu_offload`, which a `ModularPipeline` does
+# not implement, and the upsample pipeline below shares this `vae` -- so offloading first leaves
+# accelerate hooks on it and the LoRA load then fails.
+pipe.load_lora_weights("Lightricks/LTX-2.5-22b-IC-LoRA-Pixel-Spatial-Upscaler", adapter_name="detailing")
+pipe.disable_lora()  # pass 1 runs on the base model
+cm.enable_auto_cpu_offload(device=device, memory_reserve_margin="20GB")
+
+# One generator across both passes, so pass 2 continues the noise stream.
+generator = torch.Generator(device).manual_seed(42)
+common = dict(prompt=prompt, frame_rate=frame_rate, generator=generator)
+
+# Pass 1: half resolution. Returns the video latents plus one keyframe slot per segment border.
+# `num_frames` is omitted, so the duration head picks the length.
+first = pipe(
+    **common, height=height // 2, width=width // 2, sigmas=DISTILLED_SIGMA_VALUES, output_type="latent"
+)
+video_latents, keyframes_latents = first.get("videos"), first.get("keyframes_latents")
+num_frames = first.get("num_frames")
+
+# Upsample the video *and* the keyframe slots. `latents_normalized=False`: `output_type="latent"`
+# already applied the latent statistics.
+upsampler = LTX2LatentUpsamplerModel.from_pretrained(
+    model_path, subfolder="latent_upsampler", dtype=torch.bfloat16
+)
+upsample_pipe = LTX2LatentUpsamplePipeline(vae=pipe.vae, latent_upsampler=upsampler)
+upsample_pipe.enable_model_cpu_offload(device=device)
+
+
+def upsample(latents):
+    return upsample_pipe(
+        latents=latents, latents_normalized=False, output_type="latent", return_dict=False
+    )[0]
+
+
+# The detailing IC-LoRA belongs to pass 2 alone, which is why this is one blockset run twice rather
+# than one call: the adapter is switched on in the seam between them. Calibrated for strength 0.5.
+pipe.enable_lora()
+pipe.set_adapters(["detailing"], adapter_weights=[0.5])
+
+# Pass 2: full resolution, seeded from pass 1 -- video, audio, the keyframe slots, and the
+# half-resolution result as the detailing adapter's in-context reference.
+pipe.vae.enable_tiling()
+out = pipe(
+    **common,
+    height=height,
+    width=width,
+    num_frames=num_frames,
+    latents=upsample(video_latents),
+    keyframes_latents=upsample(keyframes_latents),
+    audio_latents=first.get("audio"),
+    detailing_reference_latents=video_latents,
+    detailing_reference_downscale_factor=2,
+    sigmas=STAGE_2_DISTILLED_SIGMA_VALUES,
+    noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0],
+    output_type="np",
+)
+
+encode_video(
+    out.get("videos")[0],
+    fps=frame_rate,
+    audio=out.get("audio")[0].float().cpu(),
+    audio_sample_rate=pipe.vocoder.config.output_sampling_rate,
+    output_path="ltx2_5_dfr.mp4",
+)
+```
+
+`height` and `width` are the output resolution and must be divisible by twice the VAE's spatial compression ratio (64 for LTX-2.5), since the base pass runs at half of each axis. Whatever `num_frames` asks for, the canvas is padded onto the segment grid internally and trimmed back before decoding, so the caller always gets the frame count it requested.
+
+DFR decodes with the convolutional `vae`, matching the reference implementation. For maximum detail fidelity, run the second pass with `output_type="latent"` and hand the latents to [`LTX2VideoDiffusionDecodePipeline`] instead.
+
 You can see the supported workflows in the docs for each blockset (e.g. [`LTX2AutoBlocks`], [`LTX25AutoBlocks`]).
 
 ## LTX2Pipeline
@@ -1086,6 +1180,14 @@ You can see the supported workflows in the docs for each blockset (e.g. [`LTX2Au
 
 [[autodoc]] LTX25AutoBlocks
 
+## LTX2DFRModularPipeline
+
+[[autodoc]] LTX2DFRModularPipeline
+
+## LTX2DFRBlocks
+
+[[autodoc]] LTX2DFRBlocks
+
 ## LTX2Guidance
 
 [[autodoc]] modular_pipelines.ltx2.guider.LTX2Guidance

src/diffusers/__init__.py

@@ -545,6 +545,8 @@
             "LTX25AutoBlocks",
             "LTX25ModularPipeline",
             "LTX2AutoBlocks",
+            "LTX2DFRBlocks",
+            "LTX2DFRModularPipeline",
             "LTX2ModularPipeline",
             "LTXAutoBlocks",
             "LTXModularPipeline",
@@ -1393,6 +1395,8 @@
             Krea2TurboAutoBlocks,
             Krea2TurboModularPipeline,
             LTX2AutoBlocks,
+            LTX2DFRBlocks,
+            LTX2DFRModularPipeline,
             LTX2ModularPipeline,
             LTX25AutoBlocks,
             LTX25ModularPipeline,

src/diffusers/models/transformers/transformer_ltx2.py

@@ -1115,8 +1115,8 @@ class LTX2VideoTransformer3DModel(
             for a given prompt.
         use_keyframes_abs_pos_embedding (`bool`, defaults to `False`):
             Whether to store a learned `(1, inner_dim)` absolute-position embedding for generated-keyframe tokens
-            (LTX-2.5.1+). When `True`, the weight is kept on the module for load/save; the regular distilled forward
-            path does not consume it until a dedicated keyframes pipeline wires it in.
+            (LTX-2.5). When `True`, tokens selected by `video_keyframes_mask` receive this embedding. The argument is
+            optional; omitting it leaves the distilled forward path unchanged.
     """
 
     _supports_gradient_checkpointing = True
@@ -1388,6 +1388,7 @@ def forward(
         use_cross_timestep: bool = False,
         attention_kwargs: dict[str, Any] | None = None,
         video_self_attention_mask: torch.Tensor | None = None,
+        video_keyframes_mask: torch.Tensor | None = None,
         return_dict: bool = True,
     ) -> torch.Tensor:
         """
@@ -1458,6 +1459,10 @@ def forward(
                 applied to the video self-attention in each transformer block. Values in `[0, 1]` where `1` means full
                 attention and `0` means masked. Used e.g. by the IC-LoRA pipeline to control attention strength between
                 noisy tokens and appended reference tokens. Audio self-attention is not affected.
+            video_keyframes_mask (`torch.Tensor`, *optional*):
+                Optional per-token marker of shape `(batch_size, num_video_tokens, 1)`, non-zero on video tokens whose
+                latent frame encodes a single pixel frame. Those tokens receive `keyframes_abs_pos_embedding`. Ignored
+                when the model was built without `use_keyframes_abs_pos_embedding`.
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether to return a dict-like structured output of type `AudioVisualModelOutput` or a tuple.
 
@@ -1509,6 +1514,11 @@ def forward(
         hidden_states = self.proj_in(hidden_states)
         audio_hidden_states = self.audio_proj_in(audio_hidden_states)
 
+        # 2.1. Mark tokens whose latent encodes a single pixel frame (causal first frame, generated keyframe slots).
+        if self.config.use_keyframes_abs_pos_embedding and video_keyframes_mask is not None:
+            marker = (video_keyframes_mask > 0).to(dtype=hidden_states.dtype)
+            hidden_states = hidden_states + marker * self.keyframes_abs_pos_embedding.to(dtype=hidden_states.dtype)
+
         # 3. Prepare timestep embeddings and modulation parameters
         timestep_cross_attn_gate_scale_factor = (
             self.config.cross_attn_timestep_scale_multiplier / self.config.timestep_scale_multiplier

src/diffusers/modular_pipelines/__init__.py

@@ -124,8 +124,10 @@
     _import_structure["ltx2"] = [
         "LTX2AutoBlocks",
         "LTX25AutoBlocks",
+        "LTX2DFRBlocks",
         "LTX2ModularPipeline",
         "LTX25ModularPipeline",
+        "LTX2DFRModularPipeline",
     ]
     _import_structure["minimax_h3"] = [
         "MiniMaxH3Blocks",
@@ -189,7 +191,14 @@
             Krea2TurboModularPipeline,
         )
         from .ltx import LTXAutoBlocks, LTXModularPipeline
-        from .ltx2 import LTX2AutoBlocks, LTX2ModularPipeline, LTX25AutoBlocks, LTX25ModularPipeline
+        from .ltx2 import (
+            LTX2AutoBlocks,
+            LTX2DFRBlocks,
+            LTX2DFRModularPipeline,
+            LTX2ModularPipeline,
+            LTX25AutoBlocks,
+            LTX25ModularPipeline,
+        )
         from .minimax_h3 import (
             MiniMaxH3Blocks,
             MiniMaxH3ModularPipeline,

src/diffusers/modular_pipelines/ltx2/__init__.py

@@ -29,7 +29,12 @@
         "LTX2InContextBlocks",
     ]
     _import_structure["modular_blocks_ltx25"] = ["LTX25AutoBlocks"]
-    _import_structure["modular_pipeline"] = ["LTX2ModularPipeline", "LTX25ModularPipeline"]
+    _import_structure["modular_blocks_ltx2_dfr"] = ["LTX2DFRBlocks"]
+    _import_structure["modular_pipeline"] = [
+        "LTX2DFRModularPipeline",
+        "LTX2ModularPipeline",
+        "LTX25ModularPipeline",
+    ]
 
 if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
     try:
@@ -45,8 +50,9 @@
             LTX2ImageToVideoBlocks,
             LTX2InContextBlocks,
         )
+        from .modular_blocks_ltx2_dfr import LTX2DFRBlocks
         from .modular_blocks_ltx25 import LTX25AutoBlocks
-        from .modular_pipeline import LTX2ModularPipeline, LTX25ModularPipeline
+        from .modular_pipeline import LTX2DFRModularPipeline, LTX2ModularPipeline, LTX25ModularPipeline
 else:
     import sys