From 09de38a8867d6eee2a3a31711bfea59dd8f2dfc5 Mon Sep 17 00:00:00 2001 From: Akshan Krithick Date: Sat, 11 Jul 2026 04:30:11 -0700 Subject: [PATCH 1/3] fix wan i2v num_videos batching, vace latent trim, config scale factors, processor config, modular dtype --- .../modular_pipelines/wan/before_denoise.py | 2 +- .../modular_pipelines/wan/modular_pipeline.py | 4 ++-- src/diffusers/pipelines/wan/image_processor.py | 12 +++++++++++- src/diffusers/pipelines/wan/pipeline_wan_i2v.py | 2 +- src/diffusers/pipelines/wan/pipeline_wan_vace.py | 6 +++--- .../pipelines/wan/pipeline_wan_video2video.py | 4 ++-- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/diffusers/modular_pipelines/wan/before_denoise.py b/src/diffusers/modular_pipelines/wan/before_denoise.py index 6b0874037b0d..f4f67965a0d8 100644 --- a/src/diffusers/modular_pipelines/wan/before_denoise.py +++ b/src/diffusers/modular_pipelines/wan/before_denoise.py @@ -252,7 +252,7 @@ def __call__(self, components: WanModularPipeline, state: PipelineState) -> Pipe self.check_inputs(components, block_state) block_state.batch_size = block_state.prompt_embeds.shape[0] - block_state.dtype = block_state.prompt_embeds.dtype + block_state.dtype = components.transformer.dtype _, seq_len, _ = block_state.prompt_embeds.shape block_state.prompt_embeds = block_state.prompt_embeds.repeat(1, block_state.num_videos_per_prompt, 1) diff --git a/src/diffusers/modular_pipelines/wan/modular_pipeline.py b/src/diffusers/modular_pipelines/wan/modular_pipeline.py index a360440c9251..5948faa68fda 100644 --- a/src/diffusers/modular_pipelines/wan/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/wan/modular_pipeline.py @@ -70,14 +70,14 @@ def patch_size_spatial(self): def vae_scale_factor_spatial(self): vae_scale_factor = 8 if hasattr(self, "vae") and self.vae is not None: - vae_scale_factor = 2 ** len(self.vae.temperal_downsample) + vae_scale_factor = self.vae.config.scale_factor_spatial return vae_scale_factor @property def vae_scale_factor_temporal(self): vae_scale_factor = 4 if hasattr(self, "vae") and self.vae is not None: - vae_scale_factor = 2 ** sum(self.vae.temperal_downsample) + vae_scale_factor = self.vae.config.scale_factor_temporal return vae_scale_factor @property diff --git a/src/diffusers/pipelines/wan/image_processor.py b/src/diffusers/pipelines/wan/image_processor.py index fa18150fcc6e..b2f026c885f6 100644 --- a/src/diffusers/pipelines/wan/image_processor.py +++ b/src/diffusers/pipelines/wan/image_processor.py @@ -68,7 +68,17 @@ def __init__( do_convert_grayscale: bool = False, fill_color: str | float | tuple[float, ...] | None = 0, ): - super().__init__() + super().__init__( + do_resize=do_resize, + vae_scale_factor=vae_scale_factor, + vae_latent_channels=vae_latent_channels, + resample=resample, + reducing_gap=reducing_gap, + do_normalize=do_normalize, + do_binarize=do_binarize, + do_convert_rgb=do_convert_rgb, + do_convert_grayscale=do_convert_grayscale, + ) if do_convert_rgb and do_convert_grayscale: raise ValueError( "`do_convert_rgb` and `do_convert_grayscale` can not both be set to `True`," diff --git a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py index 8061f67ab6b9..f3578af00fd7 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py @@ -698,7 +698,7 @@ def __call__( image_embeds = self.encode_image(image, device) else: image_embeds = self.encode_image([image, last_image], device) - image_embeds = image_embeds.repeat(batch_size, 1, 1) + image_embeds = image_embeds.repeat(batch_size * num_videos_per_prompt, 1, 1) image_embeds = image_embeds.to(transformer_dtype) # 4. Prepare timesteps diff --git a/src/diffusers/pipelines/wan/pipeline_wan_vace.py b/src/diffusers/pipelines/wan/pipeline_wan_vace.py index b0896d382d67..6c7285750f41 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_vace.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_vace.py @@ -196,8 +196,8 @@ def __init__( scheduler=scheduler, ) self.register_to_config(boundary_ratio=boundary_ratio) - self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4 - self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4 + self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8 self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds @@ -1020,8 +1020,8 @@ def __call__( self._current_timestep = None + latents = latents[:, :, num_reference_images:] if not output_type == "latent": - latents = latents[:, :, num_reference_images:] latents = latents.to(vae_dtype) latents_mean = ( torch.tensor(self.vae.config.latents_mean) diff --git a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py index 8993475a2851..a782c79b6c5c 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py @@ -214,8 +214,8 @@ def __init__( scheduler=scheduler, ) - self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4 - self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4 + self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8 self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds From c73d98b1ab71a8b8335e638fd46a7d96ee58a7c5 Mon Sep 17 00:00:00 2001 From: Akshan Krithick Date: Sat, 11 Jul 2026 05:31:55 -0700 Subject: [PATCH 2/3] keep flf2v image_embeds layout, only expand single-image by num_videos --- src/diffusers/pipelines/wan/pipeline_wan_i2v.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py index f3578af00fd7..0dd47e24a56f 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py @@ -698,7 +698,10 @@ def __call__( image_embeds = self.encode_image(image, device) else: image_embeds = self.encode_image([image, last_image], device) - image_embeds = image_embeds.repeat(batch_size * num_videos_per_prompt, 1, 1) + if last_image is None: + image_embeds = image_embeds.repeat(batch_size * num_videos_per_prompt, 1, 1) + else: + image_embeds = image_embeds.repeat(batch_size, 1, 1) image_embeds = image_embeds.to(transformer_dtype) # 4. Prepare timesteps From 6bacb6af2af21ac63c04a0ca42352224882f884f Mon Sep 17 00:00:00 2001 From: Akshan Krithick Date: Sat, 11 Jul 2026 05:41:43 -0700 Subject: [PATCH 3/3] support list image inputs in wan i2v, expand conditioning to effective batch --- src/diffusers/pipelines/wan/pipeline_wan_i2v.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py index 0dd47e24a56f..4bf7ed807ecd 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py @@ -351,8 +351,10 @@ def check_inputs( raise ValueError( "Provide either `image` or `prompt_embeds`. Cannot leave both `image` and `image_embeds` undefined." ) - if image is not None and not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image): - raise ValueError(f"`image` has to be of type `torch.Tensor` or `PIL.Image.Image` but is {type(image)}") + if image is not None and not isinstance(image, (torch.Tensor, PIL.Image.Image, list, tuple)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, or a list of them but is {type(image)}" + ) if height % 16 != 0 or width % 16 != 0: raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") @@ -453,7 +455,7 @@ def prepare_latents( latent_condition = torch.cat(latent_condition) else: latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") - latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) + latent_condition = latent_condition.repeat_interleave(batch_size // latent_condition.shape[0], dim=0) latent_condition = latent_condition.to(dtype) latent_condition = (latent_condition - latents_mean) * latents_std @@ -699,7 +701,9 @@ def __call__( else: image_embeds = self.encode_image([image, last_image], device) if last_image is None: - image_embeds = image_embeds.repeat(batch_size * num_videos_per_prompt, 1, 1) + image_embeds = image_embeds.repeat_interleave( + batch_size * num_videos_per_prompt // image_embeds.shape[0], dim=0 + ) else: image_embeds = image_embeds.repeat(batch_size, 1, 1) image_embeds = image_embeds.to(transformer_dtype)