From b7426bb5647e651f3cca259b6055f5e80d9d428e Mon Sep 17 00:00:00 2001 From: yjy415 <2471352175@qq.com> Date: Wed, 26 Aug 2026 17:52:42 +0800 Subject: [PATCH 1/4] add: Qwen-Video-Edit --- diffsynth/pipelines/qwen_video_edit.py | 347 ++++++++++++++++++ .../model_inference/Qwen-Video-Edit.py | 32 ++ 2 files changed, 379 insertions(+) create mode 100644 diffsynth/pipelines/qwen_video_edit.py create mode 100644 examples/qwen_image/model_inference/Qwen-Video-Edit.py diff --git a/diffsynth/pipelines/qwen_video_edit.py b/diffsynth/pipelines/qwen_video_edit.py new file mode 100644 index 000000000..6fac442de --- /dev/null +++ b/diffsynth/pipelines/qwen_video_edit.py @@ -0,0 +1,347 @@ +import torch, math +from typing import Union +from einops import rearrange +import numpy as np +from PIL import Image +from safetensors.torch import load_file +from tqdm import tqdm + +from ..core import ModelConfig, gradient_checkpoint_forward +from ..core.device.npu_compatible_device import get_device_type +from ..diffusion.base_pipeline import BasePipeline +from ..models.qwen_image_dit import QwenEmbedRope +from .qwen_image import QwenImageUnit_PromptEmbedder +from ..diffusion import FlowMatchScheduler + + +class QwenVideoEditRope(QwenEmbedRope): + def _expand_pos_freqs_if_needed(self, video_fhw, txt_seq_lens): + if isinstance(video_fhw, list) and video_fhw and isinstance(video_fhw[0], dict): + video_fhw = ( + max(x["frame"] + 1 for x in video_fhw), + max(x["full_height"] for x in video_fhw), + max(x["full_width"] for x in video_fhw), + ) + super()._expand_pos_freqs_if_needed(video_fhw, txt_seq_lens) + + def forward(self, video_fhw, txt_seq_lens, device): + if not video_fhw or not isinstance(video_fhw[0], dict): + return super().forward(video_fhw, txt_seq_lens, device) + self._expand_pos_freqs_if_needed(video_fhw, txt_seq_lens) + + if self.pos_freqs.device != device: + self.pos_freqs = self.pos_freqs.to(device) + self.neg_freqs = self.neg_freqs.to(device) + freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) + freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) + + video_freqs, max_index = [], 0 + for item in video_fhw: + frame, height, width = item["frame"], item["height"], item["width"] + full_height, full_width = item["full_height"], item["full_width"] + key = "grid_" + "_".join(str(item[x]) for x in ("frame", "height", "width", "h_off", "w_off", "full_height", "full_width")) + if key not in self.rope_cache: + frame_freq = freqs_pos[0][frame:frame + 1].view(1, 1, 1, -1).expand(1, height, width, -1) + if self.scale_rope: + axis_h = torch.cat([freqs_neg[1][-(full_height - full_height // 2):], freqs_pos[1][:full_height // 2]]) + axis_w = torch.cat([freqs_neg[2][-(full_width - full_width // 2):], freqs_pos[2][:full_width // 2]]) + else: + axis_h, axis_w = freqs_pos[1][:full_height], freqs_pos[2][:full_width] + height_freq = axis_h[item["h_off"]:item["h_off"] + height].view(1, height, 1, -1).expand(1, height, width, -1) + width_freq = axis_w[item["w_off"]:item["w_off"] + width].view(1, 1, width, -1).expand(1, height, width, -1) + self.rope_cache[key] = torch.cat([frame_freq, height_freq, width_freq], dim=-1).reshape(height * width, -1).contiguous() + video_freqs.append(self.rope_cache[key]) + max_index = max(full_height // 2, full_width // 2, max_index) if self.scale_rope else max(full_height, full_width, max_index) + + text_freqs = self.pos_freqs[max_index:max_index + max(txt_seq_lens)] + return torch.cat(video_freqs, dim=0), text_freqs + + +class WanToQwenProjection(torch.nn.Module): + def __init__(self, in_channels: int = 16, inner_dim: int = 3072): + super().__init__() + self.group = 1 + self.proj = torch.nn.Conv3d(in_channels, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2)) + + @torch.no_grad() + def init_from_qwen_dit(self, dit): + self.proj.weight.copy_(dit.img_in.weight.view(self.proj.out_channels, 16, 2, 2).unsqueeze(2)) + self.proj.bias.copy_(dit.img_in.bias) + + def forward(self, x): + return rearrange(self.proj(x), "B D T H W -> B (T H W) D") + + +class QwenToWanProjection(torch.nn.Module): + def __init__(self, out_channels: int = 16, inner_dim: int = 3072): + super().__init__() + self.group = 1 + self.proj = torch.nn.Linear(inner_dim, out_channels * 4) + + @torch.no_grad() + def init_from_qwen_dit(self, dit): + self.proj.load_state_dict(dit.proj_out.state_dict()) + + def forward(self, x, num_frames, tokens_h, tokens_w): + return rearrange(self.proj(x), "B (T H W) (C P Q) -> B C T (H P) (W Q)", + T=num_frames, H=tokens_h, W=tokens_w, P=2, Q=2) + + +def _factorize(value): + for rows in range(int(math.sqrt(value)), 0, -1): + if value % rows == 0: + return rows, value // rows + return 1, value + + +class QwenVideoEditPipeline(BasePipeline): + + def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): + super().__init__( + device=device, torch_dtype=torch_dtype, + height_division_factor=16, width_division_factor=16, + ) + from transformers import Qwen2Tokenizer, Qwen2VLProcessor + + self.scheduler = FlowMatchScheduler("Qwen-Image") + self.text_encoder: QwenImageTextEncoder = None + self.dit: QwenImageDiT = None + self.video_vae = None + self.tokenizer: Qwen2Tokenizer = None + self.processor: Qwen2VLProcessor = None + self.in_proj: WanToQwenProjection = None + self.out_proj: QwenToWanProjection = None + self.prompt_embedder = QwenImageUnit_PromptEmbedder() + self.in_iteration_models = ("dit", "in_proj", "out_proj") + self.units = [] + self.model_fn = model_fn_qwen_video_edit + self.compilable_models = ["dit"] + + + @staticmethod + def from_pretrained( + torch_dtype: torch.dtype = torch.bfloat16, + device: Union[str, torch.device] = get_device_type(), + model_configs: list[ModelConfig] = [], + video_vae_config: ModelConfig = None, + checkpoint: ModelConfig = None, + tokenizer_config: ModelConfig = ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"), + processor_config: ModelConfig = ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"), + vram_limit: float = None, + ): + # Initialize pipeline + pipe = QwenVideoEditPipeline(device=device, torch_dtype=torch_dtype) + configs = list(model_configs) + if video_vae_config is not None: + configs.append(video_vae_config) + model_pool = pipe.download_and_load_models(configs, vram_limit) + + # Fetch models + pipe.text_encoder = model_pool.fetch_model("qwen_image_text_encoder") + pipe.dit = model_pool.fetch_model("qwen_image_dit") + pipe.video_vae = model_pool.fetch_model("wan_video_vae") + if tokenizer_config is not None: + tokenizer_config.download_if_necessary() + from transformers import Qwen2Tokenizer + pipe.tokenizer = Qwen2Tokenizer.from_pretrained(tokenizer_config.path) + if processor_config is not None: + processor_config.download_if_necessary() + from transformers import Qwen2VLProcessor + pipe.processor = Qwen2VLProcessor.from_pretrained(processor_config.path) + + # Replace RoPE with the mosaic-aware variant + origin_rope = pipe.dit.pos_embed + pipe.dit.pos_embed = QwenVideoEditRope( + theta=origin_rope.theta, axes_dim=origin_rope.axes_dim, + scale_rope=origin_rope.scale_rope, + ).to(device) + + # Wan latent <-> Qwen token projections + inner_dim = pipe.dit.img_in.out_features + pipe.in_proj = WanToQwenProjection(in_channels=16, inner_dim=inner_dim).to(device, torch_dtype) + pipe.out_proj = QwenToWanProjection(out_channels=16, inner_dim=inner_dim).to(device, torch_dtype) + pipe.in_proj.init_from_qwen_dit(pipe.dit) + pipe.out_proj.init_from_qwen_dit(pipe.dit) + + # Load Fine-tuned weights + if checkpoint is not None: + checkpoint.download_if_necessary() + state = load_file(checkpoint.path) + pipe.in_proj.load_state_dict({k[len("in_proj."):]: v for k, v in state.items() if k.startswith("in_proj.")}) + pipe.out_proj.load_state_dict({k[len("out_proj."):]: v for k, v in state.items() if k.startswith("out_proj.")}) + dit_state = {k[len("pipe.dit."):]: v for k, v in state.items() if k.startswith("pipe.dit.")} + if dit_state: + if any("lora" in key for key in dit_state): + pipe.load_lora(pipe.dit, state_dict=dit_state, hotload=True) + else: + pipe.dit.load_state_dict(dit_state, strict=False) + + # VRAM Management + pipe.vram_management_enabled = pipe.check_vram_management_state() + return pipe + + @staticmethod + def build_preview_grid(frames, rows=3, cols=3, target_area=1024 * 1024): + """Uniformly sampled frames tiled into one grid image -- the Qwen2.5-VL + image prompt (the VL branch sees the whole video-as-grid).""" + idx = np.linspace(0, len(frames) - 1, rows * cols).round().astype(int) + tiles = [frames[i] for i in idx] + w0, h0 = tiles[0].size + scale = (target_area / (w0 * cols * h0 * rows)) ** 0.5 + tw, th = max(int(w0 * scale) // 2 * 2, 2), max(int(h0 * scale) // 2 * 2, 2) + grid = Image.new("RGB", (tw * cols, th * rows)) + for i, tile in enumerate(tiles): + grid.paste(tile.resize((tw, th), Image.BILINEAR), ((i % cols) * tw, (i // cols) * th)) + return grid + + @torch.no_grad() + def __call__( + self, + # Video + input_video: list[Image.Image] = None, + num_frames: int = 45, + max_pixels: int = 245760, + tiled: bool = None, + preview_image: Image.Image = None, + # Prompt + prompts: list[str] = [], + negative_prompt: str = " ", + cfg_scale: float = 4.0, + # Randomness + seed: int = None, + rand_device: str = "cpu", + # Steps + num_inference_steps: int = 40, + denoising_strength: float = 1.0, + # Qwen-Video-Edit + zero_cond_t: bool = False, + # Progress bar + progress_bar_cmd = tqdm, + ): + """Edit a long video with one prompt per chunk. + + Args: + input_video: list of PIL.Image frames (the full source video). + prompts: list of strings, one per chunk. + num_frames: frames per chunk (must match training: 45). + """ + # Resolve spatial dimensions + w0, h0 = input_video[0].size + scale = min(1.0, (max_pixels / (w0 * h0)) ** 0.5) + height = max(round(h0 * scale / 16), 1) * 16 + width = max(round(w0 * scale / 16), 1) * 16 + + if (w0, h0) != (width, height): + input_video = [frame.resize((width, height), Image.BILINEAR) for frame in input_video] + video = self.preprocess_video(input_video, torch_dtype=self.torch_dtype, device=self.device) + + total_frames = video.shape[2] + n_chunks = max(1, (total_frames + num_frames - 1) // num_frames) + results = [] + + for cid in range(n_chunks): + start = cid * num_frames + end = min(start + num_frames, total_frames) + chunk = video[:, :, start:end] + if chunk.shape[2] < num_frames: + pad = video[:, :, -1:].expand(1, chunk.shape[1], num_frames - chunk.shape[2], chunk.shape[3], chunk.shape[4]) + chunk = torch.cat([chunk, pad], dim=2) + prompt = prompts[cid] if cid < len(prompts) else prompts[-1] + height_chunk, width_chunk = chunk.shape[-2:] + encode_tiled = tiled if tiled is not None else (height_chunk * width_chunk >= 700_000) + + self.load_models_to_device(["video_vae"]) + ref = self.video_vae.encode([chunk[0]], device=self.device, tiled=encode_tiled).to(dtype=self.torch_dtype, device=self.device) + + if preview_image is None: + chunk_frames = [input_video[min(start + t, total_frames - 1)] for t in range(min(num_frames, end - start))] + preview = self.build_preview_grid(chunk_frames) + else: + preview = preview_image + emb = self.prompt_embedder.process(self, prompt=prompt, edit_image=preview) + neg_emb = self.prompt_embedder.process(self, prompt=negative_prompt, edit_image=preview) if cfg_scale > 1 else None + + # Scheduler + group = getattr(self.in_proj, "group", 1) + noise_seq_len = (ref.shape[2] // group) * (ref.shape[3] // 2) * (ref.shape[4] // 2) + self.scheduler.set_timesteps(num_inference_steps, denoising_strength=denoising_strength, dynamic_shift_len=noise_seq_len) + + # Denoise + self.load_models_to_device(self.in_iteration_models) + latents = self.generate_noise( + ref.shape, seed=seed, rand_device=rand_device, rand_torch_dtype=torch.float32, + device=self.device, torch_dtype=self.torch_dtype, + ) + for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps, desc=f"chunk {cid}")): + timestep = timestep[None].to(self.device, self.torch_dtype) + pred = self.model_fn( + self.dit, self.in_proj, self.out_proj, latents, ref, + emb["prompt_emb"], emb["prompt_emb_mask"], timestep, + zero_cond_t=zero_cond_t, + ) + if neg_emb is not None: + neg_pred = self.model_fn( + self.dit, self.in_proj, self.out_proj, latents, ref, + neg_emb["prompt_emb"], neg_emb["prompt_emb_mask"], timestep, + zero_cond_t=zero_cond_t, + ) + combined = neg_pred + cfg_scale * (pred - neg_pred) + pred = combined * (torch.norm(pred, dim=1, keepdim=True) / + torch.norm(combined, dim=1, keepdim=True).clamp_min(1e-6)) + latents = self.step(self.scheduler, latents=latents, progress_id=progress_id, noise_pred=pred) + + # Decode + decode_tiled = tiled if tiled is not None else (latents.shape[3] * 8) * (latents.shape[4] * 8) >= 700_000 + self.load_models_to_device(["video_vae"]) + edited = self.video_vae.decode(latents, device=self.device, tiled=decode_tiled)[0].cpu() + actual = end - start + edited = edited[:, :actual] if cid == n_chunks - 1 and actual < num_frames else edited + results.append(edited) + + self.load_models_to_device([]) + video = torch.cat(results, dim=1).unsqueeze(0) + return self.vae_output_to_video(video, pattern="B C T H W", min_value=-1, max_value=1) + + +def model_fn_qwen_video_edit( + dit, in_proj, out_proj, + latents, ref_latents, prompt_emb, prompt_mask, timestep, + zero_cond_t=False, +): + _, _, frames, height, width = latents.shape + groups, tokens_h, tokens_w = frames // in_proj.group, height // 2, width // 2 + rows, cols = _factorize(groups) + shapes = [] + + for base in (0, 1): + shapes.extend({"frame": base, "height": tokens_h, "width": tokens_w, + "h_off": (i // cols) * tokens_h, "w_off": (i % cols) * tokens_w, + "full_height": rows * tokens_h, "full_width": cols * tokens_w} for i in range(groups)) + image = torch.cat([in_proj(latents), in_proj(ref_latents)], dim=1) + image_len = image.shape[1] // 2 + timestep = timestep / 1000 + + if zero_cond_t: + timestep = torch.cat([timestep, timestep * 0], dim=0) + noise_len = sum(item["height"] * item["width"] for item in shapes[:groups]) + cond_len = sum(item["height"] * item["width"] for item in shapes[groups:]) + modulate_index = torch.tensor([[0] * noise_len + [1] * cond_len], device=image.device, dtype=torch.int) + else: + modulate_index = None + + conditioning = dit.time_text_embed( + timestep, image.dtype, + addition_t_cond=None if not dit.time_text_embed.use_additional_t_cond else + torch.tensor([0], device=image.device, dtype=torch.long),) + text = dit.txt_in(dit.txt_norm(prompt_emb)) + rotary = dit.pos_embed(shapes, prompt_mask.sum(dim=1).tolist(), device=image.device) + + for block in dit.transformer_blocks: + text, image = gradient_checkpoint_forward( + block, False, False, image=image, text=text, temb=conditioning, + image_rotary_emb=rotary, attention_mask=None, modulate_index=modulate_index) + + if zero_cond_t: + conditioning = conditioning.chunk(2, dim=0)[0] + image = dit.norm_out(image, conditioning)[:, :image_len] + return out_proj(image, groups, tokens_h, tokens_w) diff --git a/examples/qwen_image/model_inference/Qwen-Video-Edit.py b/examples/qwen_image/model_inference/Qwen-Video-Edit.py new file mode 100644 index 000000000..c0f2fbbd4 --- /dev/null +++ b/examples/qwen_image/model_inference/Qwen-Video-Edit.py @@ -0,0 +1,32 @@ +import torch +from modelscope import dataset_snapshot_download + +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="./data/example_image_dataset", + allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B/*", +) + +input_video = VideoData("data/example_image_dataset/wanvideo/Wan2.2-Animate-2-14B/video.mp4") +prompts = [ + "Transform the video into Japanese anime style with cel shading and clean line art, preserving the original dance motion and composition.", + "Apply a warm golden-hour color grading with soft cinematic lighting, keeping the dance motion and composition intact.", +] + +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"), + ], + video_vae_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"), + checkpoint=ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"), +) +video = pipe(input_video, prompts=prompts, cfg_scale=4.0, zero_cond_t=False, num_inference_steps=40, seed=42) +save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) From 96ce632dabf6c65f15d1194131d7027cde82ea4a Mon Sep 17 00:00:00 2001 From: yjy415 <2471352175@qq.com> Date: Wed, 26 Aug 2026 17:55:01 +0800 Subject: [PATCH 2/4] fix --- examples/qwen_image/model_inference/Qwen-Video-Edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/qwen_image/model_inference/Qwen-Video-Edit.py b/examples/qwen_image/model_inference/Qwen-Video-Edit.py index c0f2fbbd4..d0cb6778e 100644 --- a/examples/qwen_image/model_inference/Qwen-Video-Edit.py +++ b/examples/qwen_image/model_inference/Qwen-Video-Edit.py @@ -28,5 +28,5 @@ video_vae_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"), checkpoint=ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"), ) -video = pipe(input_video, prompts=prompts, cfg_scale=4.0, zero_cond_t=False, num_inference_steps=40, seed=42) +video = pipe(input_video, prompts=prompts, cfg_scale=4.0, num_inference_steps=40, seed=0) save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) From 18a556fa1178794d1561fca6478dbacd6c49335e Mon Sep 17 00:00:00 2001 From: yjy415 <2471352175@qq.com> Date: Fri, 28 Aug 2026 15:50:39 +0800 Subject: [PATCH 3/4] fix: Qwen-Video-Edit --- README.md | 1 + README_zh.md | 1 + diffsynth/configs/model_configs.py | 19 +- .../configs/vram_management_module_maps.py | 4 + diffsynth/models/qwen_video_edit_dit.py | 89 ++++ diffsynth/pipelines/qwen_video_edit.py | 499 +++++++++--------- .../state_dict_converters/qwen_video_edit.py | 6 + docs/en/Model_Details/Qwen-Video-Edit.md | 146 +++++ docs/en/README.md | 1 + docs/en/index.rst | 1 + docs/zh/Model_Details/Qwen-Video-Edit.md | 146 +++++ docs/zh/README.md | 1 + docs/zh/index.rst | 1 + .../model_inference/Qwen-Video-Edit.py | 32 -- .../model_inference/Qwen-Video-Edit.py | 27 + .../Qwen-Video-Edit.py | 40 ++ .../model_training/full/Qwen-Video-Edit.sh | 19 + .../full/accelerate_config_zero3.yaml | 23 + .../model_training/lora/Qwen-Video-Edit.sh | 22 + .../qwen_video_edit/model_training/train.py | 175 ++++++ .../validate_full/Qwen-Video-Edit.py | 25 + .../validate_lora/Qwen-Video-Edit.py | 23 + 22 files changed, 1028 insertions(+), 273 deletions(-) create mode 100644 diffsynth/models/qwen_video_edit_dit.py create mode 100644 diffsynth/utils/state_dict_converters/qwen_video_edit.py create mode 100644 docs/en/Model_Details/Qwen-Video-Edit.md create mode 100644 docs/zh/Model_Details/Qwen-Video-Edit.md delete mode 100644 examples/qwen_image/model_inference/Qwen-Video-Edit.py create mode 100644 examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py create mode 100644 examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py create mode 100644 examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh create mode 100644 examples/qwen_video_edit/model_training/full/accelerate_config_zero3.yaml create mode 100644 examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh create mode 100644 examples/qwen_video_edit/model_training/train.py create mode 100644 examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py create mode 100644 examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py diff --git a/README.md b/README.md index 9d9fbc040..2820dc814 100644 --- a/README.md +++ b/README.md @@ -695,6 +695,7 @@ https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/59fb2f7b-8de0-44 | Qwen-Image | [DiffSynth-Studio/Qwen-Image-In-Context-Control-Union](https://www.modelscope.cn/models/DiffSynth-Studio/Qwen-Image-In-Context-Control-Union) | [code](/examples/qwen_image/model_inference/Qwen-Image-In-Context-Control-Union.py) | [code](/examples/qwen_image/model_inference_low_vram/Qwen-Image-In-Context-Control-Union.py) | - | - | [code](/examples/qwen_image/model_training/lora/Qwen-Image-In-Context-Control-Union.sh) | [code](/examples/qwen_image/model_training/validate_lora/Qwen-Image-In-Context-Control-Union.py) | | Qwen-Image | [DiffSynth-Studio/Qwen-Image-Edit-Lowres-Fix](https://www.modelscope.cn/models/DiffSynth-Studio/Qwen-Image-Edit-Lowres-Fix) | [code](/examples/qwen_image/model_inference/Qwen-Image-Edit-Lowres-Fix.py) | [code](/examples/qwen_image/model_inference_low_vram/Qwen-Image-Edit-Lowres-Fix.py) | - | - | - | - | | Qwen-Image | [DiffSynth-Studio/Qwen-Image-i2L](https://www.modelscope.cn/models/DiffSynth-Studio/Qwen-Image-i2L) | [code](/examples/qwen_image/model_inference/Qwen-Image-i2L.py) | [code](/examples/qwen_image/model_inference_low_vram/Qwen-Image-i2L.py) | - | - | - | - | +| Qwen-Video-Edit | [yunpeng1998/Qwen-Video-Edit](https://www.modelscope.cn/models/yunpeng1998/Qwen-Video-Edit) | [code](/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py) | [code](/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py) | [code](/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh) | [code](/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py) | [code](/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh) | [code](/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py) | | Wan | [Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B) | [code](/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py) | [code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py) | [code](/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh) | [code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py) | [code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh) | [code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py) | | Wan | [Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B) | [code](/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py) | [code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py) | [code](/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh) | [code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py) | [code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh) | [code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py) | | Wan | [Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | [code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py) | [code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py) | [code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh) | [code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py) | [code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh) | [code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py) | diff --git a/README_zh.md b/README_zh.md index 09f3c5ad1..1cd1cfc53 100644 --- a/README_zh.md +++ b/README_zh.md @@ -656,6 +656,7 @@ https://github.com/Artiprocher/DiffSynth-Studio/assets/35051019/59fb2f7b-8de0-44 | Qwen-Image | [DiffSynth-Studio/Qwen-Image-In-Context-Control-Union](https://www.modelscope.cn/models/DiffSynth-Studio/Qwen-Image-In-Context-Control-Union) | [code](/examples/qwen_image/model_inference/Qwen-Image-In-Context-Control-Union.py) | [code](/examples/qwen_image/model_inference_low_vram/Qwen-Image-In-Context-Control-Union.py) | - | - | [code](/examples/qwen_image/model_training/lora/Qwen-Image-In-Context-Control-Union.sh) | [code](/examples/qwen_image/model_training/validate_lora/Qwen-Image-In-Context-Control-Union.py) | | Qwen-Image | [DiffSynth-Studio/Qwen-Image-Edit-Lowres-Fix](https://www.modelscope.cn/models/DiffSynth-Studio/Qwen-Image-Edit-Lowres-Fix) | [code](/examples/qwen_image/model_inference/Qwen-Image-Edit-Lowres-Fix.py) | [code](/examples/qwen_image/model_inference_low_vram/Qwen-Image-Edit-Lowres-Fix.py) | - | - | - | - | | Qwen-Image | [DiffSynth-Studio/Qwen-Image-i2L](https://www.modelscope.cn/models/DiffSynth-Studio/Qwen-Image-i2L) | [code](/examples/qwen_image/model_inference/Qwen-Image-i2L.py) | [code](/examples/qwen_image/model_inference_low_vram/Qwen-Image-i2L.py) | - | - | - | - | +| Qwen-Video-Edit | [yunpeng1998/Qwen-Video-Edit](https://www.modelscope.cn/models/yunpeng1998/Qwen-Video-Edit) | [code](/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py) | [code](/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py) | [code](/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh) | [code](/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py) | [code](/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh) | [code](/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py) | | Wan | [Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B) | [code](/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py) | [code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py) | [code](/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh) | [code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py) | [code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh) | [code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py) | | Wan | [Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B) | [code](/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py) | [code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py) | [code](/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh) | [code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py) | [code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh) | [code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py) | | Wan | [Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | [code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py) | [code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py) | [code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh) | [code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py) | [code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh) | [code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py) | diff --git a/diffsynth/configs/model_configs.py b/diffsynth/configs/model_configs.py index f013a258c..3de21f9bf 100644 --- a/diffsynth/configs/model_configs.py +++ b/diffsynth/configs/model_configs.py @@ -80,6 +80,23 @@ }, ] +qwen_video_edit_series = [ + { + # Example: ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors") + "model_hash": "8ae0ca4ab286d00197f08986c7fbbade", + "model_name": "qwen_video_edit_dit", + "model_class": "diffsynth.models.qwen_image_dit.QwenImageDiT", + "state_dict_converter": "diffsynth.utils.state_dict_converters.qwen_video_edit.QwenVideoEditDiTStateDictConverter", + }, + { + # Example: ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors") + "model_hash": "8ae0ca4ab286d00197f08986c7fbbade", + "model_name": "qwen_video_edit_adapter", + "model_class": "diffsynth.models.qwen_video_edit_dit.QwenVideoEditAdapter", + "state_dict_converter": "diffsynth.utils.state_dict_converters.qwen_video_edit.QwenVideoEditAdapterStateDictConverter", + }, +] + wan_series = [ { # Example: ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors") @@ -1546,7 +1563,7 @@ ] MODEL_CONFIGS = ( - stable_diffusion_xl_series + stable_diffusion_series + qwen_image_series + wan_series + flux_series + flux2_series + ernie_image_series + stable_diffusion_xl_series + stable_diffusion_series + qwen_image_series + qwen_video_edit_series + wan_series + flux_series + flux2_series + ernie_image_series + z_image_series + ltx2_series + anima_series + mova_series + joyai_image_series + boogu_image_series + ace_step_series + hidream_o1_image_series + image_metrics_series + ideogram4_series + krea2_series + lingbot_video_series + minimax_h3_series + minimax_music3_series ) diff --git a/diffsynth/configs/vram_management_module_maps.py b/diffsynth/configs/vram_management_module_maps.py index 0e836bd00..427a02eaf 100644 --- a/diffsynth/configs/vram_management_module_maps.py +++ b/diffsynth/configs/vram_management_module_maps.py @@ -62,6 +62,10 @@ "torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear", "torch.nn.Embedding": "diffsynth.core.vram.layers.AutoWrappedModule", }, + "diffsynth.models.qwen_video_edit_dit.QwenVideoEditAdapter": { + "torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear", + "torch.nn.Conv3d": "diffsynth.core.vram.layers.AutoWrappedModule", + }, "diffsynth.models.qwen_image_text_encoder.QwenImageTextEncoder": { "torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear", "torch.nn.Embedding": "diffsynth.core.vram.layers.AutoWrappedModule", diff --git a/diffsynth/models/qwen_video_edit_dit.py b/diffsynth/models/qwen_video_edit_dit.py new file mode 100644 index 000000000..33d646603 --- /dev/null +++ b/diffsynth/models/qwen_video_edit_dit.py @@ -0,0 +1,89 @@ +import torch, torch.nn as nn +from einops import rearrange + +from .qwen_image_dit import QwenEmbedRope + + +class QwenVideoEditRope(QwenEmbedRope): + """Grid-aware RoPE for Qwen-Video-Edit""" + + def _expand_pos_freqs_if_needed(self, video_fhw, txt_seq_lens): + if isinstance(video_fhw, list) and video_fhw and isinstance(video_fhw[0], dict): + video_fhw = (max(x["frame"] + 1 for x in video_fhw), + max(x["full_height"] for x in video_fhw), + max(x["full_width"] for x in video_fhw)) + super()._expand_pos_freqs_if_needed(video_fhw, txt_seq_lens) + + def forward(self, video_fhw, txt_seq_lens, device): + if not video_fhw or not isinstance(video_fhw[0], dict): + return super().forward(video_fhw, txt_seq_lens, device) + self._expand_pos_freqs_if_needed(video_fhw, txt_seq_lens) + if self.pos_freqs.device != device: + self.pos_freqs = self.pos_freqs.to(device) + self.neg_freqs = self.neg_freqs.to(device) + freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) + freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) + values = [] + max_index = 0 + for item in video_fhw: + frame, h, w = item["frame"], item["height"], item["width"] + full_h, full_w = item["full_height"], item["full_width"] + key = "grid_" + "_".join(str(item[x]) for x in + ("frame", "height", "width", "h_off", "w_off", "full_height", "full_width")) + if key not in self.rope_cache: + axis_h = freqs_pos[1][:full_h] + axis_w = freqs_pos[2][:full_w] + if self.scale_rope: + axis_h = torch.cat([freqs_neg[1][-(full_h - full_h // 2):], freqs_pos[1][:full_h // 2]]) + axis_w = torch.cat([freqs_neg[2][-(full_w - full_w // 2):], freqs_pos[2][:full_w // 2]]) + frame_freq = freqs_pos[0][frame:frame + 1].view(1, 1, 1, -1).expand(1, h, w, -1) + height_freq = axis_h[item["h_off"]:item["h_off"] + h].view(1, h, 1, -1).expand(1, h, w, -1) + width_freq = axis_w[item["w_off"]:item["w_off"] + w].view(1, 1, w, -1).expand(1, h, w, -1) + self.rope_cache[key] = torch.cat([frame_freq, height_freq, width_freq], dim=-1).reshape(h * w, -1).contiguous() + values.append(self.rope_cache[key]) + if self.scale_rope: + max_index = max(full_h // 2, full_w // 2, max_index) + else: + max_index = max(full_h, full_w, max_index) + return torch.cat(values, dim=0), self.pos_freqs[max_index:max_index + max(txt_seq_lens)] + + +class WanToQwenProjection(nn.Module): + def __init__(self, in_channels=16, inner_dim=3072): + super().__init__() + self.group = 1 + self.proj = nn.Conv3d(in_channels, inner_dim, (1, 2, 2), stride=(1, 2, 2)) + + @torch.no_grad() + def init_from_qwen_dit(self, dit): + self.proj.weight.copy_(dit.img_in.weight.view(self.proj.out_channels, self.proj.in_channels, 2, 2).unsqueeze(2)) + self.proj.bias.copy_(dit.img_in.bias) + + def forward(self, x): + return rearrange(self.proj(x), "B D T H W -> B (T H W) D") + + +class QwenToWanProjection(nn.Module): + def __init__(self, out_channels=16, inner_dim=3072): + super().__init__() + self.group = 1 + self.proj = nn.Linear(inner_dim, out_channels * 4) + + @torch.no_grad() + def init_from_qwen_dit(self, dit): + self.proj.load_state_dict(dit.proj_out.state_dict()) + + def forward(self, x, num_frames, tokens_h, tokens_w): + return rearrange(self.proj(x), "B (T H W) (C P Q) -> B C T (H P) (W Q)", T=num_frames, H=tokens_h, W=tokens_w, P=2, Q=2) + + +class QwenVideoEditAdapter(nn.Module): + def __init__(self, inner_dim=3072, in_channels=16, out_channels=16): + super().__init__() + self.in_proj = WanToQwenProjection(in_channels=in_channels, inner_dim=inner_dim) + self.out_proj = QwenToWanProjection(out_channels=out_channels, inner_dim=inner_dim) + + @torch.no_grad() + def init_from_qwen_dit(self, dit): + self.in_proj.init_from_qwen_dit(dit) + self.out_proj.init_from_qwen_dit(dit) \ No newline at end of file diff --git a/diffsynth/pipelines/qwen_video_edit.py b/diffsynth/pipelines/qwen_video_edit.py index 6fac442de..6bca64801 100644 --- a/diffsynth/pipelines/qwen_video_edit.py +++ b/diffsynth/pipelines/qwen_video_edit.py @@ -1,97 +1,17 @@ import torch, math from typing import Union -from einops import rearrange import numpy as np from PIL import Image -from safetensors.torch import load_file from tqdm import tqdm -from ..core import ModelConfig, gradient_checkpoint_forward from ..core.device.npu_compatible_device import get_device_type -from ..diffusion.base_pipeline import BasePipeline -from ..models.qwen_image_dit import QwenEmbedRope -from .qwen_image import QwenImageUnit_PromptEmbedder from ..diffusion import FlowMatchScheduler - - -class QwenVideoEditRope(QwenEmbedRope): - def _expand_pos_freqs_if_needed(self, video_fhw, txt_seq_lens): - if isinstance(video_fhw, list) and video_fhw and isinstance(video_fhw[0], dict): - video_fhw = ( - max(x["frame"] + 1 for x in video_fhw), - max(x["full_height"] for x in video_fhw), - max(x["full_width"] for x in video_fhw), - ) - super()._expand_pos_freqs_if_needed(video_fhw, txt_seq_lens) - - def forward(self, video_fhw, txt_seq_lens, device): - if not video_fhw or not isinstance(video_fhw[0], dict): - return super().forward(video_fhw, txt_seq_lens, device) - self._expand_pos_freqs_if_needed(video_fhw, txt_seq_lens) - - if self.pos_freqs.device != device: - self.pos_freqs = self.pos_freqs.to(device) - self.neg_freqs = self.neg_freqs.to(device) - freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) - freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) - - video_freqs, max_index = [], 0 - for item in video_fhw: - frame, height, width = item["frame"], item["height"], item["width"] - full_height, full_width = item["full_height"], item["full_width"] - key = "grid_" + "_".join(str(item[x]) for x in ("frame", "height", "width", "h_off", "w_off", "full_height", "full_width")) - if key not in self.rope_cache: - frame_freq = freqs_pos[0][frame:frame + 1].view(1, 1, 1, -1).expand(1, height, width, -1) - if self.scale_rope: - axis_h = torch.cat([freqs_neg[1][-(full_height - full_height // 2):], freqs_pos[1][:full_height // 2]]) - axis_w = torch.cat([freqs_neg[2][-(full_width - full_width // 2):], freqs_pos[2][:full_width // 2]]) - else: - axis_h, axis_w = freqs_pos[1][:full_height], freqs_pos[2][:full_width] - height_freq = axis_h[item["h_off"]:item["h_off"] + height].view(1, height, 1, -1).expand(1, height, width, -1) - width_freq = axis_w[item["w_off"]:item["w_off"] + width].view(1, 1, width, -1).expand(1, height, width, -1) - self.rope_cache[key] = torch.cat([frame_freq, height_freq, width_freq], dim=-1).reshape(height * width, -1).contiguous() - video_freqs.append(self.rope_cache[key]) - max_index = max(full_height // 2, full_width // 2, max_index) if self.scale_rope else max(full_height, full_width, max_index) - - text_freqs = self.pos_freqs[max_index:max_index + max(txt_seq_lens)] - return torch.cat(video_freqs, dim=0), text_freqs - - -class WanToQwenProjection(torch.nn.Module): - def __init__(self, in_channels: int = 16, inner_dim: int = 3072): - super().__init__() - self.group = 1 - self.proj = torch.nn.Conv3d(in_channels, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2)) - - @torch.no_grad() - def init_from_qwen_dit(self, dit): - self.proj.weight.copy_(dit.img_in.weight.view(self.proj.out_channels, 16, 2, 2).unsqueeze(2)) - self.proj.bias.copy_(dit.img_in.bias) - - def forward(self, x): - return rearrange(self.proj(x), "B D T H W -> B (T H W) D") - - -class QwenToWanProjection(torch.nn.Module): - def __init__(self, out_channels: int = 16, inner_dim: int = 3072): - super().__init__() - self.group = 1 - self.proj = torch.nn.Linear(inner_dim, out_channels * 4) - - @torch.no_grad() - def init_from_qwen_dit(self, dit): - self.proj.load_state_dict(dit.proj_out.state_dict()) - - def forward(self, x, num_frames, tokens_h, tokens_w): - return rearrange(self.proj(x), "B (T H W) (C P Q) -> B C T (H P) (W Q)", - T=num_frames, H=tokens_h, W=tokens_w, P=2, Q=2) - - -def _factorize(value): - for rows in range(int(math.sqrt(value)), 0, -1): - if value % rows == 0: - return rows, value // rows - return 1, value +from ..core import ModelConfig, gradient_checkpoint_forward +from ..diffusion.base_pipeline import BasePipeline, PipelineUnit +from ..models.qwen_image_dit import QwenImageDiT +from ..models.qwen_image_text_encoder import QwenImageTextEncoder +from ..models.qwen_video_edit_dit import QwenVideoEditAdapter, QwenVideoEditRope +from ..models.wan_video_vae import WanVideoVAE class QwenVideoEditPipeline(BasePipeline): @@ -100,46 +20,51 @@ def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): super().__init__( device=device, torch_dtype=torch_dtype, height_division_factor=16, width_division_factor=16, + time_division_factor=4, time_division_remainder=1, ) from transformers import Qwen2Tokenizer, Qwen2VLProcessor self.scheduler = FlowMatchScheduler("Qwen-Image") self.text_encoder: QwenImageTextEncoder = None self.dit: QwenImageDiT = None - self.video_vae = None + self.vae: WanVideoVAE = None self.tokenizer: Qwen2Tokenizer = None self.processor: Qwen2VLProcessor = None - self.in_proj: WanToQwenProjection = None - self.out_proj: QwenToWanProjection = None - self.prompt_embedder = QwenImageUnit_PromptEmbedder() - self.in_iteration_models = ("dit", "in_proj", "out_proj") - self.units = [] + self.adapter: QwenVideoEditAdapter = None + self.in_iteration_models = ("dit", "adapter") + self.units = [ + QwenVideoEditUnit_EditVideoEmbedder(), + QwenVideoEditUnit_NoiseInitializer(), + QwenVideoEditUnit_InputVideoEmbedder(), + QwenVideoEditUnit_PromptEmbedder(), + ] self.model_fn = model_fn_qwen_video_edit self.compilable_models = ["dit"] - @staticmethod def from_pretrained( torch_dtype: torch.dtype = torch.bfloat16, device: Union[str, torch.device] = get_device_type(), model_configs: list[ModelConfig] = [], - video_vae_config: ModelConfig = None, - checkpoint: ModelConfig = None, tokenizer_config: ModelConfig = ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/"), processor_config: ModelConfig = ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"), vram_limit: float = None, ): # Initialize pipeline pipe = QwenVideoEditPipeline(device=device, torch_dtype=torch_dtype) - configs = list(model_configs) - if video_vae_config is not None: - configs.append(video_vae_config) - model_pool = pipe.download_and_load_models(configs, vram_limit) + model_pool = pipe.download_and_load_models(model_configs, vram_limit) # Fetch models pipe.text_encoder = model_pool.fetch_model("qwen_image_text_encoder") - pipe.dit = model_pool.fetch_model("qwen_image_dit") - pipe.video_vae = model_pool.fetch_model("wan_video_vae") + pipe.dit = model_pool.fetch_model("qwen_video_edit_dit") + pipe.vae = model_pool.fetch_model("wan_video_vae") + pipe.adapter = model_pool.fetch_model("qwen_video_edit_adapter") + + # Size division factor derived from VAE + if pipe.vae is not None: + pipe.height_division_factor = pipe.vae.upsampling_factor * 2 + pipe.width_division_factor = pipe.vae.upsampling_factor * 2 + if tokenizer_config is not None: tokenizer_config.download_if_necessary() from transformers import Qwen2Tokenizer @@ -149,60 +74,30 @@ def from_pretrained( from transformers import Qwen2VLProcessor pipe.processor = Qwen2VLProcessor.from_pretrained(processor_config.path) - # Replace RoPE with the mosaic-aware variant + # Grid-aware RoPE origin_rope = pipe.dit.pos_embed - pipe.dit.pos_embed = QwenVideoEditRope( - theta=origin_rope.theta, axes_dim=origin_rope.axes_dim, - scale_rope=origin_rope.scale_rope, - ).to(device) - - # Wan latent <-> Qwen token projections - inner_dim = pipe.dit.img_in.out_features - pipe.in_proj = WanToQwenProjection(in_channels=16, inner_dim=inner_dim).to(device, torch_dtype) - pipe.out_proj = QwenToWanProjection(out_channels=16, inner_dim=inner_dim).to(device, torch_dtype) - pipe.in_proj.init_from_qwen_dit(pipe.dit) - pipe.out_proj.init_from_qwen_dit(pipe.dit) - - # Load Fine-tuned weights - if checkpoint is not None: - checkpoint.download_if_necessary() - state = load_file(checkpoint.path) - pipe.in_proj.load_state_dict({k[len("in_proj."):]: v for k, v in state.items() if k.startswith("in_proj.")}) - pipe.out_proj.load_state_dict({k[len("out_proj."):]: v for k, v in state.items() if k.startswith("out_proj.")}) - dit_state = {k[len("pipe.dit."):]: v for k, v in state.items() if k.startswith("pipe.dit.")} - if dit_state: - if any("lora" in key for key in dit_state): - pipe.load_lora(pipe.dit, state_dict=dit_state, hotload=True) - else: - pipe.dit.load_state_dict(dit_state, strict=False) + if not isinstance(origin_rope, QwenVideoEditRope): + pipe.dit.pos_embed = QwenVideoEditRope( + theta=origin_rope.theta, axes_dim=origin_rope.axes_dim, scale_rope=origin_rope.scale_rope, + ).to(device) # VRAM Management pipe.vram_management_enabled = pipe.check_vram_management_state() return pipe - @staticmethod - def build_preview_grid(frames, rows=3, cols=3, target_area=1024 * 1024): - """Uniformly sampled frames tiled into one grid image -- the Qwen2.5-VL - image prompt (the VL branch sees the whole video-as-grid).""" - idx = np.linspace(0, len(frames) - 1, rows * cols).round().astype(int) - tiles = [frames[i] for i in idx] - w0, h0 = tiles[0].size - scale = (target_area / (w0 * cols * h0 * rows)) ** 0.5 - tw, th = max(int(w0 * scale) // 2 * 2, 2), max(int(h0 * scale) // 2 * 2, 2) - grid = Image.new("RGB", (tw * cols, th * rows)) - for i, tile in enumerate(tiles): - grid.paste(tile.resize((tw, th), Image.BILINEAR), ((i % cols) * tw, (i // cols) * th)) - return grid @torch.no_grad() def __call__( self, # Video + edit_video: list[Image.Image] = None, input_video: list[Image.Image] = None, num_frames: int = 45, - max_pixels: int = 245760, - tiled: bool = None, - preview_image: Image.Image = None, + height: int = 384, + width: int = 640, + tiled: bool = False, + tile_size: tuple[int, int] = (30, 52), + tile_stride: tuple[int, int] = (15, 26), # Prompt prompts: list[str] = [], negative_prompt: str = " ", @@ -212,119 +107,238 @@ def __call__( rand_device: str = "cpu", # Steps num_inference_steps: int = 40, - denoising_strength: float = 1.0, # Qwen-Video-Edit zero_cond_t: bool = False, # Progress bar progress_bar_cmd = tqdm, ): - """Edit a long video with one prompt per chunk. - - Args: - input_video: list of PIL.Image frames (the full source video). - prompts: list of strings, one per chunk. - num_frames: frames per chunk (must match training: 45). - """ - # Resolve spatial dimensions - w0, h0 = input_video[0].size - scale = min(1.0, (max_pixels / (w0 * h0)) ** 0.5) - height = max(round(h0 * scale / 16), 1) * 16 - width = max(round(w0 * scale / 16), 1) * 16 - - if (w0, h0) != (width, height): - input_video = [frame.resize((width, height), Image.BILINEAR) for frame in input_video] - video = self.preprocess_video(input_video, torch_dtype=self.torch_dtype, device=self.device) - - total_frames = video.shape[2] - n_chunks = max(1, (total_frames + num_frames - 1) // num_frames) - results = [] - - for cid in range(n_chunks): - start = cid * num_frames - end = min(start + num_frames, total_frames) - chunk = video[:, :, start:end] - if chunk.shape[2] < num_frames: - pad = video[:, :, -1:].expand(1, chunk.shape[1], num_frames - chunk.shape[2], chunk.shape[3], chunk.shape[4]) - chunk = torch.cat([chunk, pad], dim=2) - prompt = prompts[cid] if cid < len(prompts) else prompts[-1] - height_chunk, width_chunk = chunk.shape[-2:] - encode_tiled = tiled if tiled is not None else (height_chunk * width_chunk >= 700_000) - - self.load_models_to_device(["video_vae"]) - ref = self.video_vae.encode([chunk[0]], device=self.device, tiled=encode_tiled).to(dtype=self.torch_dtype, device=self.device) - - if preview_image is None: - chunk_frames = [input_video[min(start + t, total_frames - 1)] for t in range(min(num_frames, end - start))] - preview = self.build_preview_grid(chunk_frames) - else: - preview = preview_image - emb = self.prompt_embedder.process(self, prompt=prompt, edit_image=preview) - neg_emb = self.prompt_embedder.process(self, prompt=negative_prompt, edit_image=preview) if cfg_scale > 1 else None - - # Scheduler - group = getattr(self.in_proj, "group", 1) - noise_seq_len = (ref.shape[2] // group) * (ref.shape[3] // 2) * (ref.shape[4] // 2) - self.scheduler.set_timesteps(num_inference_steps, denoising_strength=denoising_strength, dynamic_shift_len=noise_seq_len) + # Shape check + height, width, num_frames = self.check_resize_height_width(height, width, num_frames) + + # Scheduler + num_groups = ((num_frames - 1) // 4 + 1) // self.adapter.in_proj.group + self.scheduler.set_timesteps(num_inference_steps, dynamic_shift_len=num_groups * (height // self.height_division_factor) * (width // self.width_division_factor)) + + total_frames = len(edit_video) + num_video_chunks = (total_frames + num_frames - 1) // num_frames + num_chunks = min(len(prompts), num_video_chunks) + if len(prompts) < num_video_chunks: + print(f"Warning: only {len(prompts)} prompts provided for {num_video_chunks} chunks. " + f"The last {total_frames - num_chunks * num_frames} frames will be dropped.") + + videos = [] + for chunk_id in range(num_chunks): + inputs_posi = {"prompt": prompts[chunk_id]} + inputs_nega = {"negative_prompt": negative_prompt} + inputs_shared = { + "edit_video": edit_video, + "input_video": input_video, + "chunk_id": chunk_id, + "num_frames": num_frames, + "height": height, "width": width, + "tiled": tiled, "tile_size": tile_size, "tile_stride": tile_stride, + "cfg_scale": cfg_scale, + "seed": seed, "rand_device": rand_device, + "zero_cond_t": zero_cond_t, + } + for unit in self.units: + inputs_shared, inputs_posi, inputs_nega = self.unit_runner(unit, self, inputs_shared, inputs_posi, inputs_nega) # Denoise self.load_models_to_device(self.in_iteration_models) - latents = self.generate_noise( - ref.shape, seed=seed, rand_device=rand_device, rand_torch_dtype=torch.float32, - device=self.device, torch_dtype=self.torch_dtype, - ) - for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps, desc=f"chunk {cid}")): - timestep = timestep[None].to(self.device, self.torch_dtype) - pred = self.model_fn( - self.dit, self.in_proj, self.out_proj, latents, ref, - emb["prompt_emb"], emb["prompt_emb_mask"], timestep, - zero_cond_t=zero_cond_t, - ) - if neg_emb is not None: - neg_pred = self.model_fn( - self.dit, self.in_proj, self.out_proj, latents, ref, - neg_emb["prompt_emb"], neg_emb["prompt_emb_mask"], timestep, - zero_cond_t=zero_cond_t, - ) - combined = neg_pred + cfg_scale * (pred - neg_pred) - pred = combined * (torch.norm(pred, dim=1, keepdim=True) / - torch.norm(combined, dim=1, keepdim=True).clamp_min(1e-6)) - latents = self.step(self.scheduler, latents=latents, progress_id=progress_id, noise_pred=pred) + models = {name: getattr(self, name) for name in self.in_iteration_models} + for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps)): + timestep = timestep.unsqueeze(0).to(dtype=self.torch_dtype, device=self.device) + noise_pred_posi = self.model_fn(**models, **inputs_shared, **inputs_posi, timestep=timestep) + if cfg_scale != 1.0: + noise_pred_nega = self.model_fn(**models, **inputs_shared, **inputs_nega, timestep=timestep) + noise_pred = noise_pred_nega + cfg_scale * (noise_pred_posi - noise_pred_nega) + noise_pred = noise_pred * (torch.norm(noise_pred_posi, dim=1, keepdim=True) / torch.norm(noise_pred, dim=1, keepdim=True).clamp_min(1e-6)) + else: + noise_pred = noise_pred_posi + inputs_shared["latents"] = self.step(self.scheduler, progress_id=progress_id, noise_pred=noise_pred, **inputs_shared) # Decode - decode_tiled = tiled if tiled is not None else (latents.shape[3] * 8) * (latents.shape[4] * 8) >= 700_000 - self.load_models_to_device(["video_vae"]) - edited = self.video_vae.decode(latents, device=self.device, tiled=decode_tiled)[0].cpu() - actual = end - start - edited = edited[:, :actual] if cid == n_chunks - 1 and actual < num_frames else edited - results.append(edited) + self.load_models_to_device(["vae"]) + edited = self.vae.decode(inputs_shared["latents"], device=self.device, tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)[0].cpu() + num_valid_frames = inputs_shared["num_valid_frames"] + edited = edited[:, :num_valid_frames] + videos.append(edited) self.load_models_to_device([]) - video = torch.cat(results, dim=1).unsqueeze(0) - return self.vae_output_to_video(video, pattern="B C T H W", min_value=-1, max_value=1) + output_video = torch.cat(videos, dim=1).unsqueeze(0) + return self.vae_output_to_video(output_video, pattern="B C T H W", min_value=-1, max_value=1) + + +class QwenVideoEditUnit_EditVideoEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("edit_video", "chunk_id", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), + output_params=("edit_video_chunk", "num_valid_frames", "ref_latents"), + onload_model_names=("vae",), + ) + + @staticmethod + def encode_video_chunk(pipe: QwenVideoEditPipeline, video, chunk_id, num_frames, height, width, tiled, tile_size, tile_stride): + start = chunk_id * num_frames + end = min(start + num_frames, len(video)) + frames = [video[i].resize((width, height)) for i in range(start, end)] + padded_frames = frames + [frames[-1]] * (num_frames - len(frames)) + pipe.load_models_to_device(("vae",)) + latents = pipe.vae.encode( + pipe.preprocess_video(padded_frames), device=pipe.device, + tiled=tiled, tile_size=tile_size, tile_stride=tile_stride, + ).to(dtype=pipe.torch_dtype, device=pipe.device) + return frames, latents + + def process(self, pipe: QwenVideoEditPipeline, edit_video, chunk_id, num_frames, height, width, tiled, tile_size, tile_stride): + frames, ref_latents = self.encode_video_chunk(pipe, edit_video, chunk_id, num_frames, height, width, tiled, tile_size, tile_stride) + return {"edit_video_chunk": frames, "num_valid_frames": len(frames), "ref_latents": ref_latents} + + +class QwenVideoEditUnit_InputVideoEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("input_video", "noise", "chunk_id", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), + output_params=("latents", "input_latents"), + onload_model_names=("vae",), + ) + + def process(self, pipe: QwenVideoEditPipeline, input_video, noise, chunk_id, num_frames, height, width, tiled, tile_size, tile_stride): + if input_video is None or not pipe.scheduler.training: + return {} + _, input_latents = QwenVideoEditUnit_EditVideoEmbedder.encode_video_chunk( + pipe, input_video, chunk_id, num_frames, height, width, tiled, tile_size, tile_stride) + return {"latents": noise, "input_latents": input_latents} + + +class QwenVideoEditUnit_NoiseInitializer(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("ref_latents", "seed", "rand_device"), + output_params=("noise", "latents"), + ) + + def process(self, pipe: QwenVideoEditPipeline, ref_latents, seed, rand_device): + noise = pipe.generate_noise( + ref_latents.shape, seed=seed, rand_device=rand_device, rand_torch_dtype=torch.float32, + device=pipe.device, torch_dtype=pipe.torch_dtype, + ) + if pipe.scheduler.training: + return {"noise": noise} + return {"noise": noise, "latents": noise} + + +class QwenVideoEditUnit_PromptEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + seperate_cfg=True, + input_params_posi={"prompt": "prompt"}, + input_params_nega={"prompt": "negative_prompt"}, + input_params=("edit_video_chunk",), + output_params=("prompt_emb", "prompt_emb_mask"), + onload_model_names=("text_encoder",), + ) + + @staticmethod + def build_preview_grid(frames, rows=3, cols=3, target_area=1024 * 1024): + sample_indices = np.linspace(0, len(frames) - 1, rows * cols).round().astype(int) + tiles = [frames[i] for i in sample_indices] + tile_w, tile_h = tiles[0].size + scale = (target_area / (tile_w * cols * tile_h * rows)) ** 0.5 + grid_tile_w = max(int(tile_w * scale) // 2 * 2, 2) + grid_tile_h = max(int(tile_h * scale) // 2 * 2, 2) + grid = Image.new("RGB", (grid_tile_w * cols, grid_tile_h * rows)) + for i, tile in enumerate(tiles): + grid.paste(tile.resize((grid_tile_w, grid_tile_h)), ((i % cols) * grid_tile_w, (i // cols) * grid_tile_h)) + return grid + + def extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + return split_result + + def process(self, pipe: QwenVideoEditPipeline, prompt, edit_video_chunk): + if pipe.text_encoder is None: + return {} + pipe.load_models_to_device(self.onload_model_names) + preview = self.build_preview_grid(edit_video_chunk) + template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + drop_idx = 64 + txt = [template.format(prompt)] + model_inputs = pipe.processor(text=txt, images=preview, padding=True, return_tensors="pt").to(pipe.device) + hidden_states = pipe.text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + )[-1] + split_hidden_states = self.extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max(e.size(0) for e in split_hidden_states) + prompt_embeds = torch.stack([ + torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states + ]) + encoder_attention_mask = torch.stack([ + torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list + ]) + prompt_embeds = prompt_embeds.to(dtype=pipe.torch_dtype, device=pipe.device) + return {"prompt_emb": prompt_embeds, "prompt_emb_mask": encoder_attention_mask} + + +def factorize_grid(value): + for rows in range(int(math.sqrt(value)), 0, -1): + if value % rows == 0: + return rows, value // rows + return 1, value def model_fn_qwen_video_edit( - dit, in_proj, out_proj, - latents, ref_latents, prompt_emb, prompt_mask, timestep, - zero_cond_t=False, + dit: QwenImageDiT = None, + adapter: QwenVideoEditAdapter = None, + latents: torch.Tensor = None, + ref_latents: torch.Tensor = None, + prompt_emb: torch.Tensor = None, + prompt_emb_mask: torch.Tensor = None, + timestep: torch.Tensor = None, + zero_cond_t: bool = False, + use_gradient_checkpointing: bool = False, + use_gradient_checkpointing_offload: bool = False, + **kwargs, ): - _, _, frames, height, width = latents.shape - groups, tokens_h, tokens_w = frames // in_proj.group, height // 2, width // 2 - rows, cols = _factorize(groups) - shapes = [] - - for base in (0, 1): - shapes.extend({"frame": base, "height": tokens_h, "width": tokens_w, - "h_off": (i // cols) * tokens_h, "w_off": (i % cols) * tokens_w, - "full_height": rows * tokens_h, "full_width": cols * tokens_w} for i in range(groups)) + in_proj, out_proj = adapter.in_proj, adapter.out_proj + _, _, num_latent_frames, latent_height, latent_width = latents.shape + assert num_latent_frames % in_proj.group == 0, \ + f"num_latent_frames ({num_latent_frames}) must be divisible by in_proj.group ({in_proj.group})" + num_groups = num_latent_frames // in_proj.group + tokens_h, tokens_w = latent_height // 2, latent_width // 2 + rows, cols = factorize_grid(num_groups) + img_shapes = [] + + for frame_index in (0, 1): # frame axis: 0 for noise latents, 1 for reference latents + for group_index in range(num_groups): + img_shapes.append({ + "frame": frame_index, + "height": tokens_h, + "width": tokens_w, + "h_off": (group_index // cols) * tokens_h, + "w_off": (group_index % cols) * tokens_w, + "full_height": rows * tokens_h, + "full_width": cols * tokens_w, + }) + image = torch.cat([in_proj(latents), in_proj(ref_latents)], dim=1) - image_len = image.shape[1] // 2 + image_seq_len = image.shape[1] // 2 timestep = timestep / 1000 if zero_cond_t: timestep = torch.cat([timestep, timestep * 0], dim=0) - noise_len = sum(item["height"] * item["width"] for item in shapes[:groups]) - cond_len = sum(item["height"] * item["width"] for item in shapes[groups:]) + noise_len = sum(item["height"] * item["width"] for item in img_shapes[:num_groups]) + cond_len = sum(item["height"] * item["width"] for item in img_shapes[num_groups:]) modulate_index = torch.tensor([[0] * noise_len + [1] * cond_len], device=image.device, dtype=torch.int) else: modulate_index = None @@ -332,16 +346,21 @@ def model_fn_qwen_video_edit( conditioning = dit.time_text_embed( timestep, image.dtype, addition_t_cond=None if not dit.time_text_embed.use_additional_t_cond else - torch.tensor([0], device=image.device, dtype=torch.long),) + torch.tensor([0]).to(device=image.device, dtype=torch.long), + ) text = dit.txt_in(dit.txt_norm(prompt_emb)) - rotary = dit.pos_embed(shapes, prompt_mask.sum(dim=1).tolist(), device=image.device) + image_rotary_emb = dit.pos_embed(img_shapes, prompt_emb_mask.sum(dim=1).tolist(), device=image.device) for block in dit.transformer_blocks: text, image = gradient_checkpoint_forward( - block, False, False, image=image, text=text, temb=conditioning, - image_rotary_emb=rotary, attention_mask=None, modulate_index=modulate_index) + block, use_gradient_checkpointing, use_gradient_checkpointing_offload, + image=image, text=text, temb=conditioning, + image_rotary_emb=image_rotary_emb, attention_mask=None, modulate_index=modulate_index, + ) if zero_cond_t: conditioning = conditioning.chunk(2, dim=0)[0] - image = dit.norm_out(image, conditioning)[:, :image_len] - return out_proj(image, groups, tokens_h, tokens_w) + image = dit.norm_out(image, conditioning)[:, :image_seq_len] + output = out_proj(image, num_groups, tokens_h, tokens_w) + assert output.shape == latents.shape, f"Output shape {output.shape} != latents shape {latents.shape}" + return output diff --git a/diffsynth/utils/state_dict_converters/qwen_video_edit.py b/diffsynth/utils/state_dict_converters/qwen_video_edit.py new file mode 100644 index 000000000..7070eb5eb --- /dev/null +++ b/diffsynth/utils/state_dict_converters/qwen_video_edit.py @@ -0,0 +1,6 @@ +def QwenVideoEditDiTStateDictConverter(state_dict): + return {k[len("pipe.dit."):]: state_dict[k] for k in state_dict if k.startswith("pipe.dit.")} + + +def QwenVideoEditAdapterStateDictConverter(state_dict): + return {k: state_dict[k] for k in state_dict if k.startswith(("in_proj.", "out_proj."))} diff --git a/docs/en/Model_Details/Qwen-Video-Edit.md b/docs/en/Model_Details/Qwen-Video-Edit.md new file mode 100644 index 000000000..d265f9605 --- /dev/null +++ b/docs/en/Model_Details/Qwen-Video-Edit.md @@ -0,0 +1,146 @@ +# Qwen-Video-Edit + +Qwen-Video-Edit is a video editing model based on the Qwen-Image architecture. The model takes an input video and a text prompt, and generates an edited video that matches the prompt description. It uses QwenImageDiT as the core DiT backbone, combined with Wan2.1 VAE for video encoding/decoding, and a QwenVideoEditAdapter to project video features into the DiT feature space. + +## Installation + +Before using this project for model inference and training, please install DiffSynth-Studio first. + +```shell +git clone https://github.com/modelscope/DiffSynth-Studio.git +cd DiffSynth-Studio +pip install -e . +``` + +For more information about installation, please refer to [Install Dependencies](../Pipeline_Usage/Setup.md). + +## Quick Start + +Run the following code to quickly load the [yunpeng1998/Qwen-Video-Edit](https://www.modelscope.cn/models/yunpeng1998/Qwen-Video-Edit) model and perform inference. VRAM management is enabled, and the framework will automatically control model parameter loading based on remaining VRAM. + +```python +import torch +from modelscope import dataset_snapshot_download +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="qwen_video_edit/Qwen-Video-Edit/*" +) + +edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4") +prompts = [ + "Transform the video into Japanese anime style", +] +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0) +save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) +``` + +## Model Overview + +| Model ID | Inference | Low VRAM Inference | Full Training | Validation After Full Training | LoRA Training | Validation After LoRA Training | +| - | - | - | - | - | - | - | +| [yunpeng1998/Qwen-Video-Edit](https://www.modelscope.cn/models/yunpeng1998/Qwen-Video-Edit) | [code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py) | [code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py) | [code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh) | [code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py) | [code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh) | [code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py) | + +## Model Inference + +Models are loaded via `QwenVideoEditPipeline.from_pretrained`, see [Loading Models](../Pipeline_Usage/Model_Inference.md#loading-models). + +Input parameters for `QwenVideoEditPipeline` inference include: + +* `edit_video`: Input video, i.e., the source video to be edited. Type is `list[PIL.Image.Image]`, loaded via `VideoData`. +* `num_frames`: Number of video frames, default is 45. The model processes video in 45-frame chunks, each chunk corresponds to one prompt in the `prompts` list. +* `height`: Video height, must be a multiple of 16. +* `width`: Video width, must be a multiple of 16. +* `tiled`: Whether to enable VAE tiling inference, default is `False`. Setting to `True` can significantly reduce VRAM usage during VAE encoding/decoding stages, producing slight errors and slightly longer inference time. +* `tile_size`: Tile size during VAE encoding/decoding stages, default is `(30, 52)`, only effective when `tiled=True`. +* `tile_stride`: Tile stride during VAE encoding/decoding stages, default is `(15, 26)`, only effective when `tiled=True`, must be less than or equal to `tile_size`. +* `prompts`: List of prompts, each element corresponds to the editing instruction for one chunk. +* `negative_prompt`: Negative prompt describing content that should not appear in the video, default value is `" "`. +* `cfg_scale`: Classifier-free guidance parameter, default value is 4. When set to 1, it no longer takes effect. +* `seed`: Random seed. Default is `None`, meaning completely random. +* `rand_device`: Computing device for generating random Gaussian noise matrix, default is `"cpu"`. When set to `cuda`, different GPUs will produce different generation results. +* `num_inference_steps`: Number of inference steps, default value is 40. +* `zero_cond_t`: Whether to zero out condition features at timestep t=0. +* `progress_bar_cmd`: Progress bar, default is `tqdm.tqdm`. Can be disabled by setting to `lambda x:x`. + +## Model Training + +Qwen-Video-Edit is trained through [`examples/qwen_video_edit/model_training/train.py`](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/train.py), and the script parameters include: + +* General Training Parameters + * Dataset Basic Configuration + * `--dataset_base_path`: Root directory of the dataset. + * `--dataset_metadata_path`: Metadata file path of the dataset. + * `--dataset_repeat`: Number of times the dataset is repeated in each epoch. + * `--dataset_num_workers`: Number of processes for each DataLoader. + * `--data_file_keys`: Field names to be loaded from metadata, separated by `,`. For Qwen-Video-Edit, set to `"input_video,video"`, where `input_video` is the condition video (source video), and `video` is the target video. + * Model Loading Configuration + * `--model_paths`: Paths of models to be loaded. JSON format. + * `--model_id_with_origin_paths`: Model IDs with original paths, e.g., `"yunpeng1998/Qwen-Video-Edit:360P/step-30000.safetensors"`. Separated by commas. + * `--extra_inputs`: Extra input parameters required by the model Pipeline, separated by `,`. + * `--fp8_models`: Models loaded in FP8 format, consistent with `--model_paths` or `--model_id_with_origin_paths` format. Currently only supports models whose parameters are not updated by gradients. + * `--quant_options`: Dynamically quantize loaded models. Semicolon-separated entries, each `:[/]`. + * Training Basic Configuration + * `--learning_rate`: Learning rate. + * `--num_epochs`: Number of epochs. + * `--trainable_models`: Trainable models, e.g., `dit`, `adapter`. + * `--find_unused_parameters`: Whether there are unused parameters in DDP training, needs to be enabled to avoid errors in multi-GPU training. + * `--weight_decay`: Weight decay size, see [torch.optim.AdamW](https://docs.pytorch.org/docs/stable/generated/torch.optim.AdamW.html). + * `--task`: Training task, default is `sft`. + * Output Configuration + * `--output_path`: Model saving path. + * `--remove_prefix_in_ckpt`: Remove prefix in the state dict of the model file. + * `--save_steps`: Interval of training steps to save the model. If this parameter is left blank, the model is saved once per epoch. + * LoRA Configuration + * `--lora_base_model`: Which model to add LoRA to. + * `--lora_target_modules`: Which layers to add LoRA to. + * `--lora_rank`: Rank of LoRA. + * `--lora_checkpoint`: Path of the LoRA checkpoint. If this path is provided, LoRA will be loaded from this checkpoint. + * `--preset_lora_path`: Preset LoRA checkpoint path. If this path is provided, this LoRA will be loaded in the form of being merged into the base model. + * `--preset_lora_model`: Model that the preset LoRA is merged into, e.g., `dit`. + * Gradient Configuration + * `--use_gradient_checkpointing`: Whether to enable gradient checkpointing. + * `--use_gradient_checkpointing_offload`: Whether to offload gradient checkpointing to memory. + * `--gradient_accumulation_steps`: Number of gradient accumulation steps. + * Video Width/Height Configuration + * `--height`: Height of the video. + * `--width`: Width of the video. + * `--num_frames`: Number of video frames, default is 45. + * `--max_pixels`: Maximum pixel area of video frames. +* Qwen-Video-Edit Specific Parameters + * `--tokenizer_path`: Path of the tokenizer, leave blank to automatically download from remote. + * `--processor_path`: Path of the processor, leave blank to automatically download from remote. + * `--zero_cond_t`: Whether to zero out condition features at timestep t=0. + +We have built a sample video dataset for your testing. You can download this dataset with the following command: + +```shell +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "qwen_video_edit/Qwen-Video-Edit/*" --local_dir ./data/diffsynth_example_dataset +``` + +We have written recommended training scripts for the model, please refer to the table in the "Model Overview" section above. For how to write model training scripts, please refer to [Model Training](../Pipeline_Usage/Model_Training.md); for more advanced training algorithms, please refer to [Training Framework Detailed Explanation](https://github.com/modelscope/DiffSynth-Studio/tree/main/docs/en/Training/). diff --git a/docs/en/README.md b/docs/en/README.md index b052605c6..dbd41fcaa 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -44,6 +44,7 @@ This section introduces the Diffusion models supported by `DiffSynth-Studio`. So * [FLUX.1](./Model_Details/FLUX.md) * [Wan](./Model_Details/Wan.md) * [Qwen-Image](./Model_Details/Qwen-Image.md) +* [Qwen-Video-Edit](./Model_Details/Qwen-Video-Edit.md) * [FLUX.2](./Model_Details/FLUX2.md) * [Z-Image](./Model_Details/Z-Image.md) * [Anima](./Model_Details/Anima.md) diff --git a/docs/en/index.rst b/docs/en/index.rst index c57ef7fe4..032b1b488 100644 --- a/docs/en/index.rst +++ b/docs/en/index.rst @@ -28,6 +28,7 @@ Welcome to DiffSynth-Studio's Documentation Model_Details/FLUX Model_Details/Wan Model_Details/Qwen-Image + Model_Details/Qwen-Video-Edit Model_Details/FLUX2 Model_Details/Z-Image Model_Details/Anima diff --git a/docs/zh/Model_Details/Qwen-Video-Edit.md b/docs/zh/Model_Details/Qwen-Video-Edit.md new file mode 100644 index 000000000..8d1a66b94 --- /dev/null +++ b/docs/zh/Model_Details/Qwen-Video-Edit.md @@ -0,0 +1,146 @@ +# Qwen-Video-Edit + +Qwen-Video-Edit 是基于 Qwen-Image 架构的视频编辑模型。该模型接收一段输入视频和文本提示词，生成符合提示词描述的编辑后视频。模型采用 QwenImageDiT 作为核心 DiT 主干，结合 Wan2.1 VAE 进行视频编解码，并通过 QwenVideoEditAdapter 将视频特征投影到 DiT 的特征空间中。 + +## 安装 + +在使用本项目进行模型推理和训练前，请先安装 DiffSynth-Studio。 + +```shell +git clone https://github.com/modelscope/DiffSynth-Studio.git +cd DiffSynth-Studio +pip install -e . +``` + +更多关于安装的信息，请参考[安装依赖](../Pipeline_Usage/Setup.md)。 + +## 快速开始 + +运行以下代码可以快速加载 [yunpeng1998/Qwen-Video-Edit](https://www.modelscope.cn/models/yunpeng1998/Qwen-Video-Edit) 模型并进行推理。显存管理已启动，框架会自动根据剩余显存控制模型参数的加载。 + +```python +import torch +from modelscope import dataset_snapshot_download +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="qwen_video_edit/Qwen-Video-Edit/*" +) + +edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4") +prompts = [ + "Transform the video into Japanese anime style", +] +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0) +save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) +``` + +## 模型总览 + +|模型 ID|推理|低显存推理|全量训练|全量训练后验证|LoRA 训练|LoRA 训练后验证| +|-|-|-|-|-|-|-| +|[yunpeng1998/Qwen-Video-Edit](https://www.modelscope.cn/models/yunpeng1998/Qwen-Video-Edit)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py)| + +## 模型推理 + +模型通过 `QwenVideoEditPipeline.from_pretrained` 加载，详见[加载模型](../Pipeline_Usage/Model_Inference.md#加载模型)。 + +`QwenVideoEditPipeline` 推理的输入参数包括： + +* `edit_video`: 输入视频，即待编辑的源视频。类型为 `list[PIL.Image.Image]`，通过 `VideoData` 加载。 +* `num_frames`: 视频帧数，默认值为 45。模型以 45 帧为一个 chunk 进行处理，每个 chunk 对应 `prompts` 列表中的一条提示词。 +* `height`: 视频高度，需保证高度为 16 的倍数。 +* `width`: 视频宽度，需保证宽度为 16 的倍数。 +* `tiled`: 是否启用 VAE 分块推理，默认为 `False`。设置为 `True` 时可显著减少 VAE 编解码阶段的显存占用，会产生少许误差，以及少量推理时间延长。 +* `tile_size`: VAE 编解码阶段的分块大小，默认为 `(30, 52)`，仅在 `tiled=True` 时生效。 +* `tile_stride`: VAE 编解码阶段的分块步长，默认为 `(15, 26)`，仅在 `tiled=True` 时生效，需保证其数值小于或等于 `tile_size`。 +* `prompts`: 提示词列表，每个元素对应一个 chunk 的编辑指令。 +* `negative_prompt`: 负向提示词，描述画面中不应该出现的内容，默认值为 `" "`。 +* `cfg_scale`: Classifier-free guidance 的参数，默认值为 4，当设置为 1 时不再生效。 +* `seed`: 随机种子。默认为 `None`，即完全随机。 +* `rand_device`: 生成随机高斯噪声矩阵的计算设备，默认为 `"cpu"`。当设置为 `cuda` 时，在不同 GPU 上会导致不同的生成结果。 +* `num_inference_steps`: 推理次数，默认值为 40。 +* `zero_cond_t`: 是否在时间步 t=0 时将条件特征置零。 +* `progress_bar_cmd`: 进度条，默认为 `tqdm.tqdm`。可通过设置为 `lambda x:x` 来屏蔽进度条。 + +## 模型训练 + +Qwen-Video-Edit 通过 [`examples/qwen_video_edit/model_training/train.py`](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/qwen_video_edit/model_training/train.py) 进行训练，脚本的参数包括： + +* 通用训练参数 + * 数据集基础配置 + * `--dataset_base_path`: 数据集的根目录。 + * `--dataset_metadata_path`: 数据集的元数据文件路径。 + * `--dataset_repeat`: 每个 epoch 中数据集重复的次数。 + * `--dataset_num_workers`: 每个 Dataloader 的进程数量。 + * `--data_file_keys`: 元数据中需要加载的字段名称，以 `,` 分隔。Qwen-Video-Edit 需要设置为 `"input_video,video"`，其中 `input_video` 是条件视频（源视频），`video` 是目标视频。 + * 模型加载配置 + * `--model_paths`: 要加载的模型路径。JSON 格式。 + * `--model_id_with_origin_paths`: 带原始路径的模型 ID，例如 `"yunpeng1998/Qwen-Video-Edit:360P/step-30000.safetensors"`。用逗号分隔。 + * `--extra_inputs`: 模型 Pipeline 所需的额外输入参数，以 `,` 分隔。 + * `--fp8_models`：以 FP8 格式加载的模型，格式与 `--model_paths` 或 `--model_id_with_origin_paths` 一致，目前仅支持参数不被梯度更新的模型。 + * `--quant_options`：对加载的模型进行动态量化。以 `;` 分隔多个条目，每个为 `<模型字符串>:[/]`。 + * 训练基础配置 + * `--learning_rate`: 学习率。 + * `--num_epochs`: 轮数（Epoch）。 + * `--trainable_models`: 可训练的模型，例如 `dit`、`adapter`。 + * `--find_unused_parameters`: DDP 训练中是否存在未使用的参数，需开启这一设置避免在多 GPU 训练中报错。 + * `--weight_decay`：权重衰减大小，详见 [torch.optim.AdamW](https://docs.pytorch.org/docs/stable/generated/torch.optim.AdamW.html)。 + * `--task`: 训练任务，默认为 `sft`。 + * 输出配置 + * `--output_path`: 模型保存路径。 + * `--remove_prefix_in_ckpt`: 在模型文件的 state dict 中移除前缀。 + * `--save_steps`: 保存模型的训练步数间隔，若此参数留空，则每个 epoch 保存一次。 + * LoRA 配置 + * `--lora_base_model`: LoRA 添加到哪个模型上。 + * `--lora_target_modules`: LoRA 添加到哪些层上。 + * `--lora_rank`: LoRA 的秩（Rank）。 + * `--lora_checkpoint`: LoRA 检查点的路径。如果提供此路径，LoRA 将从此检查点加载。 + * `--preset_lora_path`: 预置 LoRA 检查点路径，如果提供此路径，这一 LoRA 将会以融入基础模型的形式加载。 + * `--preset_lora_model`: 预置 LoRA 融入的模型，例如 `dit`。 + * 梯度配置 + * `--use_gradient_checkpointing`: 是否启用 gradient checkpointing。 + * `--use_gradient_checkpointing_offload`: 是否将 gradient checkpointing 卸载到内存中。 + * `--gradient_accumulation_steps`: 梯度累积步数。 + * 视频宽高配置 + * `--height`: 视频的高度。 + * `--width`: 视频的宽度。 + * `--num_frames`: 视频的帧数，默认为 45。 + * `--max_pixels`: 视频帧的最大像素面积。 +* Qwen-Video-Edit 专有参数 + * `--tokenizer_path`: tokenizer 的路径，留空则自动从远程下载。 + * `--processor_path`: processor 的路径，留空则自动从远程下载。 + * `--zero_cond_t`: 是否在时间步 t=0 时将条件特征置零。 + +我们构建了一个样例视频数据集，以方便您进行测试，通过以下命令可以下载这个数据集： + +```shell +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "qwen_video_edit/Qwen-Video-Edit/*" --local_dir ./data/diffsynth_example_dataset +``` + +我们为模型编写了推荐的训练脚本，请参考前文"模型总览"中的表格。关于如何编写模型训练脚本，请参考[模型训练](../Pipeline_Usage/Model_Training.md)；更多高阶训练算法，请参考[训练框架详解](https://github.com/modelscope/DiffSynth-Studio/tree/main/docs/zh/Training/)。 diff --git a/docs/zh/README.md b/docs/zh/README.md index 585c9d698..2f31b88c4 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -44,6 +44,7 @@ graph LR; * [FLUX.1](./Model_Details/FLUX.md) * [Wan](./Model_Details/Wan.md) * [Qwen-Image](./Model_Details/Qwen-Image.md) +* [Qwen-Video-Edit](./Model_Details/Qwen-Video-Edit.md) * [FLUX.2](./Model_Details/FLUX2.md) * [Z-Image](./Model_Details/Z-Image.md) * [Anima](./Model_Details/Anima.md) diff --git a/docs/zh/index.rst b/docs/zh/index.rst index d7a6aed7b..6661d9638 100644 --- a/docs/zh/index.rst +++ b/docs/zh/index.rst @@ -28,6 +28,7 @@ Model_Details/FLUX Model_Details/Wan Model_Details/Qwen-Image + Model_Details/Qwen-Video-Edit Model_Details/FLUX2 Model_Details/Z-Image Model_Details/Anima diff --git a/examples/qwen_image/model_inference/Qwen-Video-Edit.py b/examples/qwen_image/model_inference/Qwen-Video-Edit.py deleted file mode 100644 index d0cb6778e..000000000 --- a/examples/qwen_image/model_inference/Qwen-Video-Edit.py +++ /dev/null @@ -1,32 +0,0 @@ -import torch -from modelscope import dataset_snapshot_download - -from diffsynth.core import ModelConfig -from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline -from diffsynth.utils.data import VideoData, save_video - - -dataset_snapshot_download( - "DiffSynth-Studio/diffsynth_example_dataset", - local_dir="./data/example_image_dataset", - allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B/*", -) - -input_video = VideoData("data/example_image_dataset/wanvideo/Wan2.2-Animate-2-14B/video.mp4") -prompts = [ - "Transform the video into Japanese anime style with cel shading and clean line art, preserving the original dance motion and composition.", - "Apply a warm golden-hour color grading with soft cinematic lighting, keeping the dance motion and composition intact.", -] - -pipe = QwenVideoEditPipeline.from_pretrained( - torch_dtype=torch.bfloat16, - device="cuda", - model_configs=[ - ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), - ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"), - ], - video_vae_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"), - checkpoint=ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"), -) -video = pipe(input_video, prompts=prompts, cfg_scale=4.0, num_inference_steps=40, seed=0) -save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) diff --git a/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py b/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py new file mode 100644 index 000000000..7a88a75d4 --- /dev/null +++ b/examples/qwen_video_edit/model_inference/Qwen-Video-Edit.py @@ -0,0 +1,27 @@ +import torch +from modelscope import dataset_snapshot_download +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="qwen_video_edit/Qwen-Video-Edit/*" +) + +edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4") +prompts = [ + "Transform the video into Japanese anime style", +] +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"), + ], +) +video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0) +save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) diff --git a/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py b/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py new file mode 100644 index 000000000..48b8bdd88 --- /dev/null +++ b/examples/qwen_video_edit/model_inference_low_vram/Qwen-Video-Edit.py @@ -0,0 +1,40 @@ +import torch +from modelscope import dataset_snapshot_download +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="qwen_video_edit/Qwen-Video-Edit/*" +) + +edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4") + +prompts = [ + "Transform the video into Japanese anime style", +] +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0) +save_video(video, "video_Qwen-Video-Edit.mp4", fps=16) diff --git a/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh b/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh new file mode 100644 index 000000000..ba939c77d --- /dev/null +++ b/examples/qwen_video_edit/model_training/full/Qwen-Video-Edit.sh @@ -0,0 +1,19 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "qwen_video_edit/Qwen-Video-Edit/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/qwen_video_edit/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit \ + --dataset_metadata_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/metadata.json \ + --data_file_keys "video,input_video" \ + --height 640 \ + --width 384 \ + --num_frames 45 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "yunpeng1998/Qwen-Video-Edit:360P/step-30000.safetensors,Qwen/Qwen-Image:text_encoder/model*.safetensors,Wan-AI/Wan2.1-T2V-1.3B:Wan2.1_VAE.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Qwen-Video-Edit_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --zero_cond_t \ + --find_unused_parameters diff --git a/examples/qwen_video_edit/model_training/full/accelerate_config_zero3.yaml b/examples/qwen_video_edit/model_training/full/accelerate_config_zero3.yaml new file mode 100644 index 000000000..e6a8d2733 --- /dev/null +++ b/examples/qwen_video_edit/model_training/full/accelerate_config_zero3.yaml @@ -0,0 +1,23 @@ +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + gradient_accumulation_steps: 1 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +enable_cpu_affinity: false +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh b/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh new file mode 100644 index 000000000..aac8ccf24 --- /dev/null +++ b/examples/qwen_video_edit/model_training/lora/Qwen-Video-Edit.sh @@ -0,0 +1,22 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "qwen_video_edit/Qwen-Video-Edit/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/qwen_video_edit/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit \ + --dataset_metadata_path data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/metadata.json \ + --data_file_keys "video,input_video" \ + --height 640 \ + --width 384 \ + --num_frames 45 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "yunpeng1998/Qwen-Video-Edit:360P/step-30000.safetensors,Qwen/Qwen-Image:text_encoder/model*.safetensors,Wan-AI/Wan2.1-T2V-1.3B:Wan2.1_VAE.pth" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Qwen-Video-Edit_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,add_q_proj,add_k_proj,add_v_proj,to_out.0,to_add_out,img_mlp.net.2,img_mod.1,txt_mlp.net.2,txt_mod.1" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --zero_cond_t \ + --dataset_num_workers 8 \ + --find_unused_parameters diff --git a/examples/qwen_video_edit/model_training/train.py b/examples/qwen_video_edit/model_training/train.py new file mode 100644 index 000000000..f17fbd4c4 --- /dev/null +++ b/examples/qwen_video_edit/model_training/train.py @@ -0,0 +1,175 @@ +import torch, os, argparse, accelerate, warnings +from diffsynth.core import UnifiedDataset, ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.diffusion import * +from diffsynth.core.data.operators import * +os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +class QwenVideoEditTrainingModule(DiffusionTrainingModule): + def __init__( + self, + model_paths=None, model_id_with_origin_paths=None, + tokenizer_path=None, processor_path=None, + trainable_models=None, + lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None, + preset_lora_path=None, preset_lora_model=None, + use_gradient_checkpointing=True, + use_gradient_checkpointing_offload=False, + extra_inputs=None, + fp8_models=None, + offload_models=None, + quant_options=None, + resume_from_checkpoint=None, remove_prefix_in_ckpt=None, + device="cpu", + task="sft", + zero_cond_t=False, + max_timestep_boundary=1.0, + min_timestep_boundary=0.0, + ): + super().__init__() + # Load models + model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device) + tokenizer_config = ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="tokenizer/") if tokenizer_path is None else ModelConfig(tokenizer_path) + processor_config = ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/") if processor_path is None else ModelConfig(processor_path) + self.pipe = QwenVideoEditPipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config, processor_config=processor_config) + self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model) + self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt) + + # Training mode + self.switch_pipe_to_training_mode( + self.pipe, trainable_models, + lora_base_model, lora_target_modules, lora_rank, lora_checkpoint, + preset_lora_path, preset_lora_model, + task=task, + ) + + # Store other configs + self.use_gradient_checkpointing = use_gradient_checkpointing + self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload + self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else [] + self.fp8_models = fp8_models + self.task = task + self.zero_cond_t = zero_cond_t + self.max_timestep_boundary = max_timestep_boundary + self.min_timestep_boundary = min_timestep_boundary + self.task_to_loss = { + "sft:data_process": lambda pipe, *args: args, + "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + } + + def get_pipeline_inputs(self, data): + inputs_posi = {"prompt": data["prompt"]} + inputs_nega = {"negative_prompt": ""} + input_video = data["input_video"] + num_frames = len(input_video) + inputs_shared = { + "edit_video": input_video, + "input_video": data["video"], + "chunk_id": 0, + "num_frames": num_frames, + "height": input_video[0].size[1], + "width": input_video[0].size[0], + "tiled": False, + "tile_size": (30, 52), + "tile_stride": (15, 26), + "cfg_scale": 1, + "rand_device": self.pipe.device, + "use_gradient_checkpointing": self.use_gradient_checkpointing, + "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload, + "zero_cond_t": self.zero_cond_t, + "max_timestep_boundary": self.max_timestep_boundary, + "min_timestep_boundary": self.min_timestep_boundary, + } + inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared) + return inputs_shared, inputs_posi, inputs_nega + + def forward(self, data, inputs=None): + if inputs is None: inputs = self.get_pipeline_inputs(data) + inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype) + for unit in self.pipe.units: + inputs = self.pipe.unit_runner(unit, self.pipe, *inputs) + loss = self.task_to_loss[self.task](self.pipe, *inputs) + return loss + + +def qwen_video_edit_parser(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser = add_general_config(parser) + parser = add_video_size_config(parser) + parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.") + parser.add_argument("--processor_path", type=str, default=None, help="Path to the processor. If provided, the processor will be used for image editing.") + parser.add_argument("--zero_cond_t", default=False, action="store_true", help="A special parameter introduced by Qwen-Image-Edit-2511. Please enable it for this model.") + parser.add_argument("--max_timestep_boundary", type=float, default=1.0, help="Max timestep boundary.") + parser.add_argument("--min_timestep_boundary", type=float, default=0.0, help="Min timestep boundary.") + parser.add_argument("--initialize_model_on_cpu", default=False, action="store_true", help="Whether to initialize models on CPU.") + return parser + + +if __name__ == "__main__": + parser = qwen_video_edit_parser() + args = parser.parse_args() + accelerator = accelerate.Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)], + ) + dataset = UnifiedDataset( + base_path=args.dataset_base_path, + metadata_path=args.dataset_metadata_path, + repeat=args.dataset_repeat, + data_file_keys=args.data_file_keys.split(","), + main_data_operator=UnifiedDataset.default_video_operator( + base_path=args.dataset_base_path, + max_pixels=args.max_pixels, + height=args.height, + width=args.width, + height_division_factor=16, + width_division_factor=16, + num_frames=args.num_frames, + time_division_factor=4, + time_division_remainder=1, + ), + ) + model = QwenVideoEditTrainingModule( + model_paths=args.model_paths, + model_id_with_origin_paths=args.model_id_with_origin_paths, + tokenizer_path=args.tokenizer_path, + processor_path=args.processor_path, + trainable_models=args.trainable_models, + lora_base_model=args.lora_base_model, + lora_target_modules=args.lora_target_modules, + lora_rank=args.lora_rank, + lora_checkpoint=args.lora_checkpoint, + preset_lora_path=args.preset_lora_path, + preset_lora_model=args.preset_lora_model, + use_gradient_checkpointing=args.use_gradient_checkpointing, + use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload, + extra_inputs=args.extra_inputs, + fp8_models=args.fp8_models, + offload_models=args.offload_models, + quant_options=args.quant_options, + resume_from_checkpoint=args.resume_from_checkpoint, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + task=args.task, + device="cpu" if (args.initialize_model_on_cpu or args.enable_model_cpu_offload) else accelerator.device, + zero_cond_t=args.zero_cond_t, + max_timestep_boundary=args.max_timestep_boundary, + min_timestep_boundary=args.min_timestep_boundary, + ) + model_logger = ModelLogger( + args.output_path, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + enable_tensorboard_log=args.enable_tensorboard_log, + enable_swanlab_log=args.enable_swanlab_log, + swanlab_project=args.swanlab_project, + enable_wandb_log=args.enable_wandb_log, + wandb_project=args.wandb_project, + enable_csv_log=args.enable_csv_log, + ) + launcher_map = { + "sft:data_process": launch_data_process_task, + "sft": launch_training_task, + "sft:train": launch_training_task, + } + launcher_map[args.task](accelerator, dataset, model, model_logger, args=args) diff --git a/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py b/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py new file mode 100644 index 000000000..cd5208cf0 --- /dev/null +++ b/examples/qwen_video_edit/model_training/validate_full/Qwen-Video-Edit.py @@ -0,0 +1,25 @@ +import torch + +from diffsynth import load_state_dict +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + +edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4") +prompts = [ + "Transform the video into Japanese anime style", +] +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"), + ], +) +state_dict = load_state_dict("models/train/Qwen-Video-Edit_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) + +video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0) +save_video(video, "video_Qwen-Video-Edit-full.mp4", fps=16) diff --git a/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py b/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py new file mode 100644 index 000000000..36bffa780 --- /dev/null +++ b/examples/qwen_video_edit/model_training/validate_lora/Qwen-Video-Edit.py @@ -0,0 +1,23 @@ +import torch + +from diffsynth.core import ModelConfig +from diffsynth.pipelines.qwen_video_edit import QwenVideoEditPipeline +from diffsynth.utils.data import VideoData, save_video + +edit_video = VideoData("data/diffsynth_example_dataset/qwen_video_edit/Qwen-Video-Edit/source.mp4") +prompts = [ + "Transform the video into Japanese anime style", +] +pipe = QwenVideoEditPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="yunpeng1998/Qwen-Video-Edit", origin_file_pattern="360P/step-30000.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="Wan2.1_VAE.pth"), + ], +) +pipe.load_lora(pipe.dit, "models/train/Qwen-Video-Edit_lora/epoch-4.safetensors") + +video = pipe(edit_video=edit_video, prompts=prompts, height=640, width=384, num_frames=45, cfg_scale=4.0, num_inference_steps=40, seed=0) +save_video(video, "video_Qwen-Video-Edit-lora.mp4", fps=16) From febe3ff13208a2f59e7c0fc6f3fe356abe8d47e6 Mon Sep 17 00:00:00 2001 From: yjy415 <2471352175@qq.com> Date: Fri, 28 Aug 2026 16:51:12 +0800 Subject: [PATCH 4/4] fix: PromptEmbedder --- diffsynth/pipelines/qwen_video_edit.py | 56 +++++++++++--------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/diffsynth/pipelines/qwen_video_edit.py b/diffsynth/pipelines/qwen_video_edit.py index 6bca64801..ddda7cff2 100644 --- a/diffsynth/pipelines/qwen_video_edit.py +++ b/diffsynth/pipelines/qwen_video_edit.py @@ -240,8 +240,14 @@ def __init__(self): onload_model_names=("text_encoder",), ) - @staticmethod - def build_preview_grid(frames, rows=3, cols=3, target_area=1024 * 1024): + def extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + return split_result + + def build_preview_grid(self, frames, rows=3, cols=3, target_area=1024 * 1024): sample_indices = np.linspace(0, len(frames) - 1, rows * cols).round().astype(int) tiles = [frames[i] for i in sample_indices] tile_w, tile_h = tiles[0].size @@ -253,39 +259,26 @@ def build_preview_grid(frames, rows=3, cols=3, target_area=1024 * 1024): grid.paste(tile.resize((grid_tile_w, grid_tile_h)), ((i % cols) * grid_tile_w, (i // cols) * grid_tile_h)) return grid - def extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): - bool_mask = mask.bool() - valid_lengths = bool_mask.sum(dim=1) - selected = hidden_states[bool_mask] - split_result = torch.split(selected, valid_lengths.tolist(), dim=0) - return split_result + def encode_prompt_edit(self, pipe: QwenVideoEditPipeline, prompt, edit_image): + template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + drop_idx = 64 + txt = [template.format(e) for e in prompt] + model_inputs = pipe.processor(text=txt, images=edit_image, padding=True, return_tensors="pt").to(pipe.device) + hidden_states = pipe.text_encoder(input_ids=model_inputs.input_ids, attention_mask=model_inputs.attention_mask, pixel_values=model_inputs.pixel_values, image_grid_thw=model_inputs.image_grid_thw, output_hidden_states=True,)[-1] + split_hidden_states = self.extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + return split_hidden_states - def process(self, pipe: QwenVideoEditPipeline, prompt, edit_video_chunk): + def process(self, pipe: QwenVideoEditPipeline, prompt, edit_video_chunk) -> dict: if pipe.text_encoder is None: return {} pipe.load_models_to_device(self.onload_model_names) - preview = self.build_preview_grid(edit_video_chunk) - template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" - drop_idx = 64 - txt = [template.format(prompt)] - model_inputs = pipe.processor(text=txt, images=preview, padding=True, return_tensors="pt").to(pipe.device) - hidden_states = pipe.text_encoder( - input_ids=model_inputs.input_ids, - attention_mask=model_inputs.attention_mask, - pixel_values=model_inputs.pixel_values, - image_grid_thw=model_inputs.image_grid_thw, - output_hidden_states=True, - )[-1] - split_hidden_states = self.extract_masked_hidden(hidden_states, model_inputs.attention_mask) - split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + preview_image = self.build_preview_grid(edit_video_chunk) + split_hidden_states = self.encode_prompt_edit(pipe, [prompt], preview_image) attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] - max_seq_len = max(e.size(0) for e in split_hidden_states) - prompt_embeds = torch.stack([ - torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states - ]) - encoder_attention_mask = torch.stack([ - torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list - ]) + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack([torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states]) + encoder_attention_mask = torch.stack([torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list]) prompt_embeds = prompt_embeds.to(dtype=pipe.torch_dtype, device=pipe.device) return {"prompt_emb": prompt_embeds, "prompt_emb_mask": encoder_attention_mask} @@ -312,8 +305,6 @@ def model_fn_qwen_video_edit( ): in_proj, out_proj = adapter.in_proj, adapter.out_proj _, _, num_latent_frames, latent_height, latent_width = latents.shape - assert num_latent_frames % in_proj.group == 0, \ - f"num_latent_frames ({num_latent_frames}) must be divisible by in_proj.group ({in_proj.group})" num_groups = num_latent_frames // in_proj.group tokens_h, tokens_w = latent_height // 2, latent_width // 2 rows, cols = factorize_grid(num_groups) @@ -362,5 +353,4 @@ def model_fn_qwen_video_edit( conditioning = conditioning.chunk(2, dim=0)[0] image = dit.norm_out(image, conditioning)[:, :image_seq_len] output = out_proj(image, num_groups, tokens_h, tokens_w) - assert output.shape == latents.shape, f"Output shape {output.shape} != latents shape {latents.shape}" return output