├── .gitignore ├── assets ├── boat.png ├── city.png ├── tennis.png ├── snowboard.png └── helicopter.png ├── configs ├── cogvideox_default.yaml ├── wan_default.yaml ├── hunyuan_video_default.yaml ├── cogvideox_alg.yaml ├── wan_alg.yaml └── hunyuan_video_alg.yaml ├── requirements.txt ├── run.py ├── lp_utils.py ├── readme.md ├── pipeline_wan_image2video_lowpass.py ├── pipeline_cogvideox_image2video_lowpass.py └── pipeline_hunyuan_video_image2video_lowpass.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | .vscode/ -------------------------------------------------------------------------------- /assets/boat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/choi403/ALG/HEAD/assets/boat.png -------------------------------------------------------------------------------- /assets/city.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/choi403/ALG/HEAD/assets/city.png -------------------------------------------------------------------------------- /assets/tennis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/choi403/ALG/HEAD/assets/tennis.png -------------------------------------------------------------------------------- /assets/snowboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/choi403/ALG/HEAD/assets/snowboard.png -------------------------------------------------------------------------------- /assets/helicopter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/choi403/ALG/HEAD/assets/helicopter.png -------------------------------------------------------------------------------- /configs/cogvideox_default.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | path: "THUDM/CogVideoX-5b-I2V" 3 | dtype: "bfloat16" 4 | 5 | generation: 6 | height: null 7 | width: null 8 | num_frames: 49 9 | num_inference_steps: 50 10 | guidance_scale: 6.0 11 | 12 | alg: 13 | use_low_pass_guidance: False 14 | 15 | video: 16 | fps: 12 -------------------------------------------------------------------------------- /configs/wan_default.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | path: "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" 3 | dtype: "bfloat16" 4 | 5 | generation: 6 | num_frames: 81 7 | num_inference_steps: 50 8 | guidance_scale: 5.0 9 | height: 480 10 | width: 832 11 | 12 | alg: 13 | use_low_pass_guidance: False 14 | 15 | video: 16 | fps: 16 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==1.3.0 2 | huggingface-hub 3 | imageio-ffmpeg 4 | open_clip_torch 5 | openai-clip 6 | opencv-python 7 | peft==0.15.0 8 | sentencepiece 9 | torchvision 10 | transformers==4.48.1 11 | xformers==0.0.29.post1 12 | av==12.0.0 13 | diffusers @ git+https://github.com/huggingface/diffusers.git@be2fb77dc164083bf8f033874b066c96bc6752b8 -------------------------------------------------------------------------------- /configs/hunyuan_video_default.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | path: "hunyuanvideo-community/HunyuanVideo-I2V" 3 | dtype: "bfloat16" 4 | flow_shift: 7.0 #7.0 if i2v_stable else 17.0 5 | flow_reverse: false 6 | 7 | generation: 8 | num_frames: 129 9 | num_inference_steps: 50 10 | guidance_scale: 6.0 11 | i2v_stable: true 12 | true_cfg_scale: 1.0 13 | 14 | alg: 15 | use_low_pass_guidance: True 16 | 17 | video: 18 | resolution: 360p 19 | fps: 30 -------------------------------------------------------------------------------- /configs/cogvideox_alg.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | path: "THUDM/CogVideoX-5b-I2V" 3 | dtype: "bfloat16" 4 | 5 | generation: 6 | height: null 7 | width: null 8 | num_frames: 49 9 | num_inference_steps: 50 10 | guidance_scale: 6.0 11 | 12 | alg: 13 | use_low_pass_guidance: True 14 | 15 | lp_filter_type: "down_up" 16 | lp_filter_in_latent: True 17 | 18 | lp_blur_sigma: null 19 | lp_blur_kernel_size: null 20 | lp_resize_factor: 0.25 21 | 22 | lp_strength_schedule_type: "interval" 23 | schedule_blur_kernel_size: False 24 | 25 | schedule_interval_start_time: 0.0 26 | schedule_interval_end_time: 0.04 27 | 28 | schedule_linear_start_weight: null 29 | schedule_linear_end_weight: null 30 | schedule_linear_end_time: null 31 | 32 | video: 33 | fps: 12 -------------------------------------------------------------------------------- /configs/wan_alg.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | path: "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" 3 | dtype: "bfloat16" 4 | 5 | generation: 6 | num_frames: 81 7 | num_inference_steps: 50 8 | guidance_scale: 5.0 9 | height: 480 10 | width: 832 11 | 12 | alg: 13 | use_low_pass_guidance: True 14 | 15 | lp_filter_type: "down_up" 16 | lp_filter_in_latent: True 17 | 18 | lp_blur_sigma: null 19 | lp_blur_kernel_size: null 20 | lp_resize_factor: 0.4 21 | 22 | lp_strength_schedule_type: "interval" 23 | schedule_blur_kernel_size: False 24 | 25 | schedule_interval_start_time: 0.0 26 | schedule_interval_end_time: 0.20 27 | 28 | schedule_linear_start_weight: null 29 | schedule_linear_end_weight: null 30 | schedule_linear_end_time: null 31 | 32 | video: 33 | fps: 16 -------------------------------------------------------------------------------- /configs/hunyuan_video_alg.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | path: "hunyuanvideo-community/HunyuanVideo-I2V" 3 | dtype: "bfloat16" 4 | flow_shift: 7.0 #7.0 if i2v_stable else 17.0 5 | flow_reverse: false 6 | 7 | generation: 8 | num_frames: 129 9 | num_inference_steps: 50 10 | guidance_scale: 6.0 11 | i2v_stable: true 12 | true_cfg_scale: 1.0 13 | 14 | alg: 15 | use_low_pass_guidance: True 16 | 17 | lp_filter_type: "down_up" 18 | lp_filter_in_latent: True 19 | 20 | lp_blur_sigma: null 21 | lp_blur_kernel_size: null 22 | lp_resize_factor: 0.625 23 | 24 | lp_strength_schedule_type: "interval" 25 | schedule_blur_kernel_size: False 26 | 27 | schedule_interval_start_time: 0.0 28 | schedule_interval_end_time: 0.04 29 | 30 | schedule_linear_start_weight: null 31 | schedule_linear_end_weight: null 32 | schedule_linear_end_time: null 33 | 34 | video: 35 | resolution: 360p 36 | fps: 30 -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import argparse 3 | import torch 4 | import torchvision 5 | from PIL import Image 6 | import logging 7 | import sys 8 | 9 | # --- Diffusers and Transformers Imports --- 10 | from diffusers import AutoencoderKLWan, UniPCMultistepScheduler, HunyuanVideoTransformer3DModel, FlowMatchEulerDiscreteScheduler 11 | from diffusers.utils import load_image 12 | from transformers import CLIPVisionModel 13 | 14 | # --- Low-pass Pipelines --- 15 | from pipeline_wan_image2video_lowpass import WanImageToVideoPipeline 16 | from pipeline_cogvideox_image2video_lowpass import CogVideoXImageToVideoPipeline 17 | from pipeline_hunyuan_video_image2video_lowpass import HunyuanVideoImageToVideoPipeline 18 | 19 | from lp_utils import get_hunyuan_video_size 20 | 21 | # --- Basic Logging Setup --- 22 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', stream=sys.stdout) 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | def main(args): 27 | # 1. Configuration 28 | IMAGE_PATH = args.image_path 29 | PROMPT = args.prompt 30 | OUTPUT_PATH = args.output_path 31 | MODEL_CACHE_DIR = args.model_cache_dir 32 | 33 | with open(args.config, 'r') as f: 34 | config = yaml.safe_load(f) 35 | 36 | model_path = config['model']['path'] 37 | model_dtype_str = config['model']['dtype'] 38 | model_dtype = getattr(torch, model_dtype_str) 39 | 40 | device = "cuda" if torch.cuda.is_available() else "cpu" 41 | 42 | logger.info(f"Using device: {device}") 43 | 44 | # 2. Pipeline preparation 45 | if "Wan" in model_path: 46 | image_encoder = CLIPVisionModel.from_pretrained(model_path, 47 | subfolder="image_encoder", 48 | torch_dtype=torch.float32, 49 | cache_dir=MODEL_CACHE_DIR 50 | ) 51 | vae = AutoencoderKLWan.from_pretrained(model_path, 52 | subfolder="vae", 53 | torch_dtype=torch.float32, 54 | cache_dir=MODEL_CACHE_DIR 55 | ) 56 | pipe = WanImageToVideoPipeline.from_pretrained(model_path, 57 | vae=vae, 58 | image_encoder=image_encoder, 59 | torch_dtype=model_dtype, 60 | cache_dir=MODEL_CACHE_DIR 61 | ) 62 | # Recommended setup (See https://github.com/huggingface/diffusers/blob/3c8b67b3711b668a6e7867e08b54280e51454eb5/src/diffusers/pipelines/wan/pipeline_wan.py#L58C13-L58C23) 63 | pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=3.0 if config['generation']['height'] == '480' else 5.0) 64 | elif "CogVideoX" in model_path: 65 | pipe = CogVideoXImageToVideoPipeline.from_pretrained( 66 | model_path, 67 | torch_dtype=model_dtype, 68 | cache_dir=MODEL_CACHE_DIR 69 | ) 70 | elif "HunyuanVideo" in model_path: 71 | transformer = HunyuanVideoTransformer3DModel.from_pretrained( 72 | model_path, 73 | subfolder="transformer", 74 | torch_dtype=torch.bfloat16, 75 | cache_dir=MODEL_CACHE_DIR 76 | ) 77 | pipe = HunyuanVideoImageToVideoPipeline.from_pretrained( 78 | model_path, transformer=transformer, 79 | torch_dtype=torch.float16, 80 | cache_dir=MODEL_CACHE_DIR 81 | ) 82 | pipe.scheduler = FlowMatchEulerDiscreteScheduler.from_config( 83 | pipe.scheduler.config, 84 | flow_shift= config['model']['flow_shift'], 85 | invert_sigmas = config['model']['flow_reverse'] 86 | ) 87 | pipe.to(device) 88 | 89 | logger.info("Pipeline loaded successfully.") 90 | 91 | # 3. Prepare inputs 92 | input_image = load_image(Image.open(IMAGE_PATH)) 93 | 94 | generator = torch.Generator(device=device).manual_seed(42) 95 | 96 | pipe_kwargs = { 97 | "image": input_image, 98 | "prompt": PROMPT, 99 | "generator": generator, 100 | } 101 | 102 | params_from_config = {**config.get('generation', {}), **config.get('alg', {})} 103 | 104 | for key, value in params_from_config.items(): 105 | if value is not None: 106 | pipe_kwargs[key] = value 107 | 108 | logger.info("Starting video generation...") 109 | log_subset = {k: v for k, v in pipe_kwargs.items() if k not in ['image', 'generator']} 110 | logger.info(f"Pipeline arguments: {log_subset}") 111 | 112 | if "HunyuanVideo" in model_path: 113 | pipe_kwargs["height"], pipe_kwargs["width"] = get_hunyuan_video_size(config['video']['resolution'], input_image) 114 | 115 | # 4. Generate video 116 | video_output = pipe(**pipe_kwargs) 117 | video_frames = video_output.frames[0] # Output is a list containing a list of PIL Images 118 | logger.info(f"Video generation complete. Received {len(video_frames)} frames.") 119 | 120 | # 5. Save video 121 | video_tensors = [torchvision.transforms.functional.to_tensor(frame) for frame in video_frames] 122 | video_tensor = torch.stack(video_tensors) # Shape: (T, C, H, W) 123 | video_tensor = video_tensor.permute(0, 2, 3, 1) # Shape: (T, H, W, C) for write_video 124 | video_tensor = (video_tensor * 255).clamp(0, 255).to(torch.uint8).cpu() 125 | 126 | logger.info(f"Saving video to: {OUTPUT_PATH}") 127 | torchvision.io.write_video( 128 | OUTPUT_PATH, 129 | video_tensor, 130 | fps=config['video']['fps'], 131 | video_codec='h264', 132 | options={'crf': '18', 'preset': 'slow'} 133 | ) 134 | logger.info("Video saved successfully. Run complete.") 135 | 136 | 137 | if __name__ == '__main__': 138 | parser = argparse.ArgumentParser(description="Arguments") 139 | parser.add_argument("--config", type=str, default="./configs/hunyuan_video_alg.yaml") 140 | parser.add_argument("--image_path", type=str, default="./assets/a red double decker bus driving down a street.jpg") 141 | parser.add_argument("--prompt", type=str, default="a red double decker bus driving down a street") 142 | parser.add_argument("--output_path", type=str, default="output.mp4") 143 | parser.add_argument("--model_cache_dir", type=str, default=None) 144 | args = parser.parse_args() 145 | 146 | main(args) -------------------------------------------------------------------------------- /lp_utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn.functional as F 4 | import torchvision.transforms.functional as tvF 5 | import numpy as np 6 | 7 | 8 | def apply_low_pass_filter( 9 | tensor: torch.Tensor, 10 | filter_type: str, 11 | # Gaussian Blur Params 12 | blur_sigma: float, 13 | blur_kernel_size: float, # Can be float (relative) or int (absolute) 14 | # Down/Up Sampling Params 15 | resize_factor: float, 16 | ): 17 | """ 18 | Applies the specified low-pass filtering operation to the input tensor. 19 | Handles 4D ([B, C, H, W]) and 5D ([B, C, F, H, W]) tensors by temporarily 20 | reshaping 5D tensors for spatial filtering. 21 | """ 22 | # --- Early Exits for No-Op Cases --- 23 | if filter_type == "none": 24 | return tensor 25 | if filter_type == "down_up" and resize_factor == 1.0: 26 | return tensor 27 | if filter_type == "gaussian_blur" and blur_sigma == 0: 28 | return tensor 29 | 30 | # --- Reshape 5D tensor for spatial filtering --- 31 | is_5d = tensor.ndim == 5 32 | if is_5d: 33 | B, C, K, H, W = tensor.shape 34 | # Flatten frames into batch dimension using view 35 | tensor = tensor.view(B * K, C, H, W) 36 | else: 37 | B, C, H, W = tensor.shape 38 | 39 | # --- Apply Selected Filter --- 40 | if filter_type == "gaussian_blur": 41 | if isinstance(blur_kernel_size, float): 42 | kernel_val = max(int(blur_kernel_size * H), 1) 43 | else: 44 | kernel_val = int(blur_kernel_size) 45 | if kernel_val % 2 == 0: 46 | kernel_val += 1 47 | tensor = tvF.gaussian_blur(tensor, kernel_size=[kernel_val, kernel_val], sigma=[blur_sigma, blur_sigma]) 48 | 49 | elif filter_type == "down_up": 50 | h0, w0 = tensor.shape[-2:] 51 | h1 = max(1, int(round(h0 * resize_factor))) 52 | w1 = max(1, int(round(w0 * resize_factor))) 53 | tensor = F.interpolate(tensor, size=(h1, w1), mode="bilinear", align_corners=False, antialias=True) 54 | tensor = F.interpolate(tensor, size=(h0, w0), mode="bilinear", align_corners=False, antialias=True) 55 | 56 | # --- Restore original 5D shape if necessary --- 57 | if is_5d: 58 | tensor = tensor.view(B, C, K, H, W) 59 | 60 | return tensor 61 | 62 | 63 | def get_lp_strength( 64 | step_index: int, 65 | total_steps: int, 66 | lp_strength_schedule_type: str, 67 | # Interval params 68 | schedule_interval_start_time: float, 69 | schedule_interval_end_time: float, 70 | # Linear params 71 | schedule_linear_start_weight: float, 72 | schedule_linear_end_weight: float, 73 | schedule_linear_end_time: float, 74 | # Exponential params 75 | schedule_exp_decay_rate: float, 76 | ) -> float: 77 | """ 78 | Calculates the low-pass guidance strength multiplier for the current timestep 79 | based on the specified schedule. 80 | """ 81 | step_norm = step_index / max(total_steps - 1, 1) 82 | 83 | if lp_strength_schedule_type == "linear": 84 | schedule_duration_fraction = schedule_linear_end_time 85 | if schedule_duration_fraction <= 0: 86 | return schedule_linear_start_weight 87 | if step_norm >= schedule_duration_fraction: 88 | current_strength = schedule_linear_end_weight 89 | else: 90 | progress = step_norm / schedule_duration_fraction 91 | current_strength = schedule_linear_start_weight * (1 - progress) + schedule_linear_end_weight * progress 92 | return current_strength 93 | 94 | elif lp_strength_schedule_type == "interval": 95 | if schedule_interval_start_time <= step_norm <= schedule_interval_end_time: 96 | return 1.0 97 | else: 98 | return 0.0 99 | 100 | elif lp_strength_schedule_type == "exponential": 101 | decay_rate = schedule_exp_decay_rate 102 | if decay_rate < 0: 103 | print(f"Warning: Negative exponential_decay_rate ({decay_rate}) is unusual. Using abs value.") 104 | decay_rate = abs(decay_rate) 105 | return math.exp(-decay_rate * step_norm) 106 | 107 | elif lp_strength_schedule_type == "none": 108 | return 1.0 109 | else: 110 | print(f"Warning: Unknown lp_strength_schedule_type '{lp_strength_schedule_type}'. Using constant strength 1.0.") 111 | return 1.0 112 | 113 | def _generate_crop_size_list(base_size=256, patch_size=32, max_ratio=4.0): 114 | """generate crop size list (HunyuanVideo) 115 | 116 | Args: 117 | base_size (int, optional): the base size for generate bucket. Defaults to 256. 118 | patch_size (int, optional): the stride to generate bucket. Defaults to 32. 119 | max_ratio (float, optional): th max ratio for h or w based on base_size . Defaults to 4.0. 120 | 121 | Returns: 122 | list: generate crop size list 123 | """ 124 | num_patches = round((base_size / patch_size) ** 2) 125 | assert max_ratio >= 1.0 126 | crop_size_list = [] 127 | wp, hp = num_patches, 1 128 | while wp > 0: 129 | if max(wp, hp) / min(wp, hp) <= max_ratio: 130 | crop_size_list.append((wp * patch_size, hp * patch_size)) 131 | if (hp + 1) * wp <= num_patches: 132 | hp += 1 133 | else: 134 | wp -= 1 135 | return crop_size_list 136 | 137 | def _get_closest_ratio(height: float, width: float, ratios: list, buckets: list): 138 | """get the closest ratio in the buckets (HunyuanVideo) 139 | 140 | Args: 141 | height (float): video height 142 | width (float): video width 143 | ratios (list): video aspect ratio 144 | buckets (list): buckets generate by `generate_crop_size_list` 145 | 146 | Returns: 147 | the closest ratio in the buckets and the corresponding ratio 148 | """ 149 | aspect_ratio = float(height) / float(width) 150 | diff_ratios = ratios - aspect_ratio 151 | 152 | if aspect_ratio >= 1: 153 | indices = [(index, x) for index, x in enumerate(diff_ratios) if x <= 0] 154 | else: 155 | indices = [(index, x) for index, x in enumerate(diff_ratios) if x > 0] 156 | 157 | closest_ratio_id = min(indices, key=lambda pair: abs(pair[1]))[0] 158 | closest_size = buckets[closest_ratio_id] 159 | closest_ratio = ratios[closest_ratio_id] 160 | 161 | return closest_size, closest_ratio 162 | 163 | def get_hunyuan_video_size(i2v_resolution, input_image): 164 | """ 165 | Map to target height and width based on resolution for HunyuanVideo 166 | 167 | Args: 168 | height (float): video height 169 | width (float): video width 170 | ratios (list): video aspect ratio 171 | buckets (list): buckets generate by `generate_crop_size_list` 172 | 173 | Returns: 174 | the closest ratio in the buckets and the corresponding ratio 175 | """ 176 | if i2v_resolution == "720p": 177 | bucket_hw_base_size = 960 178 | elif i2v_resolution == "540p": 179 | bucket_hw_base_size = 720 180 | elif i2v_resolution == "360p": 181 | bucket_hw_base_size = 480 182 | 183 | origin_size = input_image.size 184 | 185 | crop_size_list = _generate_crop_size_list(bucket_hw_base_size, 32) 186 | aspect_ratios = np.array([round(float(h)/float(w), 5) for h, w in crop_size_list]) 187 | closest_size, _ = _get_closest_ratio(origin_size[1], origin_size[0], aspect_ratios, crop_size_list) 188 | target_height, target_width = closest_size 189 | return target_height, target_width -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Enhancing Motion Dynamics of Image-to-Video Models via Adaptive Low-Pass Guidance 2 | 3 | [`Project Page`](https://choi403.github.io/ALG/) | [`arXiv`](https://arxiv.org/abs/2506.08456) | [`Gallery`](https://choi403.github.io/ALG/gallery/) 4 | 5 | Official implementation for [Enhancing Motion Dynamics of Image-to-Video Models via Adaptive Low-Pass Guidance](https://arxiv.org/abs/2506.08456) 6 |
7 | June Suk Choi, 8 | Kyungmin Lee, 9 | Sihyun Yu, 10 | Yisol Choi, 11 | Jinwoo Shin, 12 | Kimin Lee 13 | 14 | https://github.com/user-attachments/assets/a1faada7-624a-4259-8b40-dcef50700346 15 | 16 | **Summary**: We propose **Adaptive Low-pass Guidance (ALG)**, a simple yet effective sampling method for pre-trained Image-to-Video (I2V) models. ALG mitigates the common issue of motion suppression by adaptively applying low-pass filtering to the conditioning image during the early stages of the denoising process. This encourages the generation of more dynamic videos without compromising the visual quality or fidelity to the input image. 17 | 18 | ## 1. Setup 19 | ```bash 20 | conda create -n alg python=3.11 -y 21 | conda activate alg 22 | pip install -r requirements.txt # We recommend using torch version 2.5.1 and CUDA version 12.2 for the best compatibility. 23 | ``` 24 | 25 | ## 2. How to Run 26 | 27 | You can use the main script `run.py` to generate videos using our method. Configuration files are located in `./configs`. 28 | 29 | ### Basic Usage 30 | 31 | You can generate a video using the following command with your image file and prompt. 32 | 33 | ```bash 34 | python run.py \ 35 | --config [PATH_TO_CONFIG_FILE] \ 36 | --image_path [PATH_TO_INPUT_IMAGE] \ 37 | --prompt "[YOUR_PROMPT]" \ 38 | --output_path [PATH_TO_SAVE_VIDEO] 39 | ``` 40 | 41 | ### Examples 42 | We include a few example images in the asset folder, coupled with their corresponding prompts below. 43 | 44 | **Generate a video with ALG enabled (more dynamic)** 45 | ```bash 46 | python run.py \ 47 | --config ./configs/wan_alg.yaml \ 48 | --image_path ./assets/city.png \ 49 | --prompt "A car chase through narrow city streets at night." \ 50 | --output_path city_alg.mp4 51 | ``` 52 | 53 | **Generate a video without ALG (more static)** 54 | ```bash 55 | python run.py \ 56 | --config ./configs/wan_default.yaml \ 57 | --image_path ./assets/city.png \ 58 | --prompt "A car chase through narrow city streets at night." \ 59 | --output_path city_baseline.mp4 60 | ``` 61 | 62 | **Example prompts** 63 | ``` 64 | city.png: "A car chase through narrow city streets at night." 65 | snowboard.png: "A snowboarder doing a backflip off a jump." 66 | boat.png: "A group of people whitewater rafting in a canyon." 67 | helicopter.png: "A helicopter hovering over a rescue site." 68 | tennis.png: "A man swinging a tennis racquet at a tennis ball." 69 | ``` 70 | 71 | ## Configuration 72 | 73 | All generation and ALG parameters are defined in a single yaml config file (e.g., `config/wan_alg.yaml`). 74 | 75 | ### Model configuration 76 | ```yaml 77 | # configs/cogvideox_alg.yaml 78 | 79 | model: 80 | path: "THUDM/CogVideoX-5b-I2V" # Hugging Face model path 81 | dtype: "bfloat16" # Dtype for the model (e.g., float16, bfloat16, float32) 82 | 83 | generation: 84 | height: null # Output video height (null for model default) 85 | width: null # Output video width (null for model default) 86 | num_frames: 49 # Number of frames to generate 87 | num_inference_steps: 50 # Denoising steps 88 | guidance_scale: 6.0 # Classifier-Free Guidance scale 89 | 90 | video: 91 | fps: 12 # FPS for the output video file 92 | ``` 93 | 94 | ### ALG configuration (low-pass filtering) 95 | * `use_low_pass_guidance` (`bool`): Enable (`true`) or disable ALG for inference. 96 | 97 | * **Filter Settings**: Low-pass filtering characteristics. 98 | 99 | * `lp_filter_type` (`str`): Specifies the type of low-pass filter to use. 100 | * `"down_up"`: (Recommended) Bilinearly downsamples the image by `lp_resize_factor` and then upsamples it back to the original size. 101 | * `"gaussian_blur"`: Applies Gaussian blur. 102 | 103 | * `lp_filter_in_latent` (`bool`): Determines whether the filter is applied in pixel space or latent space. 104 | * `true`: (Recommended) The filter is applied to the image's latent representation after it has been encoded by the VAE. 105 | * `false`: The filter is applied directly to the RGB image *before* it is encoded by the VAE. 106 | 107 | * `lp_resize_factor` (`float`): (for `"down_up"`) 108 | * The factor by which to downsample the image (e.g., `0.25` means resizing to 25% of the original dimensions). Smaller value means stronget low-pass filtering, and potentially more motion. 109 | 110 | * `lp_blur_sigma` (`float`): (for `"gaussian_blur"`) 111 | * The standard deviation (sigma) for the Gaussian kernel. Larger values result in a stronger blur. 112 | 113 | * `lp_blur_kernel_size` (`float` | `int`): (for `"gaussian_blur"`) 114 | * The size of the blurring kernel. If a float, it's interpreted as a fraction of the image height. 115 | 116 | * **Adaptive Scheduling**: Controls how the strength of the low-pass filter changes over the denoising timesteps. 117 | 118 | * `lp_strength_schedule_type` (`str`): The scheduling strategy. Strength is a multiplier from 0.0 (off) to 1.0 (full). 119 | * `"interval"`: (Recommended) Applies the filter at full strength (`1.0`) for a specified portion of the denoising process and turns it off (`0.0`) for the rest. 120 | * `"linear"`: Linearly decays the filter strength from a starting value to an ending value. 121 | * `"exponential"`: Exponentially decays the filter strength from the beginning. 122 | * `"none"`: Applies filter at a constant strength throughout. 123 | 124 | * Parameters for `"interval"` schedule: 125 | * `schedule_interval_start_time` (`float`): The point to turn the filter on, as a fraction of total steps [`0.0`,`1.0`]. `0.0` is the first step. 126 | * `schedule_interval_end_time` (`float`): The point to turn the filter off. With 50 steps, `0.06` means the filter is active for the first `50 * 0.06 = 3` steps. 127 | 128 | * Parameters for `"linear"` schedule: 129 | * `schedule_linear_start_weight` (`float`): The filter strength at the first timestep (usually `1.0`). 130 | * `schedule_linear_end_weight` (`float`): The final filter strength to decay towards (usually `0.0`). 131 | * `schedule_linear_end_time` (`float`): The point in the process (as a fraction of total steps) at which the `end_weight` is reached. The strength remains at `end_weight` after this point. 132 | 133 | * Parameters for `"exponential"` schedule: 134 | * `schedule_exp_decay_rate` (`float`): The decay rate `r` for the formula `strength = exp(-r * time_fraction)`. Higher values cause strength to decay more quickly. 135 | 136 | * `schedule_blur_kernel_size` (`bool`): If `true` and using a scheduler with the `"gaussian_blur"` filter, the blur kernel size will also be scaled down along with the filter strength. 137 | 138 | ## 3. Supported Models 139 | 140 | We provide implementations and configurations for the following models: 141 | 142 | * **[CogVideoX](https://huggingface.co/THUDM/CogVideoX-5b-I2V)**: `THUDM/CogVideoX-5b-I2V` 143 | * **[Wan 2.1](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P-Diffusers)**: `Wan-AI/Wan2.1-I2V-14B-480P-Diffusers` 144 | * **[HunyuanVideo](https://huggingface.co/tencent/HunyuanVideo-I2V)**: `tencent/HunyuanVideo-I2V` 145 | * [LTX-Video](https://huggingface.co/Lightricks/LTX-Video): `Lightricks/LTX-Video` (Not available yet, coming soon!) 146 | 147 | We plan to add ALG implementation for LTX-Video as soon as possible! 148 | 149 | You can create new configuration files for these models by modifying the `model.path` and adjusting the `generation` and `alg` parameters accordingly. Example configs are provided in the `./configs` directory. 150 | 151 | ## 4. More Examples 152 | 153 | For more qualitative results and video comparisons, please visit the **[Gallery](https://choi403.github.io/ALG/gallery/)** on our project page. 154 | 155 | ## Acknowledgement 156 | 157 | This code is built upon [Hugging Face Diffusers](https://github.com/huggingface/diffusers) library. We thank the authors of the open-source Image-to-Video models used in our work for making their code and models publicly available. 158 | 159 | ## BibTeX 160 | 161 | If you find our work useful for your research, please consider citing our paper: 162 | 163 | ```bibtex 164 | @article{choi2025alg, 165 | title={Enhancing Motion Dynamics of Image-to-Video Models via Adaptive Low-Pass Guidance}, 166 | author={Choi, June Suk and Lee, Kyungmin and Yu, Sihyun and Choi, Yisol and Shin, Jinwoo and Lee, Kimin}, 167 | year={2025}, 168 | journal={arXiv preprint arXiv:2506.08456}, 169 | } 170 | ``` 171 | -------------------------------------------------------------------------------- /pipeline_wan_image2video_lowpass.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import html 16 | from typing import Any, Callable, Dict, List, Optional, Tuple, Union 17 | 18 | import PIL 19 | import regex as re 20 | import torch 21 | from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT5EncoderModel 22 | 23 | from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback 24 | from diffusers.image_processor import PipelineImageInput 25 | from diffusers.loaders import WanLoraLoaderMixin 26 | from diffusers.models import AutoencoderKLWan, WanTransformer3DModel 27 | from diffusers.schedulers import FlowMatchEulerDiscreteScheduler 28 | from diffusers.utils import is_ftfy_available, is_torch_xla_available, logging, replace_example_docstring 29 | from diffusers.utils.torch_utils import randn_tensor 30 | from diffusers.video_processor import VideoProcessor 31 | from diffusers.pipelines.pipeline_utils import DiffusionPipeline 32 | from diffusers.pipelines.wan.pipeline_output import WanPipelineOutput 33 | 34 | import lp_utils 35 | 36 | if is_torch_xla_available(): 37 | import torch_xla.core.xla_model as xm 38 | 39 | XLA_AVAILABLE = True 40 | else: 41 | XLA_AVAILABLE = False 42 | 43 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 44 | 45 | if is_ftfy_available(): 46 | import ftfy 47 | 48 | EXAMPLE_DOC_STRING = """ 49 | Examples: 50 | ```python 51 | >>> import torch 52 | >>> import numpy as np 53 | >>> from diffusers import AutoencoderKLWan, WanImageToVideoPipeline 54 | >>> from diffusers.utils import export_to_video, load_image 55 | >>> from transformers import CLIPVisionModel 56 | 57 | >>> # Available models: Wan-AI/Wan2.1-I2V-14B-480P-Diffusers, Wan-AI/Wan2.1-I2V-14B-720P-Diffusers 58 | >>> model_id = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" 59 | >>> image_encoder = CLIPVisionModel.from_pretrained( 60 | ... model_id, subfolder="image_encoder", torch_dtype=torch.float32 61 | ... ) 62 | >>> vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32) 63 | >>> pipe = WanImageToVideoPipeline.from_pretrained( 64 | ... model_id, vae=vae, image_encoder=image_encoder, torch_dtype=torch.bfloat16 65 | ... ) 66 | >>> pipe.to("cuda") 67 | 68 | >>> image = load_image( 69 | ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" 70 | ... ) 71 | >>> max_area = 480 * 832 72 | >>> aspect_ratio = image.height / image.width 73 | >>> mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1] 74 | >>> height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value 75 | >>> width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value 76 | >>> image = image.resize((width, height)) 77 | >>> prompt = ( 78 | ... "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in " 79 | ... "the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot." 80 | ... ) 81 | >>> negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" 82 | 83 | >>> output = pipe( 84 | ... image=image, 85 | ... prompt=prompt, 86 | ... negative_prompt=negative_prompt, 87 | ... height=height, 88 | ... width=width, 89 | ... num_frames=81, 90 | ... guidance_scale=5.0, 91 | ... ).frames[0] 92 | >>> export_to_video(output, "output.mp4", fps=16) 93 | ``` 94 | """ 95 | 96 | 97 | def basic_clean(text): 98 | text = ftfy.fix_text(text) 99 | text = html.unescape(html.unescape(text)) 100 | return text.strip() 101 | 102 | 103 | def whitespace_clean(text): 104 | text = re.sub(r"\s+", " ", text) 105 | text = text.strip() 106 | return text 107 | 108 | 109 | def prompt_clean(text): 110 | text = whitespace_clean(basic_clean(text)) 111 | return text 112 | 113 | 114 | # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents 115 | def retrieve_latents( 116 | encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" 117 | ): 118 | if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": 119 | return encoder_output.latent_dist.sample(generator) 120 | elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": 121 | return encoder_output.latent_dist.mode() 122 | elif hasattr(encoder_output, "latents"): 123 | return encoder_output.latents 124 | else: 125 | raise AttributeError("Could not access latents of provided encoder_output") 126 | 127 | 128 | class WanImageToVideoPipeline(DiffusionPipeline, WanLoraLoaderMixin): 129 | r""" 130 | Pipeline for image-to-video generation using Wan. 131 | 132 | This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods 133 | implemented for all pipelines (downloading, saving, running on a particular device, etc.). 134 | 135 | Args: 136 | tokenizer ([`T5Tokenizer`]): 137 | Tokenizer from [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5Tokenizer), 138 | specifically the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant. 139 | text_encoder ([`T5EncoderModel`]): 140 | [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically 141 | the [google/umt5-xxl](https://huggingface.co/google/umt5-xxl) variant. 142 | image_encoder ([`CLIPVisionModel`]): 143 | [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPVisionModel), specifically 144 | the 145 | [clip-vit-huge-patch14](https://github.com/mlfoundations/open_clip/blob/main/docs/PRETRAINED.md#vit-h14-xlm-roberta-large) 146 | variant. 147 | transformer ([`WanTransformer3DModel`]): 148 | Conditional Transformer to denoise the input latents. 149 | scheduler ([`UniPCMultistepScheduler`]): 150 | A scheduler to be used in combination with `transformer` to denoise the encoded image latents. 151 | vae ([`AutoencoderKLWan`]): 152 | Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. 153 | """ 154 | 155 | model_cpu_offload_seq = "text_encoder->image_encoder->transformer->vae" 156 | _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] 157 | 158 | def __init__( 159 | self, 160 | tokenizer: AutoTokenizer, 161 | text_encoder: UMT5EncoderModel, 162 | image_encoder: CLIPVisionModel, 163 | image_processor: CLIPImageProcessor, 164 | transformer: WanTransformer3DModel, 165 | vae: AutoencoderKLWan, 166 | scheduler: FlowMatchEulerDiscreteScheduler, 167 | ): 168 | super().__init__() 169 | 170 | self.register_modules( 171 | vae=vae, 172 | text_encoder=text_encoder, 173 | tokenizer=tokenizer, 174 | image_encoder=image_encoder, 175 | transformer=transformer, 176 | scheduler=scheduler, 177 | image_processor=image_processor, 178 | ) 179 | 180 | self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4 181 | self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 182 | self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) 183 | self.image_processor = image_processor 184 | 185 | def _get_t5_prompt_embeds( 186 | self, 187 | prompt: Union[str, List[str]] = None, 188 | num_videos_per_prompt: int = 1, 189 | max_sequence_length: int = 512, 190 | device: Optional[torch.device] = None, 191 | dtype: Optional[torch.dtype] = None, 192 | ): 193 | device = device or self._execution_device 194 | dtype = dtype or self.text_encoder.dtype 195 | 196 | prompt = [prompt] if isinstance(prompt, str) else prompt 197 | prompt = [prompt_clean(u) for u in prompt] 198 | batch_size = len(prompt) 199 | 200 | text_inputs = self.tokenizer( 201 | prompt, 202 | padding="max_length", 203 | max_length=max_sequence_length, 204 | truncation=True, 205 | add_special_tokens=True, 206 | return_attention_mask=True, 207 | return_tensors="pt", 208 | ) 209 | text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask 210 | seq_lens = mask.gt(0).sum(dim=1).long() 211 | 212 | prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state 213 | prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) 214 | prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] 215 | prompt_embeds = torch.stack( 216 | [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0 217 | ) 218 | 219 | # duplicate text embeddings for each generation per prompt, using mps friendly method 220 | _, seq_len, _ = prompt_embeds.shape 221 | prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) 222 | prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) 223 | 224 | return prompt_embeds 225 | 226 | def encode_image( 227 | self, 228 | image: PipelineImageInput, 229 | device: Optional[torch.device] = None, 230 | ): 231 | device = device or self._execution_device 232 | image = self.image_processor(images=image, return_tensors="pt").to(device) 233 | image_embeds = self.image_encoder(**image, output_hidden_states=True) 234 | return image_embeds.hidden_states[-2] 235 | 236 | # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt 237 | def encode_prompt( 238 | self, 239 | prompt: Union[str, List[str]], 240 | negative_prompt: Optional[Union[str, List[str]]] = None, 241 | do_classifier_free_guidance: bool = True, 242 | num_videos_per_prompt: int = 1, 243 | prompt_embeds: Optional[torch.Tensor] = None, 244 | negative_prompt_embeds: Optional[torch.Tensor] = None, 245 | max_sequence_length: int = 226, 246 | device: Optional[torch.device] = None, 247 | dtype: Optional[torch.dtype] = None, 248 | ): 249 | r""" 250 | Encodes the prompt into text encoder hidden states. 251 | 252 | Args: 253 | prompt (`str` or `List[str]`, *optional*): 254 | prompt to be encoded 255 | negative_prompt (`str` or `List[str]`, *optional*): 256 | The prompt or prompts not to guide the image generation. If not defined, one has to pass 257 | `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is 258 | less than `1`). 259 | do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): 260 | Whether to use classifier free guidance or not. 261 | num_videos_per_prompt (`int`, *optional*, defaults to 1): 262 | Number of videos that should be generated per prompt. torch device to place the resulting embeddings on 263 | prompt_embeds (`torch.Tensor`, *optional*): 264 | Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not 265 | provided, text embeddings will be generated from `prompt` input argument. 266 | negative_prompt_embeds (`torch.Tensor`, *optional*): 267 | Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt 268 | weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input 269 | argument. 270 | device: (`torch.device`, *optional*): 271 | torch device 272 | dtype: (`torch.dtype`, *optional*): 273 | torch dtype 274 | """ 275 | device = device or self._execution_device 276 | 277 | prompt = [prompt] if isinstance(prompt, str) else prompt 278 | if prompt is not None: 279 | batch_size = len(prompt) 280 | else: 281 | batch_size = prompt_embeds.shape[0] 282 | 283 | if prompt_embeds is None: 284 | prompt_embeds = self._get_t5_prompt_embeds( 285 | prompt=prompt, 286 | num_videos_per_prompt=num_videos_per_prompt, 287 | max_sequence_length=max_sequence_length, 288 | device=device, 289 | dtype=dtype, 290 | ) 291 | 292 | if do_classifier_free_guidance and negative_prompt_embeds is None: 293 | negative_prompt = negative_prompt or "" 294 | negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt 295 | 296 | if prompt is not None and type(prompt) is not type(negative_prompt): 297 | raise TypeError( 298 | f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" 299 | f" {type(prompt)}." 300 | ) 301 | elif batch_size != len(negative_prompt): 302 | raise ValueError( 303 | f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" 304 | f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" 305 | " the batch size of `prompt`." 306 | ) 307 | 308 | negative_prompt_embeds = self._get_t5_prompt_embeds( 309 | prompt=negative_prompt, 310 | num_videos_per_prompt=num_videos_per_prompt, 311 | max_sequence_length=max_sequence_length, 312 | device=device, 313 | dtype=dtype, 314 | ) 315 | 316 | return prompt_embeds, negative_prompt_embeds 317 | 318 | def check_inputs( 319 | self, 320 | prompt, 321 | negative_prompt, 322 | image, 323 | height, 324 | width, 325 | prompt_embeds=None, 326 | negative_prompt_embeds=None, 327 | image_embeds=None, 328 | callback_on_step_end_tensor_inputs=None, 329 | ): 330 | if image is not None and image_embeds is not None: 331 | raise ValueError( 332 | f"Cannot forward both `image`: {image} and `image_embeds`: {image_embeds}. Please make sure to" 333 | " only forward one of the two." 334 | ) 335 | if image is None and image_embeds is None: 336 | raise ValueError( 337 | "Provide either `image` or `prompt_embeds`. Cannot leave both `image` and `image_embeds` undefined." 338 | ) 339 | if image is not None and not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image): 340 | raise ValueError(f"`image` has to be of type `torch.Tensor` or `PIL.Image.Image` but is {type(image)}") 341 | if height % 16 != 0 or width % 16 != 0: 342 | raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") 343 | 344 | if callback_on_step_end_tensor_inputs is not None and not all( 345 | k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs 346 | ): 347 | raise ValueError( 348 | f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" 349 | ) 350 | 351 | if prompt is not None and prompt_embeds is not None: 352 | raise ValueError( 353 | f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" 354 | " only forward one of the two." 355 | ) 356 | elif negative_prompt is not None and negative_prompt_embeds is not None: 357 | raise ValueError( 358 | f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`: {negative_prompt_embeds}. Please make sure to" 359 | " only forward one of the two." 360 | ) 361 | elif prompt is None and prompt_embeds is None: 362 | raise ValueError( 363 | "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." 364 | ) 365 | elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): 366 | raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") 367 | elif negative_prompt is not None and ( 368 | not isinstance(negative_prompt, str) and not isinstance(negative_prompt, list) 369 | ): 370 | raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}") 371 | 372 | def prepare_latents( 373 | self, 374 | image: PipelineImageInput, 375 | batch_size: int, 376 | num_channels_latents: int = 16, 377 | height: int = 480, 378 | width: int = 832, 379 | num_frames: int = 81, 380 | dtype: Optional[torch.dtype] = None, 381 | device: Optional[torch.device] = None, 382 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, 383 | latents: Optional[torch.Tensor] = None, 384 | last_image: Optional[torch.Tensor] = None, 385 | ) -> Tuple[torch.Tensor, torch.Tensor]: 386 | num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 387 | latent_height = height // self.vae_scale_factor_spatial 388 | latent_width = width // self.vae_scale_factor_spatial 389 | 390 | shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width) 391 | if isinstance(generator, list) and len(generator) != batch_size: 392 | raise ValueError( 393 | f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" 394 | f" size of {batch_size}. Make sure the batch size matches the length of the generators." 395 | ) 396 | 397 | if latents is None: 398 | latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) 399 | else: 400 | latents = latents.to(device=device, dtype=dtype) 401 | 402 | image = image.unsqueeze(2) 403 | if last_image is None: 404 | video_condition = torch.cat( 405 | [image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 1, height, width)], dim=2 406 | ) 407 | else: 408 | last_image = last_image.unsqueeze(2) 409 | video_condition = torch.cat( 410 | [image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 2, height, width), last_image], 411 | dim=2, 412 | ) 413 | video_condition = video_condition.to(device=device, dtype=self.vae.dtype) 414 | 415 | latents_mean = ( 416 | torch.tensor(self.vae.config.latents_mean) 417 | .view(1, self.vae.config.z_dim, 1, 1, 1) 418 | .to(latents.device, latents.dtype) 419 | ) 420 | latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( 421 | latents.device, latents.dtype 422 | ) 423 | 424 | if isinstance(generator, list): 425 | latent_condition = [ 426 | retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") for _ in generator 427 | ] 428 | latent_condition = torch.cat(latent_condition) 429 | else: 430 | latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") 431 | latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) 432 | 433 | latent_condition = latent_condition.to(dtype) 434 | latent_condition = (latent_condition - latents_mean) * latents_std 435 | 436 | mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) 437 | 438 | if last_image is None: 439 | mask_lat_size[:, :, list(range(1, num_frames))] = 0 440 | else: 441 | mask_lat_size[:, :, list(range(1, num_frames - 1))] = 0 442 | first_frame_mask = mask_lat_size[:, :, 0:1] 443 | first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal) 444 | mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) 445 | mask_lat_size = mask_lat_size.view(batch_size, -1, self.vae_scale_factor_temporal, latent_height, latent_width) 446 | mask_lat_size = mask_lat_size.transpose(1, 2) 447 | mask_lat_size = mask_lat_size.to(latent_condition.device) 448 | 449 | return latents, torch.concat([mask_lat_size, latent_condition], dim=1) 450 | 451 | def prepare_lp( 452 | self, 453 | # --- Filter Selection & Strength --- 454 | lp_filter_type: str, 455 | lp_blur_sigma: float, 456 | lp_blur_kernel_size: float, 457 | lp_resize_factor: float, 458 | # --- Contextual Info --- 459 | generator: torch.Generator, 460 | num_frames: int, 461 | use_low_pass_guidance: bool, 462 | lp_filter_in_latent: bool, 463 | # --- Inputs to filter --- 464 | orig_image_latents: torch.Tensor, 465 | orig_image_tensor: torch.Tensor, 466 | ) -> Optional[torch.Tensor]: 467 | """ 468 | Prepares a low-pass filtered version of the initial image condition for guidance. (Wan 2.1) 469 | The resulting low-pass filtered latents are padded to match the required number of frames and temporal 470 | patch size for the transformer model. 471 | 472 | Args: 473 | lp_filter_type (`str`): The type of low-pass filter to apply, e.g., 'gaussian_blur', 'down_up'. 474 | lp_blur_sigma (`float`): The sigma value for the Gaussian blur filter. 475 | lp_blur_kernel_size (`float`): The kernel size for the Gaussian blur filter. 476 | lp_resize_factor (`float`): The resizing factor for the 'down_up' filter. 477 | generator (`torch.Generator`): A random generator, used for VAE sampling when filtering in image space. 478 | num_frames (`int`): The target number of frames for the final video, used to determine padding. 479 | use_low_pass_guidance (`bool`): If `False`, the function returns `None` immediately. 480 | lp_filter_in_latent (`bool`): If `True`, filtering is applied in latent space. Otherwise, in image space. 481 | orig_image_latents (`torch.Tensor`): The VAE-encoded latents of the original image. Used when 482 | `lp_filter_in_latent` is `True`. Shape: `(batch_size, num_frames_padded, channels, height, width)`. 483 | orig_image_tensor (`torch.Tensor`): The preprocessed original image tensor (RGB). Used when 484 | `lp_filter_in_latent` is `False`. Shape: `(batch_size, channels, height, width)`. 485 | 486 | Returns: 487 | `Optional[torch.Tensor]`: A tensor containing the low-pass filtered image latents, correctly shaped and 488 | padded for the transformer, or `None` if `use_low_pass_guidance` is `False`. 489 | """ 490 | if not use_low_pass_guidance: 491 | return None 492 | 493 | if not lp_filter_in_latent: 494 | # --- Filter in Image (RGB) Space --- 495 | image_lp = lp_utils.apply_low_pass_filter( 496 | orig_image_tensor, 497 | filter_type=lp_filter_type, 498 | blur_sigma=lp_blur_sigma, 499 | blur_kernel_size=lp_blur_kernel_size, 500 | resize_factor=lp_resize_factor, 501 | ) 502 | image_lp_vae_input = image_lp.unsqueeze(2) 503 | 504 | batch_size, _, height, width = orig_image_tensor.shape 505 | latent_height = height // self.vae_scale_factor_spatial 506 | latent_width = width // self.vae_scale_factor_spatial 507 | 508 | # --- Zero padding --- 509 | video_condition = torch.cat( 510 | [ 511 | image_lp_vae_input, 512 | image_lp_vae_input.new_zeros( 513 | image_lp_vae_input.shape[0], image_lp_vae_input.shape[1], num_frames - 1, height, width 514 | ), 515 | ], 516 | dim=2, 517 | ) 518 | latents_mean = ( 519 | torch.tensor(self.vae.config.latents_mean) 520 | .view(1, self.vae.config.z_dim, 1, 1, 1) 521 | .to(image_lp.device, image_lp.dtype) 522 | ) 523 | latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view( 524 | 1, self.vae.config.z_dim, 1, 1, 1 525 | ).to(image_lp.device, image_lp.dtype) 526 | encoded_lp = self.vae.encode(video_condition).latent_dist.sample(generator=generator) 527 | latent_condition = (encoded_lp - latents_mean) * latents_std 528 | 529 | mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) 530 | mask_lat_size[:, :, list(range(1, num_frames))] = 0 531 | first_frame_mask = mask_lat_size[:, :, 0:1] 532 | first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal) 533 | mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) 534 | mask_lat_size = mask_lat_size.view( 535 | batch_size, -1, self.vae_scale_factor_temporal, latent_height, latent_width 536 | ) 537 | mask_lat_size = mask_lat_size.transpose(1, 2) 538 | mask_lat_size = mask_lat_size.to(latent_condition.device) 539 | 540 | lp_image_latents = torch.concat([mask_lat_size, latent_condition], dim=1) 541 | else: 542 | lp_image_latents = lp_utils.apply_low_pass_filter( 543 | orig_image_latents, 544 | filter_type=lp_filter_type, 545 | blur_sigma=lp_blur_sigma, 546 | blur_kernel_size=lp_blur_kernel_size, 547 | resize_factor=lp_resize_factor, 548 | ) 549 | # Ensure the temporal dimension is divisible by the transformer's temporal patch size. 550 | if self.transformer.config.patch_size is not None: 551 | remainder = lp_image_latents.size(1) % self.transformer.config.patch_size[0] 552 | if remainder != 0: 553 | num_to_prepend = self.transformer.config.patch_size[0] - remainder 554 | num_to_prepend = min(num_to_prepend, lp_image_latents.shape[1]) 555 | first_frames_to_prepend = lp_image_latents[:, :num_to_prepend, ...] 556 | lp_image_latents = torch.cat([first_frames_to_prepend, lp_image_latents], dim=1) 557 | 558 | lp_image_latents = lp_image_latents.to(dtype=orig_image_latents.dtype) 559 | return lp_image_latents 560 | 561 | @property 562 | def guidance_scale(self): 563 | return self._guidance_scale 564 | 565 | @property 566 | def do_classifier_free_guidance(self): 567 | return self._guidance_scale > 1 568 | 569 | @property 570 | def num_timesteps(self): 571 | return self._num_timesteps 572 | 573 | @property 574 | def current_timestep(self): 575 | return self._current_timestep 576 | 577 | @property 578 | def interrupt(self): 579 | return self._interrupt 580 | 581 | @property 582 | def attention_kwargs(self): 583 | return self._attention_kwargs 584 | 585 | @torch.no_grad() 586 | @replace_example_docstring(EXAMPLE_DOC_STRING) 587 | def __call__( 588 | self, 589 | image: PipelineImageInput, 590 | prompt: Union[str, List[str]] = None, 591 | negative_prompt: Union[str, List[str]] = None, 592 | height: int = 480, 593 | width: int = 832, 594 | num_frames: int = 81, 595 | num_inference_steps: int = 50, 596 | guidance_scale: float = 5.0, 597 | num_videos_per_prompt: Optional[int] = 1, 598 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, 599 | latents: Optional[torch.Tensor] = None, 600 | prompt_embeds: Optional[torch.Tensor] = None, 601 | negative_prompt_embeds: Optional[torch.Tensor] = None, 602 | image_embeds: Optional[torch.Tensor] = None, 603 | last_image: Optional[torch.Tensor] = None, 604 | output_type: Optional[str] = "np", 605 | return_dict: bool = True, 606 | attention_kwargs: Optional[Dict[str, Any]] = None, 607 | callback_on_step_end: Optional[ 608 | Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] 609 | ] = None, 610 | callback_on_step_end_tensor_inputs: List[str] = ["latents"], 611 | max_sequence_length: int = 512, 612 | use_low_pass_guidance: bool = False, 613 | lp_filter_type: str = "none", # {'gaussian_blur', 'down_up'} 614 | lp_filter_in_latent: bool = False, # When set to True, low-pass filter is done after encoder. If False, low-pass filter is applied to image directly before encoder. 615 | lp_blur_sigma: float = 15.0, # Used with 'gaussian_blur'. Gaussian filter sigma value. 616 | lp_blur_kernel_size: float = 0.02734375, # Used with 'gaussian_blur'. Gaussian filter size. When set to int, used directly as kernel size. When set to float, H * `lp_blur_kernel_size` is used as kernel size. 617 | lp_resize_factor: float = 0.25, # Used with 'down_up'. Image is bilinearly downsized to (`lp_resize_factor` * WIDTH, `lp_resize_factor` * HEIGHT) and then back to original. 618 | 619 | lp_strength_schedule_type: str = "none", # Scheduling type for low-pass filtering strength. Options: {"none", "linear", "interval", "exponential"} 620 | schedule_blur_kernel_size: bool = False, # If True, schedule blur kernel size as well. Otherwise, fix to initial value. 621 | 622 | 623 | # --- Constant Interval Scheduling Params for LP Strength --- 624 | schedule_interval_start_time: float = 0.0, # Starting timestep for interval scheduling 625 | schedule_interval_end_time: float = 0.05, # Ending timestep for interval scheduling 626 | 627 | # --- Linear Scheduling Params for LP Strength --- 628 | schedule_linear_start_weight: float = 1.0, # Starting LP weight for linear scheduling at t=T (step 0) 629 | schedule_linear_end_weight: float = 0.0, # Ending LP weight for linear scheduling at t=T * schedule_linear_end_time 630 | schedule_linear_end_time: float = 0.5, # Timestep fraction at which schedule_linear_end is reached 631 | 632 | # --- Exponential Scheduling Params for LP Strength --- 633 | schedule_exp_decay_rate: float = 10.0, # Decay rate for 'exponential' schedule. Higher values decay faster. Strength = exp(-rate * time_fraction). 634 | ): 635 | r""" 636 | The call function to the pipeline for generation. 637 | 638 | Args: 639 | image (`PipelineImageInput`): 640 | The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. 641 | prompt (`str` or `List[str]`, *optional*): 642 | The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. 643 | instead. 644 | negative_prompt (`str` or `List[str]`, *optional*): 645 | The prompt or prompts not to guide the image generation. If not defined, one has to pass 646 | `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is 647 | less than `1`). 648 | height (`int`, defaults to `480`): 649 | The height of the generated video. 650 | width (`int`, defaults to `832`): 651 | The width of the generated video. 652 | num_frames (`int`, defaults to `81`): 653 | The number of frames in the generated video. 654 | num_inference_steps (`int`, defaults to `50`): 655 | The number of denoising steps. More denoising steps usually lead to a higher quality image at the 656 | expense of slower inference. 657 | guidance_scale (`float`, defaults to `5.0`): 658 | Guidance scale as defined in [Classifier-Free Diffusion 659 | Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. 660 | of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting 661 | `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to 662 | the text `prompt`, usually at the expense of lower image quality. 663 | num_videos_per_prompt (`int`, *optional*, defaults to 1): 664 | The number of images to generate per prompt. 665 | generator (`torch.Generator` or `List[torch.Generator]`, *optional*): 666 | A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make 667 | generation deterministic. 668 | latents (`torch.Tensor`, *optional*): 669 | Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image 670 | generation. Can be used to tweak the same generation with different prompts. If not provided, a latents 671 | tensor is generated by sampling using the supplied random `generator`. 672 | prompt_embeds (`torch.Tensor`, *optional*): 673 | Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not 674 | provided, text embeddings are generated from the `prompt` input argument. 675 | negative_prompt_embeds (`torch.Tensor`, *optional*): 676 | Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not 677 | provided, text embeddings are generated from the `negative_prompt` input argument. 678 | image_embeds (`torch.Tensor`, *optional*): 679 | Pre-generated image embeddings. Can be used to easily tweak image inputs (weighting). If not provided, 680 | image embeddings are generated from the `image` input argument. 681 | output_type (`str`, *optional*, defaults to `"np"`): 682 | The output format of the generated image. Choose between `PIL.Image` or `np.array`. 683 | return_dict (`bool`, *optional*, defaults to `True`): 684 | Whether or not to return a [`WanPipelineOutput`] instead of a plain tuple. 685 | attention_kwargs (`dict`, *optional*): 686 | A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under 687 | `self.processor` in 688 | [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). 689 | callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): 690 | A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of 691 | each denoising step during the inference. with the following arguments: `callback_on_step_end(self: 692 | DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a 693 | list of all tensors as specified by `callback_on_step_end_tensor_inputs`. 694 | callback_on_step_end_tensor_inputs (`List`, *optional*): 695 | The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list 696 | will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the 697 | `._callback_tensor_inputs` attribute of your pipeline class. 698 | max_sequence_length (`int`, *optional*, defaults to `512`): 699 | The maximum sequence length of the prompt. 700 | use_low_pass_guidance (`bool`, *optional*, defaults to `False`): 701 | Whether to use low-pass guidance. This can help to improve the temporal consistency of the generated 702 | video. 703 | lp_filter_type (`str`, *optional*, defaults to `"none"`): 704 | The type of low-pass filter to apply. Can be one of `gaussian_blur` or `down_up`. 705 | lp_filter_in_latent (`bool`, *optional*, defaults to `False`): 706 | If `True`, the low-pass filter is applied to the latent representation of the image. If `False`, it is 707 | applied to the image in pixel space before encoding. 708 | lp_blur_sigma (`float`, *optional*, defaults to `15.0`): 709 | The sigma value for the Gaussian blur filter. Only used if `lp_filter_type` is `gaussian_blur`. 710 | lp_blur_kernel_size (`float`, *optional*, defaults to `0.02734375`): 711 | The kernel size for the Gaussian blur filter. If an `int`, it's used directly. If a `float`, the kernel 712 | size is calculated as `height * lp_blur_kernel_size`. Only used if `lp_filter_type` is `gaussian_blur`. 713 | lp_resize_factor (`float`, *optional*, defaults to `0.25`): 714 | The resize factor for the down-sampling and up-sampling filter. Only used if `lp_filter_type` is 715 | `down_up`. 716 | lp_strength_schedule_type (`str`, *optional*, defaults to `"none"`): 717 | The scheduling type for the low-pass filter strength. Can be one of `none`, `linear`, `interval`, or 718 | `exponential`. 719 | schedule_blur_kernel_size (`bool`, *optional*, defaults to `False`): 720 | If `True`, the blur kernel size is also scheduled along with the strength. Otherwise, it remains fixed. 721 | schedule_interval_start_time (`float`, *optional*, defaults to `0.0`): 722 | The starting timestep fraction for interval scheduling. Only used if `lp_strength_schedule_type` is 723 | `interval`. 724 | schedule_interval_end_time (`float`, *optional*, defaults to `0.05`): 725 | The ending timestep fraction for interval scheduling. Only used if `lp_strength_schedule_type` is 726 | `interval`. 727 | schedule_linear_start_weight (`float`, *optional*, defaults to `1.0`): 728 | The starting weight for the low-pass filter strength in a linear schedule. Corresponds to the first 729 | timestep. Only used if `lp_strength_schedule_type` is `linear`. 730 | schedule_linear_end_weight (`float`, *optional*, defaults to `0.0`): 731 | The ending weight for the low-pass filter strength in a linear schedule. Only used if 732 | `lp_strength_schedule_type` is `linear`. 733 | schedule_linear_end_time (`float`, *optional*, defaults to `0.5`): 734 | The timestep fraction at which `schedule_linear_end_weight` is reached in a linear schedule. Only used 735 | if `lp_strength_schedule_type` is `linear`. 736 | schedule_exp_decay_rate (`float`, *optional*, defaults to `10.0`): 737 | The decay rate for the exponential schedule. Higher values lead to faster decay. Only used if 738 | `lp_strength_schedule_type` is `exponential`. 739 | 740 | Examples: 741 | 742 | Returns: 743 | [`~WanPipelineOutput`] or `tuple`: 744 | If `return_dict` is `True`, [`WanPipelineOutput`] is returned, otherwise a `tuple` is returned where 745 | the first element is a list with the generated images and the second element is a list of `bool`s 746 | indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content. 747 | """ 748 | if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): 749 | callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs 750 | 751 | # 1. Check inputs. Raise error if not correct 752 | self.check_inputs( 753 | prompt, 754 | negative_prompt, 755 | image, 756 | height, 757 | width, 758 | prompt_embeds, 759 | negative_prompt_embeds, 760 | image_embeds, 761 | callback_on_step_end_tensor_inputs, 762 | ) 763 | 764 | if num_frames % self.vae_scale_factor_temporal != 1: 765 | logger.warning( 766 | f"`num_frames - 1` has to be divisible by {self.vae_scale_factor_temporal}. Rounding to the nearest number." 767 | ) 768 | num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1 769 | num_frames = max(num_frames, 1) 770 | 771 | self._guidance_scale = guidance_scale 772 | self._attention_kwargs = attention_kwargs 773 | self._current_timestep = None 774 | self._interrupt = False 775 | 776 | device = self._execution_device 777 | 778 | # 2. Define call parameters 779 | if prompt is not None and isinstance(prompt, str): 780 | batch_size = 1 781 | elif prompt is not None and isinstance(prompt, list): 782 | batch_size = len(prompt) 783 | else: 784 | batch_size = prompt_embeds.shape[0] 785 | 786 | # 3. Encode input prompt 787 | prompt_embeds, negative_prompt_embeds = self.encode_prompt( 788 | prompt=prompt, 789 | negative_prompt=negative_prompt, 790 | do_classifier_free_guidance=self.do_classifier_free_guidance, 791 | num_videos_per_prompt=num_videos_per_prompt, 792 | prompt_embeds=prompt_embeds, 793 | negative_prompt_embeds=negative_prompt_embeds, 794 | max_sequence_length=max_sequence_length, 795 | device=device, 796 | ) 797 | 798 | # Encode image embedding 799 | transformer_dtype = self.transformer.dtype 800 | prompt_embeds = prompt_embeds.to(transformer_dtype) 801 | if negative_prompt_embeds is not None: 802 | negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype) 803 | 804 | if image_embeds is None: 805 | if last_image is None: 806 | image_embeds = self.encode_image(image, device) 807 | else: 808 | image_embeds = self.encode_image([image, last_image], device) 809 | dup_b, l, d = image_embeds.shape 810 | image_embeds = image_embeds.reshape(-1, 2 * l, d) 811 | image_embeds = image_embeds.repeat(batch_size, 1, 1) 812 | image_embeds = image_embeds.to(transformer_dtype) 813 | 814 | # 4. Prepare timesteps 815 | self.scheduler.set_timesteps(num_inference_steps, device=device) 816 | timesteps = self.scheduler.timesteps 817 | 818 | # 5. Prepare latent variables 819 | num_channels_latents = self.vae.config.z_dim 820 | image = self.video_processor.preprocess(image, height=height, width=width).to(device, dtype=torch.float32) 821 | if last_image is not None: 822 | last_image = self.video_processor.preprocess(last_image, height=height, width=width).to( 823 | device, dtype=torch.float32 824 | ) 825 | latents, condition = self.prepare_latents( 826 | image, 827 | batch_size * num_videos_per_prompt, 828 | num_channels_latents, 829 | height, 830 | width, 831 | num_frames, 832 | torch.float32, 833 | device, 834 | generator, 835 | latents, 836 | last_image, 837 | ) 838 | 839 | # 6. Denoising loop 840 | num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order 841 | self._num_timesteps = len(timesteps) 842 | 843 | with self.progress_bar(total=num_inference_steps) as progress_bar: 844 | for i, t in enumerate(timesteps): 845 | if self.interrupt: 846 | continue 847 | 848 | self._current_timestep = t 849 | 850 | if self.do_classifier_free_guidance and use_low_pass_guidance: # low-pass filtering 851 | lp_strength = lp_utils.get_lp_strength( 852 | step_index=i, 853 | total_steps=num_inference_steps, 854 | lp_strength_schedule_type=lp_strength_schedule_type, 855 | schedule_interval_start_time=schedule_interval_start_time, 856 | schedule_interval_end_time=schedule_interval_end_time, 857 | schedule_linear_start_weight=schedule_linear_start_weight, 858 | schedule_linear_end_weight=schedule_linear_end_weight, 859 | schedule_linear_end_time=schedule_linear_end_time, 860 | schedule_exp_decay_rate=schedule_exp_decay_rate, 861 | ) 862 | 863 | modulated_lp_blur_sigma = lp_blur_sigma * lp_strength 864 | modulated_lp_blur_kernel_size = ( 865 | lp_blur_kernel_size * lp_strength if schedule_blur_kernel_size else lp_blur_kernel_size 866 | ) 867 | modulated_lp_resize_factor = 1.0 - (1.0 - lp_resize_factor) * lp_strength 868 | 869 | lp_image_latents = self.prepare_lp( 870 | lp_filter_type=lp_filter_type, 871 | lp_blur_sigma=modulated_lp_blur_sigma, 872 | lp_blur_kernel_size=modulated_lp_blur_kernel_size, 873 | lp_resize_factor=modulated_lp_resize_factor, 874 | generator=generator, 875 | num_frames=num_frames, 876 | use_low_pass_guidance=use_low_pass_guidance, 877 | lp_filter_in_latent=lp_filter_in_latent, 878 | orig_image_latents=condition, 879 | orig_image_tensor=image, 880 | ) 881 | 882 | if lp_strength == 0.0: # equivalent to vanilla 883 | latent_model_input = torch.cat([latents] * 2) 884 | latent_model_input = torch.cat( 885 | [latent_model_input, torch.cat([condition, condition], dim=0)], dim=1 886 | ).to(transformer_dtype) 887 | concat_prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) 888 | else: # three passes 889 | latent_model_input = torch.cat([latents] * 3) 890 | img_cond = torch.cat([condition, lp_image_latents, lp_image_latents], dim=0) 891 | latent_model_input = torch.cat([latent_model_input, img_cond], dim=1).to(transformer_dtype) 892 | concat_prompt_embeds = torch.cat( 893 | [negative_prompt_embeds, negative_prompt_embeds, prompt_embeds], dim=0 894 | ) 895 | 896 | elif self.do_classifier_free_guidance: # no low-pass filtering 897 | latent_model_input = torch.cat([latents] * 2) 898 | latent_model_input = torch.cat( 899 | [latent_model_input, torch.cat([condition, condition], dim=0)], dim=1 900 | ).to(transformer_dtype) 901 | concat_prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) 902 | 903 | timestep = t.expand(latent_model_input.shape[0]) 904 | concat_image_embeds = ( 905 | image_embeds.repeat(latent_model_input.shape[0], 1, 1) 906 | if image_embeds.shape[0] != latent_model_input.shape[0] 907 | else image_embeds 908 | ) 909 | 910 | noise_pred = self.transformer( 911 | hidden_states=latent_model_input, 912 | timestep=timestep, 913 | encoder_hidden_states=concat_prompt_embeds, 914 | encoder_hidden_states_image=concat_image_embeds, 915 | attention_kwargs=attention_kwargs, 916 | return_dict=False, 917 | )[0] 918 | 919 | if noise_pred.shape[0] == 3: # three chunks 920 | noise_pred_uncond_init, noise_pred_uncond, noise_pred_text = noise_pred.chunk(3) 921 | noise_pred = noise_pred_uncond_init + guidance_scale * (noise_pred_text - noise_pred_uncond) 922 | else: 923 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) 924 | noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) 925 | 926 | # compute the previous noisy sample x_t -> x_t-1 927 | latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] 928 | 929 | if callback_on_step_end is not None: 930 | callback_kwargs = {} 931 | for k in callback_on_step_end_tensor_inputs: 932 | callback_kwargs[k] = locals()[k] 933 | callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) 934 | 935 | latents = callback_outputs.pop("latents", latents) 936 | prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) 937 | negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) 938 | 939 | # call the callback, if provided 940 | if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): 941 | progress_bar.update() 942 | 943 | if XLA_AVAILABLE: 944 | xm.mark_step() 945 | 946 | self._current_timestep = None 947 | 948 | if not output_type == "latent": 949 | latents = latents.to(self.vae.dtype) 950 | latents_mean = ( 951 | torch.tensor(self.vae.config.latents_mean) 952 | .view(1, self.vae.config.z_dim, 1, 1, 1) 953 | .to(latents.device, latents.dtype) 954 | ) 955 | latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( 956 | latents.device, latents.dtype 957 | ) 958 | latents = latents / latents_std + latents_mean 959 | video = self.vae.decode(latents, return_dict=False)[0] 960 | video = self.video_processor.postprocess_video(video, output_type=output_type) 961 | else: 962 | video = latents 963 | 964 | # Offload all models 965 | self.maybe_free_model_hooks() 966 | 967 | if not return_dict: 968 | return (video,) 969 | 970 | return WanPipelineOutput(frames=video) 971 | -------------------------------------------------------------------------------- /pipeline_cogvideox_image2video_lowpass.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The CogVideoX team, Tsinghua University & ZhipuAI and The HuggingFace Team. 2 | # All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import inspect 17 | import math 18 | from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Set 19 | 20 | import PIL 21 | import torch 22 | import torch.nn.functional as F 23 | import torchvision.transforms.functional as tvF 24 | from transformers import T5EncoderModel, T5Tokenizer 25 | 26 | from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback 27 | from diffusers.image_processor import PipelineImageInput 28 | from diffusers.loaders import CogVideoXLoraLoaderMixin 29 | from diffusers.models import AutoencoderKLCogVideoX, CogVideoXTransformer3DModel 30 | from diffusers.models.embeddings import get_3d_rotary_pos_embed 31 | from diffusers.pipelines.pipeline_utils import DiffusionPipeline 32 | from diffusers.schedulers import CogVideoXDDIMScheduler, CogVideoXDPMScheduler 33 | from diffusers.utils import ( 34 | is_torch_xla_available, 35 | logging, 36 | replace_example_docstring, 37 | ) 38 | from diffusers.utils.torch_utils import randn_tensor 39 | from diffusers.video_processor import VideoProcessor 40 | 41 | from diffusers.pipelines.cogvideo.pipeline_output import CogVideoXPipelineOutput 42 | 43 | import lp_utils 44 | 45 | if is_torch_xla_available(): 46 | import torch_xla.core.xla_model as xm 47 | 48 | XLA_AVAILABLE = True 49 | else: 50 | XLA_AVAILABLE = False 51 | 52 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 53 | 54 | 55 | EXAMPLE_DOC_STRING = """ 56 | Examples: 57 | ```py 58 | >>> import torch 59 | >>> from diffusers import CogVideoXImageToVideoPipeline 60 | >>> from diffusers.utils import export_to_video, load_image 61 | 62 | >>> pipe = CogVideoXImageToVideoPipeline.from_pretrained("THUDM/CogVideoX-5b-I2V", torch_dtype=torch.bfloat16) 63 | >>> pipe.to("cuda") 64 | 65 | >>> prompt = "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot." 66 | >>> image = load_image( 67 | ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" 68 | ... ) 69 | >>> video = pipe(image, prompt, use_dynamic_cfg=True) 70 | >>> export_to_video(video.frames[0], "output.mp4", fps=8) 71 | ``` 72 | """ 73 | 74 | 75 | # Similar to diffusers.pipelines.hunyuandit.pipeline_hunyuandit.get_resize_crop_region_for_grid 76 | def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): 77 | tw = tgt_width 78 | th = tgt_height 79 | h, w = src 80 | r = h / w 81 | if r > (th / tw): 82 | resize_height = th 83 | resize_width = int(round(th / h * w)) 84 | else: 85 | resize_width = tw 86 | resize_height = int(round(tw / w * h)) 87 | 88 | crop_top = int(round((th - resize_height) / 2.0)) 89 | crop_left = int(round((tw - resize_width) / 2.0)) 90 | 91 | return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) 92 | 93 | 94 | # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps 95 | def retrieve_timesteps( 96 | scheduler, 97 | num_inference_steps: Optional[int] = None, 98 | device: Optional[Union[str, torch.device]] = None, 99 | timesteps: Optional[List[int]] = None, 100 | sigmas: Optional[List[float]] = None, 101 | **kwargs, 102 | ): 103 | r""" 104 | Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles 105 | custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. 106 | 107 | Args: 108 | scheduler (`SchedulerMixin`): 109 | The scheduler to get timesteps from. 110 | num_inference_steps (`int`): 111 | The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` 112 | must be `None`. 113 | device (`str` or `torch.device`, *optional*): 114 | The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. 115 | timesteps (`List[int]`, *optional*): 116 | Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, 117 | `num_inference_steps` and `sigmas` must be `None`. 118 | sigmas (`List[float]`, *optional*): 119 | Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, 120 | `num_inference_steps` and `timesteps` must be `None`. 121 | 122 | Returns: 123 | `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the 124 | second element is the number of inference steps. 125 | """ 126 | if timesteps is not None and sigmas is not None: 127 | raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") 128 | if timesteps is not None: 129 | accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) 130 | if not accepts_timesteps: 131 | raise ValueError( 132 | f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" 133 | f" timestep schedules. Please check whether you are using the correct scheduler." 134 | ) 135 | scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) 136 | timesteps = scheduler.timesteps 137 | num_inference_steps = len(timesteps) 138 | elif sigmas is not None: 139 | accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) 140 | if not accept_sigmas: 141 | raise ValueError( 142 | f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" 143 | f" sigmas schedules. Please check whether you are using the correct scheduler." 144 | ) 145 | scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) 146 | timesteps = scheduler.timesteps 147 | num_inference_steps = len(timesteps) 148 | else: 149 | scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) 150 | timesteps = scheduler.timesteps 151 | return timesteps, num_inference_steps 152 | 153 | 154 | # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents 155 | def retrieve_latents( 156 | encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" 157 | ): 158 | if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": 159 | return encoder_output.latent_dist.sample(generator) 160 | elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": 161 | return encoder_output.latent_dist.mode() 162 | elif hasattr(encoder_output, "latents"): 163 | return encoder_output.latents 164 | else: 165 | raise AttributeError("Could not access latents of provided encoder_output") 166 | 167 | 168 | class CogVideoXImageToVideoPipeline(DiffusionPipeline, CogVideoXLoraLoaderMixin): 169 | r""" 170 | Pipeline for image-to-video generation using CogVideoX. 171 | 172 | This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the 173 | library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) 174 | 175 | Args: 176 | vae ([`AutoencoderKL`]): 177 | Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. 178 | text_encoder ([`T5EncoderModel`]): 179 | Frozen text-encoder. CogVideoX uses 180 | [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel); specifically the 181 | [t5-v1_1-xxl](https://huggingface.co/PixArt-alpha/PixArt-alpha/tree/main/t5-v1_1-xxl) variant. 182 | tokenizer (`T5Tokenizer`): 183 | Tokenizer of class 184 | [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). 185 | transformer ([`CogVideoXTransformer3DModel`]): 186 | A text conditioned `CogVideoXTransformer3DModel` to denoise the encoded video latents. 187 | scheduler ([`SchedulerMixin`]): 188 | A scheduler to be used in combination with `transformer` to denoise the encoded video latents. 189 | """ 190 | 191 | _optional_components = [] 192 | model_cpu_offload_seq = "text_encoder->transformer->vae" 193 | 194 | _callback_tensor_inputs = [ 195 | "latents", 196 | "prompt_embeds", 197 | "negative_prompt_embeds", 198 | ] 199 | 200 | def __init__( 201 | self, 202 | tokenizer: T5Tokenizer, 203 | text_encoder: T5EncoderModel, 204 | vae: AutoencoderKLCogVideoX, 205 | transformer: CogVideoXTransformer3DModel, 206 | scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], 207 | ): 208 | super().__init__() 209 | 210 | self.register_modules( 211 | tokenizer=tokenizer, 212 | text_encoder=text_encoder, 213 | vae=vae, 214 | transformer=transformer, 215 | scheduler=scheduler, 216 | ) 217 | self.vae_scale_factor_spatial = ( 218 | 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 219 | ) 220 | self.vae_scale_factor_temporal = ( 221 | self.vae.config.temporal_compression_ratio if getattr(self, "vae", None) else 4 222 | ) 223 | self.vae_scaling_factor_image = self.vae.config.scaling_factor if getattr(self, "vae", None) else 0.7 224 | 225 | self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) 226 | 227 | # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds 228 | def _get_t5_prompt_embeds( 229 | self, 230 | prompt: Union[str, List[str]] = None, 231 | num_videos_per_prompt: int = 1, 232 | max_sequence_length: int = 226, 233 | device: Optional[torch.device] = None, 234 | dtype: Optional[torch.dtype] = None, 235 | ): 236 | device = device or self._execution_device 237 | dtype = dtype or self.text_encoder.dtype 238 | 239 | prompt = [prompt] if isinstance(prompt, str) else prompt 240 | batch_size = len(prompt) 241 | 242 | text_inputs = self.tokenizer( 243 | prompt, 244 | padding="max_length", 245 | max_length=max_sequence_length, 246 | truncation=True, 247 | add_special_tokens=True, 248 | return_tensors="pt", 249 | ) 250 | text_input_ids = text_inputs.input_ids 251 | untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids 252 | 253 | if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): 254 | removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) 255 | logger.warning( 256 | "The following part of your input was truncated because `max_sequence_length` is set to " 257 | f" {max_sequence_length} tokens: {removed_text}" 258 | ) 259 | 260 | prompt_embeds = self.text_encoder(text_input_ids.to(device))[0] 261 | prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) 262 | 263 | # duplicate text embeddings for each generation per prompt, using mps friendly method 264 | _, seq_len, _ = prompt_embeds.shape 265 | prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) 266 | prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) 267 | 268 | return prompt_embeds 269 | 270 | # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt 271 | def encode_prompt( 272 | self, 273 | prompt: Union[str, List[str]], 274 | negative_prompt: Optional[Union[str, List[str]]] = None, 275 | do_classifier_free_guidance: bool = True, 276 | num_videos_per_prompt: int = 1, 277 | prompt_embeds: Optional[torch.Tensor] = None, 278 | negative_prompt_embeds: Optional[torch.Tensor] = None, 279 | max_sequence_length: int = 226, 280 | device: Optional[torch.device] = None, 281 | dtype: Optional[torch.dtype] = None, 282 | ): 283 | r""" 284 | Encodes the prompt into text encoder hidden states. 285 | 286 | Args: 287 | prompt (`str` or `List[str]`, *optional*): 288 | prompt to be encoded 289 | negative_prompt (`str` or `List[str]`, *optional*): 290 | The prompt or prompts not to guide the image generation. If not defined, one has to pass 291 | `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is 292 | less than `1`). 293 | do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): 294 | Whether to use classifier free guidance or not. 295 | num_videos_per_prompt (`int`, *optional*, defaults to 1): 296 | Number of videos that should be generated per prompt. torch device to place the resulting embeddings on 297 | prompt_embeds (`torch.Tensor`, *optional*): 298 | Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not 299 | provided, text embeddings will be generated from `prompt` input argument. 300 | negative_prompt_embeds (`torch.Tensor`, *optional*): 301 | Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt 302 | weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input 303 | argument. 304 | device: (`torch.device`, *optional*): 305 | torch device 306 | dtype: (`torch.dtype`, *optional*): 307 | torch dtype 308 | """ 309 | device = device or self._execution_device 310 | 311 | prompt = [prompt] if isinstance(prompt, str) else prompt 312 | if prompt is not None: 313 | batch_size = len(prompt) 314 | else: 315 | batch_size = prompt_embeds.shape[0] 316 | 317 | if prompt_embeds is None: 318 | prompt_embeds = self._get_t5_prompt_embeds( 319 | prompt=prompt, 320 | num_videos_per_prompt=num_videos_per_prompt, 321 | max_sequence_length=max_sequence_length, 322 | device=device, 323 | dtype=dtype, 324 | ) 325 | 326 | if do_classifier_free_guidance and negative_prompt_embeds is None: 327 | negative_prompt = negative_prompt or "" 328 | negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt 329 | 330 | if prompt is not None and type(prompt) is not type(negative_prompt): 331 | raise TypeError( 332 | f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" 333 | f" {type(prompt)}." 334 | ) 335 | elif batch_size != len(negative_prompt): 336 | raise ValueError( 337 | f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" 338 | f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" 339 | " the batch size of `prompt`." 340 | ) 341 | 342 | negative_prompt_embeds = self._get_t5_prompt_embeds( 343 | prompt=negative_prompt, 344 | num_videos_per_prompt=num_videos_per_prompt, 345 | max_sequence_length=max_sequence_length, 346 | device=device, 347 | dtype=dtype, 348 | ) 349 | 350 | return prompt_embeds, negative_prompt_embeds 351 | 352 | def prepare_latents( 353 | self, 354 | image: torch.Tensor, 355 | batch_size: int = 1, 356 | num_channels_latents: int = 16, 357 | num_frames: int = 13, 358 | height: int = 60, 359 | width: int = 90, 360 | dtype: Optional[torch.dtype] = None, 361 | device: Optional[torch.device] = None, 362 | generator: Optional[torch.Generator] = None, 363 | latents: Optional[torch.Tensor] = None, 364 | ): 365 | if isinstance(generator, list) and len(generator) != batch_size: 366 | raise ValueError( 367 | f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" 368 | f" size of {batch_size}. Make sure the batch size matches the length of the generators." 369 | ) 370 | 371 | num_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 372 | shape = ( 373 | batch_size, 374 | num_frames, 375 | num_channels_latents, 376 | height // self.vae_scale_factor_spatial, 377 | width // self.vae_scale_factor_spatial, 378 | ) 379 | 380 | # For CogVideoX1.5, the latent should add 1 for padding (Not use) 381 | if self.transformer.config.patch_size_t is not None: 382 | shape = shape[:1] + (shape[1] + shape[1] % self.transformer.config.patch_size_t,) + shape[2:] 383 | 384 | image = image.unsqueeze(2) # [B, C, F, H, W] 385 | 386 | if isinstance(generator, list): 387 | image_latents = [ 388 | retrieve_latents(self.vae.encode(image[i].unsqueeze(0)), generator[i]) for i in range(batch_size) 389 | ] 390 | else: 391 | image_latents = [retrieve_latents(self.vae.encode(img.unsqueeze(0)), generator) for img in image] 392 | 393 | image_latents = torch.cat(image_latents, dim=0).to(dtype).permute(0, 2, 1, 3, 4) # [B, F, C, H, W] 394 | 395 | if not self.vae.config.invert_scale_latents: 396 | image_latents = self.vae_scaling_factor_image * image_latents 397 | else: 398 | # This is awkward but required because the CogVideoX team forgot to multiply the 399 | # scaling factor during training :) 400 | image_latents = 1 / self.vae_scaling_factor_image * image_latents 401 | 402 | padding_shape = ( 403 | batch_size, 404 | num_frames - 1, 405 | num_channels_latents, 406 | height // self.vae_scale_factor_spatial, 407 | width // self.vae_scale_factor_spatial, 408 | ) 409 | 410 | latent_padding = torch.zeros(padding_shape, device=device, dtype=dtype) 411 | image_latents = torch.cat([image_latents, latent_padding], dim=1) 412 | 413 | # Select the first frame along the second dimension 414 | if self.transformer.config.patch_size_t is not None: 415 | first_frame = image_latents[:, : image_latents.size(1) % self.transformer.config.patch_size_t, ...] 416 | image_latents = torch.cat([first_frame, image_latents], dim=1) 417 | 418 | if latents is None: 419 | latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) 420 | else: 421 | latents = latents.to(device) 422 | 423 | # scale the initial noise by the standard deviation required by the scheduler 424 | latents = latents * self.scheduler.init_noise_sigma 425 | return latents, image_latents 426 | 427 | # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.decode_latents 428 | def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: 429 | latents = latents.permute(0, 2, 1, 3, 4) # [batch_size, num_channels, num_frames, height, width] 430 | latents = 1 / self.vae_scaling_factor_image * latents 431 | 432 | frames = self.vae.decode(latents).sample 433 | return frames 434 | 435 | # Copied from diffusers.pipelines.animatediff.pipeline_animatediff_video2video.AnimateDiffVideoToVideoPipeline.get_timesteps 436 | def get_timesteps(self, num_inference_steps, timesteps, strength, device): 437 | # get the original timestep using init_timestep 438 | init_timestep = min(int(num_inference_steps * strength), num_inference_steps) 439 | 440 | t_start = max(num_inference_steps - init_timestep, 0) 441 | timesteps = timesteps[t_start * self.scheduler.order :] 442 | 443 | return timesteps, num_inference_steps - t_start 444 | 445 | # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs 446 | def prepare_extra_step_kwargs(self, generator, eta): 447 | # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature 448 | # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. 449 | # eta corresponds to η in DDIM paper: https://huggingface.co/papers/2010.02502 450 | # and should be between [0, 1] 451 | 452 | accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) 453 | extra_step_kwargs = {} 454 | if accepts_eta: 455 | extra_step_kwargs["eta"] = eta 456 | 457 | # check if the scheduler accepts generator 458 | accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) 459 | if accepts_generator: 460 | extra_step_kwargs["generator"] = generator 461 | return extra_step_kwargs 462 | 463 | def check_inputs( 464 | self, 465 | image, 466 | prompt, 467 | height, 468 | width, 469 | negative_prompt, 470 | callback_on_step_end_tensor_inputs, 471 | latents=None, 472 | prompt_embeds=None, 473 | negative_prompt_embeds=None, 474 | ): 475 | if ( 476 | not isinstance(image, torch.Tensor) 477 | and not isinstance(image, PIL.Image.Image) 478 | and not isinstance(image, list) 479 | ): 480 | raise ValueError( 481 | "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" 482 | f" {type(image)}" 483 | ) 484 | 485 | if height % 8 != 0 or width % 8 != 0: 486 | raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") 487 | 488 | if callback_on_step_end_tensor_inputs is not None and not all( 489 | k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs 490 | ): 491 | raise ValueError( 492 | f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" 493 | ) 494 | if prompt is not None and prompt_embeds is not None: 495 | raise ValueError( 496 | f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" 497 | " only forward one of the two." 498 | ) 499 | elif prompt is None and prompt_embeds is None: 500 | raise ValueError( 501 | "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." 502 | ) 503 | elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): 504 | raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") 505 | 506 | if prompt is not None and negative_prompt_embeds is not None: 507 | raise ValueError( 508 | f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" 509 | f" {negative_prompt_embeds}. Please make sure to only forward one of the two." 510 | ) 511 | 512 | if negative_prompt is not None and negative_prompt_embeds is not None: 513 | raise ValueError( 514 | f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" 515 | f" {negative_prompt_embeds}. Please make sure to only forward one of the two." 516 | ) 517 | 518 | if prompt_embeds is not None and negative_prompt_embeds is not None: 519 | if prompt_embeds.shape != negative_prompt_embeds.shape: 520 | raise ValueError( 521 | "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" 522 | f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" 523 | f" {negative_prompt_embeds.shape}." 524 | ) 525 | 526 | # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.fuse_qkv_projections 527 | def fuse_qkv_projections(self) -> None: 528 | r"""Enables fused QKV projections.""" 529 | self.fusing_transformer = True 530 | self.transformer.fuse_qkv_projections() 531 | 532 | # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.unfuse_qkv_projections 533 | def unfuse_qkv_projections(self) -> None: 534 | r"""Disable QKV projection fusion if enabled.""" 535 | if not self.fusing_transformer: 536 | logger.warning("The Transformer was not initially fused for QKV projections. Doing nothing.") 537 | else: 538 | self.transformer.unfuse_qkv_projections() 539 | self.fusing_transformer = False 540 | 541 | # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._prepare_rotary_positional_embeddings 542 | def _prepare_rotary_positional_embeddings( 543 | self, 544 | height: int, 545 | width: int, 546 | num_frames: int, 547 | device: torch.device, 548 | ) -> Tuple[torch.Tensor, torch.Tensor]: 549 | grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) 550 | grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) 551 | 552 | p = self.transformer.config.patch_size 553 | p_t = self.transformer.config.patch_size_t 554 | 555 | base_size_width = self.transformer.config.sample_width // p 556 | base_size_height = self.transformer.config.sample_height // p 557 | 558 | if p_t is None: 559 | # CogVideoX 1.0 560 | grid_crops_coords = get_resize_crop_region_for_grid( 561 | (grid_height, grid_width), base_size_width, base_size_height 562 | ) 563 | freqs_cos, freqs_sin = get_3d_rotary_pos_embed( 564 | embed_dim=self.transformer.config.attention_head_dim, 565 | crops_coords=grid_crops_coords, 566 | grid_size=(grid_height, grid_width), 567 | temporal_size=num_frames, 568 | device=device, 569 | ) 570 | else: 571 | # CogVideoX 1.5 572 | base_num_frames = (num_frames + p_t - 1) // p_t 573 | 574 | freqs_cos, freqs_sin = get_3d_rotary_pos_embed( 575 | embed_dim=self.transformer.config.attention_head_dim, 576 | crops_coords=None, 577 | grid_size=(grid_height, grid_width), 578 | temporal_size=base_num_frames, 579 | grid_type="slice", 580 | max_size=(base_size_height, base_size_width), 581 | device=device, 582 | ) 583 | 584 | return freqs_cos, freqs_sin 585 | 586 | def prepare_lp( 587 | self, 588 | # --- Filter Selection & Strength --- 589 | lp_filter_type: str, 590 | lp_blur_sigma: float, 591 | lp_blur_kernel_size: float, 592 | lp_resize_factor: float, 593 | # --- Contextual Info --- 594 | generator: torch.Generator, 595 | num_frames: int, 596 | use_low_pass_guidance: bool, 597 | lp_filter_in_latent: bool, 598 | # --- Inputs to filter --- 599 | orig_image_latents: torch.Tensor, # Shape [B, F_padded, C, H, W] 600 | orig_image_tensor: torch.Tensor # Shape [B, C, H_orig, W_orig] (preprocessed RGB) 601 | ) -> torch.Tensor | None: 602 | """ 603 | Prepares a low-pass filtered version of the initial image condition for guidance. (CogVideoX) 604 | The resulting low-pass filtered latents are padded to match the required number of frames and temporal 605 | patch size for the transformer model. 606 | 607 | Args: 608 | lp_filter_type (`str`): The type of low-pass filter to apply, e.g., 'gaussian_blur', 'down_up'. 609 | lp_blur_sigma (`float`): The sigma value for the Gaussian blur filter. 610 | lp_blur_kernel_size (`float`): The kernel size for the Gaussian blur filter. 611 | lp_resize_factor (`float`): The resizing factor for the 'down_up' filter. 612 | generator (`torch.Generator`): A random generator, used for VAE sampling when filtering in image space. 613 | num_frames (`int`): The target number of frames for the final video, used to determine padding. 614 | use_low_pass_guidance (`bool`): If `False`, the function returns `None` immediately. 615 | lp_filter_in_latent (`bool`): If `True`, filtering is applied in latent space. Otherwise, in image space. 616 | orig_image_latents (`torch.Tensor`): The VAE-encoded latents of the original image. Used when 617 | `lp_filter_in_latent` is `True`. Shape: `(batch_size, num_frames_padded, channels, height, width)`. 618 | orig_image_tensor (`torch.Tensor`): The preprocessed original image tensor (RGB). Used when 619 | `lp_filter_in_latent` is `False`. Shape: `(batch_size, channels, height, width)`. 620 | 621 | Returns: 622 | `Optional[torch.Tensor]`: A tensor containing the low-pass filtered image latents, correctly shaped and 623 | padded for the transformer, or `None` if `use_low_pass_guidance` is `False`. 624 | """ 625 | if not use_low_pass_guidance: 626 | return None 627 | 628 | if not lp_filter_in_latent: 629 | # --- Filter in Image (RGB) Space --- 630 | 631 | # 1. Apply the filter to the original 4D RGB tensor. 632 | image_lp = lp_utils.apply_low_pass_filter( 633 | orig_image_tensor, # Should be [B, C, H, W] 634 | filter_type=lp_filter_type, 635 | blur_sigma=lp_blur_sigma, 636 | blur_kernel_size=lp_blur_kernel_size, 637 | resize_factor=lp_resize_factor, 638 | ) 639 | # image_lp: [B, C, H, W] 640 | 641 | # 2. Add the frame dimension BEFORE encoding 642 | image_lp_vae_input = image_lp.unsqueeze(2) # Shape: [B, C, 1, H, W] 643 | 644 | # 3. Encode the 5D tensor 645 | encoded_lp = self.vae.encode(image_lp_vae_input).latent_dist.sample(generator=generator) 646 | 647 | if not self.vae.config.invert_scale_latents: 648 | encoded_lp = self.vae_scaling_factor_image * encoded_lp 649 | else: 650 | encoded_lp = 1 / self.vae_scaling_factor_image * encoded_lp 651 | 652 | encoded_lp = encoded_lp.permute(0, 2, 1, 3, 4) 653 | 654 | # Calculate required latent frames based on output num_frames 655 | padded_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 656 | 657 | # Pad with zeros if needed 658 | current_frames = encoded_lp.shape[1] # Should be 1 here 659 | if padded_frames > current_frames: 660 | batch_size, _, latent_channels, latent_height, latent_width = encoded_lp.shape 661 | padding_shape = ( 662 | batch_size, 663 | padded_frames - current_frames, 664 | latent_channels, 665 | latent_height, 666 | latent_width, 667 | ) 668 | lp_padding = torch.zeros(padding_shape, device=encoded_lp.device, dtype=encoded_lp.dtype) 669 | lp_image_latents = torch.cat([encoded_lp, lp_padding], dim=1) 670 | else: 671 | lp_image_latents = encoded_lp[:, :padded_frames, ...] 672 | 673 | if self.transformer.config.patch_size_t is not None: 674 | remainder = lp_image_latents.size(1) % self.transformer.config.patch_size_t 675 | if remainder != 0: 676 | num_to_prepend = self.transformer.config.patch_size_t - remainder 677 | # Ensure num_to_prepend doesn't exceed available frames if F=1 initially 678 | num_to_prepend = min(num_to_prepend, lp_image_latents.shape[1]) 679 | first_frames_to_prepend = lp_image_latents[:, :num_to_prepend, ...] 680 | lp_image_latents = torch.cat([first_frames_to_prepend, lp_image_latents], dim=1) 681 | 682 | else: 683 | # --- Filter in Latent Space --- 684 | orig_image_latents_perm = orig_image_latents.permute(0, 2, 1, 3, 4).contiguous() 685 | lp_image_latents = lp_utils.apply_low_pass_filter( 686 | orig_image_latents_perm, # Input has shape [B, C, F_padded, H, W] 687 | filter_type=lp_filter_type, 688 | blur_sigma=lp_blur_sigma, 689 | blur_kernel_size=lp_blur_kernel_size, 690 | resize_factor=lp_resize_factor, 691 | ) 692 | lp_image_latents = lp_image_latents.permute(0, 2, 1, 3, 4).contiguous() 693 | if self.transformer.config.patch_size_t is not None: 694 | remainder = lp_image_latents.size(1) % self.transformer.config.patch_size_t 695 | if remainder != 0: 696 | num_to_prepend = self.transformer.config.patch_size_t - remainder 697 | num_to_prepend = min(num_to_prepend, lp_image_latents.shape[1]) 698 | first_frames_to_prepend = lp_image_latents[:, :num_to_prepend, ...] 699 | lp_image_latents = torch.cat([first_frames_to_prepend, lp_image_latents], dim=1) 700 | 701 | lp_image_latents = lp_image_latents.to(dtype=orig_image_latents.dtype) 702 | 703 | return lp_image_latents 704 | 705 | @property 706 | def guidance_scale(self): 707 | return self._guidance_scale 708 | 709 | @property 710 | def num_timesteps(self): 711 | return self._num_timesteps 712 | 713 | @property 714 | def attention_kwargs(self): 715 | return self._attention_kwargs 716 | 717 | @property 718 | def current_timestep(self): 719 | return self._current_timestep 720 | 721 | @property 722 | def interrupt(self): 723 | return self._interrupt 724 | 725 | @torch.no_grad() 726 | @replace_example_docstring(EXAMPLE_DOC_STRING) 727 | def __call__( 728 | self, 729 | image: PipelineImageInput, 730 | prompt: Optional[Union[str, List[str]]] = None, 731 | negative_prompt: Optional[Union[str, List[str]]] = None, 732 | height: Optional[int] = None, 733 | width: Optional[int] = None, 734 | num_frames: int = 49, 735 | num_inference_steps: int = 50, 736 | timesteps: Optional[List[int]] = None, 737 | guidance_scale: float = 6.0, 738 | use_dynamic_cfg: bool = False, 739 | num_videos_per_prompt: int = 1, 740 | eta: float = 0.0, 741 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, 742 | latents: Optional[torch.FloatTensor] = None, 743 | prompt_embeds: Optional[torch.FloatTensor] = None, 744 | negative_prompt_embeds: Optional[torch.FloatTensor] = None, 745 | output_type: str = "pil", 746 | return_dict: bool = True, 747 | attention_kwargs: Optional[Dict[str, Any]] = None, 748 | callback_on_step_end: Optional[ 749 | Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] 750 | ] = None, 751 | callback_on_step_end_tensor_inputs: List[str] = ["latents"], 752 | max_sequence_length: int = 226, 753 | use_low_pass_guidance: bool = False, 754 | lp_filter_type: str = "none", # {'gaussian_blur', 'down_up'} 755 | lp_filter_in_latent: bool = False, # When set to True, low-pass filter is done after encoder. If False, low-pass filter is applied to image directly before encoder. 756 | lp_blur_sigma: float = 15.0, # Used with 'gaussian_blur'. Gaussian filter sigma value. 757 | lp_blur_kernel_size: float = 0.02734375, # Used with 'gaussian_blur'. Gaussian filter size. When set to int, used directly as kernel size. When set to float, H * `lp_blur_kernel_size` is used as kernel size. 758 | lp_resize_factor: float = 0.25, # Used with 'down_up'. Image is bilinearly downsized to (`lp_resize_factor` * WIDTH, `lp_resize_factor` * HEIGHT) and then back to original. 759 | 760 | lp_strength_schedule_type: str = "none", # Scheduling type for low-pass filtering strength. Options: {"none", "linear", "interval", "exponential"} 761 | schedule_blur_kernel_size: bool = False, # If True, schedule blur kernel size as well. Otherwise, fix to initial value. 762 | 763 | # --- Constant Interval Scheduling Params for LP Strength --- 764 | schedule_interval_start_time: float = 0.0, # Starting timestep for interval scheduling 765 | schedule_interval_end_time: float = 0.05, # Ending timestep for interval scheduling 766 | 767 | # --- Linear Scheduling Params for LP Strength --- 768 | schedule_linear_start_weight: float = 1.0, # Starting LP weight for linear scheduling at t=T (step 0) 769 | schedule_linear_end_weight: float = 0.0, # Ending LP weight for linear scheduling at t=T * schedule_linear_end_time 770 | schedule_linear_end_time: float = 0.5, # Timestep fraction at which schedule_linear_end is reached 771 | 772 | # --- Exponential Scheduling Params for LP Strength --- 773 | schedule_exp_decay_rate: float = 10.0, # Decay rate for 'exponential' schedule. Higher values decay faster. Strength = exp(-rate * time_fraction). 774 | ) -> Union[CogVideoXPipelineOutput, Tuple]: 775 | """ 776 | Function invoked when calling the pipeline for generation. 777 | 778 | Args: 779 | image (`PipelineImageInput`): 780 | The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. 781 | prompt (`str` or `List[str]`, *optional*): 782 | The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. 783 | instead. 784 | negative_prompt (`str` or `List[str]`, *optional*): 785 | The prompt or prompts not to guide the image generation. If not defined, one has to pass 786 | `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is 787 | less than `1`). 788 | height (`int`, *optional*, defaults to self.transformer.config.sample_height * self.vae_scale_factor_spatial): 789 | The height in pixels of the generated image. This is set to 480 by default for the best results. 790 | width (`int`, *optional*, defaults to self.transformer.config.sample_height * self.vae_scale_factor_spatial): 791 | The width in pixels of the generated image. This is set to 720 by default for the best results. 792 | num_frames (`int`, defaults to `48`): 793 | Number of frames to generate. Must be divisible by self.vae_scale_factor_temporal. Generated video will 794 | contain 1 extra frame because CogVideoX is conditioned with (num_seconds * fps + 1) frames where 795 | num_seconds is 6 and fps is 8. However, since videos can be saved at any fps, the only condition that 796 | needs to be satisfied is that of divisibility mentioned above. 797 | num_inference_steps (`int`, *optional*, defaults to 50): 798 | The number of denoising steps. More denoising steps usually lead to a higher quality image at the 799 | expense of slower inference. 800 | timesteps (`List[int]`, *optional*): 801 | Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument 802 | in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is 803 | passed will be used. Must be in descending order. 804 | guidance_scale (`float`, *optional*, defaults to 7.0): 805 | Guidance scale as defined in [Classifier-Free Diffusion 806 | Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. 807 | of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting 808 | `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to 809 | the text `prompt`, usually at the expense of lower image quality. 810 | num_videos_per_prompt (`int`, *optional*, defaults to 1): 811 | The number of videos to generate per prompt. 812 | generator (`torch.Generator` or `List[torch.Generator]`, *optional*): 813 | One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) 814 | to make generation deterministic. 815 | latents (`torch.FloatTensor`, *optional*): 816 | Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image 817 | generation. Can be used to tweak the same generation with different prompts. If not provided, a latents 818 | tensor will ge generated by sampling using the supplied random `generator`. 819 | prompt_embeds (`torch.FloatTensor`, *optional*): 820 | Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not 821 | provided, text embeddings will be generated from `prompt` input argument. 822 | negative_prompt_embeds (`torch.FloatTensor`, *optional*): 823 | Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt 824 | weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input 825 | argument. 826 | output_type (`str`, *optional*, defaults to `"pil"`): 827 | The output format of the generate image. Choose between 828 | [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. 829 | return_dict (`bool`, *optional*, defaults to `True`): 830 | Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead 831 | of a plain tuple. 832 | attention_kwargs (`dict`, *optional*): 833 | A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under 834 | `self.processor` in 835 | [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). 836 | callback_on_step_end (`Callable`, *optional*): 837 | A function that calls at the end of each denoising steps during the inference. The function is called 838 | with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, 839 | callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by 840 | `callback_on_step_end_tensor_inputs`. 841 | callback_on_step_end_tensor_inputs (`List`, *optional*): 842 | The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list 843 | will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the 844 | `._callback_tensor_inputs` attribute of your pipeline class. 845 | max_sequence_length (`int`, defaults to `226`): 846 | Maximum sequence length in encoded prompt. Must be consistent with 847 | `self.transformer.config.max_text_seq_length` otherwise may lead to poor results. 848 | use_low_pass_guidance (`bool`, *optional*, defaults to `False`): 849 | Whether to use low-pass guidance. This can help to improve the temporal consistency of the generated 850 | video. 851 | lp_filter_type (`str`, *optional*, defaults to `"none"`): 852 | The type of low-pass filter to apply. Can be one of `gaussian_blur` or `down_up`. 853 | lp_filter_in_latent (`bool`, *optional*, defaults to `False`): 854 | If `True`, the low-pass filter is applied to the latent representation of the image. If `False`, it is 855 | applied to the image in pixel space before encoding. 856 | lp_blur_sigma (`float`, *optional*, defaults to `15.0`): 857 | The sigma value for the Gaussian blur filter. Only used if `lp_filter_type` is `gaussian_blur`. 858 | lp_blur_kernel_size (`float`, *optional*, defaults to `0.02734375`): 859 | The kernel size for the Gaussian blur filter. If an `int`, it's used directly. If a `float`, the kernel 860 | size is calculated as `height * lp_blur_kernel_size`. Only used if `lp_filter_type` is `gaussian_blur`. 861 | lp_resize_factor (`float`, *optional*, defaults to `0.25`): 862 | The resize factor for the down-sampling and up-sampling filter. Only used if `lp_filter_type` is 863 | `down_up`. 864 | lp_strength_schedule_type (`str`, *optional*, defaults to `"none"`): 865 | The scheduling type for the low-pass filter strength. Can be one of `none`, `linear`, `interval`, or 866 | `exponential`. 867 | schedule_blur_kernel_size (`bool`, *optional*, defaults to `False`): 868 | If `True`, the blur kernel size is also scheduled along with the strength. Otherwise, it remains fixed. 869 | schedule_interval_start_time (`float`, *optional*, defaults to `0.0`): 870 | The starting timestep fraction for interval scheduling. Only used if `lp_strength_schedule_type` is 871 | `interval`. 872 | schedule_interval_end_time (`float`, *optional*, defaults to `0.05`): 873 | The ending timestep fraction for interval scheduling. Only used if `lp_strength_schedule_type` is 874 | `interval`. 875 | schedule_linear_start_weight (`float`, *optional*, defaults to `1.0`): 876 | The starting weight for the low-pass filter strength in a linear schedule. Corresponds to the first 877 | timestep. Only used if `lp_strength_schedule_type` is `linear`. 878 | schedule_linear_end_weight (`float`, *optional*, defaults to `0.0`): 879 | The ending weight for the low-pass filter strength in a linear schedule. Only used if 880 | `lp_strength_schedule_type` is `linear`. 881 | schedule_linear_end_time (`float`, *optional*, defaults to `0.5`): 882 | The timestep fraction at which `schedule_linear_end_weight` is reached in a linear schedule. Only used 883 | if `lp_strength_schedule_type` is `linear`. 884 | schedule_exp_decay_rate (`float`, *optional*, defaults to `10.0`): 885 | The decay rate for the exponential schedule. Higher values lead to faster decay. Only used if 886 | `lp_strength_schedule_type` is `exponential`. 887 | 888 | Examples: 889 | 890 | Returns: 891 | [`~pipelines.cogvideo.pipeline_output.CogVideoXPipelineOutput`] or `tuple`: 892 | [`~pipelines.cogvideo.pipeline_output.CogVideoXPipelineOutput`] if `return_dict` is True, otherwise a 893 | `tuple`. When returning a tuple, the first element is a list with the generated images. 894 | """ 895 | 896 | if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): 897 | callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs 898 | 899 | height = height or self.transformer.config.sample_height * self.vae_scale_factor_spatial 900 | width = width or self.transformer.config.sample_width * self.vae_scale_factor_spatial 901 | num_frames = num_frames or self.transformer.config.sample_frames 902 | 903 | num_videos_per_prompt = 1 904 | 905 | # 1. Check inputs. Raise error if not correct 906 | self.check_inputs( 907 | image=image, 908 | prompt=prompt, 909 | height=height, 910 | width=width, 911 | negative_prompt=negative_prompt, 912 | callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, 913 | latents=latents, 914 | prompt_embeds=prompt_embeds, 915 | negative_prompt_embeds=negative_prompt_embeds, 916 | ) 917 | self._guidance_scale = guidance_scale 918 | self._current_timestep = None 919 | self._attention_kwargs = attention_kwargs 920 | self._interrupt = False 921 | 922 | # 2. Default call parameters 923 | if prompt is not None and isinstance(prompt, str): 924 | batch_size = 1 925 | elif prompt is not None and isinstance(prompt, list): 926 | batch_size = len(prompt) 927 | else: 928 | batch_size = prompt_embeds.shape[0] 929 | 930 | device = self._execution_device 931 | 932 | # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) 933 | # of the Imagen paper: https://huggingface.co/papers/2205.11487 . `guidance_scale = 1` 934 | # corresponds to doing no classifier free guidance. 935 | do_classifier_free_guidance = guidance_scale > 1.0 936 | 937 | # 3. Encode input prompt 938 | prompt_embeds, negative_prompt_embeds = self.encode_prompt( 939 | prompt=prompt, 940 | negative_prompt=negative_prompt, 941 | do_classifier_free_guidance=do_classifier_free_guidance, 942 | num_videos_per_prompt=num_videos_per_prompt, 943 | prompt_embeds=prompt_embeds, 944 | negative_prompt_embeds=negative_prompt_embeds, 945 | max_sequence_length=max_sequence_length, 946 | device=device, 947 | ) 948 | if do_classifier_free_guidance and use_low_pass_guidance: 949 | prompt_embeds_orig = prompt_embeds 950 | prompt_embeds = torch.cat([negative_prompt_embeds, negative_prompt_embeds, prompt_embeds_orig], dim=0) 951 | prompt_embeds_init = torch.cat([negative_prompt_embeds, prompt_embeds_orig], dim=0) 952 | elif do_classifier_free_guidance: 953 | prompt_embeds_orig = prompt_embeds 954 | prompt_embeds_init = torch.cat([negative_prompt_embeds, prompt_embeds_orig], dim=0) 955 | prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds_orig], dim=0) 956 | 957 | # 4. Prepare timesteps 958 | timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) 959 | self._num_timesteps = len(timesteps) 960 | 961 | # 5. Prepare latents 962 | latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 963 | # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t 964 | patch_size_t = self.transformer.config.patch_size_t 965 | additional_frames = 0 966 | if patch_size_t is not None and latent_frames % patch_size_t != 0: 967 | additional_frames = patch_size_t - latent_frames % patch_size_t 968 | num_frames += additional_frames * self.vae_scale_factor_temporal 969 | image_tensor = self.video_processor.preprocess(image, height=height, width=width).to( 970 | device, dtype=prompt_embeds.dtype 971 | ) 972 | 973 | latent_channels = self.transformer.config.in_channels // 2 974 | latents, image_latents = self.prepare_latents( 975 | image_tensor, 976 | batch_size * num_videos_per_prompt, 977 | latent_channels, 978 | num_frames, 979 | height, 980 | width, 981 | prompt_embeds.dtype, 982 | device, 983 | generator, 984 | latents, 985 | ) 986 | 987 | # 6. Prepare extra step kwargs 988 | extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) 989 | 990 | # 7. Create rotary embeds if required 991 | image_rotary_emb = ( 992 | self._prepare_rotary_positional_embeddings(height, width, latents.size(1), device) 993 | if self.transformer.config.use_rotary_positional_embeddings 994 | else None 995 | ) 996 | 997 | # 8. Create ofs embeds if required 998 | ofs_emb = None if self.transformer.config.ofs_embed_dim is None else latents.new_full((1,), fill_value=2.0) 999 | 1000 | # 9. Denoising loop 1001 | num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) 1002 | 1003 | with self.progress_bar(total=num_inference_steps) as progress_bar: 1004 | old_pred_original_sample = None 1005 | for i, t in enumerate(timesteps): 1006 | if self.interrupt: 1007 | continue 1008 | 1009 | self._current_timestep = t 1010 | 1011 | if not use_low_pass_guidance: 1012 | two_pass = True 1013 | 1014 | # Low-pass version input 1015 | if do_classifier_free_guidance and use_low_pass_guidance: 1016 | # Timestep scheduled low-pass filter strength ([0, 1] range) 1017 | lp_strength = lp_utils.get_lp_strength( 1018 | step_index=i, 1019 | total_steps=num_inference_steps, 1020 | lp_strength_schedule_type=lp_strength_schedule_type, 1021 | schedule_interval_start_time=schedule_interval_start_time, 1022 | schedule_interval_end_time=schedule_interval_end_time, 1023 | schedule_linear_start_weight=schedule_linear_start_weight, 1024 | schedule_linear_end_weight=schedule_linear_end_weight, 1025 | schedule_linear_end_time=schedule_linear_end_time, 1026 | schedule_exp_decay_rate=schedule_exp_decay_rate, 1027 | ) 1028 | 1029 | two_pass = (lp_strength == 0 or not use_low_pass_guidance) 1030 | 1031 | if lp_strength_schedule_type == 'exponential' and lp_strength < 0.1: # Rounding for exponential (for performance) 1032 | two_pass = True 1033 | 1034 | modulated_lp_blur_sigma = lp_blur_sigma * lp_strength 1035 | if schedule_blur_kernel_size: 1036 | modulated_lp_blur_kernel_size = lp_blur_kernel_size * lp_strength # Kernel size also scales down 1037 | else: 1038 | modulated_lp_blur_kernel_size = lp_blur_kernel_size 1039 | 1040 | modulated_lp_resize_factor = 1.0 - (1.0 - lp_resize_factor) * lp_strength 1041 | 1042 | # low-pass filter 1043 | lp_image_latents = self.prepare_lp( 1044 | # --- Filter Selection & Strength (Modulated) --- 1045 | lp_filter_type=lp_filter_type, 1046 | lp_blur_sigma=modulated_lp_blur_sigma, 1047 | lp_blur_kernel_size=modulated_lp_blur_kernel_size, 1048 | lp_resize_factor=modulated_lp_resize_factor, 1049 | # --- Contextual Info --- 1050 | generator=generator, 1051 | num_frames=num_frames, 1052 | use_low_pass_guidance=use_low_pass_guidance, 1053 | lp_filter_in_latent=lp_filter_in_latent, 1054 | # --- Inputs to filter --- 1055 | orig_image_latents=image_latents, 1056 | orig_image_tensor=image_tensor 1057 | ) 1058 | 1059 | # latent_model_input = torch.cat([latents] * 2) 1060 | if two_pass: 1061 | latent_model_input = torch.cat([latents] * 2) 1062 | else: 1063 | latent_model_input = torch.cat([latents] * 3) 1064 | 1065 | latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) 1066 | # latent_model_input = torch.cat([latent_model_input, torch.cat([lp_image_latents] * 2, dim=0)], dim=2) 1067 | if two_pass: 1068 | latent_model_input = torch.cat([latent_model_input, torch.cat([lp_image_latents] * 2, dim=0)], dim=2) 1069 | else: 1070 | latent_model_input = torch.cat([latent_model_input, torch.cat([image_latents,lp_image_latents,lp_image_latents], dim=0)], dim=2) 1071 | 1072 | elif do_classifier_free_guidance: 1073 | latent_model_input = torch.cat([latents] * 2) 1074 | latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) 1075 | latent_model_input = torch.cat([latent_model_input, torch.cat([image_latents] * 2, dim=0)], dim=2) 1076 | else: 1077 | latent_model_input = latents 1078 | latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) 1079 | latent_model_input = torch.cat([latent_model_input, image_latents], dim=2) 1080 | 1081 | timestep = t.expand(latent_model_input.shape[0]) 1082 | noise_pred = self.transformer( 1083 | hidden_states=latent_model_input, 1084 | encoder_hidden_states=prompt_embeds_init if two_pass else prompt_embeds, 1085 | timestep=timestep, 1086 | ofs=ofs_emb, 1087 | image_rotary_emb=image_rotary_emb, 1088 | attention_kwargs=attention_kwargs, 1089 | return_dict=False, 1090 | )[0] 1091 | noise_pred = noise_pred.float() 1092 | 1093 | # 12. Combine noise predictions with scheduled weights (triple pass) 1094 | if use_low_pass_guidance and do_classifier_free_guidance: 1095 | if two_pass: 1096 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) 1097 | noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) 1098 | else: 1099 | noise_pred_uncond_init, noise_pred_uncond, noise_pred_text = noise_pred.chunk(3) 1100 | noise_pred = ( 1101 | noise_pred_uncond_init + guidance_scale * (noise_pred_text - noise_pred_uncond) 1102 | ) 1103 | elif do_classifier_free_guidance: 1104 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) 1105 | if use_dynamic_cfg: 1106 | self._guidance_scale = 1 + guidance_scale * ( 1107 | (1 - math.cos(math.pi * ((num_inference_steps - t.item()) / num_inference_steps) ** 5.0)) / 2 1108 | ) 1109 | noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) 1110 | # compute the previous noisy sample x_t -> x_t-1 1111 | if not isinstance(self.scheduler, CogVideoXDPMScheduler): 1112 | latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] 1113 | else: 1114 | latents, old_pred_original_sample = self.scheduler.step( 1115 | noise_pred, 1116 | old_pred_original_sample, 1117 | t, 1118 | timesteps[i - 1] if i > 0 else None, 1119 | latents, 1120 | **extra_step_kwargs, 1121 | return_dict=False, 1122 | ) 1123 | latents = latents.to(prompt_embeds.dtype) 1124 | 1125 | # call the callback, if provided 1126 | if callback_on_step_end is not None: 1127 | callback_kwargs = {} 1128 | for k in callback_on_step_end_tensor_inputs: 1129 | callback_kwargs[k] = locals()[k] 1130 | callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) 1131 | 1132 | latents = callback_outputs.pop("latents", latents) 1133 | prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) 1134 | negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) 1135 | 1136 | if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): 1137 | progress_bar.update() 1138 | 1139 | if XLA_AVAILABLE: 1140 | xm.mark_step() 1141 | 1142 | self._current_timestep = None 1143 | 1144 | if not output_type == "latent": 1145 | # Discard any padding frames that were added for CogVideoX 1.5 1146 | latents = latents[:, additional_frames:] 1147 | video = self.decode_latents(latents) 1148 | video = self.video_processor.postprocess_video(video=video, output_type=output_type) 1149 | else: 1150 | video = latents 1151 | 1152 | # Offload all models 1153 | self.maybe_free_model_hooks() 1154 | 1155 | if not return_dict: 1156 | return (video,) 1157 | 1158 | return CogVideoXPipelineOutput(frames=video) 1159 | -------------------------------------------------------------------------------- /pipeline_hunyuan_video_image2video_lowpass.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HunyuanVideo Team and The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import inspect 16 | from typing import Any, Callable, Dict, List, Optional, Tuple, Union 17 | 18 | import numpy as np 19 | import PIL.Image 20 | import torch 21 | from transformers import ( 22 | CLIPImageProcessor, 23 | CLIPTextModel, 24 | CLIPTokenizer, 25 | LlamaTokenizerFast, 26 | LlavaForConditionalGeneration, 27 | ) 28 | 29 | from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback 30 | from diffusers.loaders import HunyuanVideoLoraLoaderMixin 31 | from diffusers.models import AutoencoderKLHunyuanVideo, HunyuanVideoTransformer3DModel 32 | from diffusers.schedulers import FlowMatchEulerDiscreteScheduler 33 | from diffusers.utils import is_torch_xla_available, logging, replace_example_docstring 34 | from diffusers.utils.torch_utils import randn_tensor 35 | from diffusers.video_processor import VideoProcessor 36 | from diffusers.pipelines.pipeline_utils import DiffusionPipeline 37 | from diffusers.pipelines.hunyuan_video.pipeline_output import HunyuanVideoPipelineOutput 38 | import math 39 | import torchvision.transforms.functional as tvF 40 | import torch.nn.functional as F 41 | 42 | import lp_utils 43 | 44 | if is_torch_xla_available(): 45 | import torch_xla.core.xla_model as xm 46 | 47 | XLA_AVAILABLE = True 48 | else: 49 | XLA_AVAILABLE = False 50 | 51 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 52 | 53 | 54 | EXAMPLE_DOC_STRING = """ 55 | Examples: 56 | ```python 57 | >>> import torch 58 | >>> from diffusers import HunyuanVideoImageToVideoPipeline, HunyuanVideoTransformer3DModel 59 | >>> from diffusers.utils import load_image, export_to_video 60 | 61 | >>> # Available checkpoints: hunyuanvideo-community/HunyuanVideo-I2V, hunyuanvideo-community/HunyuanVideo-I2V-33ch 62 | >>> model_id = "hunyuanvideo-community/HunyuanVideo-I2V" 63 | >>> transformer = HunyuanVideoTransformer3DModel.from_pretrained( 64 | ... model_id, subfolder="transformer", torch_dtype=torch.bfloat16 65 | ... ) 66 | >>> pipe = HunyuanVideoImageToVideoPipeline.from_pretrained( 67 | ... model_id, transformer=transformer, torch_dtype=torch.float16 68 | ... ) 69 | >>> pipe.vae.enable_tiling() 70 | >>> pipe.to("cuda") 71 | 72 | >>> prompt = "A man with short gray hair plays a red electric guitar." 73 | >>> image = load_image( 74 | ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png" 75 | ... ) 76 | 77 | >>> # If using hunyuanvideo-community/HunyuanVideo-I2V 78 | >>> output = pipe(image=image, prompt=prompt, guidance_scale=6.0).frames[0] 79 | 80 | >>> # If using hunyuanvideo-community/HunyuanVideo-I2V-33ch 81 | >>> output = pipe(image=image, prompt=prompt, guidance_scale=1.0, true_cfg_scale=1.0).frames[0] 82 | 83 | >>> export_to_video(output, "output.mp4", fps=15) 84 | ``` 85 | """ 86 | 87 | 88 | DEFAULT_PROMPT_TEMPLATE = { 89 | "template": ( 90 | "<|start_header_id|>system<|end_header_id|>\n\n\nDescribe the video by detailing the following aspects according to the reference image: " 91 | "1. The main content and theme of the video." 92 | "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." 93 | "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." 94 | "4. background environment, light, style and atmosphere." 95 | "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n" 96 | "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" 97 | "<|start_header_id|>assistant<|end_header_id|>\n\n" 98 | ), 99 | "crop_start": 103, 100 | "image_emb_start": 5, 101 | "image_emb_end": 581, 102 | "image_emb_len": 576, 103 | "double_return_token_id": 271, 104 | } 105 | 106 | 107 | def _expand_input_ids_with_image_tokens( 108 | text_input_ids, 109 | prompt_attention_mask, 110 | max_sequence_length, 111 | image_token_index, 112 | image_emb_len, 113 | image_emb_start, 114 | image_emb_end, 115 | pad_token_id, 116 | ): 117 | special_image_token_mask = text_input_ids == image_token_index 118 | num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1) 119 | batch_indices, non_image_indices = torch.where(text_input_ids != image_token_index) 120 | 121 | max_expanded_length = max_sequence_length + (num_special_image_tokens.max() * (image_emb_len - 1)) 122 | new_token_positions = torch.cumsum((special_image_token_mask * (image_emb_len - 1) + 1), -1) - 1 123 | text_to_overwrite = new_token_positions[batch_indices, non_image_indices] 124 | 125 | expanded_input_ids = torch.full( 126 | (text_input_ids.shape[0], max_expanded_length), 127 | pad_token_id, 128 | dtype=text_input_ids.dtype, 129 | device=text_input_ids.device, 130 | ) 131 | expanded_input_ids[batch_indices, text_to_overwrite] = text_input_ids[batch_indices, non_image_indices] 132 | expanded_input_ids[batch_indices, image_emb_start:image_emb_end] = image_token_index 133 | 134 | expanded_attention_mask = torch.zeros( 135 | (text_input_ids.shape[0], max_expanded_length), 136 | dtype=prompt_attention_mask.dtype, 137 | device=prompt_attention_mask.device, 138 | ) 139 | attn_batch_indices, attention_indices = torch.where(expanded_input_ids != pad_token_id) 140 | expanded_attention_mask[attn_batch_indices, attention_indices] = 1.0 141 | expanded_attention_mask = expanded_attention_mask.to(prompt_attention_mask.dtype) 142 | position_ids = (expanded_attention_mask.cumsum(-1) - 1).masked_fill_((expanded_attention_mask == 0), 1) 143 | 144 | return { 145 | "input_ids": expanded_input_ids, 146 | "attention_mask": expanded_attention_mask, 147 | "position_ids": position_ids, 148 | } 149 | 150 | 151 | 152 | def retrieve_timesteps( 153 | scheduler, 154 | num_inference_steps: Optional[int] = None, 155 | device: Optional[Union[str, torch.device]] = None, 156 | timesteps: Optional[List[int]] = None, 157 | sigmas: Optional[List[float]] = None, 158 | **kwargs, 159 | ): 160 | r""" 161 | Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles 162 | custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. 163 | 164 | Args: 165 | scheduler (`SchedulerMixin`): 166 | The scheduler to get timesteps from. 167 | num_inference_steps (`int`): 168 | The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` 169 | must be `None`. 170 | device (`str` or `torch.device`, *optional*): 171 | The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. 172 | timesteps (`List[int]`, *optional*): 173 | Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, 174 | `num_inference_steps` and `sigmas` must be `None`. 175 | sigmas (`List[float]`, *optional*): 176 | Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, 177 | `num_inference_steps` and `timesteps` must be `None`. 178 | 179 | Returns: 180 | `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the 181 | second element is the number of inference steps. 182 | """ 183 | if timesteps is not None and sigmas is not None: 184 | raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") 185 | if timesteps is not None: 186 | accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) 187 | if not accepts_timesteps: 188 | raise ValueError( 189 | f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" 190 | f" timestep schedules. Please check whether you are using the correct scheduler." 191 | ) 192 | scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) 193 | timesteps = scheduler.timesteps 194 | num_inference_steps = len(timesteps) 195 | elif sigmas is not None: 196 | accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) 197 | if not accept_sigmas: 198 | raise ValueError( 199 | f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" 200 | f" sigmas schedules. Please check whether you are using the correct scheduler." 201 | ) 202 | scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) 203 | timesteps = scheduler.timesteps 204 | num_inference_steps = len(timesteps) 205 | else: 206 | scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) 207 | timesteps = scheduler.timesteps 208 | return timesteps, num_inference_steps 209 | 210 | 211 | def retrieve_latents( 212 | encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" 213 | ): 214 | if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": 215 | return encoder_output.latent_dist.sample(generator) 216 | elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": 217 | return encoder_output.latent_dist.mode() 218 | elif hasattr(encoder_output, "latents"): 219 | return encoder_output.latents 220 | else: 221 | raise AttributeError("Could not access latents of provided encoder_output") 222 | 223 | 224 | class HunyuanVideoImageToVideoPipeline(DiffusionPipeline, HunyuanVideoLoraLoaderMixin): 225 | r""" 226 | Pipeline for image-to-video generation using HunyuanVideo. 227 | 228 | This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods 229 | implemented for all pipelines (downloading, saving, running on a particular device, etc.). 230 | 231 | Args: 232 | text_encoder ([`LlavaForConditionalGeneration`]): 233 | [Llava Llama3-8B](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers). 234 | tokenizer (`LlamaTokenizer`): 235 | Tokenizer from [Llava Llama3-8B](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers). 236 | transformer ([`HunyuanVideoTransformer3DModel`]): 237 | Conditional Transformer to denoise the encoded image latents. 238 | scheduler ([`FlowMatchEulerDiscreteScheduler`]): 239 | A scheduler to be used in combination with `transformer` to denoise the encoded image latents. 240 | vae ([`AutoencoderKLHunyuanVideo`]): 241 | Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. 242 | text_encoder_2 ([`CLIPTextModel`]): 243 | [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically 244 | the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. 245 | tokenizer_2 (`CLIPTokenizer`): 246 | Tokenizer of class 247 | [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). 248 | """ 249 | 250 | model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae" 251 | _callback_tensor_inputs = ["latents", "prompt_embeds"] 252 | 253 | def __init__( 254 | self, 255 | text_encoder: LlavaForConditionalGeneration, 256 | tokenizer: LlamaTokenizerFast, 257 | transformer: HunyuanVideoTransformer3DModel, 258 | vae: AutoencoderKLHunyuanVideo, 259 | scheduler: FlowMatchEulerDiscreteScheduler, 260 | text_encoder_2: CLIPTextModel, 261 | tokenizer_2: CLIPTokenizer, 262 | image_processor: CLIPImageProcessor, 263 | ): 264 | super().__init__() 265 | 266 | self.register_modules( 267 | vae=vae, 268 | text_encoder=text_encoder, 269 | tokenizer=tokenizer, 270 | transformer=transformer, 271 | scheduler=scheduler, 272 | text_encoder_2=text_encoder_2, 273 | tokenizer_2=tokenizer_2, 274 | image_processor=image_processor, 275 | ) 276 | 277 | self.vae_scaling_factor = self.vae.config.scaling_factor if getattr(self, "vae", None) else 0.476986 278 | self.vae_scale_factor_temporal = self.vae.temporal_compression_ratio if getattr(self, "vae", None) else 4 279 | self.vae_scale_factor_spatial = self.vae.spatial_compression_ratio if getattr(self, "vae", None) else 8 280 | self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) 281 | 282 | def _get_llama_prompt_embeds( 283 | self, 284 | image: torch.Tensor, 285 | prompt: Union[str, List[str]], 286 | prompt_template: Dict[str, Any], 287 | num_videos_per_prompt: int = 1, 288 | device: Optional[torch.device] = None, 289 | dtype: Optional[torch.dtype] = None, 290 | max_sequence_length: int = 256, 291 | num_hidden_layers_to_skip: int = 2, 292 | image_embed_interleave: int = 2, 293 | ) -> Tuple[torch.Tensor, torch.Tensor]: 294 | device = device or self._execution_device 295 | dtype = dtype or self.text_encoder.dtype 296 | 297 | prompt = [prompt] if isinstance(prompt, str) else prompt 298 | prompt = [prompt_template["template"].format(p) for p in prompt] 299 | 300 | crop_start = prompt_template.get("crop_start", None) 301 | 302 | image_emb_len = prompt_template.get("image_emb_len", 576) 303 | image_emb_start = prompt_template.get("image_emb_start", 5) 304 | image_emb_end = prompt_template.get("image_emb_end", 581) 305 | double_return_token_id = prompt_template.get("double_return_token_id", 271) 306 | 307 | if crop_start is None: 308 | prompt_template_input = self.tokenizer( 309 | prompt_template["template"], 310 | padding="max_length", 311 | return_tensors="pt", 312 | return_length=False, 313 | return_overflowing_tokens=False, 314 | return_attention_mask=False, 315 | ) 316 | crop_start = prompt_template_input["input_ids"].shape[-1] 317 | # Remove <|start_header_id|>, <|end_header_id|>, assistant, <|eot_id|>, and placeholder {} 318 | crop_start -= 5 319 | 320 | max_sequence_length += crop_start 321 | text_inputs = self.tokenizer( 322 | prompt, 323 | max_length=max_sequence_length, 324 | padding="max_length", 325 | truncation=True, 326 | return_tensors="pt", 327 | return_length=False, 328 | return_overflowing_tokens=False, 329 | return_attention_mask=True, 330 | ) 331 | text_input_ids = text_inputs.input_ids.to(device=device) 332 | prompt_attention_mask = text_inputs.attention_mask.to(device=device) 333 | 334 | image_embeds = self.image_processor(image, return_tensors="pt").pixel_values.to(device) 335 | 336 | image_token_index = self.text_encoder.config.image_token_index 337 | pad_token_id = self.text_encoder.config.pad_token_id 338 | expanded_inputs = _expand_input_ids_with_image_tokens( 339 | text_input_ids, 340 | prompt_attention_mask, 341 | max_sequence_length, 342 | image_token_index, 343 | image_emb_len, 344 | image_emb_start, 345 | image_emb_end, 346 | pad_token_id, 347 | ) 348 | prompt_embeds = self.text_encoder( 349 | **expanded_inputs, 350 | pixel_values=image_embeds, 351 | output_hidden_states=True, 352 | ).hidden_states[-(num_hidden_layers_to_skip + 1)] 353 | prompt_embeds = prompt_embeds.to(dtype=dtype) 354 | 355 | if crop_start is not None and crop_start > 0: 356 | text_crop_start = crop_start - 1 + image_emb_len 357 | batch_indices, last_double_return_token_indices = torch.where(text_input_ids == double_return_token_id) 358 | 359 | if last_double_return_token_indices.shape[0] == 3: 360 | # in case the prompt is too long 361 | last_double_return_token_indices = torch.cat( 362 | (last_double_return_token_indices, torch.tensor([text_input_ids.shape[-1]])) 363 | ) 364 | batch_indices = torch.cat((batch_indices, torch.tensor([0]))) 365 | 366 | last_double_return_token_indices = last_double_return_token_indices.reshape(text_input_ids.shape[0], -1)[ 367 | :, -1 368 | ] 369 | batch_indices = batch_indices.reshape(text_input_ids.shape[0], -1)[:, -1] 370 | assistant_crop_start = last_double_return_token_indices - 1 + image_emb_len - 4 371 | assistant_crop_end = last_double_return_token_indices - 1 + image_emb_len 372 | attention_mask_assistant_crop_start = last_double_return_token_indices - 4 373 | attention_mask_assistant_crop_end = last_double_return_token_indices 374 | 375 | prompt_embed_list = [] 376 | prompt_attention_mask_list = [] 377 | image_embed_list = [] 378 | image_attention_mask_list = [] 379 | 380 | for i in range(text_input_ids.shape[0]): 381 | prompt_embed_list.append( 382 | torch.cat( 383 | [ 384 | prompt_embeds[i, text_crop_start : assistant_crop_start[i].item()], 385 | prompt_embeds[i, assistant_crop_end[i].item() :], 386 | ] 387 | ) 388 | ) 389 | prompt_attention_mask_list.append( 390 | torch.cat( 391 | [ 392 | prompt_attention_mask[i, crop_start : attention_mask_assistant_crop_start[i].item()], 393 | prompt_attention_mask[i, attention_mask_assistant_crop_end[i].item() :], 394 | ] 395 | ) 396 | ) 397 | image_embed_list.append(prompt_embeds[i, image_emb_start:image_emb_end]) 398 | image_attention_mask_list.append( 399 | torch.ones(image_embed_list[-1].shape[0]).to(prompt_embeds.device).to(prompt_attention_mask.dtype) 400 | ) 401 | 402 | prompt_embed_list = torch.stack(prompt_embed_list) 403 | prompt_attention_mask_list = torch.stack(prompt_attention_mask_list) 404 | image_embed_list = torch.stack(image_embed_list) 405 | image_attention_mask_list = torch.stack(image_attention_mask_list) 406 | 407 | if 0 < image_embed_interleave < 6: 408 | image_embed_list = image_embed_list[:, ::image_embed_interleave, :] 409 | image_attention_mask_list = image_attention_mask_list[:, ::image_embed_interleave] 410 | 411 | assert ( 412 | prompt_embed_list.shape[0] == prompt_attention_mask_list.shape[0] 413 | and image_embed_list.shape[0] == image_attention_mask_list.shape[0] 414 | ) 415 | 416 | prompt_embeds = torch.cat([image_embed_list, prompt_embed_list], dim=1) 417 | prompt_attention_mask = torch.cat([image_attention_mask_list, prompt_attention_mask_list], dim=1) 418 | 419 | return prompt_embeds, prompt_attention_mask 420 | 421 | def _get_clip_prompt_embeds( 422 | self, 423 | prompt: Union[str, List[str]], 424 | num_videos_per_prompt: int = 1, 425 | device: Optional[torch.device] = None, 426 | dtype: Optional[torch.dtype] = None, 427 | max_sequence_length: int = 77, 428 | ) -> torch.Tensor: 429 | device = device or self._execution_device 430 | dtype = dtype or self.text_encoder_2.dtype 431 | 432 | prompt = [prompt] if isinstance(prompt, str) else prompt 433 | 434 | text_inputs = self.tokenizer_2( 435 | prompt, 436 | padding="max_length", 437 | max_length=max_sequence_length, 438 | truncation=True, 439 | return_tensors="pt", 440 | ) 441 | 442 | text_input_ids = text_inputs.input_ids 443 | untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids 444 | if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): 445 | removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) 446 | logger.warning( 447 | "The following part of your input was truncated because CLIP can only handle sequences up to" 448 | f" {max_sequence_length} tokens: {removed_text}" 449 | ) 450 | 451 | prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False).pooler_output 452 | return prompt_embeds 453 | 454 | def encode_prompt( 455 | self, 456 | image: torch.Tensor, 457 | prompt: Union[str, List[str]], 458 | prompt_2: Union[str, List[str]] = None, 459 | prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, 460 | num_videos_per_prompt: int = 1, 461 | prompt_embeds: Optional[torch.Tensor] = None, 462 | pooled_prompt_embeds: Optional[torch.Tensor] = None, 463 | prompt_attention_mask: Optional[torch.Tensor] = None, 464 | device: Optional[torch.device] = None, 465 | dtype: Optional[torch.dtype] = None, 466 | max_sequence_length: int = 256, 467 | image_embed_interleave: int = 2, 468 | ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 469 | if prompt_embeds is None: 470 | prompt_embeds, prompt_attention_mask = self._get_llama_prompt_embeds( 471 | image, 472 | prompt, 473 | prompt_template, 474 | num_videos_per_prompt, 475 | device=device, 476 | dtype=dtype, 477 | max_sequence_length=max_sequence_length, 478 | image_embed_interleave=image_embed_interleave, 479 | ) 480 | 481 | if pooled_prompt_embeds is None: 482 | if prompt_2 is None: 483 | prompt_2 = prompt 484 | pooled_prompt_embeds = self._get_clip_prompt_embeds( 485 | prompt, 486 | num_videos_per_prompt, 487 | device=device, 488 | dtype=dtype, 489 | max_sequence_length=77, 490 | ) 491 | 492 | return prompt_embeds, pooled_prompt_embeds, prompt_attention_mask 493 | 494 | def check_inputs( 495 | self, 496 | prompt, 497 | prompt_2, 498 | height, 499 | width, 500 | prompt_embeds=None, 501 | callback_on_step_end_tensor_inputs=None, 502 | prompt_template=None, 503 | true_cfg_scale=1.0, 504 | guidance_scale=1.0, 505 | ): 506 | if height % 16 != 0 or width % 16 != 0: 507 | raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") 508 | 509 | if callback_on_step_end_tensor_inputs is not None and not all( 510 | k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs 511 | ): 512 | raise ValueError( 513 | f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" 514 | ) 515 | 516 | if prompt is not None and prompt_embeds is not None: 517 | raise ValueError( 518 | f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" 519 | " only forward one of the two." 520 | ) 521 | elif prompt_2 is not None and prompt_embeds is not None: 522 | raise ValueError( 523 | f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" 524 | " only forward one of the two." 525 | ) 526 | elif prompt is None and prompt_embeds is None: 527 | raise ValueError( 528 | "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." 529 | ) 530 | elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): 531 | raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") 532 | elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): 533 | raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") 534 | 535 | if prompt_template is not None: 536 | if not isinstance(prompt_template, dict): 537 | raise ValueError(f"`prompt_template` has to be of type `dict` but is {type(prompt_template)}") 538 | if "template" not in prompt_template: 539 | raise ValueError( 540 | f"`prompt_template` has to contain a key `template` but only found {prompt_template.keys()}" 541 | ) 542 | 543 | if true_cfg_scale > 1.0 and guidance_scale > 1.0: 544 | logger.warning( 545 | "Both `true_cfg_scale` and `guidance_scale` are greater than 1.0. This will result in both " 546 | "classifier-free guidance and embedded-guidance to be applied. This is not recommended " 547 | "as it may lead to higher memory usage, slower inference and potentially worse results." 548 | ) 549 | 550 | def prepare_latents( 551 | self, 552 | image: torch.Tensor, 553 | batch_size: int, 554 | num_channels_latents: int = 32, 555 | height: int = 720, 556 | width: int = 1280, 557 | num_frames: int = 129, 558 | dtype: Optional[torch.dtype] = None, 559 | device: Optional[torch.device] = None, 560 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, 561 | latents: Optional[torch.Tensor] = None, 562 | image_condition_type: str = "latent_concat", 563 | i2v_stable: bool = False, 564 | ) -> torch.Tensor: 565 | if isinstance(generator, list) and len(generator) != batch_size: 566 | raise ValueError( 567 | f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" 568 | f" size of {batch_size}. Make sure the batch size matches the length of the generators." 569 | ) 570 | 571 | num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 572 | latent_height, latent_width = height // self.vae_scale_factor_spatial, width // self.vae_scale_factor_spatial 573 | shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width) 574 | 575 | image = image.unsqueeze(2) # [B, C, 1, H, W] 576 | if isinstance(generator, list): 577 | image_latents = [ 578 | retrieve_latents(self.vae.encode(image[i].unsqueeze(0)), generator[i], "argmax") 579 | for i in range(batch_size) 580 | ] 581 | else: 582 | image_latents = [retrieve_latents(self.vae.encode(img.unsqueeze(0)), generator, "argmax") for img in image] 583 | 584 | image_latents = torch.cat(image_latents, dim=0).to(dtype) * self.vae_scaling_factor 585 | 586 | if latents is None: 587 | latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) 588 | else: 589 | latents = latents.to(device=device, dtype=dtype) 590 | 591 | if i2v_stable: 592 | image_latents = image_latents.repeat(1, 1, num_latent_frames, 1, 1) 593 | t = torch.tensor([0.999]).to(device=device) 594 | latents = latents * t + image_latents * (1 - t) 595 | 596 | if image_condition_type == "token_replace": 597 | image_latents = image_latents[:, :, :1] 598 | 599 | return latents, image_latents 600 | 601 | def enable_vae_slicing(self): 602 | r""" 603 | Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to 604 | compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. 605 | """ 606 | self.vae.enable_slicing() 607 | 608 | def disable_vae_slicing(self): 609 | r""" 610 | Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to 611 | computing decoding in one step. 612 | """ 613 | self.vae.disable_slicing() 614 | 615 | def enable_vae_tiling(self): 616 | r""" 617 | Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to 618 | compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow 619 | processing larger images. 620 | """ 621 | self.vae.enable_tiling() 622 | 623 | def disable_vae_tiling(self): 624 | r""" 625 | Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to 626 | computing decoding in one step. 627 | """ 628 | self.vae.disable_tiling() 629 | 630 | @property 631 | def guidance_scale(self): 632 | return self._guidance_scale 633 | 634 | @property 635 | def num_timesteps(self): 636 | return self._num_timesteps 637 | 638 | @property 639 | def attention_kwargs(self): 640 | return self._attention_kwargs 641 | 642 | @property 643 | def current_timestep(self): 644 | return self._current_timestep 645 | 646 | @property 647 | def interrupt(self): 648 | return self._interrupt 649 | 650 | def prepare_lp( 651 | self, 652 | # --- Filter Selection & Strength --- 653 | lp_filter_type: str, 654 | lp_blur_sigma: float, 655 | lp_blur_kernel_size: float, 656 | lp_resize_factor: float, 657 | # --- Contextual Info --- 658 | generator: torch.Generator, 659 | num_frames: int, 660 | use_low_pass_guidance: bool, 661 | lp_filter_in_latent: bool, 662 | # --- Inputs to filter --- 663 | orig_image_latents: torch.Tensor, 664 | orig_image_tensor: torch.Tensor, 665 | last_image: Optional[torch.Tensor] = None, 666 | ) -> Optional[torch.Tensor]: 667 | """ 668 | Prepares a low-pass filtered version of the initial image condition for guidance. (HunyuanVideo) 669 | 670 | This function works in two modes: 671 | 1. **Filtering in Image (RGB) Space (`lp_filter_in_latent=False`)**: 672 | It applies a low-pass filter to the source image, constructs a video tensor (e.g., first frame is 673 | the filtered image, last frame is an optionally provided filtered `last_image`, and the rest are zeros), 674 | encodes this video tensor with the VAE, normalizes the result, and finally prepends a temporal mask 675 | to create a condition tensor in the format expected by the transformer (`[mask, latents]`). 676 | 2. **Filtering in Latent Space (`lp_filter_in_latent=True`)**: 677 | Directly applies the low-pass filter to the already-encoded `orig_image_latents`. 678 | 679 | Args: 680 | lp_filter_type (`str`): The type of low-pass filter to apply, e.g., 'gaussian_blur', 'down_up'. 681 | lp_blur_sigma (`float`): The sigma value for the Gaussian blur filter. 682 | lp_blur_kernel_size (`float`): The kernel size for the Gaussian blur filter. 683 | lp_resize_factor (`float`): The resizing factor for the 'down_up' filter. 684 | generator (`torch.Generator`): A random generator, used for VAE sampling when filtering in image space. 685 | num_frames (`int`): The target number of frames for the video condition tensor. 686 | use_low_pass_guidance (`bool`): If `False`, the function returns `None` immediately. 687 | lp_filter_in_latent (`bool`): If `True`, filtering is applied in latent space. Otherwise, in image space. 688 | orig_image_latents (`torch.Tensor`): The VAE-encoded latents of the original image. Used when 689 | `lp_filter_in_latent` is `True`. 690 | orig_image_tensor (`torch.Tensor`): The preprocessed original image tensor (RGB). Used when 691 | `lp_filter_in_latent` is `False`. 692 | last_image (`Optional[torch.Tensor]`, defaults to `None`): 693 | An optional image tensor for the last frame. If provided (and when filtering in image space), it will 694 | also be low-pass filtered and used as the last frame of the VAE input. 695 | 696 | Returns: 697 | `Optional[torch.Tensor]`: A tensor containing the low-pass filtered image condition ready for the 698 | transformer, or `None` if `use_low_pass_guidance` is `False`. 699 | """ 700 | if not use_low_pass_guidance: 701 | return None 702 | 703 | if not lp_filter_in_latent: 704 | # --- Filter in Image (RGB) Space --- 705 | # 1. Apply the low-pass filter to the source image(s). 706 | image_lp = lp_utils.apply_low_pass_filter( 707 | orig_image_tensor, 708 | filter_type=lp_filter_type, 709 | blur_sigma=lp_blur_sigma, 710 | blur_kernel_size=lp_blur_kernel_size, 711 | resize_factor=lp_resize_factor, 712 | ) 713 | image_lp_vae_input = image_lp.unsqueeze(2) 714 | 715 | batch_size,_,height,width = orig_image_tensor.shape 716 | latent_height = height // self.vae_scale_factor_spatial 717 | latent_width = width // self.vae_scale_factor_spatial 718 | 719 | # 2. Construct a video tensor to be encoded. This tensor has the filtered image as the first frame. 720 | # If a `last_image` is given, it's also filtered and placed at the end. Intermediate frames are black. 721 | if last_image is None: 722 | video_condition = torch.cat( 723 | [image_lp_vae_input, image_lp_vae_input.new_zeros(image_lp_vae_input.shape[0], image_lp_vae_input.shape[1], num_frames - 1, height, width)], dim=2 724 | ) 725 | else: 726 | 727 | last_image_lp = lp_utils.apply_low_pass_filter( 728 | last_image, 729 | filter_type=lp_filter_type, 730 | blur_sigma=lp_blur_sigma, 731 | blur_kernel_size=lp_blur_kernel_size, 732 | resize_factor=lp_resize_factor, 733 | ) 734 | 735 | last_image_lp = last_image_lp.unsqueeze(2) 736 | video_condition = torch.cat( 737 | [image_lp_vae_input, image_lp_vae_input.new_zeros(image_lp_vae_input.shape[0], image_lp_vae_input.shape[1], num_frames - 2, height, width), last_image_lp], 738 | dim=2, 739 | ) 740 | # 3. Encode the constructed video tensor and normalize the resulting latents. 741 | latents_mean = ( 742 | torch.tensor(self.vae.config.latents_mean) 743 | .view(1, self.vae.config.z_dim, 1, 1, 1) 744 | .to(image_lp.device, image_lp.dtype) 745 | ) 746 | latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( 747 | image_lp.device, image_lp.dtype 748 | ) 749 | encoded_lp = self.vae.encode(video_condition).latent_dist.sample(generator=generator) 750 | latent_condition = (encoded_lp - latents_mean) * latents_std 751 | 752 | # 4. Create a temporal mask. The transformer condition is `[mask, latents]`. 753 | # The mask is 1 for conditioned frames (first, and optionally last) and 0 for unconditioned frames. 754 | mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) 755 | 756 | if last_image is None: 757 | mask_lat_size[:, :, list(range(1, num_frames))] = 0 758 | else: 759 | mask_lat_size[:, :, list(range(1, num_frames - 1))] = 0 760 | first_frame_mask = mask_lat_size[:, :, 0:1] 761 | first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal) 762 | mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) 763 | mask_lat_size = mask_lat_size.view(batch_size, -1, self.vae_scale_factor_temporal, latent_height, latent_width) 764 | mask_lat_size = mask_lat_size.transpose(1, 2) 765 | mask_lat_size = mask_lat_size.to(latent_condition.device) 766 | 767 | # 5. Concatenate the mask and the normalized latents along the channel dimension. 768 | lp_image_latents = torch.concat([mask_lat_size, latent_condition], dim=1) 769 | 770 | else: 771 | # --- Filter Directly in Latent Space --- 772 | # This path assumes `orig_image_latents` is already prepared and just needs filtering. 773 | lp_image_latents = lp_utils.apply_low_pass_filter( 774 | orig_image_latents, 775 | filter_type=lp_filter_type, 776 | blur_sigma=lp_blur_sigma, 777 | blur_kernel_size=lp_blur_kernel_size, 778 | resize_factor=lp_resize_factor, 779 | ) 780 | 781 | if self.transformer.config.patch_size is not None: 782 | remainder = lp_image_latents.size(1) % self.transformer.config.patch_size 783 | if remainder != 0: 784 | num_to_prepend = self.transformer.config.patch_size - remainder 785 | num_to_prepend = min(num_to_prepend, lp_image_latents.shape[1]) 786 | first_frames_to_prepend = lp_image_latents[:, :num_to_prepend, ...] 787 | lp_image_latents = torch.cat([first_frames_to_prepend, lp_image_latents], dim=1) 788 | 789 | 790 | lp_image_latents = lp_image_latents.to(dtype=orig_image_latents.dtype) 791 | 792 | return lp_image_latents 793 | 794 | @torch.no_grad() 795 | @replace_example_docstring(EXAMPLE_DOC_STRING) 796 | def __call__( 797 | self, 798 | image: PIL.Image.Image, 799 | prompt: Union[str, List[str]] = None, 800 | prompt_2: Union[str, List[str]] = None, 801 | negative_prompt: Union[str, List[str]] = "bad quality", 802 | negative_prompt_2: Union[str, List[str]] = None, 803 | height: int = 720, 804 | width: int = 1280, 805 | num_frames: int = 129, 806 | num_inference_steps: int = 50, 807 | sigmas: List[float] = None, 808 | true_cfg_scale: float = 1.0, 809 | guidance_scale: float = 1.0, 810 | num_videos_per_prompt: Optional[int] = 1, 811 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, 812 | latents: Optional[torch.Tensor] = None, 813 | prompt_embeds: Optional[torch.Tensor] = None, 814 | pooled_prompt_embeds: Optional[torch.Tensor] = None, 815 | prompt_attention_mask: Optional[torch.Tensor] = None, 816 | negative_prompt_embeds: Optional[torch.Tensor] = None, 817 | negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, 818 | negative_prompt_attention_mask: Optional[torch.Tensor] = None, 819 | output_type: Optional[str] = "pil", 820 | return_dict: bool = True, 821 | attention_kwargs: Optional[Dict[str, Any]] = None, 822 | callback_on_step_end: Optional[ 823 | Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] 824 | ] = None, 825 | callback_on_step_end_tensor_inputs: List[str] = ["latents"], 826 | prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, 827 | max_sequence_length: int = 256, 828 | image_embed_interleave: Optional[int] = None, 829 | 830 | use_low_pass_guidance: bool = False, 831 | lp_filter_type: str = "none", # {'gaussian_blur', 'down_up'} 832 | lp_filter_in_latent: bool = False, # When set to True, low-pass filter is done after encoder. If False, low-pass filter is applied to image directly before encoder. 833 | lp_blur_sigma: float = 15.0, # Used with 'gaussian_blur'. Gaussian filter sigma value. 834 | lp_blur_kernel_size: float = 0.02734375, # Used with 'gaussian_blur'. Gaussian filter size. When set to int, used directly as kernel size. When set to float, H * `lp_blur_kernel_size` is used as kernel size. 835 | lp_resize_factor: float = 0.25, # Used with 'down_up'. Image is bilinearly downsized to (`lp_resize_factor` * WIDTH, `lp_resize_factor` * HEIGHT) and then back to original. 836 | 837 | lp_strength_schedule_type: str = "none", # Scheduling type for low-pass filtering strength. Options: {"none", "linear", "interval", "exponential"} 838 | schedule_blur_kernel_size: bool = False, # If True, schedule blur kernel size as well. Otherwise, fix to initial value. 839 | 840 | # --- Constant Interval Scheduling Params for LP Strength --- 841 | schedule_interval_start_time: float = 0.0, # Starting timestep for interval scheduling 842 | schedule_interval_end_time: float = 0.05, # Ending timestep for interval scheduling 843 | 844 | # --- Linear Scheduling Params for LP Strength --- 845 | schedule_linear_start_weight: float = 1.0, # Starting LP weight for linear scheduling at t=T (step 0) 846 | schedule_linear_end_weight: float = 0.0, # Ending LP weight for linear scheduling at t=T * schedule_linear_end_time 847 | schedule_linear_end_time: float = 0.5, # Timestep fraction at which schedule_linear_end is reached 848 | 849 | # --- Exponential Scheduling Params for LP Strength --- 850 | schedule_exp_decay_rate: float = 10.0, # Decay rate for 'exponential' schedule. Higher values decay faster. Strength = exp(-rate * time_fraction). 851 | 852 | lp_on_noisy_latent = False, 853 | enable_lp_img_embeds = False, 854 | i2v_stable= False, 855 | ): 856 | r""" 857 | The call function to the pipeline for generation. 858 | 859 | Args: 860 | prompt (`str` or `List[str]`, *optional*): 861 | The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. 862 | instead. 863 | prompt_2 (`str` or `List[str]`, *optional*): 864 | The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is 865 | will be used instead. 866 | negative_prompt (`str` or `List[str]`, *optional*): 867 | The prompt or prompts not to guide the image generation. If not defined, one has to pass 868 | `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is 869 | not greater than `1`). 870 | negative_prompt_2 (`str` or `List[str]`, *optional*): 871 | The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and 872 | `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. 873 | height (`int`, defaults to `720`): 874 | The height in pixels of the generated image. 875 | width (`int`, defaults to `1280`): 876 | The width in pixels of the generated image. 877 | num_frames (`int`, defaults to `129`): 878 | The number of frames in the generated video. 879 | num_inference_steps (`int`, defaults to `50`): 880 | The number of denoising steps. More denoising steps usually lead to a higher quality image at the 881 | expense of slower inference. 882 | sigmas (`List[float]`, *optional*): 883 | Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in 884 | their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed 885 | will be used. 886 | true_cfg_scale (`float`, *optional*, defaults to 1.0): 887 | When > 1.0 and a provided `negative_prompt`, enables true classifier-free guidance. 888 | guidance_scale (`float`, defaults to `1.0`): 889 | Guidance scale as defined in [Classifier-Free Diffusion 890 | Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. 891 | of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting 892 | `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to 893 | the text `prompt`, usually at the expense of lower image quality. Note that the only available 894 | HunyuanVideo model is CFG-distilled, which means that traditional guidance between unconditional and 895 | conditional latent is not applied. 896 | num_videos_per_prompt (`int`, *optional*, defaults to 1): 897 | The number of images to generate per prompt. 898 | generator (`torch.Generator` or `List[torch.Generator]`, *optional*): 899 | A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make 900 | generation deterministic. 901 | latents (`torch.Tensor`, *optional*): 902 | Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image 903 | generation. Can be used to tweak the same generation with different prompts. If not provided, a latents 904 | tensor is generated by sampling using the supplied random `generator`. 905 | prompt_embeds (`torch.Tensor`, *optional*): 906 | Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not 907 | provided, text embeddings are generated from the `prompt` input argument. 908 | pooled_prompt_embeds (`torch.FloatTensor`, *optional*): 909 | Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. 910 | If not provided, pooled text embeddings will be generated from `prompt` input argument. 911 | negative_prompt_embeds (`torch.FloatTensor`, *optional*): 912 | Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt 913 | weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input 914 | argument. 915 | negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): 916 | Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt 917 | weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` 918 | input argument. 919 | output_type (`str`, *optional*, defaults to `"pil"`): 920 | The output format of the generated image. Choose between `PIL.Image` or `np.array`. 921 | return_dict (`bool`, *optional*, defaults to `True`): 922 | Whether or not to return a [`HunyuanVideoPipelineOutput`] instead of a plain tuple. 923 | attention_kwargs (`dict`, *optional*): 924 | A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under 925 | `self.processor` in 926 | [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). 927 | clip_skip (`int`, *optional*): 928 | Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that 929 | the output of the pre-final layer will be used for computing the prompt embeddings. 930 | callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): 931 | A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of 932 | each denoising step during the inference. with the following arguments: `callback_on_step_end(self: 933 | DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a 934 | list of all tensors as specified by `callback_on_step_end_tensor_inputs`. 935 | prompt_template (`Dict[str, Any]`, *optional*, defaults to `DEFAULT_PROMPT_TEMPLATE`): 936 | A dictionary defining the template for constructing the LLaVA prompt. It should include keys like 937 | `"template"`, `"crop_start"`, `"image_emb_start"`, `"image_emb_end"`, `"image_emb_len"`, and 938 | `"double_return_token_id"`. 939 | max_sequence_length (`int`, *optional*, defaults to 256): 940 | The maximum sequence length for the LLaVA text encoder. 941 | image_embed_interleave (`int`, *optional*): 942 | The interleave factor for image embeddings. Defaults to 2 if `image_condition_type` is 943 | `"latent_concat"`, 4 if `"token_replace"`, otherwise 1. 944 | use_low_pass_guidance (`bool`, *optional*, defaults to `False`): 945 | Whether to use low-pass guidance. This can help to improve the temporal consistency of the generated 946 | video. 947 | lp_filter_type (`str`, *optional*, defaults to `"none"`): 948 | The type of low-pass filter to apply. Can be one of `gaussian_blur` or `down_up`. 949 | lp_filter_in_latent (`bool`, *optional*, defaults to `False`): 950 | If `True`, the low-pass filter is applied to the latent representation of the image. If `False`, it is 951 | applied to the image in pixel space before encoding. 952 | lp_blur_sigma (`float`, *optional*, defaults to `15.0`): 953 | The sigma value for the Gaussian blur filter. Only used if `lp_filter_type` is `gaussian_blur`. 954 | lp_blur_kernel_size (`float`, *optional*, defaults to `0.02734375`): 955 | The kernel size for the Gaussian blur filter. If an `int`, it's used directly. If a `float`, the kernel 956 | size is calculated as `height * lp_blur_kernel_size`. Only used if `lp_filter_type` is `gaussian_blur`. 957 | lp_resize_factor (`float`, *optional*, defaults to `0.25`): 958 | The resize factor for the down-sampling and up-sampling filter. Only used if `lp_filter_type` is 959 | `down_up`. 960 | lp_strength_schedule_type (`str`, *optional*, defaults to `"none"`): 961 | The scheduling type for the low-pass filter strength. Can be one of `none`, `linear`, `interval`, or 962 | `exponential`. 963 | schedule_blur_kernel_size (`bool`, *optional*, defaults to `False`): 964 | If `True`, the blur kernel size is also scheduled along with the strength. Otherwise, it remains fixed. 965 | schedule_interval_start_time (`float`, *optional*, defaults to `0.0`): 966 | The starting timestep fraction for interval scheduling. Only used if `lp_strength_schedule_type` is 967 | `interval`. 968 | schedule_interval_end_time (`float`, *optional*, defaults to `0.05`): 969 | The ending timestep fraction for interval scheduling. Only used if `lp_strength_schedule_type` is 970 | `interval`. 971 | schedule_linear_start_weight (`float`, *optional*, defaults to `1.0`): 972 | The starting weight for the low-pass filter strength in a linear schedule. Corresponds to the first 973 | timestep. Only used if `lp_strength_schedule_type` is `linear`. 974 | schedule_linear_end_weight (`float`, *optional*, defaults to `0.0`): 975 | The ending weight for the low-pass filter strength in a linear schedule. Only used if 976 | `lp_strength_schedule_type` is `linear`. 977 | schedule_linear_end_time (`float`, *optional*, defaults to `0.5`): 978 | The timestep fraction at which `schedule_linear_end_weight` is reached in a linear schedule. Only used 979 | if `lp_strength_schedule_type` is `linear`. 980 | schedule_exp_decay_rate (`float`, *optional*, defaults to `10.0`): 981 | The decay rate for the exponential schedule. Higher values lead to faster decay. Only used if 982 | `lp_strength_schedule_type` is `exponential`. 983 | lp_on_noisy_latent (`bool`, *optional*, defaults to `False`): 984 | If `True` and using low-pass guidance with true CFG, applies the low-pass condition to the noisy latent input 985 | when the low-pass strength is zero, instead of using the original image condition. 986 | enable_lp_img_embeds (`bool`, *optional*, defaults to `False`): 987 | Whether to apply low-pass filtering to image embeddings. 988 | i2v_stable (`bool`, *optional*, defaults to `False`): 989 | If `True`, initializes the video latents with initial image latents. 990 | 991 | Examples: 992 | 993 | Returns: 994 | [`~HunyuanVideoPipelineOutput`] or `tuple`: 995 | If `return_dict` is `True`, [`HunyuanVideoPipelineOutput`] is returned, otherwise a `tuple` is returned 996 | where the first element is a list with the generated images and the second element is a list of `bool`s 997 | indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content. 998 | """ 999 | 1000 | if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): 1001 | callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs 1002 | 1003 | # 1. Check inputs. Raise error if not correct 1004 | self.check_inputs( 1005 | prompt, 1006 | prompt_2, 1007 | height, 1008 | width, 1009 | prompt_embeds, 1010 | callback_on_step_end_tensor_inputs, 1011 | prompt_template, 1012 | true_cfg_scale, 1013 | guidance_scale, 1014 | ) 1015 | 1016 | image_condition_type = self.transformer.config.image_condition_type 1017 | has_neg_prompt = negative_prompt is not None or ( 1018 | negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None 1019 | ) 1020 | do_true_cfg = true_cfg_scale > 1 and has_neg_prompt 1021 | image_embed_interleave = ( 1022 | image_embed_interleave 1023 | if image_embed_interleave is not None 1024 | else ( 1025 | 2 if image_condition_type == "latent_concat" else 4 if image_condition_type == "token_replace" else 1 1026 | ) 1027 | ) 1028 | 1029 | self._guidance_scale = guidance_scale 1030 | self._attention_kwargs = attention_kwargs 1031 | self._current_timestep = None 1032 | self._interrupt = False 1033 | 1034 | device = self._execution_device 1035 | 1036 | # 2. Define call parameters 1037 | if prompt is not None and isinstance(prompt, str): 1038 | batch_size = 1 1039 | elif prompt is not None and isinstance(prompt, list): 1040 | batch_size = len(prompt) 1041 | else: 1042 | batch_size = prompt_embeds.shape[0] 1043 | 1044 | # 3. Prepare latent variables 1045 | vae_dtype = self.vae.dtype 1046 | image_tensor = self.video_processor.preprocess(image, height, width).to(device, vae_dtype) 1047 | 1048 | if image_condition_type == "latent_concat": 1049 | num_channels_latents = (self.transformer.config.in_channels - 1) // 2 1050 | elif image_condition_type == "token_replace": 1051 | num_channels_latents = self.transformer.config.in_channels 1052 | 1053 | latents, image_latents = self.prepare_latents( 1054 | image_tensor, 1055 | batch_size * num_videos_per_prompt, 1056 | num_channels_latents, 1057 | height, 1058 | width, 1059 | num_frames, 1060 | torch.float32, 1061 | device, 1062 | generator, 1063 | latents, 1064 | image_condition_type, 1065 | i2v_stable 1066 | ) 1067 | if image_condition_type == "latent_concat": 1068 | image_latents[:, :, 1:] = 0 1069 | mask = image_latents.new_ones(image_latents.shape[0], 1, *image_latents.shape[2:]) 1070 | mask[:, :, 1:] = 0 1071 | 1072 | # 4. Encode input prompt 1073 | transformer_dtype = self.transformer.dtype 1074 | prompt_embeds, pooled_prompt_embeds, prompt_attention_mask = self.encode_prompt( 1075 | image=image, 1076 | prompt=prompt, 1077 | prompt_2=prompt_2, 1078 | prompt_template=prompt_template, 1079 | num_videos_per_prompt=num_videos_per_prompt, 1080 | prompt_embeds=prompt_embeds, 1081 | pooled_prompt_embeds=pooled_prompt_embeds, 1082 | prompt_attention_mask=prompt_attention_mask, 1083 | device=device, 1084 | max_sequence_length=max_sequence_length, 1085 | image_embed_interleave=image_embed_interleave, 1086 | ) 1087 | prompt_embeds = prompt_embeds.to(transformer_dtype) 1088 | prompt_attention_mask = prompt_attention_mask.to(transformer_dtype) 1089 | pooled_prompt_embeds = pooled_prompt_embeds.to(transformer_dtype) 1090 | 1091 | if do_true_cfg: 1092 | black_image = PIL.Image.new("RGB", (width, height), 0) 1093 | negative_prompt_embeds, negative_pooled_prompt_embeds, negative_prompt_attention_mask = self.encode_prompt( 1094 | image=black_image, 1095 | prompt=negative_prompt, 1096 | prompt_2=negative_prompt_2, 1097 | prompt_template=prompt_template, 1098 | num_videos_per_prompt=num_videos_per_prompt, 1099 | prompt_embeds=negative_prompt_embeds, 1100 | pooled_prompt_embeds=negative_pooled_prompt_embeds, 1101 | prompt_attention_mask=negative_prompt_attention_mask, 1102 | device=device, 1103 | max_sequence_length=max_sequence_length, 1104 | image_embed_interleave=image_embed_interleave, 1105 | ) 1106 | negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype) 1107 | negative_prompt_attention_mask = negative_prompt_attention_mask.to(transformer_dtype) 1108 | negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.to(transformer_dtype) 1109 | 1110 | # 5. Prepare timesteps 1111 | sigmas = np.linspace(1.0, 0.0, num_inference_steps + 1)[:-1] if sigmas is None else sigmas 1112 | timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, sigmas=sigmas) 1113 | 1114 | # 6. Prepare guidance condition 1115 | guidance = None 1116 | if self.transformer.config.guidance_embeds: 1117 | guidance = ( 1118 | torch.tensor([guidance_scale] * latents.shape[0], dtype=transformer_dtype, device=device) * 1000.0 1119 | ) 1120 | 1121 | # 7. Denoising loop 1122 | num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order 1123 | self._num_timesteps = len(timesteps) 1124 | 1125 | with self.progress_bar(total=num_inference_steps) as progress_bar: 1126 | for i, t in enumerate(timesteps): 1127 | if self.interrupt: 1128 | continue 1129 | 1130 | self._current_timestep = t 1131 | if do_true_cfg and use_low_pass_guidance: 1132 | lp_strength = lp_utils.get_lp_strength( 1133 | step_index=i, 1134 | total_steps=num_inference_steps, 1135 | lp_strength_schedule_type=lp_strength_schedule_type, 1136 | schedule_interval_start_time=schedule_interval_start_time, 1137 | schedule_interval_end_time=schedule_interval_end_time, 1138 | schedule_linear_start_weight=schedule_linear_start_weight, 1139 | schedule_linear_end_weight=schedule_linear_end_weight, 1140 | schedule_linear_end_time=schedule_linear_end_time, 1141 | schedule_exp_decay_rate=schedule_exp_decay_rate, 1142 | ) 1143 | 1144 | modulated_lp_blur_sigma = lp_blur_sigma * lp_strength 1145 | if schedule_blur_kernel_size: 1146 | modulated_lp_blur_kernel_size = lp_blur_kernel_size * lp_strength 1147 | else: 1148 | modulated_lp_blur_kernel_size = lp_blur_kernel_size 1149 | 1150 | # No-effect resize_factor is 1.0 1151 | modulated_lp_resize_factor = 1.0 - (1.0 - lp_resize_factor) * lp_strength 1152 | 1153 | if enable_lp_img_embeds: 1154 | assert False, "Low-pass filter on image embeds is not supported in HunyuanVideo pipeline. Please set enable_lp_img_embeds = False" 1155 | 1156 | lp_image_latents = self.prepare_lp( 1157 | lp_filter_type=lp_filter_type, 1158 | lp_blur_sigma=modulated_lp_blur_sigma, 1159 | lp_blur_kernel_size=modulated_lp_blur_kernel_size, 1160 | lp_resize_factor=modulated_lp_resize_factor, 1161 | generator=generator, 1162 | num_frames=num_frames, 1163 | use_low_pass_guidance=use_low_pass_guidance, 1164 | lp_filter_in_latent=lp_filter_in_latent, 1165 | orig_image_latents=image_latents, 1166 | orig_image_tensor=image 1167 | ) 1168 | if lp_strength == 0.0 or lp_on_noisy_latent: 1169 | latent_model_input = torch.cat([latents] * 2) 1170 | img_cond = torch.cat([image_latents,image_latents], dim=0).to(transformer_dtype) 1171 | latent_model_input = torch.cat([img_cond, latent_model_input[:, :, 1:]], dim=2).to(transformer_dtype) 1172 | 1173 | concat_prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) 1174 | concat_pooled_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0) 1175 | concat_prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) 1176 | else: 1177 | latent_model_input = torch.cat([latents] * 3) 1178 | img_cond = torch.cat([image_latents,lp_image_latents,lp_image_latents], dim=0) 1179 | latent_model_input = torch.cat([img_cond, latent_model_input[:, :, 1:]], dim=2).to(transformer_dtype) 1180 | concat_prompt_embeds = torch.cat([negative_prompt_embeds,negative_prompt_embeds, prompt_embeds], dim=0) 1181 | concat_pooled_embeds = torch.cat([negative_pooled_prompt_embeds,negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0) 1182 | concat_prompt_attention_mask = torch.cat([negative_prompt_attention_mask,negative_prompt_attention_mask, prompt_attention_mask], dim=0) 1183 | elif do_true_cfg: 1184 | latent_model_input = torch.cat([latents] * 2) 1185 | img_cond = torch.cat([image_latents,image_latents], dim=0).to(transformer_dtype) 1186 | latent_model_input = torch.cat([img_cond, latent_model_input[:, :, 1:]], dim=2).to(transformer_dtype) 1187 | 1188 | concat_prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) 1189 | concat_pooled_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0) 1190 | concat_prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) 1191 | elif not use_low_pass_guidance: 1192 | latent_model_input = torch.cat([image_latents, latents[:, :, 1:]], dim=2).to(transformer_dtype) 1193 | concat_prompt_embeds = prompt_embeds 1194 | concat_pooled_embeds = pooled_prompt_embeds 1195 | concat_prompt_attention_mask = prompt_attention_mask 1196 | else: 1197 | lp_strength = lp_utils.get_lp_strength( 1198 | step_index=i, 1199 | total_steps=num_inference_steps, 1200 | lp_strength_schedule_type=lp_strength_schedule_type, 1201 | schedule_interval_start_time=schedule_interval_start_time, 1202 | schedule_interval_end_time=schedule_interval_end_time, 1203 | schedule_linear_start_weight=schedule_linear_start_weight, 1204 | schedule_linear_end_weight=schedule_linear_end_weight, 1205 | schedule_linear_end_time=schedule_linear_end_time, 1206 | schedule_exp_decay_rate=schedule_exp_decay_rate, 1207 | ) 1208 | 1209 | modulated_lp_blur_sigma = lp_blur_sigma * lp_strength 1210 | if schedule_blur_kernel_size: 1211 | modulated_lp_blur_kernel_size = lp_blur_kernel_size * lp_strength 1212 | else: 1213 | modulated_lp_blur_kernel_size = lp_blur_kernel_size 1214 | 1215 | modulated_lp_resize_factor = 1.0 - (1.0 - lp_resize_factor) * lp_strength 1216 | 1217 | if enable_lp_img_embeds: 1218 | assert False, "Low-pass filter on image embeds is not supported in HunyuanVideo pipeline. Please set enable_lp_img_embeds = False" 1219 | 1220 | lp_image_latents = self.prepare_lp( 1221 | lp_filter_type=lp_filter_type, 1222 | lp_blur_sigma=modulated_lp_blur_sigma, 1223 | lp_blur_kernel_size=modulated_lp_blur_kernel_size, 1224 | lp_resize_factor=modulated_lp_resize_factor, 1225 | generator=generator, 1226 | num_frames=num_frames, 1227 | use_low_pass_guidance=use_low_pass_guidance, 1228 | lp_filter_in_latent=lp_filter_in_latent, 1229 | orig_image_latents=image_latents, 1230 | orig_image_tensor=image 1231 | ) 1232 | latent_model_input = torch.cat([lp_image_latents, latents[:, :, 1:]], dim=2).to(transformer_dtype) 1233 | concat_prompt_embeds = prompt_embeds 1234 | concat_pooled_embeds = pooled_prompt_embeds 1235 | concat_prompt_attention_mask = prompt_attention_mask 1236 | 1237 | timestep = t.expand(latent_model_input.shape[0]).to(transformer_dtype) 1238 | latent_model_input = latent_model_input.to(transformer_dtype) 1239 | prompt_embeds = prompt_embeds.to(transformer_dtype) 1240 | prompt_attention_mask = prompt_attention_mask.to(transformer_dtype) 1241 | pooled_prompt_embeds = pooled_prompt_embeds.to(transformer_dtype) 1242 | 1243 | noise_pred = self.transformer( 1244 | hidden_states=latent_model_input, 1245 | timestep=timestep, 1246 | encoder_hidden_states=concat_prompt_embeds, 1247 | encoder_attention_mask=concat_prompt_attention_mask, 1248 | pooled_projections=concat_pooled_embeds, 1249 | guidance=guidance, 1250 | attention_kwargs=attention_kwargs, 1251 | return_dict=False, 1252 | )[0] 1253 | 1254 | if noise_pred.shape[0] == 3: 1255 | noise_pred_uncond_init, noise_pred_uncond, noise_pred_text = noise_pred.chunk(3) 1256 | noise_pred = ( 1257 | noise_pred_uncond_init + true_cfg_scale * (noise_pred_text - noise_pred_uncond) 1258 | ) 1259 | elif noise_pred.shape[0] == 2: 1260 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) 1261 | noise_pred = noise_pred_uncond + true_cfg_scale * (noise_pred_text - noise_pred_uncond) 1262 | 1263 | # compute the previous noisy sample x_t -> x_t-1 1264 | if image_condition_type == "latent_concat": 1265 | latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] 1266 | elif image_condition_type == "token_replace": 1267 | latents = latents = self.scheduler.step( 1268 | noise_pred[:, :, 1:], t, latents[:, :, 1:], return_dict=False 1269 | )[0] 1270 | latents = torch.cat([image_latents, latents], dim=2) 1271 | 1272 | if callback_on_step_end is not None: 1273 | callback_kwargs = {} 1274 | for k in callback_on_step_end_tensor_inputs: 1275 | callback_kwargs[k] = locals()[k] 1276 | callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) 1277 | 1278 | latents = callback_outputs.pop("latents", latents) 1279 | prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) 1280 | 1281 | # call the callback, if provided 1282 | if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): 1283 | progress_bar.update() 1284 | 1285 | if XLA_AVAILABLE: 1286 | xm.mark_step() 1287 | 1288 | self._current_timestep = None 1289 | 1290 | if not output_type == "latent": 1291 | latents = latents.to(self.vae.dtype) / self.vae_scaling_factor 1292 | video = self.vae.decode(latents, return_dict=False)[0] 1293 | if image_condition_type == "latent_concat": 1294 | video = video[:, :, 4:, :, :] 1295 | video = self.video_processor.postprocess_video(video, output_type=output_type) 1296 | else: 1297 | if image_condition_type == "latent_concat": 1298 | video = latents[:, :, 1:, :, :] 1299 | else: 1300 | video = latents 1301 | 1302 | # Offload all models 1303 | self.maybe_free_model_hooks() 1304 | 1305 | if not return_dict: 1306 | return (video,) 1307 | 1308 | return HunyuanVideoPipelineOutput(frames=video) 1309 | --------------------------------------------------------------------------------