├── .gitattributes ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── __init__.py ├── cascade_node.py ├── diffusers_helper ├── bucket_tools.py ├── clip_vision.py ├── dit_common.py ├── gradio │ └── progress_bar.py ├── hf_login.py ├── hunyuan.py ├── k_diffusion │ ├── uni_pc_fm.py │ └── wrapper.py ├── lora.py ├── memory.py ├── models │ └── hunyuan_video_packed.py ├── pipelines │ └── k_diffusion_hunyuan.py ├── thread_utils.py └── utils.py ├── example_workflows ├── framepack_hv_cascade.json ├── framepack_hv_example.json ├── framepack_hv_keyframe.json ├── framepack_hv_lora.json ├── framepack_hv_start_end.json └── framepack_hv_timeprompt.json ├── fp8_optimization.py ├── images ├── framepack-cascade2.mp4 └── screenshot-01.png ├── nodes.py ├── requirements.txt └── transformer_config.json /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | hf_download/ 2 | outputs/ 3 | repo/ 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | cover/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | .pybuilder/ 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | # For a library or package, you might want to ignore these files since the code is 91 | # intended to run in multiple environments; otherwise, check them in: 92 | # .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # UV 102 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 103 | # This is especially recommended for binary packages to ensure reproducibility, and is more 104 | # commonly ignored for libraries. 105 | #uv.lock 106 | 107 | # poetry 108 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 109 | # This is especially recommended for binary packages to ensure reproducibility, and is more 110 | # commonly ignored for libraries. 111 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 112 | #poetry.lock 113 | 114 | # pdm 115 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 116 | #pdm.lock 117 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 118 | # in version control. 119 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 120 | .pdm.toml 121 | .pdm-python 122 | .pdm-build/ 123 | 124 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 125 | __pypackages__/ 126 | 127 | # Celery stuff 128 | celerybeat-schedule 129 | celerybeat.pid 130 | 131 | # SageMath parsed files 132 | *.sage.py 133 | 134 | # Environments 135 | .env 136 | .venv 137 | env/ 138 | venv/ 139 | ENV/ 140 | env.bak/ 141 | venv.bak/ 142 | 143 | # Spyder project settings 144 | .spyderproject 145 | .spyproject 146 | 147 | # Rope project settings 148 | .ropeproject 149 | 150 | # mkdocs documentation 151 | /site 152 | 153 | # mypy 154 | .mypy_cache/ 155 | .dmypy.json 156 | dmypy.json 157 | 158 | # Pyre type checker 159 | .pyre/ 160 | 161 | # pytype static type analyzer 162 | .pytype/ 163 | 164 | # Cython debug symbols 165 | cython_debug/ 166 | 167 | # PyCharm 168 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 169 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 170 | # and can be added to the global gitignore or merged into this file. For a more nuclear 171 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 172 | .idea/ 173 | 174 | # Ruff stuff: 175 | .ruff_cache/ 176 | 177 | # PyPI configuration file 178 | .pypirc 179 | demo_gradio.py -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "autocast", 4 | "denoise", 5 | "hunyuan", 6 | "Lantents", 7 | "musubi", 8 | "pooler", 9 | "poolers", 10 | "teacache", 11 | "unet", 12 | "unipc", 13 | "vecs" 14 | ] 15 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Notice: This is a fork by nirvash. This repository is a test version for keyframe support, based on original ComfyUI-FramePackWrapper. 2 | Original repository (kijai): https://github.com/kijai/ComfyUI-FramePackWrapper** 3 | 4 | ![screenshot](images/screenshot-01.png) 5 | 6 | ## Abstract 7 | - FramePack を ComfyUI で利用するためのカスタムノードです. 8 | kijai 氏のオリジナルに加え, いくつかの機能を追加しています. 9 | - This is a custom node for using FramePack in ComfyUI. 10 | In addition to the original by kijai, several features have been added. 11 | 12 | ## How to use 13 | - [EasyWanVideo](https://github.com/Zuntan03/EasyWanVideo) を利用することで必要な ComfyUI 環境や動画作成に必要なモデルデータを簡単にセットアップすることができます 14 | - [EasyWanVideo](https://github.com/Zuntan03/EasyWanVideo) allows you to easily set up the necessary ComfyUI environment and model files for video generation. 15 | - ComfyUI を使い慣れている方は custom_nodes フォルダにこのリポジトリをチェックアウトして利用してください 16 | - If you're already familiar with ComfyUI, you can simply check out this repository into your custom_nodes folder and start using it. 17 | - [example_workflows](./example_workflows) にサンプルのワークフローが含まれていますので, そちらを参考にしてください 18 | - Sample workflows are included in [example_workflows](./example_workflows), so please refer to them. 19 | 20 | ## Start - End 21 | https://github.com/user-attachments/assets/d4af1e9b-904f-41aa-8a00-4306ed4ff4b0 22 | - 開始画像に加えて, 終了画像を指定することができます 23 | - In addition to the start image, you can specify an end image. 24 | 25 | ## Keyframe 26 | https://github.com/user-attachments/assets/23e777e5-dd49-444f-bccf-69b4d00625a2 27 | 28 | https://github.com/user-attachments/assets/dbc4444e-5e6d-41ad-b1ff-801f27ca86cf 29 | - キーフレームを指定することで, 画像の変化を制御することができます 30 | - By specifying keyframes, you can control the changes in the image. 31 | 32 | ## Cascade Sampler 33 | https://github.com/user-attachments/assets/7491220b-49b5-4cfe-984d-ea7f71a55610 34 | 35 | From left to right: 36 | - 1: Entire 5-second clip generated at once 37 | - 2: Split generation for 1 section 38 | - 3: Split generation for 2 sections 39 | - 4: Final result generated with 3 samplers. 40 | 41 | - 動画を段階的に生成することができます. 気に入った前段の生成結果が得られたら、その続きを生成することができます 42 | - You can generate a video in stages. If you are satisfied with the previous stage, you can generate the next stage. 43 | # LoRA 44 | - musubi-tuner で作成した学習データのみ対応 45 | - LoRA 適用時はモデルは bf16、base precision bf16 を指定 (fp8 だと LoRA 適用効果がみられませんでした) 46 | ![image](https://github.com/user-attachments/assets/f1574fc2-2bcc-40e2-be4d-819e942f6af5) 47 | 48 | ## Feature 49 | - Set end frame 50 | - Assign weighted keyframes 51 | - Use different prompts per section 52 | - FramePackCascadeSampler can be cascaded for multi-stage processing 53 | 54 | # ComfyUI Wrapper for [FramePack by lllyasviel](https://lllyasviel.github.io/frame_pack_gitpage/) 55 | 56 | # WORK IN PROGRESS 57 | 58 | Mostly working, took some liberties to make it run faster. 59 | 60 | Uses all the native models for text encoders, VAE and sigclip: 61 | 62 | https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files 63 | 64 | https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main 65 | 66 | And the transformer model itself is either autodownloaded from here: 67 | 68 | https://huggingface.co/lllyasviel/FramePackI2V_HY/tree/main 69 | 70 | to `ComfyUI\models\diffusers\lllyasviel\FramePackI2V_HY` 71 | 72 | Or from single file, in `ComfyUI\models\diffusion_models`: 73 | 74 | https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors 75 | https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors 76 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS 2 | 3 | __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] -------------------------------------------------------------------------------- /cascade_node.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn.functional as F 4 | import gc 5 | import numpy as np 6 | import math 7 | from tqdm import tqdm 8 | import re 9 | 10 | from accelerate import init_empty_weights 11 | from accelerate.utils import set_module_tensor_to_device 12 | 13 | import folder_paths 14 | import comfy.model_management as mm 15 | from comfy.utils import load_torch_file, ProgressBar, common_upscale 16 | import comfy.model_base 17 | import comfy.latent_formats 18 | from comfy.cli_args import args, LatentPreviewMethod 19 | 20 | script_directory = os.path.dirname(os.path.abspath(__file__)) 21 | vae_scaling_factor = 0.476986 22 | 23 | from .diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked 24 | from .diffusers_helper.memory import DynamicSwapInstaller, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation 25 | from .diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan 26 | from .diffusers_helper.utils import crop_or_pad_yield_mask 27 | from .diffusers_helper.bucket_tools import find_nearest_bucket 28 | 29 | class HyVideoModel(comfy.model_base.BaseModel): 30 | def __init__(self, *args, **kwargs): 31 | super().__init__(*args, **kwargs) 32 | self.pipeline = {} 33 | self.load_device = mm.get_torch_device() 34 | 35 | def __getitem__(self, k): 36 | return self.pipeline[k] 37 | 38 | def __setitem__(self, k, v): 39 | self.pipeline[k] = v 40 | 41 | class HyVideoModelConfig: 42 | def __init__(self, dtype): 43 | self.unet_config = {} 44 | self.unet_extra_config = {} 45 | self.latent_format = comfy.latent_formats.HunyuanVideo 46 | self.latent_format.latent_channels = 16 47 | self.manual_cast_dtype = dtype 48 | self.sampling_settings = {"multiplier": 1.0} 49 | self.memory_usage_factor = 2.0 50 | self.unet_config["disable_unet_model_creation"] = True 51 | 52 | class FramePackCascadeSampler: 53 | @classmethod 54 | def INPUT_TYPES(s): 55 | return { 56 | "required": { 57 | "model": ("FramePackMODEL",), 58 | "positive": ("CONDITIONING",), 59 | "negative": ("CONDITIONING",), 60 | "image_embeds": ("CLIP_VISION_OUTPUT", ), 61 | "steps": ("INT", {"default": 30, "min": 1}), 62 | "use_teacache": ("BOOLEAN", {"default": True, "tooltip": "Use teacache for faster sampling."}), 63 | "teacache_rel_l1_thresh": ("FLOAT", {"default": 0.15, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "The threshold for the relative L1 loss."}), 64 | "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 30.0, "step": 0.01}), 65 | "guidance_scale": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 32.0, "step": 0.01}), 66 | "shift": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1000.0, "step": 0.01}), 67 | "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), 68 | "latent_window_size": ("INT", {"default": 9, "min": 1, "max": 33, "step": 1, "tooltip": "The size of the latent window to use for sampling."}), 69 | "total_second_length": ("FLOAT", {"default": 5, "min": 1, "max": 120, "step": 0.1, "tooltip": "The total length of the video in seconds."}), 70 | "gpu_memory_preservation": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 128.0, "step": 0.1, "tooltip": "The amount of GPU memory to preserve."}), 71 | "sampler": (["unipc_bh1", "unipc_bh2"], 72 | { 73 | "default": 'unipc_bh1' 74 | }), 75 | }, 76 | "optional": { 77 | "start_latent": ("LATENT", {"tooltip": "init Latents to use for image2video"} ), 78 | "end_latent": ("LATENT", {"tooltip": "end Latents to use for last frame"} ), 79 | "keyframes": ("LATENT", {"tooltip": "init Lantents to use for image2video keyframes"} ), 80 | "keyframe_indices": ("LIST", {"tooltip": "section index for each keyframe (e.g. [0, 3, 5])"}), 81 | "initial_samples": ("LATENT", {"tooltip": "init Latents to use for video2video"} ), 82 | "denoise_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), 83 | "positive_keyframes": ("LIST", {"tooltip": "List of positive CONDITIONING for keyframes"}), 84 | "positive_keyframe_indices": ("LIST", {"tooltip": "Section indices for each positive_keyframe"}), 85 | "keyframe_weight": ("FLOAT", {"default": 1.5, "min": 0.1, "max": 10.0, "step": 0.1, "tooltip": "Keyframe multiplier: How much to emphasize the latent at keyframe positions."}), 86 | "section_start": ("INT", {"default": 0, "min": 0}), 87 | "section_count": ("INT", {"default": -1, "min": -1, "tooltip": "-1または未指定で全セクションを処理"}), 88 | "history_latents": ("LATENT", ), 89 | "total_generated_latent_frames": ("INT", {"default": 0, "min": 0, "tooltip": "total generated frames"}), 90 | } 91 | } 92 | 93 | RETURN_TYPES = ("LATENT", 94 | "FramePackMODEL", 95 | "CONDITIONING", 96 | "CONDITIONING", 97 | "CLIP_VISION_OUTPUT", 98 | "LATENT", 99 | "LATENT", 100 | "FLOAT", 101 | "INT", 102 | "INT") 103 | RETURN_NAMES = ("samples", 104 | "model", 105 | "positive", 106 | "negative", 107 | "image_embeds", 108 | "start_latent", 109 | "history_latents", 110 | "total_second_length", 111 | "next_section_start", 112 | "total_generated_latent_frames") 113 | FUNCTION = "process" 114 | CATEGORY = "FramePackWrapper" 115 | 116 | def process(self, model, shift, positive, negative, latent_window_size, use_teacache, total_second_length, teacache_rel_l1_thresh, image_embeds, steps, cfg, 117 | guidance_scale, seed, sampler, gpu_memory_preservation, 118 | start_latent=None, initial_samples=None, keyframes=None, end_latent=None, denoise_strength=1.0, keyframe_indices=None, 119 | positive_keyframes=None, positive_keyframe_indices=None, keyframe_weight=2.0, force_keyframe=False, 120 | section_start=0, section_count=-1, history_latents=None, total_generated_latent_frames=None): 121 | 122 | 123 | # process start 124 | total_latent_sections = total_second_length * 30 / (latent_window_size * 4) 125 | total_latent_sections = int(max(round(total_latent_sections), 1)) 126 | print("total_latent_sections: ", total_latent_sections) 127 | force_keyframe = False 128 | 129 | transformer = model["transformer"] 130 | base_dtype = model["dtype"] 131 | 132 | device = mm.get_torch_device() 133 | offload_device = mm.unet_offload_device() 134 | 135 | mm.unload_all_models() 136 | mm.cleanup_models() 137 | mm.soft_empty_cache() 138 | 139 | original_start_latent = start_latent 140 | start_latent = start_latent["samples"] * vae_scaling_factor 141 | if initial_samples is not None: 142 | initial_samples = initial_samples["samples"] * vae_scaling_factor 143 | if keyframes is not None: 144 | keyframes = keyframes["samples"] * vae_scaling_factor 145 | print(f"keyframes shape: {keyframes.shape}") 146 | if end_latent is not None: 147 | end_latent = end_latent["samples"] * vae_scaling_factor 148 | print("start_latent", start_latent.shape) 149 | B, C, T, H, W = start_latent.shape 150 | 151 | print(f"[FramePackSampler] device: {device}") 152 | print(f"[FramePackSampler] start_latent device: {start_latent.device}") 153 | if keyframes is not None: 154 | print(f"[FramePackSampler] keyframes device: {keyframes.device}") 155 | if end_latent is not None: 156 | print(f"[FramePackSampler] end_latent device: {end_latent.device}") 157 | print(f"[FramePackSampler] positive[0][0] device: {positive[0][0].device}") 158 | print(f"[FramePackSampler] negative[0][0] device: {negative[0][0].device}") 159 | 160 | image_encoder_last_hidden_state = image_embeds["last_hidden_state"].to(base_dtype).to(device) 161 | 162 | llama_vec = positive[0][0].to(base_dtype).to(device) 163 | llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512) 164 | clip_l_pooler = positive[0][1]["pooled_output"].to(base_dtype).to(device) 165 | cached_keyframe_vecs = [] 166 | cached_keyframe_masks = [] 167 | cached_keyframe_poolers = [] 168 | if positive_keyframes is not None: 169 | for kf in positive_keyframes: 170 | v = kf[0][0].to(base_dtype).to(device) 171 | v, m = crop_or_pad_yield_mask(v, length=512) 172 | p = kf[0][1]["pooled_output"].to(base_dtype).to(device) 173 | cached_keyframe_vecs.append(v) 174 | cached_keyframe_masks.append(m) 175 | cached_keyframe_poolers.append(p) 176 | 177 | if not math.isclose(cfg, 1.0): 178 | llama_vec_n = negative[0][0].to(base_dtype) 179 | clip_l_pooler_n = negative[0][1]["pooled_output"].to(base_dtype).to(device) 180 | else: 181 | llama_vec_n = torch.zeros_like(llama_vec, device=device) 182 | clip_l_pooler_n = torch.zeros_like(clip_l_pooler, device=device) 183 | 184 | llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512) 185 | llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) 186 | 187 | 188 | # Sampling 189 | 190 | rnd = torch.Generator("cpu").manual_seed(seed) 191 | 192 | num_frames = latent_window_size * 4 - 3 193 | 194 | if history_latents is None: 195 | print("[FramePackSampler] initializing new history_latents") 196 | history_latents = torch.zeros(size=(1, 16, 1 + 2 + 16, H, W), dtype=torch.float32).cpu() 197 | else: 198 | print("[FramePackSampler] using previous history_latents") 199 | history_latents = history_latents["samples"] # 必ず1+2+16フレームを含む 200 | print(f"[FramePackSampler] previous history shape: {history_latents.shape}") 201 | print(f"[FramePackSampler] previous history means: {history_latents[:,:,:3,:,:].mean().item():.4f}") 202 | real_history_latents = None 203 | 204 | # nodes.py準拠: inで受け取った値を使う 205 | # 初回は0、2回目以降は前段から受け継ぐ 206 | if total_generated_latent_frames is None: 207 | total_generated_latent_frames = 0 208 | 209 | latent_paddings_list = list(reversed(range(total_latent_sections))) 210 | latent_paddings = latent_paddings_list.copy() # Create a copy for iteration 211 | 212 | comfy_model = HyVideoModel( 213 | HyVideoModelConfig(base_dtype), 214 | model_type=comfy.model_base.ModelType.FLOW, 215 | device=device, 216 | ) 217 | 218 | patcher = comfy.model_patcher.ModelPatcher(comfy_model, device, torch.device("cpu")) 219 | from latent_preview import prepare_callback 220 | callback = prepare_callback(patcher, steps) 221 | 222 | move_model_to_device_with_memory_preservation(transformer, target_device=device, preserved_memory_gb=gpu_memory_preservation) 223 | 224 | if total_latent_sections > 4: 225 | # In theory the latent_paddings should follow the above sequence, but it seems that duplicating some 226 | # items looks better than expanding it when total_latent_sections > 4 227 | # One can try to remove below trick and just 228 | # use `latent_paddings = list(reversed(range(total_latent_sections)))` to compare 229 | latent_paddings = [3] + [2] * (total_latent_sections - 3) + [1, 0] 230 | latent_paddings_list = latent_paddings.copy() 231 | 232 | print(f"[FramePackSampler] initial section_start: {section_start}, section_count: {section_count}") 233 | # セクション範囲の制御 234 | if section_count == -1: 235 | section_count = total_latent_sections - section_start 236 | section_end = min(section_start + section_count, total_latent_sections) 237 | next_section = section_start + section_count 238 | print(f"[FramePackSampler] calculated section_count: {section_count}, section_end: {section_end}, next_section: {next_section}") 239 | for section_no in range(section_start, section_end): 240 | latent_padding = latent_paddings[section_no] 241 | print(f"latent_padding: {latent_padding}") 242 | print(f"section no: {section_no}") 243 | is_last_section = latent_padding == 0 244 | latent_padding_size = latent_padding * latent_window_size 245 | 246 | print(f'latent_padding_size = {latent_padding_size}, is_last_section = {is_last_section}') 247 | 248 | total_size = sum([1, latent_padding_size, latent_window_size, 1, 2, 16]) 249 | print(f"[FramePackSampler] section {section_no}: indices components: pre=1, padding={latent_padding_size}, window={latent_window_size}, post=1, 2x=2, 4x=16") 250 | print(f"[FramePackSampler] section {section_no}: total indices size: {total_size}") 251 | 252 | indices = torch.arange(0, total_size).unsqueeze(0) 253 | clean_latent_indices_pre, blank_indices, latent_indices, clean_latent_indices_post, clean_latent_2x_indices, clean_latent_4x_indices = indices.split([1, latent_padding_size, latent_window_size, 1, 2, 16], dim=1) 254 | 255 | print(f"[FramePackSampler] section {section_no}: latent_indices: {latent_indices.shape}, values={latent_indices.tolist()}") 256 | clean_latent_indices = torch.cat([clean_latent_indices_pre, clean_latent_indices_post], dim=1) 257 | 258 | # clean_latents_pre を keyframes からセクションごとに取得。なければ start_latent 259 | total_sections = len(latent_paddings) 260 | forward_section_no = total_sections - 1 - section_no 261 | current_keyframe = start_latent.to(history_latents) 262 | idx_current = 0 263 | next_idx = None 264 | if keyframes is not None and keyframes.shape[2] > 0 and keyframe_indices is not None and len(keyframe_indices) > 0: 265 | if forward_section_no < keyframe_indices[0]: 266 | # 先頭より前の区間: start_latent→最初のキーフレーム 267 | current_keyframe = start_latent.to(history_latents) 268 | idx_current = 0 269 | next_idx = keyframe_indices[0] 270 | elif forward_section_no >= keyframe_indices[-1]: 271 | # 最後のキーフレーム以降: 最後のキーフレーム→末尾 272 | current_keyframe = keyframes[:, :, -1:, :, :].to(history_latents) 273 | idx_current = keyframe_indices[-1] 274 | next_idx = total_sections - 1 275 | else: 276 | for i in range(1, len(keyframe_indices)): 277 | if keyframe_indices[i-1] <= forward_section_no < keyframe_indices[i]: 278 | current_keyframe = keyframes[:, :, i-1:i, :, :].to(history_latents) 279 | idx_current = keyframe_indices[i-1] 280 | next_idx = keyframe_indices[i] 281 | break 282 | # t計算: 分母が0（同じキーフレームindexが複数回設定など）の場合はt=1.0で回避 283 | width = next_idx - idx_current if next_idx is not None else 1 284 | if width == 0: 285 | t = 1.0 286 | else: 287 | t = (next_idx - forward_section_no) / width 288 | weight = 1.0 + (keyframe_weight - 1.0) * t 289 | clean_latents_pre = current_keyframe * weight 290 | print(f"[FramePackSampler] forward_section_no={forward_section_no}: use keyframe {idx_current} (next {next_idx}), weight={weight:.2f}, t={t:.2f}") 291 | else: 292 | clean_latents_pre = start_latent.to(history_latents) 293 | print(f"keyframes is None: uses start_latent") 294 | clean_latents_post, clean_latents_2x, clean_latents_4x = history_latents[:, :, :1 + 2 + 16, :, :].split([1, 2, 16], dim=2) 295 | print(f"[FramePackSampler] section {section_no}: clean_latents sizes: post={clean_latents_post.shape}, 2x={clean_latents_2x.shape}, 4x={clean_latents_4x.shape}") 296 | # end_latent対応: 最初のセクションでclean_latents_postをend_latentで差し替え 297 | if section_no == 0 and end_latent is not None: 298 | print(f"[FramePackSampler] end_latent is set. Overwriting clean_latents_post. old shape: {clean_latents_post.shape}, new shape: {end_latent.shape}") 299 | clean_latents_post = end_latent.to(clean_latents_post) 300 | clean_latents = torch.cat([clean_latents_pre, clean_latents_post], dim=2) 301 | print(f"[FramePackSampler] section {section_no}: final clean_latents shape: {clean_latents.shape}") 302 | 303 | #vid2vid 304 | 305 | if initial_samples is not None: 306 | total_length = initial_samples.shape[2] 307 | 308 | # Get the max padding value for normalization 309 | max_padding = max(latent_paddings_list) 310 | 311 | if is_last_section: 312 | # Last section should capture the end of the sequence 313 | start_idx = max(0, total_length - latent_window_size) 314 | else: 315 | # Calculate windows that distribute more evenly across the sequence 316 | # This normalizes the padding values to create appropriate spacing 317 | if max_padding > 0: # Avoid division by zero 318 | progress = (max_padding - latent_padding) / max_padding 319 | start_idx = int(progress * max(0, total_length - latent_window_size)) 320 | else: 321 | start_idx = 0 322 | 323 | end_idx = min(start_idx + latent_window_size, total_length) 324 | print(f"start_idx: {start_idx}, end_idx: {end_idx}, total_length: {total_length}") 325 | input_init_latents = initial_samples[:, :, start_idx:end_idx, :, :].to(device) 326 | 327 | 328 | # セクションごとのpositiveを選択 329 | section_positive = positive 330 | use_keyframe_positive = False 331 | current_llama_vec = llama_vec 332 | current_llama_attention_mask = llama_attention_mask 333 | current_clip_l_pooler = clip_l_pooler 334 | if positive_keyframes is not None and positive_keyframe_indices is not None and len(positive_keyframes) > 0: 335 | total_sections = len(latent_paddings) 336 | forward_section_no = total_sections - 1 - section_no 337 | kf_idx = None 338 | for i, idx in enumerate(positive_keyframe_indices): 339 | if forward_section_no <= idx: 340 | kf_idx = i 341 | break 342 | if kf_idx is not None: 343 | section_positive = positive_keyframes[kf_idx] 344 | use_keyframe_positive = True 345 | current_llama_vec = cached_keyframe_vecs[kf_idx] 346 | current_llama_attention_mask = cached_keyframe_masks[kf_idx] 347 | current_clip_l_pooler = cached_keyframe_poolers[kf_idx] 348 | print(f"[FramePackSampler] section {section_no} (forward {forward_section_no}): use positive_keyframe {kf_idx} (user index {positive_keyframe_indices[kf_idx]})") 349 | else: 350 | # forward_section_no が最後のキーフレームindexより大きい場合は最終キーフレームを使う 351 | section_positive = positive_keyframes[-1] 352 | use_keyframe_positive = True 353 | current_llama_vec = cached_keyframe_vecs[-1] 354 | current_llama_attention_mask = cached_keyframe_masks[-1] 355 | current_clip_l_pooler = cached_keyframe_poolers[-1] 356 | print(f"[FramePackSampler] section {section_no} (forward {forward_section_no}): use last positive_keyframe (user index {positive_keyframe_indices[-1]})") 357 | print(f"[FramePackSampler] section {section_no}: section_positive[0][0].shape = {section_positive[0][0].shape}") 358 | 359 | if use_teacache: 360 | transformer.initialize_teacache(enable_teacache=True, num_steps=steps, rel_l1_thresh=teacache_rel_l1_thresh) 361 | else: 362 | transformer.initialize_teacache(enable_teacache=False) 363 | 364 | print(f"[FramePackSampler] section {section_no}: starting generation...") 365 | with torch.autocast(device_type=mm.get_autocast_device(device), dtype=base_dtype, enabled=True): 366 | generated_latents = sample_hunyuan( 367 | transformer=transformer, 368 | sampler=sampler, 369 | initial_latent=input_init_latents if initial_samples is not None else None, 370 | strength=denoise_strength, 371 | width=W * 8, 372 | height=H * 8, 373 | frames=num_frames, 374 | real_guidance_scale=cfg, 375 | distilled_guidance_scale=guidance_scale, 376 | guidance_rescale=0, 377 | shift=shift if shift != 0 else None, 378 | num_inference_steps=steps, 379 | generator=rnd, 380 | prompt_embeds=current_llama_vec, 381 | prompt_embeds_mask=current_llama_attention_mask, 382 | prompt_poolers=current_clip_l_pooler, 383 | negative_prompt_embeds=llama_vec_n, 384 | negative_prompt_embeds_mask=llama_attention_mask_n, 385 | negative_prompt_poolers=clip_l_pooler_n, 386 | device=device, 387 | dtype=base_dtype, 388 | image_embeddings=image_encoder_last_hidden_state, 389 | latent_indices=latent_indices, 390 | clean_latents=clean_latents, 391 | clean_latent_indices=clean_latent_indices, 392 | clean_latents_2x=clean_latents_2x, 393 | clean_latent_2x_indices=clean_latent_2x_indices, 394 | clean_latents_4x=clean_latents_4x, 395 | clean_latent_4x_indices=clean_latent_4x_indices, 396 | callback=callback, 397 | ) 398 | print(f"[FramePackSampler] section {section_no}: generated shape: {generated_latents.shape}") 399 | if section_no > 0: 400 | print(f"[FramePackSampler] section {section_no}: first frame values: {generated_latents[:,:,0,:,:].mean().item():.4f}") 401 | 402 | # キーフレーム強制オプションが有効な場合、current_keyframe（weightなし）で上書き 403 | # うまく前後がつながらないので蓋してます 404 | if force_keyframe and (keyframes is not None and keyframe_indices is not None): 405 | if section_no in keyframe_indices: 406 | print(f"[FramePackSampler] section {section_no}: blend first frame with keyframe (no weight, 50%)") 407 | generated_latents[:, :, 0:1, :, :] = current_keyframe.to(generated_latents) 408 | 409 | if is_last_section: 410 | generated_latents = torch.cat([start_latent.to(generated_latents), generated_latents], dim=2) 411 | 412 | print(f"[FramePackSampler] section {section_no}: final output shape: {generated_latents.shape}, frames: {generated_latents.shape[2]}") 413 | 414 | total_generated_latent_frames += int(generated_latents.shape[2]) 415 | history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2) 416 | 417 | real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :] 418 | 419 | print(f"[FramePackSampler] section {section_no}: history after cat: {history_latents.shape}") 420 | print(f"[FramePackSampler] section {section_no}: section output shape: {generated_latents.shape}") 421 | 422 | if is_last_section: 423 | break 424 | # forループ終了 425 | 426 | transformer.to(offload_device) 427 | mm.soft_empty_cache() 428 | 429 | # バイパス: 既に全セクション分生成済みの場合に real_history_latentsを返す 430 | if real_history_latents is None: 431 | print(f"[FramePackSampler] bypass: all sections already generated (history_latents.shape={history_latents.shape})") 432 | real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :] 433 | 434 | # 次のセクション番号を計算 435 | next_section_start = section_start + section_count 436 | print(f"[FramePackSampler] returning next_section_start: {next_section_start} (current: {section_start}, count: {section_count})") 437 | 438 | return ( 439 | {"samples": real_history_latents / vae_scaling_factor}, 440 | model, # MODEL 441 | positive, # CONDITIONING 442 | negative, # CONDITIONING 443 | image_embeds, # CLIP_VISION_OUTPUT 444 | original_start_latent, # LATENT（オリジナルをそのまま返す） 445 | {"samples": history_latents}, # LATENT（必要なバッファを含むhistory） 446 | total_second_length, # FLOAT 447 | next_section_start, # next_section_start (INT) 448 | total_generated_latent_frames, # INT（累積生成フレーム数） 449 | ) 450 | -------------------------------------------------------------------------------- /diffusers_helper/bucket_tools.py: -------------------------------------------------------------------------------- 1 | bucket_options = { 2 | 240: [ 3 | (224, 336), 4 | (256, 304), 5 | (288, 272), 6 | (320, 240), 7 | (352, 208), 8 | (384, 176), 9 | ], 10 | 320: [ 11 | (256, 576), 12 | (288, 544), 13 | (320, 512), 14 | (352, 480), 15 | (384, 448), 16 | (416, 416), 17 | (448, 384), 18 | (480, 352), 19 | (512, 320), 20 | (544, 288), 21 | (576, 256), 22 | ], 23 | 640: [ 24 | (416, 960), 25 | (448, 864), 26 | (480, 832), 27 | (512, 768), 28 | (544, 704), 29 | (576, 672), 30 | (608, 640), 31 | (640, 608), 32 | (672, 576), 33 | (704, 544), 34 | (768, 512), 35 | (832, 480), 36 | (864, 448), 37 | (960, 416), 38 | ], 39 | } 40 | 41 | 42 | def find_nearest_bucket(h, w, resolution=640): 43 | min_metric = float("inf") 44 | best_bucket = None 45 | for bucket_h, bucket_w in bucket_options[resolution]: 46 | metric = abs(h * bucket_w - w * bucket_h) 47 | if metric <= min_metric: 48 | min_metric = metric 49 | best_bucket = (bucket_h, bucket_w) 50 | print(f"Best bucket: {best_bucket}") 51 | return best_bucket 52 | 53 | -------------------------------------------------------------------------------- /diffusers_helper/clip_vision.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def hf_clip_vision_encode(image, feature_extractor, image_encoder): 5 | assert isinstance(image, np.ndarray) 6 | assert image.ndim == 3 and image.shape[2] == 3 7 | assert image.dtype == np.uint8 8 | 9 | preprocessed = feature_extractor.preprocess(images=image, return_tensors="pt").to(device=image_encoder.device, dtype=image_encoder.dtype) 10 | image_encoder_output = image_encoder(**preprocessed) 11 | 12 | return image_encoder_output 13 | -------------------------------------------------------------------------------- /diffusers_helper/dit_common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import accelerate.accelerator 3 | 4 | from diffusers.models.normalization import RMSNorm, LayerNorm, FP32LayerNorm, AdaLayerNormContinuous 5 | 6 | 7 | accelerate.accelerator.convert_outputs_to_fp32 = lambda x: x 8 | 9 | 10 | def LayerNorm_forward(self, x): 11 | return torch.nn.functional.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps).to(x) 12 | 13 | 14 | LayerNorm.forward = LayerNorm_forward 15 | torch.nn.LayerNorm.forward = LayerNorm_forward 16 | 17 | 18 | def FP32LayerNorm_forward(self, x): 19 | origin_dtype = x.dtype 20 | return torch.nn.functional.layer_norm( 21 | x.float(), 22 | self.normalized_shape, 23 | self.weight.float() if self.weight is not None else None, 24 | self.bias.float() if self.bias is not None else None, 25 | self.eps, 26 | ).to(origin_dtype) 27 | 28 | 29 | FP32LayerNorm.forward = FP32LayerNorm_forward 30 | 31 | 32 | def RMSNorm_forward(self, hidden_states): 33 | input_dtype = hidden_states.dtype 34 | variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) 35 | hidden_states = hidden_states * torch.rsqrt(variance + self.eps) 36 | 37 | if self.weight is None: 38 | return hidden_states.to(input_dtype) 39 | 40 | return hidden_states.to(input_dtype) * self.weight.to(input_dtype) 41 | 42 | 43 | RMSNorm.forward = RMSNorm_forward 44 | 45 | 46 | def AdaLayerNormContinuous_forward(self, x, conditioning_embedding): 47 | emb = self.linear(self.silu(conditioning_embedding)) 48 | scale, shift = emb.chunk(2, dim=1) 49 | x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :] 50 | return x 51 | 52 | 53 | AdaLayerNormContinuous.forward = AdaLayerNormContinuous_forward 54 | -------------------------------------------------------------------------------- /diffusers_helper/gradio/progress_bar.py: -------------------------------------------------------------------------------- 1 | progress_html = ''' 2 |
3 |
4 |
5 | 6 |
7 | *text* 8 |
9 | ''' 10 | 11 | css = ''' 12 | .loader-container { 13 | display: flex; /* Use flex to align items horizontally */ 14 | align-items: center; /* Center items vertically within the container */ 15 | white-space: nowrap; /* Prevent line breaks within the container */ 16 | } 17 | 18 | .loader { 19 | border: 8px solid #f3f3f3; /* Light grey */ 20 | border-top: 8px solid #3498db; /* Blue */ 21 | border-radius: 50%; 22 | width: 30px; 23 | height: 30px; 24 | animation: spin 2s linear infinite; 25 | } 26 | 27 | @keyframes spin { 28 | 0% { transform: rotate(0deg); } 29 | 100% { transform: rotate(360deg); } 30 | } 31 | 32 | /* Style the progress bar */ 33 | progress { 34 | appearance: none; /* Remove default styling */ 35 | height: 20px; /* Set the height of the progress bar */ 36 | border-radius: 5px; /* Round the corners of the progress bar */ 37 | background-color: #f3f3f3; /* Light grey background */ 38 | width: 100%; 39 | vertical-align: middle !important; 40 | } 41 | 42 | /* Style the progress bar container */ 43 | .progress-container { 44 | margin-left: 20px; 45 | margin-right: 20px; 46 | flex-grow: 1; /* Allow the progress container to take up remaining space */ 47 | } 48 | 49 | /* Set the color of the progress bar fill */ 50 | progress::-webkit-progress-value { 51 | background-color: #3498db; /* Blue color for the fill */ 52 | } 53 | 54 | progress::-moz-progress-bar { 55 | background-color: #3498db; /* Blue color for the fill in Firefox */ 56 | } 57 | 58 | /* Style the text on the progress bar */ 59 | progress::after { 60 | content: attr(value '%'); /* Display the progress value followed by '%' */ 61 | position: absolute; 62 | top: 50%; 63 | left: 50%; 64 | transform: translate(-50%, -50%); 65 | color: white; /* Set text color */ 66 | font-size: 14px; /* Set font size */ 67 | } 68 | 69 | /* Style other texts */ 70 | .loader-container > span { 71 | margin-left: 5px; /* Add spacing between the progress bar and the text */ 72 | } 73 | 74 | .no-generating-animation > .generating { 75 | display: none !important; 76 | } 77 | 78 | ''' 79 | 80 | 81 | def make_progress_bar_html(number, text): 82 | return progress_html.replace('*number*', str(number)).replace('*text*', text) 83 | 84 | 85 | def make_progress_bar_css(): 86 | return css 87 | -------------------------------------------------------------------------------- /diffusers_helper/hf_login.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def login(token): 5 | from huggingface_hub import login 6 | import time 7 | 8 | while True: 9 | try: 10 | login(token) 11 | print('HF login ok.') 12 | break 13 | except Exception as e: 14 | print(f'HF login failed: {e}. Retrying') 15 | time.sleep(0.5) 16 | 17 | 18 | hf_token = os.environ.get('HF_TOKEN', None) 19 | 20 | if hf_token is not None: 21 | login(hf_token) 22 | -------------------------------------------------------------------------------- /diffusers_helper/hunyuan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video import DEFAULT_PROMPT_TEMPLATE 4 | 5 | @torch.no_grad() 6 | def encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2, max_length=256): 7 | assert isinstance(prompt, str) 8 | 9 | prompt = [prompt] 10 | 11 | # LLAMA 12 | 13 | prompt_llama = [DEFAULT_PROMPT_TEMPLATE["template"].format(p) for p in prompt] 14 | crop_start = DEFAULT_PROMPT_TEMPLATE["crop_start"] 15 | 16 | llama_inputs = tokenizer( 17 | prompt_llama, 18 | padding="max_length", 19 | max_length=max_length + crop_start, 20 | truncation=True, 21 | return_tensors="pt", 22 | return_length=False, 23 | return_overflowing_tokens=False, 24 | return_attention_mask=True, 25 | ) 26 | 27 | llama_input_ids = llama_inputs.input_ids.to(text_encoder.device) 28 | llama_attention_mask = llama_inputs.attention_mask.to(text_encoder.device) 29 | llama_attention_length = int(llama_attention_mask.sum()) 30 | 31 | llama_outputs = text_encoder( 32 | input_ids=llama_input_ids, 33 | attention_mask=llama_attention_mask, 34 | output_hidden_states=True, 35 | ) 36 | 37 | llama_vec = llama_outputs.hidden_states[-3][:, crop_start:llama_attention_length] 38 | # llama_vec_remaining = llama_outputs.hidden_states[-3][:, llama_attention_length:] 39 | llama_attention_mask = llama_attention_mask[:, crop_start:llama_attention_length] 40 | 41 | assert torch.all(llama_attention_mask.bool()) 42 | 43 | # CLIP 44 | 45 | clip_l_input_ids = tokenizer_2( 46 | prompt, 47 | padding="max_length", 48 | max_length=77, 49 | truncation=True, 50 | return_overflowing_tokens=False, 51 | return_length=False, 52 | return_tensors="pt", 53 | ).input_ids 54 | clip_l_pooler = text_encoder_2(clip_l_input_ids.to(text_encoder_2.device), output_hidden_states=False).pooler_output 55 | 56 | return llama_vec, clip_l_pooler 57 | 58 | 59 | @torch.no_grad() 60 | def vae_decode_fake(latents): 61 | latent_rgb_factors = [ 62 | [-0.0395, -0.0331, 0.0445], 63 | [0.0696, 0.0795, 0.0518], 64 | [0.0135, -0.0945, -0.0282], 65 | [0.0108, -0.0250, -0.0765], 66 | [-0.0209, 0.0032, 0.0224], 67 | [-0.0804, -0.0254, -0.0639], 68 | [-0.0991, 0.0271, -0.0669], 69 | [-0.0646, -0.0422, -0.0400], 70 | [-0.0696, -0.0595, -0.0894], 71 | [-0.0799, -0.0208, -0.0375], 72 | [0.1166, 0.1627, 0.0962], 73 | [0.1165, 0.0432, 0.0407], 74 | [-0.2315, -0.1920, -0.1355], 75 | [-0.0270, 0.0401, -0.0821], 76 | [-0.0616, -0.0997, -0.0727], 77 | [0.0249, -0.0469, -0.1703] 78 | ] # From comfyui 79 | 80 | latent_rgb_factors_bias = [0.0259, -0.0192, -0.0761] 81 | 82 | weight = torch.tensor(latent_rgb_factors, device=latents.device, dtype=latents.dtype).transpose(0, 1)[:, :, None, None, None] 83 | bias = torch.tensor(latent_rgb_factors_bias, device=latents.device, dtype=latents.dtype) 84 | 85 | images = torch.nn.functional.conv3d(latents, weight, bias=bias, stride=1, padding=0, dilation=1, groups=1) 86 | images = images.clamp(0.0, 1.0) 87 | 88 | return images 89 | 90 | 91 | @torch.no_grad() 92 | def vae_decode(latents, vae, image_mode=False): 93 | latents = latents / vae.config.scaling_factor 94 | 95 | if not image_mode: 96 | image = vae.decode(latents.to(device=vae.device, dtype=vae.dtype)).sample 97 | else: 98 | latents = latents.to(device=vae.device, dtype=vae.dtype).unbind(2) 99 | image = [vae.decode(l.unsqueeze(2)).sample for l in latents] 100 | image = torch.cat(image, dim=2) 101 | 102 | return image 103 | 104 | 105 | @torch.no_grad() 106 | def vae_encode(image, vae): 107 | latents = vae.encode(image.to(device=vae.device, dtype=vae.dtype)).latent_dist.sample() 108 | latents = latents * vae.config.scaling_factor 109 | return latents 110 | -------------------------------------------------------------------------------- /diffusers_helper/k_diffusion/uni_pc_fm.py: -------------------------------------------------------------------------------- 1 | # Better Flow Matching UniPC by Lvmin Zhang 2 | # (c) 2025 3 | # CC BY-SA 4.0 4 | # Attribution-ShareAlike 4.0 International Licence 5 | 6 | 7 | import torch 8 | from comfy.utils import ProgressBar 9 | from tqdm.auto import trange 10 | 11 | 12 | def expand_dims(v, dims): 13 | return v[(...,) + (None,) * (dims - 1)] 14 | 15 | 16 | class FlowMatchUniPC: 17 | def __init__(self, model, extra_args, variant='bh1'): 18 | self.model = model 19 | self.variant = variant 20 | self.extra_args = extra_args 21 | 22 | def model_fn(self, x, t): 23 | return self.model(x, t, **self.extra_args) 24 | 25 | def update_fn(self, x, model_prev_list, t_prev_list, t, order): 26 | assert order <= len(model_prev_list) 27 | dims = x.dim() 28 | 29 | t_prev_0 = t_prev_list[-1] 30 | lambda_prev_0 = - torch.log(t_prev_0) 31 | lambda_t = - torch.log(t) 32 | model_prev_0 = model_prev_list[-1] 33 | 34 | h = lambda_t - lambda_prev_0 35 | 36 | rks = [] 37 | D1s = [] 38 | for i in range(1, order): 39 | t_prev_i = t_prev_list[-(i + 1)] 40 | model_prev_i = model_prev_list[-(i + 1)] 41 | lambda_prev_i = - torch.log(t_prev_i) 42 | rk = ((lambda_prev_i - lambda_prev_0) / h)[0] 43 | rks.append(rk) 44 | D1s.append((model_prev_i - model_prev_0) / rk) 45 | 46 | rks.append(1.) 47 | rks = torch.tensor(rks, device=x.device) 48 | 49 | R = [] 50 | b = [] 51 | 52 | hh = -h[0] 53 | h_phi_1 = torch.expm1(hh) 54 | h_phi_k = h_phi_1 / hh - 1 55 | 56 | factorial_i = 1 57 | 58 | if self.variant == 'bh1': 59 | B_h = hh 60 | elif self.variant == 'bh2': 61 | B_h = torch.expm1(hh) 62 | else: 63 | raise NotImplementedError('Bad variant!') 64 | 65 | for i in range(1, order + 1): 66 | R.append(torch.pow(rks, i - 1)) 67 | b.append(h_phi_k * factorial_i / B_h) 68 | factorial_i *= (i + 1) 69 | h_phi_k = h_phi_k / hh - 1 / factorial_i 70 | 71 | R = torch.stack(R) 72 | b = torch.tensor(b, device=x.device) 73 | 74 | use_predictor = len(D1s) > 0 75 | 76 | if use_predictor: 77 | D1s = torch.stack(D1s, dim=1) 78 | if order == 2: 79 | rhos_p = torch.tensor([0.5], device=b.device) 80 | else: 81 | rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1]) 82 | else: 83 | D1s = None 84 | rhos_p = None 85 | 86 | if order == 1: 87 | rhos_c = torch.tensor([0.5], device=b.device) 88 | else: 89 | rhos_c = torch.linalg.solve(R, b) 90 | 91 | x_t_ = expand_dims(t / t_prev_0, dims) * x - expand_dims(h_phi_1, dims) * model_prev_0 92 | 93 | if use_predictor: 94 | pred_res = torch.tensordot(D1s, rhos_p, dims=([1], [0])) 95 | else: 96 | pred_res = 0 97 | 98 | x_t = x_t_ - expand_dims(B_h, dims) * pred_res 99 | model_t = self.model_fn(x_t, t) 100 | 101 | if D1s is not None: 102 | corr_res = torch.tensordot(D1s, rhos_c[:-1], dims=([1], [0])) 103 | else: 104 | corr_res = 0 105 | 106 | D1_t = (model_t - model_prev_0) 107 | x_t = x_t_ - expand_dims(B_h, dims) * (corr_res + rhos_c[-1] * D1_t) 108 | 109 | return x_t, model_t 110 | 111 | def sample(self, x, sigmas, callback=None, disable_pbar=False): 112 | order = min(3, len(sigmas) - 2) 113 | model_prev_list, t_prev_list = [], [] 114 | comfy_pbar = ProgressBar(len(sigmas)-1) 115 | for i in trange(len(sigmas) - 1, disable=disable_pbar): 116 | vec_t = sigmas[i].expand(x.shape[0]) 117 | 118 | if i == 0: 119 | model_prev_list = [self.model_fn(x, vec_t)] 120 | t_prev_list = [vec_t] 121 | elif i < order: 122 | init_order = i 123 | x, model_x = self.update_fn(x, model_prev_list, t_prev_list, vec_t, init_order) 124 | model_prev_list.append(model_x) 125 | t_prev_list.append(vec_t) 126 | else: 127 | x, model_x = self.update_fn(x, model_prev_list, t_prev_list, vec_t, order) 128 | model_prev_list.append(model_x) 129 | t_prev_list.append(vec_t) 130 | 131 | model_prev_list = model_prev_list[-order:] 132 | t_prev_list = t_prev_list[-order:] 133 | 134 | if callback is not None: 135 | callback_latent = model_prev_list[-1].detach()[0].permute(1,0,2,3) 136 | callback( 137 | i, 138 | callback_latent, 139 | None, 140 | len(sigmas) - 1 141 | ) 142 | comfy_pbar.update(1) 143 | 144 | return model_prev_list[-1] 145 | 146 | 147 | def sample_unipc(model, noise, sigmas, extra_args=None, callback=None, disable=False, variant='bh1'): 148 | assert variant in ['bh1', 'bh2'] 149 | return FlowMatchUniPC(model, extra_args=extra_args, variant=variant).sample(noise, sigmas=sigmas, callback=callback, disable_pbar=disable) 150 | -------------------------------------------------------------------------------- /diffusers_helper/k_diffusion/wrapper.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def append_dims(x, target_dims): 5 | return x[(...,) + (None,) * (target_dims - x.ndim)] 6 | 7 | 8 | def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=1.0): 9 | if guidance_rescale == 0: 10 | return noise_cfg 11 | 12 | std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) 13 | std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) 14 | noise_pred_rescaled = noise_cfg * (std_text / std_cfg) 15 | noise_cfg = guidance_rescale * noise_pred_rescaled + (1.0 - guidance_rescale) * noise_cfg 16 | return noise_cfg 17 | 18 | 19 | def fm_wrapper(transformer, t_scale=1000.0): 20 | def k_model(x, sigma, **extra_args): 21 | dtype = extra_args['dtype'] 22 | cfg_scale = extra_args['cfg_scale'] 23 | cfg_rescale = extra_args['cfg_rescale'] 24 | concat_latent = extra_args['concat_latent'] 25 | 26 | original_dtype = x.dtype 27 | sigma = sigma.float() 28 | 29 | x = x.to(dtype) 30 | timestep = (sigma * t_scale).to(dtype) 31 | 32 | if concat_latent is None: 33 | hidden_states = x 34 | else: 35 | hidden_states = torch.cat([x, concat_latent.to(x)], dim=1) 36 | 37 | pred_positive = transformer(hidden_states=hidden_states, timestep=timestep, return_dict=False, **extra_args['positive'])[0].float() 38 | 39 | if cfg_scale == 1.0: 40 | pred_negative = torch.zeros_like(pred_positive) 41 | else: 42 | pred_negative = transformer(hidden_states=hidden_states, timestep=timestep, return_dict=False, **extra_args['negative'])[0].float() 43 | 44 | pred_cfg = pred_negative + cfg_scale * (pred_positive - pred_negative) 45 | pred = rescale_noise_cfg(pred_cfg, pred_positive, guidance_rescale=cfg_rescale) 46 | 47 | x0 = x.float() - pred.float() * append_dims(sigma, x.ndim) 48 | 49 | return x0.to(dtype=original_dtype) 50 | 51 | return k_model 52 | -------------------------------------------------------------------------------- /diffusers_helper/lora.py: -------------------------------------------------------------------------------- 1 | # LoRA network module: FramePack専用（musubi tuner準拠） 2 | import math 3 | import re 4 | from typing import Dict, List, Optional, Type, Union 5 | import torch 6 | import torch.nn as nn 7 | 8 | FRAMEPACK_TARGET_REPLACE_MODULES = [ 9 | "HunyuanVideoTransformerBlock", 10 | "HunyuanVideoSingleTransformerBlock", 11 | ] 12 | 13 | class LoRAModule(torch.nn.Module): 14 | def __init__( 15 | self, 16 | lora_name, 17 | org_module: torch.nn.Module, 18 | multiplier=1.0, 19 | lora_dim=4, 20 | alpha=1, 21 | dropout=None, 22 | rank_dropout=None, 23 | module_dropout=None, 24 | split_dims: Optional[List[int]] = None, 25 | ): 26 | super().__init__() 27 | self.lora_name = lora_name 28 | 29 | if org_module.__class__.__name__ == "Conv2d": 30 | in_dim = org_module.in_channels 31 | out_dim = org_module.out_channels 32 | else: 33 | in_dim = org_module.in_features 34 | out_dim = org_module.out_features 35 | 36 | self.lora_dim = lora_dim 37 | self.split_dims = split_dims 38 | 39 | if split_dims is None: 40 | if org_module.__class__.__name__ == "Conv2d": 41 | kernel_size = org_module.kernel_size 42 | stride = org_module.stride 43 | padding = org_module.padding 44 | self.lora_down = torch.nn.Conv2d(in_dim, self.lora_dim, kernel_size, stride, padding, bias=False) 45 | self.lora_up = torch.nn.Conv2d(self.lora_dim, out_dim, (1, 1), (1, 1), bias=False) 46 | else: 47 | self.lora_down = torch.nn.Linear(in_dim, self.lora_dim, bias=False) 48 | self.lora_up = torch.nn.Linear(self.lora_dim, out_dim, bias=False) 49 | 50 | torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5)) 51 | torch.nn.init.zeros_(self.lora_up.weight) 52 | else: 53 | assert sum(split_dims) == out_dim, "sum of split_dims must be equal to out_dim" 54 | assert org_module.__class__.__name__ == "Linear", "split_dims is only supported for Linear" 55 | self.lora_down = torch.nn.ModuleList( 56 | [torch.nn.Linear(in_dim, self.lora_dim, bias=False) for _ in range(len(split_dims))] 57 | ) 58 | self.lora_up = torch.nn.ModuleList([torch.nn.Linear(self.lora_dim, split_dim, bias=False) for split_dim in split_dims]) 59 | for lora_down in self.lora_down: 60 | torch.nn.init.kaiming_uniform_(lora_down.weight, a=math.sqrt(5)) 61 | for lora_up in self.lora_up: 62 | torch.nn.init.zeros_(lora_up.weight) 63 | 64 | if isinstance(alpha, torch.Tensor): 65 | alpha_buf = alpha.detach().clone() 66 | else: 67 | alpha_buf = torch.tensor(alpha) 68 | self.register_buffer("alpha", alpha_buf) 69 | 70 | self.scale = alpha / self.lora_dim 71 | 72 | self.multiplier = multiplier 73 | self.org_module = org_module 74 | self.dropout = dropout 75 | self.rank_dropout = rank_dropout 76 | self.module_dropout = module_dropout 77 | 78 | def apply_to(self): 79 | self.org_forward = self.org_module.forward 80 | self.org_module.forward = self.forward 81 | del self.org_module 82 | 83 | def forward(self, x): 84 | org_forwarded = self.org_forward(x) 85 | if self.split_dims is None: 86 | lx = self.lora_down(x) 87 | lx = self.lora_up(lx) 88 | return org_forwarded + lx * self.multiplier * self.scale 89 | else: 90 | lxs = [lora_down(x) for lora_down in self.lora_down] 91 | lxs = [lora_up(lx) for lora_up, lx in zip(self.lora_up, lxs)] 92 | return org_forwarded + torch.cat(lxs, dim=-1) * self.multiplier * self.scale 93 | 94 | class LoRAInfModule(LoRAModule): 95 | def __init__( 96 | self, 97 | lora_name, 98 | org_module: torch.nn.Module, 99 | multiplier=1.0, 100 | lora_dim=4, 101 | alpha=1, 102 | **kwargs, 103 | ): 104 | super().__init__(lora_name, org_module, multiplier, lora_dim, alpha) 105 | self.org_module_ref = [org_module] 106 | self.enabled = True 107 | self.network = None 108 | 109 | def set_network(self, network): 110 | self.network = network 111 | 112 | def merge_to(self, sd, dtype, device, non_blocking=False): 113 | org_sd = self.org_module.state_dict() 114 | weight = org_sd["weight"] 115 | org_dtype = weight.dtype 116 | org_device = weight.device 117 | weight = weight.to(device, dtype=torch.float, non_blocking=non_blocking) 118 | if dtype is None: 119 | dtype = org_dtype 120 | if device is None: 121 | device = org_device 122 | 123 | if self.split_dims is None: 124 | down_weight = sd["lora_down.weight"].to(device, dtype=torch.float, non_blocking=non_blocking) 125 | up_weight = sd["lora_up.weight"].to(device, dtype=torch.float, non_blocking=non_blocking) 126 | if len(weight.size()) == 2: 127 | weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale 128 | elif down_weight.size()[2:4] == (1, 1): 129 | weight = ( 130 | weight 131 | + self.multiplier 132 | * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3) 133 | * self.scale 134 | ) 135 | else: 136 | conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3) 137 | weight = weight + self.multiplier * conved * self.scale 138 | org_sd["weight"] = weight.to(org_device, dtype=dtype) 139 | self.org_module.load_state_dict(org_sd) 140 | else: 141 | total_dims = sum(self.split_dims) 142 | for i in range(len(self.split_dims)): 143 | down_weight = sd[f"lora_down.{i}.weight"].to(device, torch.float, non_blocking=non_blocking) 144 | up_weight = sd[f"lora_up.{i}.weight"].to(device, torch.float, non_blocking=non_blocking) 145 | padded_up_weight = torch.zeros((total_dims, up_weight.size(0)), device=device, dtype=torch.float) 146 | padded_up_weight[sum(self.split_dims[:i]) : sum(self.split_dims[: i + 1])] = up_weight 147 | weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale 148 | org_sd["weight"] = weight.to(org_device, dtype) 149 | self.org_module.load_state_dict(org_sd) 150 | 151 | def create_arch_network_from_weights( 152 | multiplier: float, 153 | weights_sd: Dict[str, torch.Tensor], 154 | text_encoders: Optional[List[nn.Module]] = None, 155 | unet: Optional[nn.Module] = None, 156 | for_inference: bool = True, 157 | **kwargs, 158 | ): 159 | return create_network_from_weights( 160 | FRAMEPACK_TARGET_REPLACE_MODULES, multiplier, weights_sd, text_encoders, unet, for_inference, **kwargs 161 | ) 162 | 163 | def create_network_from_weights( 164 | target_replace_modules: List[str], 165 | multiplier: float, 166 | weights_sd: Dict[str, torch.Tensor], 167 | text_encoders: Optional[List[nn.Module]] = None, 168 | unet: Optional[nn.Module] = None, 169 | for_inference: bool = True, 170 | **kwargs, 171 | ): 172 | modules_dim = {} 173 | modules_alpha = {} 174 | for key, value in weights_sd.items(): 175 | if "." not in key: 176 | continue 177 | lora_name = key.split(".")[0] 178 | if "alpha" in key: 179 | modules_alpha[lora_name] = value 180 | elif "lora_down" in key: 181 | dim = value.shape[0] 182 | modules_dim[lora_name] = dim 183 | module_class = LoRAInfModule if for_inference else LoRAModule 184 | network = LoRANetwork( 185 | target_replace_modules, 186 | "lora_unet", 187 | text_encoders, 188 | unet, 189 | multiplier=multiplier, 190 | modules_dim=modules_dim, 191 | modules_alpha=modules_alpha, 192 | module_class=module_class, 193 | ) 194 | return network 195 | 196 | class LoRANetwork(torch.nn.Module): 197 | def __init__( 198 | self, 199 | target_replace_modules: List[str], 200 | prefix: str, 201 | text_encoders: Optional[List[nn.Module]], 202 | unet: nn.Module, 203 | multiplier: float = 1.0, 204 | lora_dim: int = 4, 205 | alpha: float = 1, 206 | dropout: Optional[float] = None, 207 | rank_dropout: Optional[float] = None, 208 | module_dropout: Optional[float] = None, 209 | conv_lora_dim: Optional[int] = None, 210 | conv_alpha: Optional[float] = None, 211 | module_class: Type[object] = LoRAModule, 212 | modules_dim: Optional[Dict[str, int]] = None, 213 | modules_alpha: Optional[Dict[str, int]] = None, 214 | exclude_patterns: Optional[List[str]] = None, 215 | include_patterns: Optional[List[str]] = None, 216 | verbose: Optional[bool] = False, 217 | ) -> None: 218 | super().__init__() 219 | self.multiplier = multiplier 220 | self.lora_dim = lora_dim 221 | self.alpha = alpha 222 | self.conv_lora_dim = conv_lora_dim 223 | self.conv_alpha = conv_alpha 224 | self.dropout = dropout 225 | self.rank_dropout = rank_dropout 226 | self.module_dropout = module_dropout 227 | self.target_replace_modules = target_replace_modules 228 | self.prefix = prefix 229 | self.text_encoder_loras = [] 230 | self.unet_loras, _ = self.create_modules(True, prefix, unet, target_replace_modules, module_class, modules_dim, modules_alpha, dropout, rank_dropout, module_dropout, exclude_patterns, include_patterns, verbose) 231 | 232 | def create_modules( 233 | self, 234 | is_unet: bool, 235 | pfx: str, 236 | root_module: torch.nn.Module, 237 | target_replace_mods: Optional[List[str]], 238 | module_class: Type[object], 239 | modules_dim: Optional[Dict[str, int]], 240 | modules_alpha: Optional[Dict[str, int]], 241 | dropout, 242 | rank_dropout, 243 | module_dropout, 244 | exclude_patterns, 245 | include_patterns, 246 | verbose, 247 | ): 248 | loras = [] 249 | skipped = [] 250 | for name, module in root_module.named_modules(): 251 | # exclude_patternsによる除外判定 252 | if exclude_patterns is not None: 253 | excluded = False 254 | for pattern in exclude_patterns: 255 | if re.match(pattern, name): 256 | print(f"[LoRA][exclude] skip module: {name} (pattern: {pattern})") 257 | excluded = True 258 | break 259 | if excluded: 260 | continue 261 | if target_replace_mods is None or module.__class__.__name__ in target_replace_mods: 262 | for child_name, child_module in module.named_modules(): 263 | # exclude_patternsによる除外判定（子モジュール名にも適用） 264 | if exclude_patterns is not None: 265 | excluded = False 266 | for pattern in exclude_patterns: 267 | if re.match(pattern, child_name): 268 | print(f"[LoRA][exclude] skip child module: {child_name} (pattern: {pattern})") 269 | excluded = True 270 | break 271 | if excluded: 272 | continue 273 | is_linear = child_module.__class__.__name__ == "Linear" 274 | is_conv2d = child_module.__class__.__name__ == "Conv2d" 275 | is_conv2d_1x1 = is_conv2d and child_module.kernel_size == (1, 1) 276 | if is_linear or is_conv2d: 277 | original_name = (name + "." if name else "") + child_name 278 | lora_name = f"{pfx}.{original_name}".replace(".", "_") 279 | dim = None 280 | alpha = None 281 | if modules_dim is not None: 282 | if lora_name in modules_dim: 283 | dim = modules_dim[lora_name] 284 | alpha = modules_alpha[lora_name] 285 | else: 286 | if is_linear or is_conv2d_1x1: 287 | dim = self.lora_dim 288 | alpha = self.alpha 289 | elif self.conv_lora_dim is not None: 290 | dim = self.conv_lora_dim 291 | alpha = self.conv_alpha 292 | if dim is None or dim == 0: 293 | skipped.append(lora_name) 294 | continue 295 | lora = module_class( 296 | lora_name, 297 | child_module, 298 | self.multiplier, 299 | dim, 300 | alpha, 301 | dropout=dropout, 302 | rank_dropout=rank_dropout, 303 | module_dropout=module_dropout, 304 | ) 305 | loras.append(lora) 306 | if target_replace_mods is None: 307 | break 308 | return loras, skipped 309 | 310 | def merge_to(self, text_encoders, unet, weights_sd, dtype=None, device=None, non_blocking=False): 311 | for lora in self.unet_loras: 312 | sd_for_lora = {} 313 | for key in weights_sd.keys(): 314 | if key.startswith(lora.lora_name): 315 | sd_for_lora[key[len(lora.lora_name) + 1 :]] = weights_sd[key] 316 | if len(sd_for_lora) == 0: 317 | continue 318 | lora.merge_to(sd_for_lora, dtype, device, non_blocking) -------------------------------------------------------------------------------- /diffusers_helper/memory.py: -------------------------------------------------------------------------------- 1 | # By lllyasviel 2 | 3 | 4 | import torch 5 | 6 | 7 | cpu = torch.device('cpu') 8 | gpu = torch.device(f'cuda:{torch.cuda.current_device()}') 9 | gpu_complete_modules = [] 10 | 11 | 12 | class DynamicSwapInstaller: 13 | @staticmethod 14 | def _install_module(module: torch.nn.Module, **kwargs): 15 | original_class = module.__class__ 16 | module.__dict__['forge_backup_original_class'] = original_class 17 | 18 | def hacked_get_attr(self, name: str): 19 | if '_parameters' in self.__dict__: 20 | _parameters = self.__dict__['_parameters'] 21 | if name in _parameters: 22 | p = _parameters[name] 23 | if p is None: 24 | return None 25 | if p.__class__ == torch.nn.Parameter: 26 | return torch.nn.Parameter(p.to(**kwargs), requires_grad=p.requires_grad) 27 | else: 28 | return p.to(**kwargs) 29 | if '_buffers' in self.__dict__: 30 | _buffers = self.__dict__['_buffers'] 31 | if name in _buffers: 32 | return _buffers[name].to(**kwargs) 33 | return super(original_class, self).__getattr__(name) 34 | 35 | module.__class__ = type('DynamicSwap_' + original_class.__name__, (original_class,), { 36 | '__getattr__': hacked_get_attr, 37 | }) 38 | 39 | return 40 | 41 | @staticmethod 42 | def _uninstall_module(module: torch.nn.Module): 43 | if 'forge_backup_original_class' in module.__dict__: 44 | module.__class__ = module.__dict__.pop('forge_backup_original_class') 45 | return 46 | 47 | @staticmethod 48 | def install_model(model: torch.nn.Module, **kwargs): 49 | for m in model.modules(): 50 | DynamicSwapInstaller._install_module(m, **kwargs) 51 | return 52 | 53 | @staticmethod 54 | def uninstall_model(model: torch.nn.Module): 55 | for m in model.modules(): 56 | DynamicSwapInstaller._uninstall_module(m) 57 | return 58 | 59 | 60 | def fake_diffusers_current_device(model: torch.nn.Module, target_device: torch.device): 61 | if hasattr(model, 'scale_shift_table'): 62 | model.scale_shift_table.data = model.scale_shift_table.data.to(target_device) 63 | return 64 | 65 | for k, p in model.named_modules(): 66 | if hasattr(p, 'weight'): 67 | p.to(target_device) 68 | return 69 | 70 | 71 | def get_cuda_free_memory_gb(device=None): 72 | if device is None: 73 | device = gpu 74 | 75 | memory_stats = torch.cuda.memory_stats(device) 76 | bytes_active = memory_stats['active_bytes.all.current'] 77 | bytes_reserved = memory_stats['reserved_bytes.all.current'] 78 | bytes_free_cuda, _ = torch.cuda.mem_get_info(device) 79 | bytes_inactive_reserved = bytes_reserved - bytes_active 80 | bytes_total_available = bytes_free_cuda + bytes_inactive_reserved 81 | return bytes_total_available / (1024 ** 3) 82 | 83 | 84 | def move_model_to_device_with_memory_preservation(model, target_device, preserved_memory_gb=0): 85 | print(f'Moving {model.__class__.__name__} to {target_device} with preserved memory: {preserved_memory_gb} GB') 86 | 87 | for m in model.modules(): 88 | if get_cuda_free_memory_gb(target_device) <= preserved_memory_gb: 89 | torch.cuda.empty_cache() 90 | return 91 | 92 | if hasattr(m, 'weight'): 93 | m.to(device=target_device) 94 | 95 | model.to(device=target_device) 96 | torch.cuda.empty_cache() 97 | return 98 | 99 | 100 | def offload_model_from_device_for_memory_preservation(model, target_device, preserved_memory_gb=0): 101 | print(f'Offloading {model.__class__.__name__} from {target_device} to preserve memory: {preserved_memory_gb} GB') 102 | 103 | for m in model.modules(): 104 | if get_cuda_free_memory_gb(target_device) >= preserved_memory_gb: 105 | torch.cuda.empty_cache() 106 | return 107 | 108 | if hasattr(m, 'weight'): 109 | m.to(device=cpu) 110 | 111 | model.to(device=cpu) 112 | torch.cuda.empty_cache() 113 | return 114 | 115 | 116 | def unload_complete_models(*args): 117 | for m in gpu_complete_modules + list(args): 118 | m.to(device=cpu) 119 | print(f'Unloaded {m.__class__.__name__} as complete.') 120 | 121 | gpu_complete_modules.clear() 122 | torch.cuda.empty_cache() 123 | return 124 | 125 | 126 | def load_model_as_complete(model, target_device, unload=True): 127 | if unload: 128 | unload_complete_models() 129 | 130 | model.to(device=target_device) 131 | print(f'Loaded {model.__class__.__name__} to {target_device} as complete.') 132 | 133 | gpu_complete_modules.append(model) 134 | return 135 | -------------------------------------------------------------------------------- /diffusers_helper/models/hunyuan_video_packed.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Optional, Tuple, Union 2 | 3 | import torch 4 | import einops 5 | import torch.nn as nn 6 | import numpy as np 7 | 8 | from diffusers.loaders import FromOriginalModelMixin 9 | from diffusers.configuration_utils import ConfigMixin, register_to_config 10 | from diffusers.loaders import PeftAdapterMixin 11 | from diffusers.utils import logging 12 | from diffusers.models.attention import FeedForward 13 | from diffusers.models.attention_processor import Attention 14 | from diffusers.models.embeddings import TimestepEmbedding, Timesteps, PixArtAlphaTextProjection 15 | from diffusers.models.modeling_outputs import Transformer2DModelOutput 16 | from diffusers.models.modeling_utils import ModelMixin 17 | from ...diffusers_helper.dit_common import LayerNorm 18 | 19 | 20 | enabled_backends = [] 21 | 22 | if torch.backends.cuda.flash_sdp_enabled(): 23 | enabled_backends.append("flash") 24 | if torch.backends.cuda.math_sdp_enabled(): 25 | enabled_backends.append("math") 26 | if torch.backends.cuda.mem_efficient_sdp_enabled(): 27 | enabled_backends.append("mem_efficient") 28 | if torch.backends.cuda.cudnn_sdp_enabled(): 29 | enabled_backends.append("cudnn") 30 | 31 | try: 32 | # raise NotImplementedError 33 | from flash_attn import flash_attn_varlen_func, flash_attn_func 34 | except: 35 | flash_attn_varlen_func = None 36 | flash_attn_func = None 37 | 38 | try: 39 | # raise NotImplementedError 40 | from sageattention import sageattn_varlen, sageattn 41 | except: 42 | sageattn_varlen = None 43 | sageattn = None 44 | 45 | 46 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 47 | 48 | 49 | def pad_for_3d_conv(x, kernel_size): 50 | b, c, t, h, w = x.shape 51 | pt, ph, pw = kernel_size 52 | pad_t = (pt - (t % pt)) % pt 53 | pad_h = (ph - (h % ph)) % ph 54 | pad_w = (pw - (w % pw)) % pw 55 | return torch.nn.functional.pad(x, (0, pad_w, 0, pad_h, 0, pad_t), mode='replicate') 56 | 57 | 58 | def center_down_sample_3d(x, kernel_size): 59 | # pt, ph, pw = kernel_size 60 | # cp = (pt * ph * pw) // 2 61 | # xp = einops.rearrange(x, 'b c (t pt) (h ph) (w pw) -> (pt ph pw) b c t h w', pt=pt, ph=ph, pw=pw) 62 | # xc = xp[cp] 63 | # return xc 64 | return torch.nn.functional.avg_pool3d(x, kernel_size, stride=kernel_size) 65 | 66 | 67 | def get_cu_seqlens(text_mask, img_len): 68 | batch_size = text_mask.shape[0] 69 | text_len = text_mask.sum(dim=1) 70 | max_len = text_mask.shape[1] + img_len 71 | 72 | cu_seqlens = torch.zeros([2 * batch_size + 1], dtype=torch.int32, device="cuda") 73 | 74 | for i in range(batch_size): 75 | s = text_len[i] + img_len 76 | s1 = i * max_len + s 77 | s2 = (i + 1) * max_len 78 | cu_seqlens[2 * i + 1] = s1 79 | cu_seqlens[2 * i + 2] = s2 80 | 81 | return cu_seqlens 82 | 83 | 84 | def apply_rotary_emb_transposed(x, freqs_cis): 85 | cos, sin = freqs_cis.unsqueeze(-2).chunk(2, dim=-1) 86 | x_real, x_imag = x.unflatten(-1, (-1, 2)).unbind(-1) 87 | x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) 88 | out = x.float() * cos + x_rotated.float() * sin 89 | out = out.to(x) 90 | return out 91 | 92 | 93 | def attn_varlen_func(q, k, v, cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv, attention_mode='sdpa'): 94 | if cu_seqlens_q is None and cu_seqlens_kv is None and max_seqlen_q is None and max_seqlen_kv is None: 95 | if attention_mode == "sageattn": 96 | x = sageattn(q, k, v, tensor_layout='NHD') 97 | if attention_mode == "flash_attn": 98 | x = flash_attn_func(q, k, v) 99 | elif attention_mode == "sdpa": 100 | x = torch.nn.functional.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)).transpose(1, 2) 101 | return x 102 | 103 | # batch_size = q.shape[0] 104 | # q = q.view(q.shape[0] * q.shape[1], *q.shape[2:]) 105 | # k = k.view(k.shape[0] * k.shape[1], *k.shape[2:]) 106 | # v = v.view(v.shape[0] * v.shape[1], *v.shape[2:]) 107 | # if sageattn_varlen is not None: 108 | # x = sageattn_varlen(q, k, v, cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv) 109 | # elif flash_attn_varlen_func is not None: 110 | # x = flash_attn_varlen_func(q, k, v, cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv) 111 | # else: 112 | # raise NotImplementedError('No Attn Installed!') 113 | # x = x.view(batch_size, max_seqlen_q, *x.shape[2:]) 114 | # return x 115 | 116 | 117 | class HunyuanAttnProcessorFlashAttnDouble: 118 | def __init__(self, attention_mode): 119 | self.attention_mode = attention_mode 120 | def __call__(self, attn, hidden_states, encoder_hidden_states, attention_mask, image_rotary_emb): 121 | cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv = attention_mask 122 | 123 | query = attn.to_q(hidden_states) 124 | key = attn.to_k(hidden_states) 125 | value = attn.to_v(hidden_states) 126 | 127 | query = query.unflatten(2, (attn.heads, -1)) 128 | key = key.unflatten(2, (attn.heads, -1)) 129 | value = value.unflatten(2, (attn.heads, -1)) 130 | 131 | query = attn.norm_q(query) 132 | key = attn.norm_k(key) 133 | 134 | query = apply_rotary_emb_transposed(query, image_rotary_emb) 135 | key = apply_rotary_emb_transposed(key, image_rotary_emb) 136 | 137 | encoder_query = attn.add_q_proj(encoder_hidden_states) 138 | encoder_key = attn.add_k_proj(encoder_hidden_states) 139 | encoder_value = attn.add_v_proj(encoder_hidden_states) 140 | 141 | encoder_query = encoder_query.unflatten(2, (attn.heads, -1)) 142 | encoder_key = encoder_key.unflatten(2, (attn.heads, -1)) 143 | encoder_value = encoder_value.unflatten(2, (attn.heads, -1)) 144 | 145 | encoder_query = attn.norm_added_q(encoder_query) 146 | encoder_key = attn.norm_added_k(encoder_key) 147 | 148 | query = torch.cat([query, encoder_query], dim=1) 149 | key = torch.cat([key, encoder_key], dim=1) 150 | value = torch.cat([value, encoder_value], dim=1) 151 | 152 | hidden_states = attn_varlen_func(query, key, value, cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv, self.attention_mode) 153 | hidden_states = hidden_states.flatten(-2) 154 | 155 | txt_length = encoder_hidden_states.shape[1] 156 | hidden_states, encoder_hidden_states = hidden_states[:, :-txt_length], hidden_states[:, -txt_length:] 157 | 158 | hidden_states = attn.to_out[0](hidden_states) 159 | hidden_states = attn.to_out[1](hidden_states) 160 | encoder_hidden_states = attn.to_add_out(encoder_hidden_states) 161 | 162 | return hidden_states, encoder_hidden_states 163 | 164 | 165 | class HunyuanAttnProcessorFlashAttnSingle: 166 | def __init__(self, attention_mode): 167 | self.attention_mode = attention_mode 168 | def __call__(self, attn, hidden_states, encoder_hidden_states, attention_mask, image_rotary_emb): 169 | cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv = attention_mask 170 | 171 | hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) 172 | 173 | query = attn.to_q(hidden_states) 174 | key = attn.to_k(hidden_states) 175 | value = attn.to_v(hidden_states) 176 | 177 | query = query.unflatten(2, (attn.heads, -1)) 178 | key = key.unflatten(2, (attn.heads, -1)) 179 | value = value.unflatten(2, (attn.heads, -1)) 180 | 181 | query = attn.norm_q(query) 182 | key = attn.norm_k(key) 183 | 184 | txt_length = encoder_hidden_states.shape[1] 185 | 186 | query = torch.cat([apply_rotary_emb_transposed(query[:, :-txt_length], image_rotary_emb), query[:, -txt_length:]], dim=1) 187 | key = torch.cat([apply_rotary_emb_transposed(key[:, :-txt_length], image_rotary_emb), key[:, -txt_length:]], dim=1) 188 | 189 | hidden_states = attn_varlen_func(query, key, value, cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv, self.attention_mode) 190 | hidden_states = hidden_states.flatten(-2) 191 | 192 | hidden_states, encoder_hidden_states = hidden_states[:, :-txt_length], hidden_states[:, -txt_length:] 193 | 194 | return hidden_states, encoder_hidden_states 195 | 196 | 197 | class CombinedTimestepGuidanceTextProjEmbeddings(nn.Module): 198 | def __init__(self, embedding_dim, pooled_projection_dim): 199 | super().__init__() 200 | 201 | self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0) 202 | self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) 203 | self.guidance_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) 204 | self.text_embedder = PixArtAlphaTextProjection(pooled_projection_dim, embedding_dim, act_fn="silu") 205 | 206 | def forward(self, timestep, guidance, pooled_projection): 207 | timesteps_proj = self.time_proj(timestep) 208 | timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=pooled_projection.dtype)) 209 | 210 | guidance_proj = self.time_proj(guidance) 211 | guidance_emb = self.guidance_embedder(guidance_proj.to(dtype=pooled_projection.dtype)) 212 | 213 | time_guidance_emb = timesteps_emb + guidance_emb 214 | 215 | pooled_projections = self.text_embedder(pooled_projection) 216 | conditioning = time_guidance_emb + pooled_projections 217 | 218 | return conditioning 219 | 220 | 221 | class CombinedTimestepTextProjEmbeddings(nn.Module): 222 | def __init__(self, embedding_dim, pooled_projection_dim): 223 | super().__init__() 224 | 225 | self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0) 226 | self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) 227 | self.text_embedder = PixArtAlphaTextProjection(pooled_projection_dim, embedding_dim, act_fn="silu") 228 | 229 | def forward(self, timestep, pooled_projection): 230 | timesteps_proj = self.time_proj(timestep) 231 | timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=pooled_projection.dtype)) 232 | 233 | pooled_projections = self.text_embedder(pooled_projection) 234 | 235 | conditioning = timesteps_emb + pooled_projections 236 | 237 | return conditioning 238 | 239 | 240 | class HunyuanVideoAdaNorm(nn.Module): 241 | def __init__(self, in_features: int, out_features: Optional[int] = None) -> None: 242 | super().__init__() 243 | 244 | out_features = out_features or 2 * in_features 245 | self.linear = nn.Linear(in_features, out_features) 246 | self.nonlinearity = nn.SiLU() 247 | 248 | def forward( 249 | self, temb: torch.Tensor 250 | ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 251 | temb = self.linear(self.nonlinearity(temb)) 252 | gate_msa, gate_mlp = temb.chunk(2, dim=-1) 253 | gate_msa, gate_mlp = gate_msa.unsqueeze(1), gate_mlp.unsqueeze(1) 254 | return gate_msa, gate_mlp 255 | 256 | 257 | class HunyuanVideoIndividualTokenRefinerBlock(nn.Module): 258 | def __init__( 259 | self, 260 | num_attention_heads: int, 261 | attention_head_dim: int, 262 | mlp_width_ratio: str = 4.0, 263 | mlp_drop_rate: float = 0.0, 264 | attention_bias: bool = True, 265 | ) -> None: 266 | super().__init__() 267 | 268 | hidden_size = num_attention_heads * attention_head_dim 269 | 270 | self.norm1 = LayerNorm(hidden_size, elementwise_affine=True, eps=1e-6) 271 | self.attn = Attention( 272 | query_dim=hidden_size, 273 | cross_attention_dim=None, 274 | heads=num_attention_heads, 275 | dim_head=attention_head_dim, 276 | bias=attention_bias, 277 | ) 278 | 279 | self.norm2 = LayerNorm(hidden_size, elementwise_affine=True, eps=1e-6) 280 | self.ff = FeedForward(hidden_size, mult=mlp_width_ratio, activation_fn="linear-silu", dropout=mlp_drop_rate) 281 | 282 | self.norm_out = HunyuanVideoAdaNorm(hidden_size, 2 * hidden_size) 283 | 284 | def forward( 285 | self, 286 | hidden_states: torch.Tensor, 287 | temb: torch.Tensor, 288 | attention_mask: Optional[torch.Tensor] = None, 289 | ) -> torch.Tensor: 290 | norm_hidden_states = self.norm1(hidden_states) 291 | 292 | attn_output = self.attn( 293 | hidden_states=norm_hidden_states, 294 | encoder_hidden_states=None, 295 | attention_mask=attention_mask, 296 | ) 297 | 298 | gate_msa, gate_mlp = self.norm_out(temb) 299 | hidden_states = hidden_states + attn_output * gate_msa 300 | 301 | ff_output = self.ff(self.norm2(hidden_states)) 302 | hidden_states = hidden_states + ff_output * gate_mlp 303 | 304 | return hidden_states 305 | 306 | 307 | class HunyuanVideoIndividualTokenRefiner(nn.Module): 308 | def __init__( 309 | self, 310 | num_attention_heads: int, 311 | attention_head_dim: int, 312 | num_layers: int, 313 | mlp_width_ratio: float = 4.0, 314 | mlp_drop_rate: float = 0.0, 315 | attention_bias: bool = True, 316 | ) -> None: 317 | super().__init__() 318 | 319 | self.refiner_blocks = nn.ModuleList( 320 | [ 321 | HunyuanVideoIndividualTokenRefinerBlock( 322 | num_attention_heads=num_attention_heads, 323 | attention_head_dim=attention_head_dim, 324 | mlp_width_ratio=mlp_width_ratio, 325 | mlp_drop_rate=mlp_drop_rate, 326 | attention_bias=attention_bias, 327 | ) 328 | for _ in range(num_layers) 329 | ] 330 | ) 331 | 332 | def forward( 333 | self, 334 | hidden_states: torch.Tensor, 335 | temb: torch.Tensor, 336 | attention_mask: Optional[torch.Tensor] = None, 337 | ) -> None: 338 | self_attn_mask = None 339 | if attention_mask is not None: 340 | batch_size = attention_mask.shape[0] 341 | seq_len = attention_mask.shape[1] 342 | attention_mask = attention_mask.to(hidden_states.device).bool() 343 | self_attn_mask_1 = attention_mask.view(batch_size, 1, 1, seq_len).repeat(1, 1, seq_len, 1) 344 | self_attn_mask_2 = self_attn_mask_1.transpose(2, 3) 345 | self_attn_mask = (self_attn_mask_1 & self_attn_mask_2).bool() 346 | self_attn_mask[:, :, :, 0] = True 347 | 348 | for block in self.refiner_blocks: 349 | hidden_states = block(hidden_states, temb, self_attn_mask) 350 | 351 | return hidden_states 352 | 353 | 354 | class HunyuanVideoTokenRefiner(nn.Module): 355 | def __init__( 356 | self, 357 | in_channels: int, 358 | num_attention_heads: int, 359 | attention_head_dim: int, 360 | num_layers: int, 361 | mlp_ratio: float = 4.0, 362 | mlp_drop_rate: float = 0.0, 363 | attention_bias: bool = True, 364 | ) -> None: 365 | super().__init__() 366 | 367 | hidden_size = num_attention_heads * attention_head_dim 368 | 369 | self.time_text_embed = CombinedTimestepTextProjEmbeddings( 370 | embedding_dim=hidden_size, pooled_projection_dim=in_channels 371 | ) 372 | self.proj_in = nn.Linear(in_channels, hidden_size, bias=True) 373 | self.token_refiner = HunyuanVideoIndividualTokenRefiner( 374 | num_attention_heads=num_attention_heads, 375 | attention_head_dim=attention_head_dim, 376 | num_layers=num_layers, 377 | mlp_width_ratio=mlp_ratio, 378 | mlp_drop_rate=mlp_drop_rate, 379 | attention_bias=attention_bias, 380 | ) 381 | 382 | def forward( 383 | self, 384 | hidden_states: torch.Tensor, 385 | timestep: torch.LongTensor, 386 | attention_mask: Optional[torch.LongTensor] = None, 387 | ) -> torch.Tensor: 388 | if attention_mask is None: 389 | pooled_projections = hidden_states.mean(dim=1) 390 | else: 391 | original_dtype = hidden_states.dtype 392 | mask_float = attention_mask.float().unsqueeze(-1) 393 | pooled_projections = (hidden_states * mask_float).sum(dim=1) / mask_float.sum(dim=1) 394 | pooled_projections = pooled_projections.to(original_dtype) 395 | 396 | temb = self.time_text_embed(timestep, pooled_projections) 397 | hidden_states = self.proj_in(hidden_states) 398 | hidden_states = self.token_refiner(hidden_states, temb, attention_mask) 399 | 400 | return hidden_states 401 | 402 | 403 | class HunyuanVideoRotaryPosEmbed(nn.Module): 404 | def __init__(self, rope_dim, theta): 405 | super().__init__() 406 | self.DT, self.DY, self.DX = rope_dim 407 | self.theta = theta 408 | 409 | @torch.no_grad() 410 | def get_frequency(self, dim, pos): 411 | T, H, W = pos.shape 412 | freqs = 1.0 / (self.theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=pos.device)[: (dim // 2)] / dim)) 413 | freqs = torch.outer(freqs, pos.reshape(-1)).unflatten(-1, (T, H, W)).repeat_interleave(2, dim=0) 414 | return freqs.cos(), freqs.sin() 415 | 416 | @torch.no_grad() 417 | def forward_inner(self, frame_indices, height, width, device): 418 | GT, GY, GX = torch.meshgrid( 419 | frame_indices.to(device=device, dtype=torch.float32), 420 | torch.arange(0, height, device=device, dtype=torch.float32), 421 | torch.arange(0, width, device=device, dtype=torch.float32), 422 | indexing="ij" 423 | ) 424 | 425 | FCT, FST = self.get_frequency(self.DT, GT) 426 | FCY, FSY = self.get_frequency(self.DY, GY) 427 | FCX, FSX = self.get_frequency(self.DX, GX) 428 | 429 | result = torch.cat([FCT, FCY, FCX, FST, FSY, FSX], dim=0) 430 | 431 | return result.to(device) 432 | 433 | @torch.no_grad() 434 | def forward(self, frame_indices, height, width, device): 435 | frame_indices = frame_indices.unbind(0) 436 | results = [self.forward_inner(f, height, width, device) for f in frame_indices] 437 | results = torch.stack(results, dim=0) 438 | return results 439 | 440 | 441 | class AdaLayerNormZero(nn.Module): 442 | def __init__(self, embedding_dim: int, norm_type="layer_norm", bias=True): 443 | super().__init__() 444 | self.silu = nn.SiLU() 445 | self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=bias) 446 | if norm_type == "layer_norm": 447 | self.norm = LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6) 448 | else: 449 | raise ValueError(f"unknown norm_type {norm_type}") 450 | 451 | def forward( 452 | self, 453 | x: torch.Tensor, 454 | emb: Optional[torch.Tensor] = None, 455 | ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 456 | emb = emb.unsqueeze(-2) 457 | emb = self.linear(self.silu(emb)) 458 | shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=-1) 459 | x = self.norm(x) * (1 + scale_msa) + shift_msa 460 | return x, gate_msa, shift_mlp, scale_mlp, gate_mlp 461 | 462 | 463 | class AdaLayerNormZeroSingle(nn.Module): 464 | def __init__(self, embedding_dim: int, norm_type="layer_norm", bias=True): 465 | super().__init__() 466 | 467 | self.silu = nn.SiLU() 468 | self.linear = nn.Linear(embedding_dim, 3 * embedding_dim, bias=bias) 469 | if norm_type == "layer_norm": 470 | self.norm = LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6) 471 | else: 472 | raise ValueError(f"unknown norm_type {norm_type}") 473 | 474 | def forward( 475 | self, 476 | x: torch.Tensor, 477 | emb: Optional[torch.Tensor] = None, 478 | ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 479 | emb = emb.unsqueeze(-2) 480 | emb = self.linear(self.silu(emb)) 481 | shift_msa, scale_msa, gate_msa = emb.chunk(3, dim=-1) 482 | x = self.norm(x) * (1 + scale_msa) + shift_msa 483 | return x, gate_msa 484 | 485 | 486 | class AdaLayerNormContinuous(nn.Module): 487 | def __init__( 488 | self, 489 | embedding_dim: int, 490 | conditioning_embedding_dim: int, 491 | elementwise_affine=True, 492 | eps=1e-5, 493 | bias=True, 494 | norm_type="layer_norm", 495 | ): 496 | super().__init__() 497 | self.silu = nn.SiLU() 498 | self.linear = nn.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias) 499 | if norm_type == "layer_norm": 500 | self.norm = LayerNorm(embedding_dim, eps, elementwise_affine, bias) 501 | else: 502 | raise ValueError(f"unknown norm_type {norm_type}") 503 | 504 | def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor: 505 | emb = emb.unsqueeze(-2) 506 | emb = self.linear(self.silu(emb)) 507 | scale, shift = emb.chunk(2, dim=-1) 508 | x = self.norm(x) * (1 + scale) + shift 509 | return x 510 | 511 | 512 | class HunyuanVideoSingleTransformerBlock(nn.Module): 513 | def __init__( 514 | self, 515 | num_attention_heads: int, 516 | attention_head_dim: int, 517 | mlp_ratio: float = 4.0, 518 | qk_norm: str = "rms_norm", 519 | attention_mode: str = "sdpa", 520 | ) -> None: 521 | super().__init__() 522 | 523 | hidden_size = num_attention_heads * attention_head_dim 524 | mlp_dim = int(hidden_size * mlp_ratio) 525 | 526 | self.attn = Attention( 527 | query_dim=hidden_size, 528 | cross_attention_dim=None, 529 | dim_head=attention_head_dim, 530 | heads=num_attention_heads, 531 | out_dim=hidden_size, 532 | bias=True, 533 | processor=HunyuanAttnProcessorFlashAttnSingle(attention_mode), 534 | qk_norm=qk_norm, 535 | eps=1e-6, 536 | pre_only=True, 537 | ) 538 | 539 | self.norm = AdaLayerNormZeroSingle(hidden_size, norm_type="layer_norm") 540 | self.proj_mlp = nn.Linear(hidden_size, mlp_dim) 541 | self.act_mlp = nn.GELU(approximate="tanh") 542 | self.proj_out = nn.Linear(hidden_size + mlp_dim, hidden_size) 543 | 544 | def forward( 545 | self, 546 | hidden_states: torch.Tensor, 547 | encoder_hidden_states: torch.Tensor, 548 | temb: torch.Tensor, 549 | attention_mask: Optional[torch.Tensor] = None, 550 | image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, 551 | ) -> torch.Tensor: 552 | text_seq_length = encoder_hidden_states.shape[1] 553 | hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) 554 | 555 | residual = hidden_states 556 | 557 | # 1. Input normalization 558 | norm_hidden_states, gate = self.norm(hidden_states, emb=temb) 559 | mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)) 560 | 561 | norm_hidden_states, norm_encoder_hidden_states = ( 562 | norm_hidden_states[:, :-text_seq_length, :], 563 | norm_hidden_states[:, -text_seq_length:, :], 564 | ) 565 | 566 | # 2. Attention 567 | attn_output, context_attn_output = self.attn( 568 | hidden_states=norm_hidden_states, 569 | encoder_hidden_states=norm_encoder_hidden_states, 570 | attention_mask=attention_mask, 571 | image_rotary_emb=image_rotary_emb, 572 | ) 573 | attn_output = torch.cat([attn_output, context_attn_output], dim=1) 574 | 575 | # 3. Modulation and residual connection 576 | hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) 577 | hidden_states = gate * self.proj_out(hidden_states) 578 | hidden_states = hidden_states + residual 579 | 580 | hidden_states, encoder_hidden_states = ( 581 | hidden_states[:, :-text_seq_length, :], 582 | hidden_states[:, -text_seq_length:, :], 583 | ) 584 | return hidden_states, encoder_hidden_states 585 | 586 | 587 | class HunyuanVideoTransformerBlock(nn.Module): 588 | def __init__( 589 | self, 590 | num_attention_heads: int, 591 | attention_head_dim: int, 592 | mlp_ratio: float, 593 | qk_norm: str = "rms_norm", 594 | attention_mode: str = "sdpa", 595 | ) -> None: 596 | super().__init__() 597 | 598 | hidden_size = num_attention_heads * attention_head_dim 599 | 600 | self.norm1 = AdaLayerNormZero(hidden_size, norm_type="layer_norm") 601 | self.norm1_context = AdaLayerNormZero(hidden_size, norm_type="layer_norm") 602 | 603 | self.attn = Attention( 604 | query_dim=hidden_size, 605 | cross_attention_dim=None, 606 | added_kv_proj_dim=hidden_size, 607 | dim_head=attention_head_dim, 608 | heads=num_attention_heads, 609 | out_dim=hidden_size, 610 | context_pre_only=False, 611 | bias=True, 612 | processor=HunyuanAttnProcessorFlashAttnDouble(attention_mode), 613 | qk_norm=qk_norm, 614 | eps=1e-6, 615 | ) 616 | 617 | self.norm2 = LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) 618 | self.ff = FeedForward(hidden_size, mult=mlp_ratio, activation_fn="gelu-approximate") 619 | 620 | self.norm2_context = LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) 621 | self.ff_context = FeedForward(hidden_size, mult=mlp_ratio, activation_fn="gelu-approximate") 622 | 623 | def forward( 624 | self, 625 | hidden_states: torch.Tensor, 626 | encoder_hidden_states: torch.Tensor, 627 | temb: torch.Tensor, 628 | attention_mask: Optional[torch.Tensor] = None, 629 | freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, 630 | ) -> Tuple[torch.Tensor, torch.Tensor]: 631 | # 1. Input normalization 632 | norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) 633 | norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context(encoder_hidden_states, emb=temb) 634 | 635 | # 2. Joint attention 636 | attn_output, context_attn_output = self.attn( 637 | hidden_states=norm_hidden_states, 638 | encoder_hidden_states=norm_encoder_hidden_states, 639 | attention_mask=attention_mask, 640 | image_rotary_emb=freqs_cis, 641 | ) 642 | 643 | # 3. Modulation and residual connection 644 | hidden_states = hidden_states + attn_output * gate_msa 645 | encoder_hidden_states = encoder_hidden_states + context_attn_output * c_gate_msa 646 | 647 | norm_hidden_states = self.norm2(hidden_states) 648 | norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) 649 | 650 | norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp 651 | norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp) + c_shift_mlp 652 | 653 | # 4. Feed-forward 654 | ff_output = self.ff(norm_hidden_states) 655 | context_ff_output = self.ff_context(norm_encoder_hidden_states) 656 | 657 | hidden_states = hidden_states + gate_mlp * ff_output 658 | encoder_hidden_states = encoder_hidden_states + c_gate_mlp * context_ff_output 659 | 660 | return hidden_states, encoder_hidden_states 661 | 662 | 663 | class ClipVisionProjection(nn.Module): 664 | def __init__(self, in_channels, out_channels): 665 | super().__init__() 666 | self.up = nn.Linear(in_channels, out_channels * 3) 667 | self.down = nn.Linear(out_channels * 3, out_channels) 668 | 669 | def forward(self, x): 670 | projected_x = self.down(nn.functional.silu(self.up(x))) 671 | return projected_x 672 | 673 | 674 | class HunyuanVideoPatchEmbed(nn.Module): 675 | def __init__(self, patch_size, in_chans, embed_dim): 676 | super().__init__() 677 | self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) 678 | 679 | 680 | class HunyuanVideoPatchEmbedForCleanLatents(nn.Module): 681 | def __init__(self, inner_dim): 682 | super().__init__() 683 | self.proj = nn.Conv3d(16, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2)) 684 | self.proj_2x = nn.Conv3d(16, inner_dim, kernel_size=(2, 4, 4), stride=(2, 4, 4)) 685 | self.proj_4x = nn.Conv3d(16, inner_dim, kernel_size=(4, 8, 8), stride=(4, 8, 8)) 686 | 687 | @torch.no_grad() 688 | def initialize_weight_from_another_conv3d(self, another_layer): 689 | weight = another_layer.weight.detach().clone() 690 | bias = another_layer.bias.detach().clone() 691 | 692 | sd = { 693 | 'proj.weight': weight.clone(), 694 | 'proj.bias': bias.clone(), 695 | 'proj_2x.weight': einops.repeat(weight, 'b c t h w -> b c (t tk) (h hk) (w wk)', tk=2, hk=2, wk=2) / 8.0, 696 | 'proj_2x.bias': bias.clone(), 697 | 'proj_4x.weight': einops.repeat(weight, 'b c t h w -> b c (t tk) (h hk) (w wk)', tk=4, hk=4, wk=4) / 64.0, 698 | 'proj_4x.bias': bias.clone(), 699 | } 700 | 701 | sd = {k: v.clone() for k, v in sd.items()} 702 | 703 | self.load_state_dict(sd) 704 | return 705 | 706 | 707 | class HunyuanVideoTransformer3DModelPacked(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin): 708 | @register_to_config 709 | def __init__( 710 | self, 711 | in_channels: int = 16, 712 | out_channels: int = 16, 713 | num_attention_heads: int = 24, 714 | attention_head_dim: int = 128, 715 | num_layers: int = 20, 716 | num_single_layers: int = 40, 717 | num_refiner_layers: int = 2, 718 | mlp_ratio: float = 4.0, 719 | patch_size: int = 2, 720 | patch_size_t: int = 1, 721 | qk_norm: str = "rms_norm", 722 | guidance_embeds: bool = True, 723 | text_embed_dim: int = 4096, 724 | pooled_projection_dim: int = 768, 725 | rope_theta: float = 256.0, 726 | rope_axes_dim: Tuple[int] = (16, 56, 56), 727 | has_image_proj=False, 728 | image_proj_dim=1152, 729 | has_clean_x_embedder=False, 730 | attention_mode="sdpa", 731 | ) -> None: 732 | super().__init__() 733 | 734 | inner_dim = num_attention_heads * attention_head_dim 735 | out_channels = out_channels or in_channels 736 | 737 | # 1. Latent and condition embedders 738 | self.x_embedder = HunyuanVideoPatchEmbed((patch_size_t, patch_size, patch_size), in_channels, inner_dim) 739 | self.context_embedder = HunyuanVideoTokenRefiner( 740 | text_embed_dim, num_attention_heads, attention_head_dim, num_layers=num_refiner_layers 741 | ) 742 | self.time_text_embed = CombinedTimestepGuidanceTextProjEmbeddings(inner_dim, pooled_projection_dim) 743 | 744 | self.clean_x_embedder = None 745 | self.image_projection = None 746 | 747 | # 2. RoPE 748 | self.rope = HunyuanVideoRotaryPosEmbed(rope_axes_dim, rope_theta) 749 | 750 | # 3. Dual stream transformer blocks 751 | self.transformer_blocks = nn.ModuleList( 752 | [ 753 | HunyuanVideoTransformerBlock( 754 | num_attention_heads, attention_head_dim, mlp_ratio=mlp_ratio, qk_norm=qk_norm, attention_mode=attention_mode 755 | ) 756 | for _ in range(num_layers) 757 | ] 758 | ) 759 | 760 | # 4. Single stream transformer blocks 761 | self.single_transformer_blocks = nn.ModuleList( 762 | [ 763 | HunyuanVideoSingleTransformerBlock( 764 | num_attention_heads, attention_head_dim, mlp_ratio=mlp_ratio, qk_norm=qk_norm, attention_mode=attention_mode 765 | ) 766 | for _ in range(num_single_layers) 767 | ] 768 | ) 769 | 770 | # 5. Output projection 771 | self.norm_out = AdaLayerNormContinuous(inner_dim, inner_dim, elementwise_affine=False, eps=1e-6) 772 | self.proj_out = nn.Linear(inner_dim, patch_size_t * patch_size * patch_size * out_channels) 773 | 774 | self.inner_dim = inner_dim 775 | self.use_gradient_checkpointing = False 776 | self.enable_teacache = False 777 | 778 | if has_image_proj: 779 | self.install_image_projection(image_proj_dim) 780 | 781 | if has_clean_x_embedder: 782 | self.install_clean_x_embedder() 783 | 784 | self.high_quality_fp32_output_for_inference = False 785 | 786 | def install_image_projection(self, in_channels): 787 | self.image_projection = ClipVisionProjection(in_channels=in_channels, out_channels=self.inner_dim) 788 | self.config['has_image_proj'] = True 789 | self.config['image_proj_dim'] = in_channels 790 | 791 | def install_clean_x_embedder(self): 792 | self.clean_x_embedder = HunyuanVideoPatchEmbedForCleanLatents(self.inner_dim) 793 | self.config['has_clean_x_embedder'] = True 794 | 795 | def enable_gradient_checkpointing(self): 796 | self.use_gradient_checkpointing = True 797 | print('self.use_gradient_checkpointing = True') 798 | 799 | def disable_gradient_checkpointing(self): 800 | self.use_gradient_checkpointing = False 801 | print('self.use_gradient_checkpointing = False') 802 | 803 | def initialize_teacache(self, enable_teacache=True, num_steps=25, rel_l1_thresh=0.15): 804 | self.enable_teacache = enable_teacache 805 | self.cnt = 0 806 | self.num_steps = num_steps 807 | self.rel_l1_thresh = rel_l1_thresh # 0.1 for 1.6x speedup, 0.15 for 2.1x speedup 808 | self.accumulated_rel_l1_distance = 0 809 | self.previous_modulated_input = None 810 | self.previous_residual = None 811 | self.teacache_rescale_func = np.poly1d([7.33226126e+02, -4.01131952e+02, 6.75869174e+01, -3.14987800e+00, 9.61237896e-02]) 812 | 813 | def gradient_checkpointing_method(self, block, *args): 814 | if self.use_gradient_checkpointing: 815 | result = torch.utils.checkpoint.checkpoint(block, *args, use_reentrant=False) 816 | else: 817 | result = block(*args) 818 | return result 819 | 820 | def process_input_hidden_states( 821 | self, 822 | latents, latent_indices=None, 823 | clean_latents=None, clean_latent_indices=None, 824 | clean_latents_2x=None, clean_latent_2x_indices=None, 825 | clean_latents_4x=None, clean_latent_4x_indices=None 826 | ): 827 | hidden_states = self.gradient_checkpointing_method(self.x_embedder.proj, latents) 828 | B, C, T, H, W = hidden_states.shape 829 | 830 | if latent_indices is None: 831 | latent_indices = torch.arange(0, T).unsqueeze(0).expand(B, -1) 832 | 833 | hidden_states = hidden_states.flatten(2).transpose(1, 2) 834 | 835 | rope_freqs = self.rope(frame_indices=latent_indices, height=H, width=W, device=hidden_states.device) 836 | rope_freqs = rope_freqs.flatten(2).transpose(1, 2) 837 | 838 | if clean_latents is not None and clean_latent_indices is not None: 839 | clean_latents = clean_latents.to(hidden_states) 840 | clean_latents = self.gradient_checkpointing_method(self.clean_x_embedder.proj, clean_latents) 841 | clean_latents = clean_latents.flatten(2).transpose(1, 2) 842 | 843 | clean_latent_rope_freqs = self.rope(frame_indices=clean_latent_indices, height=H, width=W, device=clean_latents.device) 844 | clean_latent_rope_freqs = clean_latent_rope_freqs.flatten(2).transpose(1, 2) 845 | 846 | hidden_states = torch.cat([clean_latents, hidden_states], dim=1) 847 | rope_freqs = torch.cat([clean_latent_rope_freqs, rope_freqs], dim=1) 848 | 849 | if clean_latents_2x is not None and clean_latent_2x_indices is not None: 850 | clean_latents_2x = clean_latents_2x.to(hidden_states) 851 | clean_latents_2x = pad_for_3d_conv(clean_latents_2x, (2, 4, 4)) 852 | clean_latents_2x = self.gradient_checkpointing_method(self.clean_x_embedder.proj_2x, clean_latents_2x) 853 | clean_latents_2x = clean_latents_2x.flatten(2).transpose(1, 2) 854 | 855 | clean_latent_2x_rope_freqs = self.rope(frame_indices=clean_latent_2x_indices, height=H, width=W, device=clean_latents_2x.device) 856 | clean_latent_2x_rope_freqs = pad_for_3d_conv(clean_latent_2x_rope_freqs, (2, 2, 2)) 857 | clean_latent_2x_rope_freqs = center_down_sample_3d(clean_latent_2x_rope_freqs, (2, 2, 2)) 858 | clean_latent_2x_rope_freqs = clean_latent_2x_rope_freqs.flatten(2).transpose(1, 2) 859 | 860 | hidden_states = torch.cat([clean_latents_2x, hidden_states], dim=1) 861 | rope_freqs = torch.cat([clean_latent_2x_rope_freqs, rope_freqs], dim=1) 862 | 863 | if clean_latents_4x is not None and clean_latent_4x_indices is not None: 864 | clean_latents_4x = clean_latents_4x.to(hidden_states) 865 | clean_latents_4x = pad_for_3d_conv(clean_latents_4x, (4, 8, 8)) 866 | clean_latents_4x = self.gradient_checkpointing_method(self.clean_x_embedder.proj_4x, clean_latents_4x) 867 | clean_latents_4x = clean_latents_4x.flatten(2).transpose(1, 2) 868 | 869 | clean_latent_4x_rope_freqs = self.rope(frame_indices=clean_latent_4x_indices, height=H, width=W, device=clean_latents_4x.device) 870 | clean_latent_4x_rope_freqs = pad_for_3d_conv(clean_latent_4x_rope_freqs, (4, 4, 4)) 871 | clean_latent_4x_rope_freqs = center_down_sample_3d(clean_latent_4x_rope_freqs, (4, 4, 4)) 872 | clean_latent_4x_rope_freqs = clean_latent_4x_rope_freqs.flatten(2).transpose(1, 2) 873 | 874 | hidden_states = torch.cat([clean_latents_4x, hidden_states], dim=1) 875 | rope_freqs = torch.cat([clean_latent_4x_rope_freqs, rope_freqs], dim=1) 876 | 877 | return hidden_states, rope_freqs 878 | 879 | def forward( 880 | self, 881 | hidden_states, timestep, encoder_hidden_states, encoder_attention_mask, pooled_projections, guidance, 882 | latent_indices=None, 883 | clean_latents=None, clean_latent_indices=None, 884 | clean_latents_2x=None, clean_latent_2x_indices=None, 885 | clean_latents_4x=None, clean_latent_4x_indices=None, 886 | image_embeddings=None, 887 | attention_kwargs=None, return_dict=True 888 | ): 889 | 890 | if attention_kwargs is None: 891 | attention_kwargs = {} 892 | 893 | batch_size, num_channels, num_frames, height, width = hidden_states.shape 894 | p, p_t = self.config['patch_size'], self.config['patch_size_t'] 895 | post_patch_num_frames = num_frames // p_t 896 | post_patch_height = height // p 897 | post_patch_width = width // p 898 | original_context_length = post_patch_num_frames * post_patch_height * post_patch_width 899 | 900 | hidden_states, rope_freqs = self.process_input_hidden_states(hidden_states, latent_indices, clean_latents, clean_latent_indices, clean_latents_2x, clean_latent_2x_indices, clean_latents_4x, clean_latent_4x_indices) 901 | 902 | temb = self.gradient_checkpointing_method(self.time_text_embed, timestep, guidance, pooled_projections) 903 | encoder_hidden_states = self.gradient_checkpointing_method(self.context_embedder, encoder_hidden_states, timestep, encoder_attention_mask) 904 | 905 | if self.image_projection is not None: 906 | assert image_embeddings is not None, 'You must use image embeddings!' 907 | extra_encoder_hidden_states = self.gradient_checkpointing_method(self.image_projection, image_embeddings) 908 | extra_attention_mask = torch.ones((batch_size, extra_encoder_hidden_states.shape[1]), dtype=encoder_attention_mask.dtype, device=encoder_attention_mask.device) 909 | 910 | # must cat before (not after) encoder_hidden_states, due to attn masking 911 | encoder_hidden_states = torch.cat([extra_encoder_hidden_states, encoder_hidden_states], dim=1) 912 | encoder_attention_mask = torch.cat([extra_attention_mask, encoder_attention_mask], dim=1) 913 | 914 | with torch.no_grad(): 915 | if batch_size == 1: 916 | # When batch size is 1, we do not need any masks or var-len funcs since cropping is mathematically same to what we want 917 | # If they are not same, then their impls are wrong. Ours are always the correct one. 918 | text_len = encoder_attention_mask.sum().item() 919 | encoder_hidden_states = encoder_hidden_states[:, :text_len] 920 | attention_mask = None, None, None, None 921 | else: 922 | img_seq_len = hidden_states.shape[1] 923 | txt_seq_len = encoder_hidden_states.shape[1] 924 | 925 | cu_seqlens_q = get_cu_seqlens(encoder_attention_mask, img_seq_len) 926 | cu_seqlens_kv = cu_seqlens_q 927 | max_seqlen_q = img_seq_len + txt_seq_len 928 | max_seqlen_kv = max_seqlen_q 929 | 930 | attention_mask = cu_seqlens_q, cu_seqlens_kv, max_seqlen_q, max_seqlen_kv 931 | 932 | if self.enable_teacache: 933 | modulated_inp = self.transformer_blocks[0].norm1(hidden_states, emb=temb)[0] 934 | 935 | if self.cnt == 0 or self.cnt == self.num_steps-1: 936 | should_calc = True 937 | self.accumulated_rel_l1_distance = 0 938 | else: 939 | curr_rel_l1 = ((modulated_inp - self.previous_modulated_input).abs().mean() / self.previous_modulated_input.abs().mean()).cpu().item() 940 | self.accumulated_rel_l1_distance += self.teacache_rescale_func(curr_rel_l1) 941 | should_calc = self.accumulated_rel_l1_distance >= self.rel_l1_thresh 942 | 943 | if should_calc: 944 | self.accumulated_rel_l1_distance = 0 945 | 946 | self.previous_modulated_input = modulated_inp 947 | self.cnt += 1 948 | 949 | if self.cnt == self.num_steps: 950 | self.cnt = 0 951 | 952 | if not should_calc: 953 | hidden_states = hidden_states + self.previous_residual 954 | else: 955 | ori_hidden_states = hidden_states.clone() 956 | 957 | for block_id, block in enumerate(self.transformer_blocks): 958 | hidden_states, encoder_hidden_states = self.gradient_checkpointing_method( 959 | block, 960 | hidden_states, 961 | encoder_hidden_states, 962 | temb, 963 | attention_mask, 964 | rope_freqs 965 | ) 966 | 967 | for block_id, block in enumerate(self.single_transformer_blocks): 968 | hidden_states, encoder_hidden_states = self.gradient_checkpointing_method( 969 | block, 970 | hidden_states, 971 | encoder_hidden_states, 972 | temb, 973 | attention_mask, 974 | rope_freqs 975 | ) 976 | 977 | self.previous_residual = hidden_states - ori_hidden_states 978 | else: 979 | for block_id, block in enumerate(self.transformer_blocks): 980 | hidden_states, encoder_hidden_states = self.gradient_checkpointing_method( 981 | block, 982 | hidden_states, 983 | encoder_hidden_states, 984 | temb, 985 | attention_mask, 986 | rope_freqs 987 | ) 988 | 989 | for block_id, block in enumerate(self.single_transformer_blocks): 990 | hidden_states, encoder_hidden_states = self.gradient_checkpointing_method( 991 | block, 992 | hidden_states, 993 | encoder_hidden_states, 994 | temb, 995 | attention_mask, 996 | rope_freqs 997 | ) 998 | 999 | hidden_states = self.gradient_checkpointing_method(self.norm_out, hidden_states, temb) 1000 | 1001 | hidden_states = hidden_states[:, -original_context_length:, :] 1002 | 1003 | if self.high_quality_fp32_output_for_inference: 1004 | hidden_states = hidden_states.to(dtype=torch.float32) 1005 | if self.proj_out.weight.dtype != torch.float32: 1006 | self.proj_out.to(dtype=torch.float32) 1007 | 1008 | hidden_states = self.gradient_checkpointing_method(self.proj_out, hidden_states) 1009 | 1010 | hidden_states = einops.rearrange(hidden_states, 'b (t h w) (c pt ph pw) -> b c (t pt) (h ph) (w pw)', 1011 | t=post_patch_num_frames, h=post_patch_height, w=post_patch_width, 1012 | pt=p_t, ph=p, pw=p) 1013 | 1014 | if return_dict: 1015 | return Transformer2DModelOutput(sample=hidden_states) 1016 | 1017 | return hidden_states, 1018 | -------------------------------------------------------------------------------- /diffusers_helper/pipelines/k_diffusion_hunyuan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import math 3 | 4 | from ..k_diffusion.uni_pc_fm import sample_unipc 5 | from ..k_diffusion.wrapper import fm_wrapper 6 | from ..utils import repeat_to_batch_size 7 | 8 | 9 | def flux_time_shift(t, mu=1.15, sigma=1.0): 10 | return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) 11 | 12 | 13 | def calculate_flux_mu(context_length, x1=256, y1=0.5, x2=4096, y2=1.15, exp_max=7.0): 14 | k = (y2 - y1) / (x2 - x1) 15 | b = y1 - k * x1 16 | mu = k * context_length + b 17 | mu = min(mu, math.log(exp_max)) 18 | return mu 19 | 20 | 21 | def get_flux_sigmas_from_mu(n, mu): 22 | sigmas = torch.linspace(1, 0, steps=n + 1) 23 | sigmas = flux_time_shift(sigmas, mu=mu) 24 | return sigmas 25 | 26 | 27 | @torch.inference_mode() 28 | def sample_hunyuan( 29 | transformer, 30 | sampler='unipc', 31 | initial_latent=None, 32 | concat_latent=None, 33 | strength=1.0, 34 | width=512, 35 | height=512, 36 | frames=16, 37 | real_guidance_scale=1.0, 38 | distilled_guidance_scale=6.0, 39 | guidance_rescale=0.0, 40 | shift=None, 41 | num_inference_steps=25, 42 | batch_size=None, 43 | generator=None, 44 | prompt_embeds=None, 45 | prompt_embeds_mask=None, 46 | prompt_poolers=None, 47 | negative_prompt_embeds=None, 48 | negative_prompt_embeds_mask=None, 49 | negative_prompt_poolers=None, 50 | dtype=torch.bfloat16, 51 | device=None, 52 | negative_kwargs=None, 53 | callback=None, 54 | **kwargs, 55 | ): 56 | device = device or transformer.device 57 | 58 | if batch_size is None: 59 | batch_size = int(prompt_embeds.shape[0]) 60 | 61 | latents = torch.randn((batch_size, 16, (frames + 3) // 4, height // 8, width // 8), generator=generator, device=generator.device).to(device=device, dtype=torch.float32) 62 | 63 | B, C, T, H, W = latents.shape 64 | seq_length = T * H * W // 4 65 | 66 | if shift is None: 67 | mu = calculate_flux_mu(seq_length, exp_max=7.0) 68 | else: 69 | mu = math.log(shift) 70 | 71 | sigmas = get_flux_sigmas_from_mu(num_inference_steps, mu).to(device) 72 | 73 | k_model = fm_wrapper(transformer) 74 | 75 | if initial_latent is not None: 76 | sigmas = sigmas * strength 77 | first_sigma = sigmas[0].to(device=device, dtype=torch.float32) 78 | initial_latent = initial_latent.to(device=device, dtype=torch.float32) 79 | latents = initial_latent.float() * (1.0 - first_sigma) + latents.float() * first_sigma 80 | 81 | if concat_latent is not None: 82 | concat_latent = concat_latent.to(latents) 83 | 84 | distilled_guidance = torch.tensor([distilled_guidance_scale * 1000.0] * batch_size).to(device=device, dtype=dtype) 85 | 86 | prompt_embeds = repeat_to_batch_size(prompt_embeds, batch_size) 87 | prompt_embeds_mask = repeat_to_batch_size(prompt_embeds_mask, batch_size) 88 | prompt_poolers = repeat_to_batch_size(prompt_poolers, batch_size) 89 | negative_prompt_embeds = repeat_to_batch_size(negative_prompt_embeds, batch_size) 90 | negative_prompt_embeds_mask = repeat_to_batch_size(negative_prompt_embeds_mask, batch_size) 91 | negative_prompt_poolers = repeat_to_batch_size(negative_prompt_poolers, batch_size) 92 | concat_latent = repeat_to_batch_size(concat_latent, batch_size) 93 | 94 | sampler_kwargs = dict( 95 | dtype=dtype, 96 | cfg_scale=real_guidance_scale, 97 | cfg_rescale=guidance_rescale, 98 | concat_latent=concat_latent, 99 | positive=dict( 100 | pooled_projections=prompt_poolers, 101 | encoder_hidden_states=prompt_embeds, 102 | encoder_attention_mask=prompt_embeds_mask, 103 | guidance=distilled_guidance, 104 | **kwargs, 105 | ), 106 | negative=dict( 107 | pooled_projections=negative_prompt_poolers, 108 | encoder_hidden_states=negative_prompt_embeds, 109 | encoder_attention_mask=negative_prompt_embeds_mask, 110 | guidance=distilled_guidance, 111 | **(kwargs if negative_kwargs is None else {**kwargs, **negative_kwargs}), 112 | ) 113 | ) 114 | 115 | if sampler == 'unipc_bh1': 116 | variant = 'bh1' 117 | elif sampler == 'unipc_bh2': 118 | variant = 'bh2' 119 | results = sample_unipc(k_model, latents, sigmas, extra_args=sampler_kwargs, disable=False, variant=variant, callback=callback) 120 | 121 | return results 122 | -------------------------------------------------------------------------------- /diffusers_helper/thread_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from threading import Thread, Lock 4 | 5 | 6 | class Listener: 7 | task_queue = [] 8 | lock = Lock() 9 | thread = None 10 | 11 | @classmethod 12 | def _process_tasks(cls): 13 | while True: 14 | task = None 15 | with cls.lock: 16 | if cls.task_queue: 17 | task = cls.task_queue.pop(0) 18 | 19 | if task is None: 20 | time.sleep(0.001) 21 | continue 22 | 23 | func, args, kwargs = task 24 | try: 25 | func(*args, **kwargs) 26 | except Exception as e: 27 | print(f"Error in listener thread: {e}") 28 | 29 | @classmethod 30 | def add_task(cls, func, *args, **kwargs): 31 | with cls.lock: 32 | cls.task_queue.append((func, args, kwargs)) 33 | 34 | if cls.thread is None: 35 | cls.thread = Thread(target=cls._process_tasks, daemon=True) 36 | cls.thread.start() 37 | 38 | 39 | def async_run(func, *args, **kwargs): 40 | Listener.add_task(func, *args, **kwargs) 41 | 42 | 43 | class FIFOQueue: 44 | def __init__(self): 45 | self.queue = [] 46 | self.lock = Lock() 47 | 48 | def push(self, item): 49 | with self.lock: 50 | self.queue.append(item) 51 | 52 | def pop(self): 53 | with self.lock: 54 | if self.queue: 55 | return self.queue.pop(0) 56 | return None 57 | 58 | def top(self): 59 | with self.lock: 60 | if self.queue: 61 | return self.queue[0] 62 | return None 63 | 64 | def next(self): 65 | while True: 66 | with self.lock: 67 | if self.queue: 68 | return self.queue.pop(0) 69 | 70 | time.sleep(0.001) 71 | 72 | 73 | class AsyncStream: 74 | def __init__(self): 75 | self.input_queue = FIFOQueue() 76 | self.output_queue = FIFOQueue() 77 | -------------------------------------------------------------------------------- /diffusers_helper/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import json 4 | import random 5 | import glob 6 | import torch 7 | import einops 8 | import numpy as np 9 | import datetime 10 | import torchvision 11 | 12 | import safetensors.torch as sf 13 | from PIL import Image 14 | 15 | 16 | def min_resize(x, m): 17 | if x.shape[0] < x.shape[1]: 18 | s0 = m 19 | s1 = int(float(m) / float(x.shape[0]) * float(x.shape[1])) 20 | else: 21 | s0 = int(float(m) / float(x.shape[1]) * float(x.shape[0])) 22 | s1 = m 23 | new_max = max(s1, s0) 24 | raw_max = max(x.shape[0], x.shape[1]) 25 | if new_max < raw_max: 26 | interpolation = cv2.INTER_AREA 27 | else: 28 | interpolation = cv2.INTER_LANCZOS4 29 | y = cv2.resize(x, (s1, s0), interpolation=interpolation) 30 | return y 31 | 32 | 33 | def d_resize(x, y): 34 | H, W, C = y.shape 35 | new_min = min(H, W) 36 | raw_min = min(x.shape[0], x.shape[1]) 37 | if new_min < raw_min: 38 | interpolation = cv2.INTER_AREA 39 | else: 40 | interpolation = cv2.INTER_LANCZOS4 41 | y = cv2.resize(x, (W, H), interpolation=interpolation) 42 | return y 43 | 44 | 45 | def resize_and_center_crop(image, target_width, target_height): 46 | if target_height == image.shape[0] and target_width == image.shape[1]: 47 | return image 48 | 49 | pil_image = Image.fromarray(image) 50 | original_width, original_height = pil_image.size 51 | scale_factor = max(target_width / original_width, target_height / original_height) 52 | resized_width = int(round(original_width * scale_factor)) 53 | resized_height = int(round(original_height * scale_factor)) 54 | resized_image = pil_image.resize((resized_width, resized_height), Image.LANCZOS) 55 | left = (resized_width - target_width) / 2 56 | top = (resized_height - target_height) / 2 57 | right = (resized_width + target_width) / 2 58 | bottom = (resized_height + target_height) / 2 59 | cropped_image = resized_image.crop((left, top, right, bottom)) 60 | return np.array(cropped_image) 61 | 62 | 63 | def resize_and_center_crop_pytorch(image, target_width, target_height): 64 | B, C, H, W = image.shape 65 | 66 | if H == target_height and W == target_width: 67 | return image 68 | 69 | scale_factor = max(target_width / W, target_height / H) 70 | resized_width = int(round(W * scale_factor)) 71 | resized_height = int(round(H * scale_factor)) 72 | 73 | resized = torch.nn.functional.interpolate(image, size=(resized_height, resized_width), mode='bilinear', align_corners=False) 74 | 75 | top = (resized_height - target_height) // 2 76 | left = (resized_width - target_width) // 2 77 | cropped = resized[:, :, top:top + target_height, left:left + target_width] 78 | 79 | return cropped 80 | 81 | 82 | def resize_without_crop(image, target_width, target_height): 83 | if target_height == image.shape[0] and target_width == image.shape[1]: 84 | return image 85 | 86 | pil_image = Image.fromarray(image) 87 | resized_image = pil_image.resize((target_width, target_height), Image.LANCZOS) 88 | return np.array(resized_image) 89 | 90 | 91 | def just_crop(image, w, h): 92 | if h == image.shape[0] and w == image.shape[1]: 93 | return image 94 | 95 | original_height, original_width = image.shape[:2] 96 | k = min(original_height / h, original_width / w) 97 | new_width = int(round(w * k)) 98 | new_height = int(round(h * k)) 99 | x_start = (original_width - new_width) // 2 100 | y_start = (original_height - new_height) // 2 101 | cropped_image = image[y_start:y_start + new_height, x_start:x_start + new_width] 102 | return cropped_image 103 | 104 | 105 | def write_to_json(data, file_path): 106 | temp_file_path = file_path + ".tmp" 107 | with open(temp_file_path, 'wt', encoding='utf-8') as temp_file: 108 | json.dump(data, temp_file, indent=4) 109 | os.replace(temp_file_path, file_path) 110 | return 111 | 112 | 113 | def read_from_json(file_path): 114 | with open(file_path, 'rt', encoding='utf-8') as file: 115 | data = json.load(file) 116 | return data 117 | 118 | 119 | def get_active_parameters(m): 120 | return {k: v for k, v in m.named_parameters() if v.requires_grad} 121 | 122 | 123 | def cast_training_params(m, dtype=torch.float32): 124 | result = {} 125 | for n, param in m.named_parameters(): 126 | if param.requires_grad: 127 | param.data = param.to(dtype) 128 | result[n] = param 129 | return result 130 | 131 | 132 | def separate_lora_AB(parameters, B_patterns=None): 133 | parameters_normal = {} 134 | parameters_B = {} 135 | 136 | if B_patterns is None: 137 | B_patterns = ['.lora_B.', '__zero__'] 138 | 139 | for k, v in parameters.items(): 140 | if any(B_pattern in k for B_pattern in B_patterns): 141 | parameters_B[k] = v 142 | else: 143 | parameters_normal[k] = v 144 | 145 | return parameters_normal, parameters_B 146 | 147 | 148 | def set_attr_recursive(obj, attr, value): 149 | attrs = attr.split(".") 150 | for name in attrs[:-1]: 151 | obj = getattr(obj, name) 152 | setattr(obj, attrs[-1], value) 153 | return 154 | 155 | 156 | def print_tensor_list_size(tensors): 157 | total_size = 0 158 | total_elements = 0 159 | 160 | if isinstance(tensors, dict): 161 | tensors = tensors.values() 162 | 163 | for tensor in tensors: 164 | total_size += tensor.nelement() * tensor.element_size() 165 | total_elements += tensor.nelement() 166 | 167 | total_size_MB = total_size / (1024 ** 2) 168 | total_elements_B = total_elements / 1e9 169 | 170 | print(f"Total number of tensors: {len(tensors)}") 171 | print(f"Total size of tensors: {total_size_MB:.2f} MB") 172 | print(f"Total number of parameters: {total_elements_B:.3f} billion") 173 | return 174 | 175 | 176 | @torch.no_grad() 177 | def batch_mixture(a, b=None, probability_a=0.5, mask_a=None): 178 | batch_size = a.size(0) 179 | 180 | if b is None: 181 | b = torch.zeros_like(a) 182 | 183 | if mask_a is None: 184 | mask_a = torch.rand(batch_size) < probability_a 185 | 186 | mask_a = mask_a.to(a.device) 187 | mask_a = mask_a.reshape((batch_size,) + (1,) * (a.dim() - 1)) 188 | result = torch.where(mask_a, a, b) 189 | return result 190 | 191 | 192 | @torch.no_grad() 193 | def zero_module(module): 194 | for p in module.parameters(): 195 | p.detach().zero_() 196 | return module 197 | 198 | 199 | @torch.no_grad() 200 | def supress_lower_channels(m, k, alpha=0.01): 201 | data = m.weight.data.clone() 202 | 203 | assert int(data.shape[1]) >= k 204 | 205 | data[:, :k] = data[:, :k] * alpha 206 | m.weight.data = data.contiguous().clone() 207 | return m 208 | 209 | 210 | def freeze_module(m): 211 | if not hasattr(m, '_forward_inside_frozen_module'): 212 | m._forward_inside_frozen_module = m.forward 213 | m.requires_grad_(False) 214 | m.forward = torch.no_grad()(m.forward) 215 | return m 216 | 217 | 218 | def get_latest_safetensors(folder_path): 219 | safetensors_files = glob.glob(os.path.join(folder_path, '*.safetensors')) 220 | 221 | if not safetensors_files: 222 | raise ValueError('No file to resume!') 223 | 224 | latest_file = max(safetensors_files, key=os.path.getmtime) 225 | latest_file = os.path.abspath(os.path.realpath(latest_file)) 226 | return latest_file 227 | 228 | 229 | def generate_random_prompt_from_tags(tags_str, min_length=3, max_length=32): 230 | tags = tags_str.split(', ') 231 | tags = random.sample(tags, k=min(random.randint(min_length, max_length), len(tags))) 232 | prompt = ', '.join(tags) 233 | return prompt 234 | 235 | 236 | def interpolate_numbers(a, b, n, round_to_int=False, gamma=1.0): 237 | numbers = a + (b - a) * (np.linspace(0, 1, n) ** gamma) 238 | if round_to_int: 239 | numbers = np.round(numbers).astype(int) 240 | return numbers.tolist() 241 | 242 | 243 | def uniform_random_by_intervals(inclusive, exclusive, n, round_to_int=False): 244 | edges = np.linspace(0, 1, n + 1) 245 | points = np.random.uniform(edges[:-1], edges[1:]) 246 | numbers = inclusive + (exclusive - inclusive) * points 247 | if round_to_int: 248 | numbers = np.round(numbers).astype(int) 249 | return numbers.tolist() 250 | 251 | 252 | def soft_append_bcthw(history, current, overlap=0): 253 | if overlap <= 0: 254 | return torch.cat([history, current], dim=2) 255 | 256 | assert history.shape[2] >= overlap, f"History length ({history.shape[2]}) must be >= overlap ({overlap})" 257 | assert current.shape[2] >= overlap, f"Current length ({current.shape[2]}) must be >= overlap ({overlap})" 258 | 259 | weights = torch.linspace(1, 0, overlap, dtype=history.dtype, device=history.device).view(1, 1, -1, 1, 1) 260 | blended = weights * history[:, :, -overlap:] + (1 - weights) * current[:, :, :overlap] 261 | output = torch.cat([history[:, :, :-overlap], blended, current[:, :, overlap:]], dim=2) 262 | 263 | return output.to(history) 264 | 265 | 266 | def save_bcthw_as_mp4(x, output_filename, fps=10): 267 | b, c, t, h, w = x.shape 268 | 269 | per_row = b 270 | for p in [6, 5, 4, 3, 2]: 271 | if b % p == 0: 272 | per_row = p 273 | break 274 | 275 | os.makedirs(os.path.dirname(os.path.abspath(os.path.realpath(output_filename))), exist_ok=True) 276 | x = torch.clamp(x.float(), -1., 1.) * 127.5 + 127.5 277 | x = x.detach().cpu().to(torch.uint8) 278 | x = einops.rearrange(x, '(m n) c t h w -> t (m h) (n w) c', n=per_row) 279 | torchvision.io.write_video(output_filename, x, fps=fps, video_codec='h264', options={'crf': '0'}) 280 | return x 281 | 282 | 283 | def save_bcthw_as_png(x, output_filename): 284 | os.makedirs(os.path.dirname(os.path.abspath(os.path.realpath(output_filename))), exist_ok=True) 285 | x = torch.clamp(x.float(), -1., 1.) * 127.5 + 127.5 286 | x = x.detach().cpu().to(torch.uint8) 287 | x = einops.rearrange(x, 'b c t h w -> c (b h) (t w)') 288 | torchvision.io.write_png(x, output_filename) 289 | return output_filename 290 | 291 | 292 | def save_bchw_as_png(x, output_filename): 293 | os.makedirs(os.path.dirname(os.path.abspath(os.path.realpath(output_filename))), exist_ok=True) 294 | x = torch.clamp(x.float(), -1., 1.) * 127.5 + 127.5 295 | x = x.detach().cpu().to(torch.uint8) 296 | x = einops.rearrange(x, 'b c h w -> c h (b w)') 297 | torchvision.io.write_png(x, output_filename) 298 | return output_filename 299 | 300 | 301 | def add_tensors_with_padding(tensor1, tensor2): 302 | if tensor1.shape == tensor2.shape: 303 | return tensor1 + tensor2 304 | 305 | shape1 = tensor1.shape 306 | shape2 = tensor2.shape 307 | 308 | new_shape = tuple(max(s1, s2) for s1, s2 in zip(shape1, shape2)) 309 | 310 | padded_tensor1 = torch.zeros(new_shape) 311 | padded_tensor2 = torch.zeros(new_shape) 312 | 313 | padded_tensor1[tuple(slice(0, s) for s in shape1)] = tensor1 314 | padded_tensor2[tuple(slice(0, s) for s in shape2)] = tensor2 315 | 316 | result = padded_tensor1 + padded_tensor2 317 | return result 318 | 319 | 320 | def print_free_mem(): 321 | torch.cuda.empty_cache() 322 | free_mem, total_mem = torch.cuda.mem_get_info(0) 323 | free_mem_mb = free_mem / (1024 ** 2) 324 | total_mem_mb = total_mem / (1024 ** 2) 325 | print(f"Free memory: {free_mem_mb:.2f} MB") 326 | print(f"Total memory: {total_mem_mb:.2f} MB") 327 | return 328 | 329 | 330 | def print_gpu_parameters(device, state_dict, log_count=1): 331 | summary = {"device": device, "keys_count": len(state_dict)} 332 | 333 | logged_params = {} 334 | for i, (key, tensor) in enumerate(state_dict.items()): 335 | if i >= log_count: 336 | break 337 | logged_params[key] = tensor.flatten()[:3].tolist() 338 | 339 | summary["params"] = logged_params 340 | 341 | print(str(summary)) 342 | return 343 | 344 | 345 | def visualize_txt_as_img(width, height, text, font_path='font/DejaVuSans.ttf', size=18): 346 | from PIL import Image, ImageDraw, ImageFont 347 | 348 | txt = Image.new("RGB", (width, height), color="white") 349 | draw = ImageDraw.Draw(txt) 350 | font = ImageFont.truetype(font_path, size=size) 351 | 352 | if text == '': 353 | return np.array(txt) 354 | 355 | # Split text into lines that fit within the image width 356 | lines = [] 357 | words = text.split() 358 | current_line = words[0] 359 | 360 | for word in words[1:]: 361 | line_with_word = f"{current_line} {word}" 362 | if draw.textbbox((0, 0), line_with_word, font=font)[2] <= width: 363 | current_line = line_with_word 364 | else: 365 | lines.append(current_line) 366 | current_line = word 367 | 368 | lines.append(current_line) 369 | 370 | # Draw the text line by line 371 | y = 0 372 | line_height = draw.textbbox((0, 0), "A", font=font)[3] 373 | 374 | for line in lines: 375 | if y + line_height > height: 376 | break # stop drawing if the next line will be outside the image 377 | draw.text((0, y), line, fill="black", font=font) 378 | y += line_height 379 | 380 | return np.array(txt) 381 | 382 | 383 | def blue_mark(x): 384 | x = x.copy() 385 | c = x[:, :, 2] 386 | b = cv2.blur(c, (9, 9)) 387 | x[:, :, 2] = ((c - b) * 16.0 + b).clip(-1, 1) 388 | return x 389 | 390 | 391 | def green_mark(x): 392 | x = x.copy() 393 | x[:, :, 2] = -1 394 | x[:, :, 0] = -1 395 | return x 396 | 397 | 398 | def frame_mark(x): 399 | x = x.copy() 400 | x[:64] = -1 401 | x[-64:] = -1 402 | x[:, :8] = 1 403 | x[:, -8:] = 1 404 | return x 405 | 406 | 407 | @torch.inference_mode() 408 | def pytorch2numpy(imgs): 409 | results = [] 410 | for x in imgs: 411 | y = x.movedim(0, -1) 412 | y = y * 127.5 + 127.5 413 | y = y.detach().float().cpu().numpy().clip(0, 255).astype(np.uint8) 414 | results.append(y) 415 | return results 416 | 417 | 418 | @torch.inference_mode() 419 | def numpy2pytorch(imgs): 420 | h = torch.from_numpy(np.stack(imgs, axis=0)).float() / 127.5 - 1.0 421 | h = h.movedim(-1, 1) 422 | return h 423 | 424 | 425 | @torch.no_grad() 426 | def duplicate_prefix_to_suffix(x, count, zero_out=False): 427 | if zero_out: 428 | return torch.cat([x, torch.zeros_like(x[:count])], dim=0) 429 | else: 430 | return torch.cat([x, x[:count]], dim=0) 431 | 432 | 433 | def weighted_mse(a, b, weight): 434 | return torch.mean(weight.float() * (a.float() - b.float()) ** 2) 435 | 436 | 437 | def clamped_linear_interpolation(x, x_min, y_min, x_max, y_max, sigma=1.0): 438 | x = (x - x_min) / (x_max - x_min) 439 | x = max(0.0, min(x, 1.0)) 440 | x = x ** sigma 441 | return y_min + x * (y_max - y_min) 442 | 443 | 444 | def expand_to_dims(x, target_dims): 445 | return x.view(*x.shape, *([1] * max(0, target_dims - x.dim()))) 446 | 447 | 448 | def repeat_to_batch_size(tensor: torch.Tensor, batch_size: int): 449 | if tensor is None: 450 | return None 451 | 452 | first_dim = tensor.shape[0] 453 | 454 | if first_dim == batch_size: 455 | return tensor 456 | 457 | if batch_size % first_dim != 0: 458 | raise ValueError(f"Cannot evenly repeat first dim {first_dim} to match batch_size {batch_size}.") 459 | 460 | repeat_times = batch_size // first_dim 461 | 462 | return tensor.repeat(repeat_times, *[1] * (tensor.dim() - 1)) 463 | 464 | 465 | def dim5(x): 466 | return expand_to_dims(x, 5) 467 | 468 | 469 | def dim4(x): 470 | return expand_to_dims(x, 4) 471 | 472 | 473 | def dim3(x): 474 | return expand_to_dims(x, 3) 475 | 476 | 477 | def crop_or_pad_yield_mask(x, length): 478 | B, F, C = x.shape 479 | device = x.device 480 | dtype = x.dtype 481 | 482 | if F < length: 483 | y = torch.zeros((B, length, C), dtype=dtype, device=device) 484 | mask = torch.zeros((B, length), dtype=torch.bool, device=device) 485 | y[:, :F, :] = x 486 | mask[:, :F] = True 487 | return y, mask 488 | 489 | return x[:, :length, :], torch.ones((B, length), dtype=torch.bool, device=device) 490 | 491 | 492 | def extend_dim(x, dim, minimal_length, zero_pad=False): 493 | original_length = int(x.shape[dim]) 494 | 495 | if original_length >= minimal_length: 496 | return x 497 | 498 | if zero_pad: 499 | padding_shape = list(x.shape) 500 | padding_shape[dim] = minimal_length - original_length 501 | padding = torch.zeros(padding_shape, dtype=x.dtype, device=x.device) 502 | else: 503 | idx = (slice(None),) * dim + (slice(-1, None),) + (slice(None),) * (len(x.shape) - dim - 1) 504 | last_element = x[idx] 505 | padding = last_element.repeat_interleave(minimal_length - original_length, dim=dim) 506 | 507 | return torch.cat([x, padding], dim=dim) 508 | 509 | 510 | def lazy_positional_encoding(t, repeats=None): 511 | if not isinstance(t, list): 512 | t = [t] 513 | 514 | from diffusers.models.embeddings import get_timestep_embedding 515 | 516 | te = torch.tensor(t) 517 | te = get_timestep_embedding(timesteps=te, embedding_dim=256, flip_sin_to_cos=True, downscale_freq_shift=0.0, scale=1.0) 518 | 519 | if repeats is None: 520 | return te 521 | 522 | te = te[:, None, :].expand(-1, repeats, -1) 523 | 524 | return te 525 | 526 | 527 | def state_dict_offset_merge(A, B, C=None): 528 | result = {} 529 | keys = A.keys() 530 | 531 | for key in keys: 532 | A_value = A[key] 533 | B_value = B[key].to(A_value) 534 | 535 | if C is None: 536 | result[key] = A_value + B_value 537 | else: 538 | C_value = C[key].to(A_value) 539 | result[key] = A_value + B_value - C_value 540 | 541 | return result 542 | 543 | 544 | def state_dict_weighted_merge(state_dicts, weights): 545 | if len(state_dicts) != len(weights): 546 | raise ValueError("Number of state dictionaries must match number of weights") 547 | 548 | if not state_dicts: 549 | return {} 550 | 551 | total_weight = sum(weights) 552 | 553 | if total_weight == 0: 554 | raise ValueError("Sum of weights cannot be zero") 555 | 556 | normalized_weights = [w / total_weight for w in weights] 557 | 558 | keys = state_dicts[0].keys() 559 | result = {} 560 | 561 | for key in keys: 562 | result[key] = state_dicts[0][key] * normalized_weights[0] 563 | 564 | for i in range(1, len(state_dicts)): 565 | state_dict_value = state_dicts[i][key].to(result[key]) 566 | result[key] += state_dict_value * normalized_weights[i] 567 | 568 | return result 569 | 570 | 571 | def group_files_by_folder(all_files): 572 | grouped_files = {} 573 | 574 | for file in all_files: 575 | folder_name = os.path.basename(os.path.dirname(file)) 576 | if folder_name not in grouped_files: 577 | grouped_files[folder_name] = [] 578 | grouped_files[folder_name].append(file) 579 | 580 | list_of_lists = list(grouped_files.values()) 581 | return list_of_lists 582 | 583 | 584 | def generate_timestamp(): 585 | now = datetime.datetime.now() 586 | timestamp = now.strftime('%y%m%d_%H%M%S') 587 | milliseconds = f"{int(now.microsecond / 1000):03d}" 588 | random_number = random.randint(0, 9999) 589 | return f"{timestamp}_{milliseconds}_{random_number}" 590 | 591 | 592 | def write_PIL_image_with_png_info(image, metadata, path): 593 | from PIL.PngImagePlugin import PngInfo 594 | 595 | png_info = PngInfo() 596 | for key, value in metadata.items(): 597 | png_info.add_text(key, value) 598 | 599 | image.save(path, "PNG", pnginfo=png_info) 600 | return image 601 | 602 | 603 | def torch_safe_save(content, path): 604 | torch.save(content, path + '_tmp') 605 | os.replace(path + '_tmp', path) 606 | return path 607 | 608 | 609 | def move_optimizer_to_device(optimizer, device): 610 | for state in optimizer.state.values(): 611 | for k, v in state.items(): 612 | if isinstance(v, torch.Tensor): 613 | state[k] = v.to(device) 614 | -------------------------------------------------------------------------------- /example_workflows/framepack_hv_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "ce2cb810-7775-4564-8928-dd5bed1053cd", 3 | "revision": 0, 4 | "last_node_id": 55, 5 | "last_link_id": 130, 6 | "nodes": [ 7 | { 8 | "id": 12, 9 | "type": "VAELoader", 10 | "pos": [ 11 | 570.5363159179688, 12 | -282.70068359375 13 | ], 14 | "size": [ 15 | 469.0488586425781, 16 | 58 17 | ], 18 | "flags": {}, 19 | "order": 0, 20 | "mode": 0, 21 | "inputs": [], 22 | "outputs": [ 23 | { 24 | "name": "VAE", 25 | "type": "VAE", 26 | "links": [ 27 | 22, 28 | 62 29 | ] 30 | } 31 | ], 32 | "properties": { 33 | "cnr_id": "comfy-core", 34 | "ver": "0.3.28", 35 | "Node name for S&R": "VAELoader" 36 | }, 37 | "widgets_values": [ 38 | "hyvid\\hunyuan_video_vae_bf16_repack.safetensors" 39 | ], 40 | "color": "#322", 41 | "bgcolor": "#533" 42 | }, 43 | { 44 | "id": 33, 45 | "type": "VAEDecodeTiled", 46 | "pos": [ 47 | 2181.271484375, 48 | -292.61370849609375 49 | ], 50 | "size": [ 51 | 315, 52 | 150 53 | ], 54 | "flags": {}, 55 | "order": 16, 56 | "mode": 0, 57 | "inputs": [ 58 | { 59 | "name": "samples", 60 | "type": "LATENT", 61 | "link": 85 62 | }, 63 | { 64 | "name": "vae", 65 | "type": "VAE", 66 | "link": 62 67 | } 68 | ], 69 | "outputs": [ 70 | { 71 | "name": "IMAGE", 72 | "type": "IMAGE", 73 | "links": [ 74 | 96 75 | ] 76 | } 77 | ], 78 | "properties": { 79 | "cnr_id": "comfy-core", 80 | "ver": "0.3.28", 81 | "Node name for S&R": "VAEDecodeTiled" 82 | }, 83 | "widgets_values": [ 84 | 256, 85 | 64, 86 | 64, 87 | 8 88 | ], 89 | "color": "#322", 90 | "bgcolor": "#533" 91 | }, 92 | { 93 | "id": 20, 94 | "type": "VAEEncode", 95 | "pos": [ 96 | 1329.880859375, 97 | 701.230224609375 98 | ], 99 | "size": [ 100 | 210, 101 | 46 102 | ], 103 | "flags": {}, 104 | "order": 14, 105 | "mode": 0, 106 | "inputs": [ 107 | { 108 | "name": "pixels", 109 | "type": "IMAGE", 110 | "link": 117 111 | }, 112 | { 113 | "name": "vae", 114 | "type": "VAE", 115 | "link": 22 116 | } 117 | ], 118 | "outputs": [ 119 | { 120 | "name": "LATENT", 121 | "type": "LATENT", 122 | "links": [ 123 | 86 124 | ] 125 | } 126 | ], 127 | "properties": { 128 | "cnr_id": "comfy-core", 129 | "ver": "0.3.28", 130 | "Node name for S&R": "VAEEncode" 131 | }, 132 | "widgets_values": [], 133 | "color": "#322", 134 | "bgcolor": "#533" 135 | }, 136 | { 137 | "id": 17, 138 | "type": "CLIPVisionEncode", 139 | "pos": [ 140 | 1228.9832763671875, 141 | 525.7402954101562 142 | ], 143 | "size": [ 144 | 380.4000244140625, 145 | 78 146 | ], 147 | "flags": {}, 148 | "order": 13, 149 | "mode": 0, 150 | "inputs": [ 151 | { 152 | "name": "clip_vision", 153 | "type": "CLIP_VISION", 154 | "link": 18 155 | }, 156 | { 157 | "name": "image", 158 | "type": "IMAGE", 159 | "link": 116 160 | } 161 | ], 162 | "outputs": [ 163 | { 164 | "name": "CLIP_VISION_OUTPUT", 165 | "type": "CLIP_VISION_OUTPUT", 166 | "links": [ 167 | 83 168 | ] 169 | } 170 | ], 171 | "properties": { 172 | "cnr_id": "comfy-core", 173 | "ver": "0.3.28", 174 | "Node name for S&R": "CLIPVisionEncode" 175 | }, 176 | "widgets_values": [ 177 | "center" 178 | ], 179 | "color": "#233", 180 | "bgcolor": "#355" 181 | }, 182 | { 183 | "id": 18, 184 | "type": "CLIPVisionLoader", 185 | "pos": [ 186 | 511.98028564453125, 187 | 530.965576171875 188 | ], 189 | "size": [ 190 | 388.87139892578125, 191 | 58 192 | ], 193 | "flags": {}, 194 | "order": 1, 195 | "mode": 0, 196 | "inputs": [], 197 | "outputs": [ 198 | { 199 | "name": "CLIP_VISION", 200 | "type": "CLIP_VISION", 201 | "links": [ 202 | 18 203 | ] 204 | } 205 | ], 206 | "properties": { 207 | "cnr_id": "comfy-core", 208 | "ver": "0.3.28", 209 | "Node name for S&R": "CLIPVisionLoader" 210 | }, 211 | "widgets_values": [ 212 | "sigclip_vision_patch14_384.safetensors" 213 | ], 214 | "color": "#2a363b", 215 | "bgcolor": "#3f5159" 216 | }, 217 | { 218 | "id": 27, 219 | "type": "FramePackTorchCompileSettings", 220 | "pos": [ 221 | 528.2340087890625, 222 | -143.91505432128906 223 | ], 224 | "size": [ 225 | 531.5999755859375, 226 | 202 227 | ], 228 | "flags": {}, 229 | "order": 2, 230 | "mode": 0, 231 | "inputs": [], 232 | "outputs": [ 233 | { 234 | "name": "torch_compile_args", 235 | "type": "FRAMEPACKCOMPILEARGS", 236 | "links": [] 237 | } 238 | ], 239 | "properties": { 240 | "aux_id": "lllyasviel/FramePack", 241 | "ver": "0e5fe5d7ca13c76fb8e13708f4b92e7c7a34f20c", 242 | "Node name for S&R": "FramePackTorchCompileSettings" 243 | }, 244 | "widgets_values": [ 245 | "inductor", 246 | false, 247 | "default", 248 | false, 249 | 64, 250 | true, 251 | true 252 | ] 253 | }, 254 | { 255 | "id": 23, 256 | "type": "VHS_VideoCombine", 257 | "pos": [ 258 | 2723.8759765625, 259 | -376.53961181640625 260 | ], 261 | "size": [ 262 | 908.428955078125, 263 | 1283.1883544921875 264 | ], 265 | "flags": {}, 266 | "order": 18, 267 | "mode": 0, 268 | "inputs": [ 269 | { 270 | "name": "images", 271 | "type": "IMAGE", 272 | "link": 97 273 | }, 274 | { 275 | "name": "audio", 276 | "shape": 7, 277 | "type": "AUDIO", 278 | "link": null 279 | }, 280 | { 281 | "name": "meta_batch", 282 | "shape": 7, 283 | "type": "VHS_BatchManager", 284 | "link": null 285 | }, 286 | { 287 | "name": "vae", 288 | "shape": 7, 289 | "type": "VAE", 290 | "link": null 291 | } 292 | ], 293 | "outputs": [ 294 | { 295 | "name": "Filenames", 296 | "type": "VHS_FILENAMES", 297 | "links": null 298 | } 299 | ], 300 | "properties": { 301 | "cnr_id": "comfyui-videohelpersuite", 302 | "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", 303 | "Node name for S&R": "VHS_VideoCombine" 304 | }, 305 | "widgets_values": { 306 | "frame_rate": 30, 307 | "loop_count": 0, 308 | "filename_prefix": "FramePack", 309 | "format": "video/h264-mp4", 310 | "pix_fmt": "yuv420p", 311 | "crf": 19, 312 | "save_metadata": true, 313 | "trim_to_audio": false, 314 | "pingpong": false, 315 | "save_output": false, 316 | "videopreview": { 317 | "hidden": false, 318 | "paused": false, 319 | "params": { 320 | "filename": "FramePack_00057.mp4", 321 | "subfolder": "", 322 | "type": "temp", 323 | "format": "video/h264-mp4", 324 | "frame_rate": 24, 325 | "workflow": "FramePack_00057.png", 326 | "fullpath": "N:\\AI\\ComfyUI\\temp\\FramePack_00057.mp4" 327 | } 328 | } 329 | } 330 | }, 331 | { 332 | "id": 48, 333 | "type": "GetImageSizeAndCount", 334 | "pos": [ 335 | 1266.1427001953125, 336 | 844.8764038085938 337 | ], 338 | "size": [ 339 | 277.20001220703125, 340 | 86 341 | ], 342 | "flags": {}, 343 | "order": 12, 344 | "mode": 0, 345 | "inputs": [ 346 | { 347 | "name": "image", 348 | "type": "IMAGE", 349 | "link": 125 350 | } 351 | ], 352 | "outputs": [ 353 | { 354 | "name": "image", 355 | "type": "IMAGE", 356 | "links": [ 357 | 116, 358 | 117 359 | ] 360 | }, 361 | { 362 | "label": "608 width", 363 | "name": "width", 364 | "type": "INT", 365 | "links": null 366 | }, 367 | { 368 | "label": "640 height", 369 | "name": "height", 370 | "type": "INT", 371 | "links": null 372 | }, 373 | { 374 | "label": "1 count", 375 | "name": "count", 376 | "type": "INT", 377 | "links": null 378 | } 379 | ], 380 | "properties": { 381 | "cnr_id": "comfyui-kjnodes", 382 | "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", 383 | "Node name for S&R": "GetImageSizeAndCount" 384 | }, 385 | "widgets_values": [] 386 | }, 387 | { 388 | "id": 15, 389 | "type": "ConditioningZeroOut", 390 | "pos": [ 391 | 1346.0872802734375, 392 | 263.21856689453125 393 | ], 394 | "size": [ 395 | 317.4000244140625, 396 | 26 397 | ], 398 | "flags": { 399 | "collapsed": true 400 | }, 401 | "order": 10, 402 | "mode": 0, 403 | "inputs": [ 404 | { 405 | "name": "conditioning", 406 | "type": "CONDITIONING", 407 | "link": 118 408 | } 409 | ], 410 | "outputs": [ 411 | { 412 | "name": "CONDITIONING", 413 | "type": "CONDITIONING", 414 | "links": [ 415 | 108 416 | ] 417 | } 418 | ], 419 | "properties": { 420 | "cnr_id": "comfy-core", 421 | "ver": "0.3.28", 422 | "Node name for S&R": "ConditioningZeroOut" 423 | }, 424 | "widgets_values": [], 425 | "color": "#332922", 426 | "bgcolor": "#593930" 427 | }, 428 | { 429 | "id": 39, 430 | "type": "FramePackSampler", 431 | "pos": [ 432 | 1822.7847900390625, 433 | 167.60594177246094 434 | ], 435 | "size": [ 436 | 393, 437 | 852.631591796875 438 | ], 439 | "flags": {}, 440 | "order": 15, 441 | "mode": 0, 442 | "inputs": [ 443 | { 444 | "name": "model", 445 | "type": "FramePackMODEL", 446 | "link": 129 447 | }, 448 | { 449 | "name": "positive", 450 | "type": "CONDITIONING", 451 | "link": 114 452 | }, 453 | { 454 | "name": "negative", 455 | "type": "CONDITIONING", 456 | "link": 108 457 | }, 458 | { 459 | "name": "image_embeds", 460 | "type": "CLIP_VISION_OUTPUT", 461 | "link": 83 462 | }, 463 | { 464 | "name": "start_latent", 465 | "shape": 7, 466 | "type": "LATENT", 467 | "link": 86 468 | }, 469 | { 470 | "name": "initial_samples", 471 | "shape": 7, 472 | "type": "LATENT", 473 | "link": null 474 | } 475 | ], 476 | "outputs": [ 477 | { 478 | "name": "samples", 479 | "type": "LATENT", 480 | "links": [ 481 | 85 482 | ] 483 | } 484 | ], 485 | "properties": { 486 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 487 | "ver": "8e5ec6b7f3acf88255c5d93d062079f18b43aa2b", 488 | "Node name for S&R": "FramePackSampler" 489 | }, 490 | "widgets_values": [ 491 | 30, 492 | true, 493 | 0.15, 494 | 1, 495 | 10, 496 | 0, 497 | 47, 498 | "fixed", 499 | 9, 500 | 5, 501 | 6, 502 | "unipc_bh1", 503 | 1, 504 | "" 505 | ] 506 | }, 507 | { 508 | "id": 47, 509 | "type": "CLIPTextEncode", 510 | "pos": [ 511 | 715.3054809570312, 512 | 127.73457336425781 513 | ], 514 | "size": [ 515 | 400, 516 | 200 517 | ], 518 | "flags": {}, 519 | "order": 8, 520 | "mode": 0, 521 | "inputs": [ 522 | { 523 | "name": "clip", 524 | "type": "CLIP", 525 | "link": 102 526 | } 527 | ], 528 | "outputs": [ 529 | { 530 | "name": "CONDITIONING", 531 | "type": "CONDITIONING", 532 | "links": [ 533 | 114, 534 | 118 535 | ] 536 | } 537 | ], 538 | "properties": { 539 | "cnr_id": "comfy-core", 540 | "ver": "0.3.28", 541 | "Node name for S&R": "CLIPTextEncode" 542 | }, 543 | "widgets_values": [ 544 | "old man gets up and takes a walk on the beach" 545 | ], 546 | "color": "#232", 547 | "bgcolor": "#353" 548 | }, 549 | { 550 | "id": 13, 551 | "type": "DualCLIPLoader", 552 | "pos": [ 553 | 320.9956359863281, 554 | 166.8336181640625 555 | ], 556 | "size": [ 557 | 340.2243957519531, 558 | 130 559 | ], 560 | "flags": {}, 561 | "order": 3, 562 | "mode": 0, 563 | "inputs": [], 564 | "outputs": [ 565 | { 566 | "name": "CLIP", 567 | "type": "CLIP", 568 | "links": [ 569 | 102 570 | ] 571 | } 572 | ], 573 | "properties": { 574 | "cnr_id": "comfy-core", 575 | "ver": "0.3.28", 576 | "Node name for S&R": "DualCLIPLoader" 577 | }, 578 | "widgets_values": [ 579 | "clip_l.safetensors", 580 | "llava_llama3_fp16.safetensors", 581 | "hunyuan_video", 582 | "default" 583 | ], 584 | "color": "#432", 585 | "bgcolor": "#653" 586 | }, 587 | { 588 | "id": 44, 589 | "type": "GetImageSizeAndCount", 590 | "pos": [ 591 | 2225.53564453125, 592 | -78.62096405029297 593 | ], 594 | "size": [ 595 | 277.20001220703125, 596 | 86 597 | ], 598 | "flags": {}, 599 | "order": 17, 600 | "mode": 0, 601 | "inputs": [ 602 | { 603 | "name": "image", 604 | "type": "IMAGE", 605 | "link": 96 606 | } 607 | ], 608 | "outputs": [ 609 | { 610 | "name": "image", 611 | "type": "IMAGE", 612 | "links": [ 613 | 97 614 | ] 615 | }, 616 | { 617 | "label": "608 width", 618 | "name": "width", 619 | "type": "INT", 620 | "links": null 621 | }, 622 | { 623 | "label": "640 height", 624 | "name": "height", 625 | "type": "INT", 626 | "links": null 627 | }, 628 | { 629 | "label": "145 count", 630 | "name": "count", 631 | "type": "INT", 632 | "links": null 633 | } 634 | ], 635 | "properties": { 636 | "cnr_id": "comfyui-kjnodes", 637 | "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", 638 | "Node name for S&R": "GetImageSizeAndCount" 639 | }, 640 | "widgets_values": [] 641 | }, 642 | { 643 | "id": 50, 644 | "type": "ImageResize+", 645 | "pos": [ 646 | 921.9315795898438, 647 | 701.9561767578125 648 | ], 649 | "size": [ 650 | 315, 651 | 218 652 | ], 653 | "flags": {}, 654 | "order": 11, 655 | "mode": 0, 656 | "inputs": [ 657 | { 658 | "name": "image", 659 | "type": "IMAGE", 660 | "link": 122 661 | }, 662 | { 663 | "name": "width", 664 | "type": "INT", 665 | "widget": { 666 | "name": "width" 667 | }, 668 | "link": 128 669 | }, 670 | { 671 | "name": "height", 672 | "type": "INT", 673 | "widget": { 674 | "name": "height" 675 | }, 676 | "link": 127 677 | } 678 | ], 679 | "outputs": [ 680 | { 681 | "name": "IMAGE", 682 | "type": "IMAGE", 683 | "links": [ 684 | 125 685 | ] 686 | }, 687 | { 688 | "name": "width", 689 | "type": "INT", 690 | "links": null 691 | }, 692 | { 693 | "name": "height", 694 | "type": "INT", 695 | "links": null 696 | } 697 | ], 698 | "properties": { 699 | "aux_id": "kijai/ComfyUI_essentials", 700 | "ver": "76e9d1e4399bd025ce8b12c290753d58f9f53e93", 701 | "Node name for S&R": "ImageResize+" 702 | }, 703 | "widgets_values": [ 704 | 512, 705 | 512, 706 | "lanczos", 707 | "stretch", 708 | "always", 709 | 0 710 | ] 711 | }, 712 | { 713 | "id": 19, 714 | "type": "LoadImage", 715 | "pos": [ 716 | 221.91770935058594, 717 | 702.9730834960938 718 | ], 719 | "size": [ 720 | 315, 721 | 314 722 | ], 723 | "flags": {}, 724 | "order": 4, 725 | "mode": 0, 726 | "inputs": [], 727 | "outputs": [ 728 | { 729 | "name": "IMAGE", 730 | "type": "IMAGE", 731 | "links": [ 732 | 122, 733 | 126 734 | ] 735 | }, 736 | { 737 | "name": "MASK", 738 | "type": "MASK", 739 | "links": null 740 | } 741 | ], 742 | "properties": { 743 | "cnr_id": "comfy-core", 744 | "ver": "0.3.28", 745 | "Node name for S&R": "LoadImage" 746 | }, 747 | "widgets_values": [ 748 | "oldman_upscaled.png", 749 | "image" 750 | ] 751 | }, 752 | { 753 | "id": 51, 754 | "type": "FramePackFindNearestBucket", 755 | "pos": [ 756 | 578.9364624023438, 757 | 773.94677734375 758 | ], 759 | "size": [ 760 | 315, 761 | 78 762 | ], 763 | "flags": {}, 764 | "order": 9, 765 | "mode": 0, 766 | "inputs": [ 767 | { 768 | "name": "image", 769 | "type": "IMAGE", 770 | "link": 126 771 | } 772 | ], 773 | "outputs": [ 774 | { 775 | "name": "width", 776 | "type": "INT", 777 | "links": [ 778 | 128 779 | ] 780 | }, 781 | { 782 | "name": "height", 783 | "type": "INT", 784 | "links": [ 785 | 127 786 | ] 787 | } 788 | ], 789 | "properties": { 790 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 791 | "ver": "4f9030a9f4c0bd67d86adf3d3dc07e37118c40bd", 792 | "Node name for S&R": "FramePackFindNearestBucket" 793 | }, 794 | "widgets_values": [ 795 | 640 796 | ] 797 | }, 798 | { 799 | "id": 54, 800 | "type": "DownloadAndLoadFramePackModel", 801 | "pos": [ 802 | 1256.5235595703125, 803 | -277.76226806640625 804 | ], 805 | "size": [ 806 | 315, 807 | 130 808 | ], 809 | "flags": {}, 810 | "order": 5, 811 | "mode": 4, 812 | "inputs": [ 813 | { 814 | "name": "compile_args", 815 | "shape": 7, 816 | "type": "FRAMEPACKCOMPILEARGS", 817 | "link": null 818 | } 819 | ], 820 | "outputs": [ 821 | { 822 | "name": "model", 823 | "type": "FramePackMODEL", 824 | "links": null 825 | } 826 | ], 827 | "properties": { 828 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 829 | "ver": "49fe507eca8246cc9d08a8093892f40c1180e88f", 830 | "Node name for S&R": "DownloadAndLoadFramePackModel" 831 | }, 832 | "widgets_values": [ 833 | "lllyasviel/FramePackI2V_HY", 834 | "bf16", 835 | "disabled", 836 | "sdpa" 837 | ] 838 | }, 839 | { 840 | "id": 55, 841 | "type": "MarkdownNote", 842 | "pos": [ 843 | 567.05908203125, 844 | -628.8865966796875 845 | ], 846 | "size": [ 847 | 459.8609619140625, 848 | 285.9714660644531 849 | ], 850 | "flags": {}, 851 | "order": 6, 852 | "mode": 0, 853 | "inputs": [], 854 | "outputs": [], 855 | "properties": {}, 856 | "widgets_values": [ 857 | "Model links:\n\n[https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors](https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors)\n\n[https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors](https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors)\n\nsigclip:\n\n[https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main](https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main)\n\ntext encoder and VAE:\n\n[https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files](https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files)" 858 | ], 859 | "color": "#432", 860 | "bgcolor": "#653" 861 | }, 862 | { 863 | "id": 52, 864 | "type": "LoadFramePackModel", 865 | "pos": [ 866 | 1253.046630859375, 867 | -82.57657623291016 868 | ], 869 | "size": [ 870 | 480.7601013183594, 871 | 130 872 | ], 873 | "flags": {}, 874 | "order": 7, 875 | "mode": 0, 876 | "inputs": [ 877 | { 878 | "name": "compile_args", 879 | "shape": 7, 880 | "type": "FRAMEPACKCOMPILEARGS", 881 | "link": null 882 | } 883 | ], 884 | "outputs": [ 885 | { 886 | "name": "model", 887 | "type": "FramePackMODEL", 888 | "links": [ 889 | 129 890 | ] 891 | } 892 | ], 893 | "properties": { 894 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 895 | "ver": "49fe507eca8246cc9d08a8093892f40c1180e88f", 896 | "Node name for S&R": "LoadFramePackModel" 897 | }, 898 | "widgets_values": [ 899 | "Hyvid\\FramePackI2V_HY_fp8_e4m3fn.safetensors", 900 | "bf16", 901 | "fp8_e4m3fn", 902 | "sdpa" 903 | ] 904 | } 905 | ], 906 | "links": [ 907 | [ 908 | 18, 909 | 18, 910 | 0, 911 | 17, 912 | 0, 913 | "CLIP_VISION" 914 | ], 915 | [ 916 | 22, 917 | 12, 918 | 0, 919 | 20, 920 | 1, 921 | "VAE" 922 | ], 923 | [ 924 | 62, 925 | 12, 926 | 0, 927 | 33, 928 | 1, 929 | "VAE" 930 | ], 931 | [ 932 | 83, 933 | 17, 934 | 0, 935 | 39, 936 | 3, 937 | "CLIP_VISION_OUTPUT" 938 | ], 939 | [ 940 | 85, 941 | 39, 942 | 0, 943 | 33, 944 | 0, 945 | "LATENT" 946 | ], 947 | [ 948 | 86, 949 | 20, 950 | 0, 951 | 39, 952 | 4, 953 | "LATENT" 954 | ], 955 | [ 956 | 96, 957 | 33, 958 | 0, 959 | 44, 960 | 0, 961 | "IMAGE" 962 | ], 963 | [ 964 | 97, 965 | 44, 966 | 0, 967 | 23, 968 | 0, 969 | "IMAGE" 970 | ], 971 | [ 972 | 102, 973 | 13, 974 | 0, 975 | 47, 976 | 0, 977 | "CLIP" 978 | ], 979 | [ 980 | 108, 981 | 15, 982 | 0, 983 | 39, 984 | 2, 985 | "CONDITIONING" 986 | ], 987 | [ 988 | 114, 989 | 47, 990 | 0, 991 | 39, 992 | 1, 993 | "CONDITIONING" 994 | ], 995 | [ 996 | 116, 997 | 48, 998 | 0, 999 | 17, 1000 | 1, 1001 | "IMAGE" 1002 | ], 1003 | [ 1004 | 117, 1005 | 48, 1006 | 0, 1007 | 20, 1008 | 0, 1009 | "IMAGE" 1010 | ], 1011 | [ 1012 | 118, 1013 | 47, 1014 | 0, 1015 | 15, 1016 | 0, 1017 | "CONDITIONING" 1018 | ], 1019 | [ 1020 | 122, 1021 | 19, 1022 | 0, 1023 | 50, 1024 | 0, 1025 | "IMAGE" 1026 | ], 1027 | [ 1028 | 125, 1029 | 50, 1030 | 0, 1031 | 48, 1032 | 0, 1033 | "IMAGE" 1034 | ], 1035 | [ 1036 | 126, 1037 | 19, 1038 | 0, 1039 | 51, 1040 | 0, 1041 | "IMAGE" 1042 | ], 1043 | [ 1044 | 127, 1045 | 51, 1046 | 1, 1047 | 50, 1048 | 2, 1049 | "INT" 1050 | ], 1051 | [ 1052 | 128, 1053 | 51, 1054 | 0, 1055 | 50, 1056 | 1, 1057 | "INT" 1058 | ], 1059 | [ 1060 | 129, 1061 | 52, 1062 | 0, 1063 | 39, 1064 | 0, 1065 | "FramePackMODEL" 1066 | ] 1067 | ], 1068 | "groups": [], 1069 | "config": {}, 1070 | "extra": { 1071 | "ds": { 1072 | "scale": 0.6727499949325823, 1073 | "offset": [ 1074 | -101.98335175553812, 1075 | 539.7905569391395 1076 | ] 1077 | }, 1078 | "frontendVersion": "1.16.7", 1079 | "VHS_latentpreview": true, 1080 | "VHS_latentpreviewrate": 0, 1081 | "VHS_MetadataImage": true, 1082 | "VHS_KeepIntermediate": true 1083 | }, 1084 | "version": 0.4 1085 | } 1086 | -------------------------------------------------------------------------------- /example_workflows/framepack_hv_start_end.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "2d185939-adfa-466d-965c-c958f3c02c02", 3 | "revision": 0, 4 | "last_node_id": 107, 5 | "last_link_id": 215, 6 | "nodes": [ 7 | { 8 | "id": 27, 9 | "type": "FramePackTorchCompileSettings", 10 | "pos": [ 11 | 528.2340087890625, 12 | -143.91505432128906 13 | ], 14 | "size": [ 15 | 531.5999755859375, 16 | 202 17 | ], 18 | "flags": {}, 19 | "order": 0, 20 | "mode": 0, 21 | "inputs": [], 22 | "outputs": [ 23 | { 24 | "name": "torch_compile_args", 25 | "type": "FRAMEPACKCOMPILEARGS", 26 | "links": [] 27 | } 28 | ], 29 | "properties": { 30 | "aux_id": "lllyasviel/FramePack", 31 | "ver": "0e5fe5d7ca13c76fb8e13708f4b92e7c7a34f20c", 32 | "Node name for S&R": "FramePackTorchCompileSettings" 33 | }, 34 | "widgets_values": [ 35 | "inductor", 36 | false, 37 | "default", 38 | false, 39 | 64, 40 | true, 41 | true 42 | ] 43 | }, 44 | { 45 | "id": 54, 46 | "type": "DownloadAndLoadFramePackModel", 47 | "pos": [ 48 | 1256.5235595703125, 49 | -277.76226806640625 50 | ], 51 | "size": [ 52 | 315, 53 | 130 54 | ], 55 | "flags": {}, 56 | "order": 1, 57 | "mode": 4, 58 | "inputs": [ 59 | { 60 | "name": "compile_args", 61 | "shape": 7, 62 | "type": "FRAMEPACKCOMPILEARGS", 63 | "link": null 64 | } 65 | ], 66 | "outputs": [ 67 | { 68 | "name": "model", 69 | "type": "FramePackMODEL", 70 | "links": null 71 | } 72 | ], 73 | "properties": { 74 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 75 | "ver": "49fe507eca8246cc9d08a8093892f40c1180e88f", 76 | "Node name for S&R": "DownloadAndLoadFramePackModel" 77 | }, 78 | "widgets_values": [ 79 | "lllyasviel/FramePackI2V_HY", 80 | "bf16", 81 | "disabled", 82 | "sdpa" 83 | ] 84 | }, 85 | { 86 | "id": 55, 87 | "type": "MarkdownNote", 88 | "pos": [ 89 | 567.05908203125, 90 | -628.8865966796875 91 | ], 92 | "size": [ 93 | 459.8609619140625, 94 | 285.9714660644531 95 | ], 96 | "flags": {}, 97 | "order": 2, 98 | "mode": 0, 99 | "inputs": [], 100 | "outputs": [], 101 | "properties": {}, 102 | "widgets_values": [ 103 | "Model links:\n\n[https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors](https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_fp8_e4m3fn.safetensors)\n\n[https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors](https://huggingface.co/Kijai/HunyuanVideo_comfy/blob/main/FramePackI2V_HY_bf16.safetensors)\n\nsigclip:\n\n[https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main](https://huggingface.co/Comfy-Org/sigclip_vision_384/tree/main)\n\ntext encoder and VAE:\n\n[https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files](https://huggingface.co/Comfy-Org/HunyuanVideo_repackaged/tree/main/split_files)" 104 | ], 105 | "color": "#432", 106 | "bgcolor": "#653" 107 | }, 108 | { 109 | "id": 52, 110 | "type": "LoadFramePackModel", 111 | "pos": [ 112 | 1253.046630859375, 113 | -82.57657623291016 114 | ], 115 | "size": [ 116 | 480.7601013183594, 117 | 130 118 | ], 119 | "flags": {}, 120 | "order": 3, 121 | "mode": 0, 122 | "inputs": [ 123 | { 124 | "name": "compile_args", 125 | "shape": 7, 126 | "type": "FRAMEPACKCOMPILEARGS", 127 | "link": null 128 | } 129 | ], 130 | "outputs": [ 131 | { 132 | "name": "model", 133 | "type": "FramePackMODEL", 134 | "links": [ 135 | 129 136 | ] 137 | } 138 | ], 139 | "properties": { 140 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 141 | "ver": "49fe507eca8246cc9d08a8093892f40c1180e88f", 142 | "Node name for S&R": "LoadFramePackModel" 143 | }, 144 | "widgets_values": [ 145 | "FramePackI2V_HY_fp8_e4m3fn.safetensors", 146 | "bf16", 147 | "fp8_e4m3fn", 148 | "sageattn" 149 | ] 150 | }, 151 | { 152 | "id": 98, 153 | "type": "Note", 154 | "pos": [ 155 | 112.81798553466797, 156 | -494.2582092285156 157 | ], 158 | "size": [ 159 | 210, 160 | 88 161 | ], 162 | "flags": {}, 163 | "order": 4, 164 | "mode": 0, 165 | "inputs": [], 166 | "outputs": [], 167 | "properties": {}, 168 | "widgets_values": [ 169 | "" 170 | ], 171 | "color": "#432", 172 | "bgcolor": "#653" 173 | }, 174 | { 175 | "id": 23, 176 | "type": "VHS_VideoCombine", 177 | "pos": [ 178 | 2558.431396484375, 179 | 787.1417846679688 180 | ], 181 | "size": [ 182 | 1287.8934326171875, 183 | 1193.2623291015625 184 | ], 185 | "flags": {}, 186 | "order": 22, 187 | "mode": 0, 188 | "inputs": [ 189 | { 190 | "name": "images", 191 | "type": "IMAGE", 192 | "link": 97 193 | }, 194 | { 195 | "name": "audio", 196 | "shape": 7, 197 | "type": "AUDIO", 198 | "link": null 199 | }, 200 | { 201 | "name": "meta_batch", 202 | "shape": 7, 203 | "type": "VHS_BatchManager", 204 | "link": null 205 | }, 206 | { 207 | "name": "vae", 208 | "shape": 7, 209 | "type": "VAE", 210 | "link": null 211 | } 212 | ], 213 | "outputs": [ 214 | { 215 | "name": "Filenames", 216 | "type": "VHS_FILENAMES", 217 | "links": null 218 | } 219 | ], 220 | "properties": { 221 | "cnr_id": "comfyui-videohelpersuite", 222 | "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", 223 | "Node name for S&R": "VHS_VideoCombine" 224 | }, 225 | "widgets_values": { 226 | "frame_rate": 24, 227 | "loop_count": 0, 228 | "filename_prefix": "FramePack", 229 | "format": "video/h264-mp4", 230 | "pix_fmt": "yuv420p", 231 | "crf": 19, 232 | "save_metadata": true, 233 | "trim_to_audio": false, 234 | "pingpong": false, 235 | "save_output": true, 236 | "videopreview": { 237 | "hidden": false, 238 | "paused": false, 239 | "params": { 240 | "filename": "FramePack_00021.mp4", 241 | "subfolder": "", 242 | "type": "output", 243 | "format": "video/h264-mp4", 244 | "frame_rate": 24, 245 | "workflow": "FramePack_00021.png", 246 | "fullpath": "C:\\Work\\comfyUI\\output\\FramePack_00021.mp4" 247 | } 248 | } 249 | } 250 | }, 251 | { 252 | "id": 15, 253 | "type": "ConditioningZeroOut", 254 | "pos": [ 255 | 1238.275390625, 256 | 176.70359802246094 257 | ], 258 | "size": [ 259 | 317.4000244140625, 260 | 26 261 | ], 262 | "flags": { 263 | "collapsed": true 264 | }, 265 | "order": 12, 266 | "mode": 0, 267 | "inputs": [ 268 | { 269 | "name": "conditioning", 270 | "type": "CONDITIONING", 271 | "link": 118 272 | } 273 | ], 274 | "outputs": [ 275 | { 276 | "name": "CONDITIONING", 277 | "type": "CONDITIONING", 278 | "links": [ 279 | 108 280 | ] 281 | } 282 | ], 283 | "properties": { 284 | "cnr_id": "comfy-core", 285 | "ver": "0.3.28", 286 | "Node name for S&R": "ConditioningZeroOut" 287 | }, 288 | "widgets_values": [], 289 | "color": "#332922", 290 | "bgcolor": "#593930" 291 | }, 292 | { 293 | "id": 81, 294 | "type": "ImageResize+", 295 | "pos": [ 296 | 830.6694946289062, 297 | 1166.654296875 298 | ], 299 | "size": [ 300 | 315, 301 | 218 302 | ], 303 | "flags": {}, 304 | "order": 14, 305 | "mode": 0, 306 | "inputs": [ 307 | { 308 | "name": "image", 309 | "type": "IMAGE", 310 | "link": 165 311 | }, 312 | { 313 | "name": "width", 314 | "type": "INT", 315 | "widget": { 316 | "name": "width" 317 | }, 318 | "link": 202 319 | }, 320 | { 321 | "name": "height", 322 | "type": "INT", 323 | "widget": { 324 | "name": "height" 325 | }, 326 | "link": 203 327 | } 328 | ], 329 | "outputs": [ 330 | { 331 | "name": "IMAGE", 332 | "type": "IMAGE", 333 | "links": [ 334 | 171 335 | ] 336 | }, 337 | { 338 | "name": "width", 339 | "type": "INT", 340 | "links": null 341 | }, 342 | { 343 | "name": "height", 344 | "type": "INT", 345 | "links": null 346 | } 347 | ], 348 | "properties": { 349 | "cnr_id": "comfyui_essentials", 350 | "ver": "76e9d1e4399bd025ce8b12c290753d58f9f53e93", 351 | "Node name for S&R": "ImageResize+", 352 | "aux_id": "kijai/ComfyUI_essentials" 353 | }, 354 | "widgets_values": [ 355 | 512, 356 | 512, 357 | "lanczos", 358 | "stretch", 359 | "always", 360 | 0 361 | ] 362 | }, 363 | { 364 | "id": 84, 365 | "type": "VAEEncode", 366 | "pos": [ 367 | 1441.838134765625, 368 | 1095.2545166015625 369 | ], 370 | "size": [ 371 | 210, 372 | 46 373 | ], 374 | "flags": {}, 375 | "order": 17, 376 | "mode": 0, 377 | "inputs": [ 378 | { 379 | "name": "pixels", 380 | "type": "IMAGE", 381 | "link": 171 382 | }, 383 | { 384 | "name": "vae", 385 | "type": "VAE", 386 | "link": 168 387 | } 388 | ], 389 | "outputs": [ 390 | { 391 | "name": "LATENT", 392 | "type": "LATENT", 393 | "links": [ 394 | 204 395 | ] 396 | } 397 | ], 398 | "properties": { 399 | "cnr_id": "comfy-core", 400 | "ver": "0.3.28", 401 | "Node name for S&R": "VAEEncode" 402 | }, 403 | "widgets_values": [], 404 | "color": "#322", 405 | "bgcolor": "#533" 406 | }, 407 | { 408 | "id": 13, 409 | "type": "DualCLIPLoader", 410 | "pos": [ 411 | -36.613277435302734, 412 | 161.11524963378906 413 | ], 414 | "size": [ 415 | 340.2243957519531, 416 | 130 417 | ], 418 | "flags": {}, 419 | "order": 5, 420 | "mode": 0, 421 | "inputs": [], 422 | "outputs": [ 423 | { 424 | "name": "CLIP", 425 | "type": "CLIP", 426 | "links": [ 427 | 102 428 | ] 429 | } 430 | ], 431 | "properties": { 432 | "cnr_id": "comfy-core", 433 | "ver": "0.3.28", 434 | "Node name for S&R": "DualCLIPLoader" 435 | }, 436 | "widgets_values": [ 437 | "clip_l.safetensors", 438 | "llava_llama3_fp8_scaled.safetensors", 439 | "hunyuan_video", 440 | "default" 441 | ], 442 | "color": "#432", 443 | "bgcolor": "#653" 444 | }, 445 | { 446 | "id": 47, 447 | "type": "CLIPTextEncode", 448 | "pos": [ 449 | 532.3970336914062, 450 | 133.50706481933594 451 | ], 452 | "size": [ 453 | 400, 454 | 200 455 | ], 456 | "flags": {}, 457 | "order": 10, 458 | "mode": 0, 459 | "inputs": [ 460 | { 461 | "name": "clip", 462 | "type": "CLIP", 463 | "link": 102 464 | } 465 | ], 466 | "outputs": [ 467 | { 468 | "name": "CONDITIONING", 469 | "type": "CONDITIONING", 470 | "links": [ 471 | 114, 472 | 118 473 | ] 474 | } 475 | ], 476 | "properties": { 477 | "cnr_id": "comfy-core", 478 | "ver": "0.3.28", 479 | "Node name for S&R": "CLIPTextEncode" 480 | }, 481 | "widgets_values": [ 482 | "A girl suddenly spins gracefully in place, making a full twirl with her skirt fluttering around her. As she finishes the spin, she smiles brightly and waves her hand towards the camera. The camera gently pans to follow her movement, slightly zooming in as she waves." 483 | ], 484 | "color": "#232", 485 | "bgcolor": "#353" 486 | }, 487 | { 488 | "id": 12, 489 | "type": "VAELoader", 490 | "pos": [ 491 | 542.1045532226562, 492 | 401.9949951171875 493 | ], 494 | "size": [ 495 | 469.0488586425781, 496 | 58 497 | ], 498 | "flags": {}, 499 | "order": 6, 500 | "mode": 0, 501 | "inputs": [], 502 | "outputs": [ 503 | { 504 | "name": "VAE", 505 | "type": "VAE", 506 | "links": [ 507 | 22, 508 | 62, 509 | 168 510 | ] 511 | } 512 | ], 513 | "properties": { 514 | "cnr_id": "comfy-core", 515 | "ver": "0.3.28", 516 | "Node name for S&R": "VAELoader" 517 | }, 518 | "widgets_values": [ 519 | "hunyuan_video_vae_bf16.safetensors" 520 | ], 521 | "color": "#322", 522 | "bgcolor": "#533" 523 | }, 524 | { 525 | "id": 19, 526 | "type": "LoadImage", 527 | "pos": [ 528 | -147.01988220214844, 529 | 949.157958984375 530 | ], 531 | "size": [ 532 | 503.868896484375, 533 | 469.1946105957031 534 | ], 535 | "flags": {}, 536 | "order": 7, 537 | "mode": 0, 538 | "inputs": [], 539 | "outputs": [ 540 | { 541 | "name": "IMAGE", 542 | "type": "IMAGE", 543 | "links": [ 544 | 122, 545 | 126 546 | ] 547 | }, 548 | { 549 | "name": "MASK", 550 | "type": "MASK", 551 | "links": null 552 | } 553 | ], 554 | "properties": { 555 | "cnr_id": "comfy-core", 556 | "ver": "0.3.28", 557 | "Node name for S&R": "LoadImage" 558 | }, 559 | "widgets_values": [ 560 | "250418_200558_452_6587.png", 561 | "image" 562 | ] 563 | }, 564 | { 565 | "id": 79, 566 | "type": "LoadImage", 567 | "pos": [ 568 | -139.5203399658203, 569 | 1537.0316162109375 570 | ], 571 | "size": [ 572 | 503.868896484375, 573 | 469.1946105957031 574 | ], 575 | "flags": {}, 576 | "order": 8, 577 | "mode": 0, 578 | "inputs": [], 579 | "outputs": [ 580 | { 581 | "name": "IMAGE", 582 | "type": "IMAGE", 583 | "links": [ 584 | 165 585 | ] 586 | }, 587 | { 588 | "name": "MASK", 589 | "type": "MASK", 590 | "links": null 591 | } 592 | ], 593 | "properties": { 594 | "cnr_id": "comfy-core", 595 | "ver": "0.3.28", 596 | "Node name for S&R": "LoadImage" 597 | }, 598 | "widgets_values": [ 599 | "250418_203022_543_3046_2.png", 600 | "image" 601 | ] 602 | }, 603 | { 604 | "id": 20, 605 | "type": "VAEEncode", 606 | "pos": [ 607 | 1614.081787109375, 608 | 541.8619995117188 609 | ], 610 | "size": [ 611 | 210, 612 | 46 613 | ], 614 | "flags": {}, 615 | "order": 18, 616 | "mode": 0, 617 | "inputs": [ 618 | { 619 | "name": "pixels", 620 | "type": "IMAGE", 621 | "link": 117 622 | }, 623 | { 624 | "name": "vae", 625 | "type": "VAE", 626 | "link": 22 627 | } 628 | ], 629 | "outputs": [ 630 | { 631 | "name": "LATENT", 632 | "type": "LATENT", 633 | "links": [ 634 | 86 635 | ] 636 | } 637 | ], 638 | "properties": { 639 | "cnr_id": "comfy-core", 640 | "ver": "0.3.28", 641 | "Node name for S&R": "VAEEncode" 642 | }, 643 | "widgets_values": [], 644 | "color": "#322", 645 | "bgcolor": "#533" 646 | }, 647 | { 648 | "id": 48, 649 | "type": "GetImageSizeAndCount", 650 | "pos": [ 651 | 1252.4576416015625, 652 | 550.1927490234375 653 | ], 654 | "size": [ 655 | 277.20001220703125, 656 | 86 657 | ], 658 | "flags": {}, 659 | "order": 15, 660 | "mode": 0, 661 | "inputs": [ 662 | { 663 | "name": "image", 664 | "type": "IMAGE", 665 | "link": 125 666 | } 667 | ], 668 | "outputs": [ 669 | { 670 | "name": "image", 671 | "type": "IMAGE", 672 | "links": [ 673 | 117 674 | ] 675 | }, 676 | { 677 | "label": "768 width", 678 | "name": "width", 679 | "type": "INT", 680 | "links": null 681 | }, 682 | { 683 | "label": "512 height", 684 | "name": "height", 685 | "type": "INT", 686 | "links": null 687 | }, 688 | { 689 | "label": "1 count", 690 | "name": "count", 691 | "type": "INT", 692 | "links": null 693 | } 694 | ], 695 | "properties": { 696 | "cnr_id": "comfyui-kjnodes", 697 | "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", 698 | "Node name for S&R": "GetImageSizeAndCount" 699 | }, 700 | "widgets_values": [] 701 | }, 702 | { 703 | "id": 50, 704 | "type": "ImageResize+", 705 | "pos": [ 706 | 848.3390502929688, 707 | 542.4176025390625 708 | ], 709 | "size": [ 710 | 315, 711 | 218 712 | ], 713 | "flags": {}, 714 | "order": 13, 715 | "mode": 0, 716 | "inputs": [ 717 | { 718 | "name": "image", 719 | "type": "IMAGE", 720 | "link": 122 721 | }, 722 | { 723 | "name": "width", 724 | "type": "INT", 725 | "widget": { 726 | "name": "width" 727 | }, 728 | "link": 128 729 | }, 730 | { 731 | "name": "height", 732 | "type": "INT", 733 | "widget": { 734 | "name": "height" 735 | }, 736 | "link": 127 737 | } 738 | ], 739 | "outputs": [ 740 | { 741 | "name": "IMAGE", 742 | "type": "IMAGE", 743 | "links": [ 744 | 125, 745 | 159 746 | ] 747 | }, 748 | { 749 | "name": "width", 750 | "type": "INT", 751 | "links": null 752 | }, 753 | { 754 | "name": "height", 755 | "type": "INT", 756 | "links": null 757 | } 758 | ], 759 | "properties": { 760 | "cnr_id": "comfyui_essentials", 761 | "ver": "76e9d1e4399bd025ce8b12c290753d58f9f53e93", 762 | "Node name for S&R": "ImageResize+", 763 | "aux_id": "kijai/ComfyUI_essentials" 764 | }, 765 | "widgets_values": [ 766 | 512, 767 | 512, 768 | "lanczos", 769 | "stretch", 770 | "always", 771 | 0 772 | ] 773 | }, 774 | { 775 | "id": 33, 776 | "type": "VAEDecodeTiled", 777 | "pos": [ 778 | 1926.964111328125, 779 | 73.85487365722656 780 | ], 781 | "size": [ 782 | 315, 783 | 150 784 | ], 785 | "flags": {}, 786 | "order": 20, 787 | "mode": 0, 788 | "inputs": [ 789 | { 790 | "name": "samples", 791 | "type": "LATENT", 792 | "link": 85 793 | }, 794 | { 795 | "name": "vae", 796 | "type": "VAE", 797 | "link": 62 798 | } 799 | ], 800 | "outputs": [ 801 | { 802 | "name": "IMAGE", 803 | "type": "IMAGE", 804 | "links": [ 805 | 96 806 | ] 807 | } 808 | ], 809 | "properties": { 810 | "cnr_id": "comfy-core", 811 | "ver": "0.3.28", 812 | "Node name for S&R": "VAEDecodeTiled" 813 | }, 814 | "widgets_values": [ 815 | 256, 816 | 64, 817 | 64, 818 | 8 819 | ], 820 | "color": "#322", 821 | "bgcolor": "#533" 822 | }, 823 | { 824 | "id": 44, 825 | "type": "GetImageSizeAndCount", 826 | "pos": [ 827 | 2326.93798828125, 828 | 212.88839721679688 829 | ], 830 | "size": [ 831 | 277.20001220703125, 832 | 86 833 | ], 834 | "flags": {}, 835 | "order": 21, 836 | "mode": 0, 837 | "inputs": [ 838 | { 839 | "name": "image", 840 | "type": "IMAGE", 841 | "link": 96 842 | } 843 | ], 844 | "outputs": [ 845 | { 846 | "name": "image", 847 | "type": "IMAGE", 848 | "links": [ 849 | 97 850 | ] 851 | }, 852 | { 853 | "label": "768 width", 854 | "name": "width", 855 | "type": "INT", 856 | "links": null 857 | }, 858 | { 859 | "label": "512 height", 860 | "name": "height", 861 | "type": "INT", 862 | "links": null 863 | }, 864 | { 865 | "label": "253 count", 866 | "name": "count", 867 | "type": "INT", 868 | "links": null 869 | } 870 | ], 871 | "properties": { 872 | "cnr_id": "comfyui-kjnodes", 873 | "ver": "8ecf5cd05e0a1012087b0da90eea9a13674668db", 874 | "Node name for S&R": "GetImageSizeAndCount" 875 | }, 876 | "widgets_values": [] 877 | }, 878 | { 879 | "id": 39, 880 | "type": "FramePackSampler", 881 | "pos": [ 882 | 1897.4779052734375, 883 | 441.9130859375 884 | ], 885 | "size": [ 886 | 558.9678344726562, 887 | 578 888 | ], 889 | "flags": {}, 890 | "order": 19, 891 | "mode": 0, 892 | "inputs": [ 893 | { 894 | "name": "model", 895 | "type": "FramePackMODEL", 896 | "link": 129 897 | }, 898 | { 899 | "name": "positive", 900 | "type": "CONDITIONING", 901 | "link": 114 902 | }, 903 | { 904 | "name": "negative", 905 | "type": "CONDITIONING", 906 | "link": 108 907 | }, 908 | { 909 | "name": "image_embeds", 910 | "type": "CLIP_VISION_OUTPUT", 911 | "link": 83 912 | }, 913 | { 914 | "name": "start_latent", 915 | "shape": 7, 916 | "type": "LATENT", 917 | "link": 86 918 | }, 919 | { 920 | "name": "end_latent", 921 | "shape": 7, 922 | "type": "LATENT", 923 | "link": 204 924 | }, 925 | { 926 | "name": "keyframes", 927 | "shape": 7, 928 | "type": "LATENT", 929 | "link": null 930 | }, 931 | { 932 | "name": "keyframe_indices", 933 | "shape": 7, 934 | "type": "LIST", 935 | "link": null 936 | }, 937 | { 938 | "name": "initial_samples", 939 | "shape": 7, 940 | "type": "LATENT", 941 | "link": null 942 | }, 943 | { 944 | "name": "positive_keyframes", 945 | "shape": 7, 946 | "type": "LIST", 947 | "link": null 948 | }, 949 | { 950 | "name": "positive_keyframe_indices", 951 | "shape": 7, 952 | "type": "LIST", 953 | "link": null 954 | } 955 | ], 956 | "outputs": [ 957 | { 958 | "name": "samples", 959 | "type": "LATENT", 960 | "links": [ 961 | 85 962 | ] 963 | } 964 | ], 965 | "properties": { 966 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 967 | "ver": "8e5ec6b7f3acf88255c5d93d062079f18b43aa2b", 968 | "Node name for S&R": "FramePackSampler" 969 | }, 970 | "widgets_values": [ 971 | 25, 972 | true, 973 | 0.15, 974 | 1, 975 | 9.000000000000002, 976 | 0, 977 | 100, 978 | "increment", 979 | 9, 980 | 4, 981 | 6, 982 | "unipc_bh1", 983 | 1.0000000000000002, 984 | 2.0000000000000004 985 | ] 986 | }, 987 | { 988 | "id": 51, 989 | "type": "FramePackFindNearestBucket", 990 | "pos": [ 991 | 478.0079040527344, 992 | 949.587890625 993 | ], 994 | "size": [ 995 | 315, 996 | 78 997 | ], 998 | "flags": {}, 999 | "order": 11, 1000 | "mode": 0, 1001 | "inputs": [ 1002 | { 1003 | "name": "image", 1004 | "type": "IMAGE", 1005 | "link": 126 1006 | } 1007 | ], 1008 | "outputs": [ 1009 | { 1010 | "name": "width", 1011 | "type": "INT", 1012 | "links": [ 1013 | 128, 1014 | 202 1015 | ] 1016 | }, 1017 | { 1018 | "name": "height", 1019 | "type": "INT", 1020 | "links": [ 1021 | 127, 1022 | 203 1023 | ] 1024 | } 1025 | ], 1026 | "properties": { 1027 | "aux_id": "kijai/ComfyUI-FramePackWrapper", 1028 | "ver": "4f9030a9f4c0bd67d86adf3d3dc07e37118c40bd", 1029 | "Node name for S&R": "FramePackFindNearestBucket" 1030 | }, 1031 | "widgets_values": [ 1032 | 640 1033 | ] 1034 | }, 1035 | { 1036 | "id": 17, 1037 | "type": "CLIPVisionEncode", 1038 | "pos": [ 1039 | 1384.66357421875, 1040 | 743.9740600585938 1041 | ], 1042 | "size": [ 1043 | 380.4000244140625, 1044 | 78 1045 | ], 1046 | "flags": {}, 1047 | "order": 16, 1048 | "mode": 0, 1049 | "inputs": [ 1050 | { 1051 | "name": "clip_vision", 1052 | "type": "CLIP_VISION", 1053 | "link": 18 1054 | }, 1055 | { 1056 | "name": "image", 1057 | "type": "IMAGE", 1058 | "link": 159 1059 | } 1060 | ], 1061 | "outputs": [ 1062 | { 1063 | "name": "CLIP_VISION_OUTPUT", 1064 | "type": "CLIP_VISION_OUTPUT", 1065 | "links": [ 1066 | 83 1067 | ] 1068 | } 1069 | ], 1070 | "properties": { 1071 | "cnr_id": "comfy-core", 1072 | "ver": "0.3.28", 1073 | "Node name for S&R": "CLIPVisionEncode" 1074 | }, 1075 | "widgets_values": [ 1076 | "center" 1077 | ], 1078 | "color": "#233", 1079 | "bgcolor": "#355" 1080 | }, 1081 | { 1082 | "id": 18, 1083 | "type": "CLIPVisionLoader", 1084 | "pos": [ 1085 | 848.22119140625, 1086 | 828.3516235351562 1087 | ], 1088 | "size": [ 1089 | 388.87139892578125, 1090 | 58 1091 | ], 1092 | "flags": {}, 1093 | "order": 9, 1094 | "mode": 0, 1095 | "inputs": [], 1096 | "outputs": [ 1097 | { 1098 | "name": "CLIP_VISION", 1099 | "type": "CLIP_VISION", 1100 | "links": [ 1101 | 18 1102 | ] 1103 | } 1104 | ], 1105 | "properties": { 1106 | "cnr_id": "comfy-core", 1107 | "ver": "0.3.28", 1108 | "Node name for S&R": "CLIPVisionLoader" 1109 | }, 1110 | "widgets_values": [ 1111 | "sigclip_vision_patch14_384.safetensors" 1112 | ], 1113 | "color": "#2a363b", 1114 | "bgcolor": "#3f5159" 1115 | } 1116 | ], 1117 | "links": [ 1118 | [ 1119 | 18, 1120 | 18, 1121 | 0, 1122 | 17, 1123 | 0, 1124 | "CLIP_VISION" 1125 | ], 1126 | [ 1127 | 22, 1128 | 12, 1129 | 0, 1130 | 20, 1131 | 1, 1132 | "VAE" 1133 | ], 1134 | [ 1135 | 62, 1136 | 12, 1137 | 0, 1138 | 33, 1139 | 1, 1140 | "VAE" 1141 | ], 1142 | [ 1143 | 83, 1144 | 17, 1145 | 0, 1146 | 39, 1147 | 3, 1148 | "CLIP_VISION_OUTPUT" 1149 | ], 1150 | [ 1151 | 85, 1152 | 39, 1153 | 0, 1154 | 33, 1155 | 0, 1156 | "LATENT" 1157 | ], 1158 | [ 1159 | 86, 1160 | 20, 1161 | 0, 1162 | 39, 1163 | 4, 1164 | "LATENT" 1165 | ], 1166 | [ 1167 | 96, 1168 | 33, 1169 | 0, 1170 | 44, 1171 | 0, 1172 | "IMAGE" 1173 | ], 1174 | [ 1175 | 97, 1176 | 44, 1177 | 0, 1178 | 23, 1179 | 0, 1180 | "IMAGE" 1181 | ], 1182 | [ 1183 | 102, 1184 | 13, 1185 | 0, 1186 | 47, 1187 | 0, 1188 | "CLIP" 1189 | ], 1190 | [ 1191 | 108, 1192 | 15, 1193 | 0, 1194 | 39, 1195 | 2, 1196 | "CONDITIONING" 1197 | ], 1198 | [ 1199 | 114, 1200 | 47, 1201 | 0, 1202 | 39, 1203 | 1, 1204 | "CONDITIONING" 1205 | ], 1206 | [ 1207 | 117, 1208 | 48, 1209 | 0, 1210 | 20, 1211 | 0, 1212 | "IMAGE" 1213 | ], 1214 | [ 1215 | 118, 1216 | 47, 1217 | 0, 1218 | 15, 1219 | 0, 1220 | "CONDITIONING" 1221 | ], 1222 | [ 1223 | 122, 1224 | 19, 1225 | 0, 1226 | 50, 1227 | 0, 1228 | "IMAGE" 1229 | ], 1230 | [ 1231 | 125, 1232 | 50, 1233 | 0, 1234 | 48, 1235 | 0, 1236 | "IMAGE" 1237 | ], 1238 | [ 1239 | 126, 1240 | 19, 1241 | 0, 1242 | 51, 1243 | 0, 1244 | "IMAGE" 1245 | ], 1246 | [ 1247 | 127, 1248 | 51, 1249 | 1, 1250 | 50, 1251 | 2, 1252 | "INT" 1253 | ], 1254 | [ 1255 | 128, 1256 | 51, 1257 | 0, 1258 | 50, 1259 | 1, 1260 | "INT" 1261 | ], 1262 | [ 1263 | 129, 1264 | 52, 1265 | 0, 1266 | 39, 1267 | 0, 1268 | "FramePackMODEL" 1269 | ], 1270 | [ 1271 | 159, 1272 | 50, 1273 | 0, 1274 | 17, 1275 | 1, 1276 | "IMAGE" 1277 | ], 1278 | [ 1279 | 165, 1280 | 79, 1281 | 0, 1282 | 81, 1283 | 0, 1284 | "IMAGE" 1285 | ], 1286 | [ 1287 | 168, 1288 | 12, 1289 | 0, 1290 | 84, 1291 | 1, 1292 | "VAE" 1293 | ], 1294 | [ 1295 | 171, 1296 | 81, 1297 | 0, 1298 | 84, 1299 | 0, 1300 | "IMAGE" 1301 | ], 1302 | [ 1303 | 202, 1304 | 51, 1305 | 0, 1306 | 81, 1307 | 1, 1308 | "INT" 1309 | ], 1310 | [ 1311 | 203, 1312 | 51, 1313 | 1, 1314 | 81, 1315 | 2, 1316 | "INT" 1317 | ], 1318 | [ 1319 | 204, 1320 | 84, 1321 | 0, 1322 | 39, 1323 | 5, 1324 | "LATENT" 1325 | ] 1326 | ], 1327 | "groups": [], 1328 | "config": {}, 1329 | "extra": { 1330 | "ds": { 1331 | "scale": 0.513158118230707, 1332 | "offset": [ 1333 | 790.7374267578125, 1334 | -475.99627685546875 1335 | ] 1336 | }, 1337 | "frontendVersion": "1.16.8", 1338 | "VHS_latentpreview": false, 1339 | "VHS_latentpreviewrate": 0, 1340 | "VHS_MetadataImage": true, 1341 | "VHS_KeepIntermediate": true 1342 | }, 1343 | "version": 0.4 1344 | } -------------------------------------------------------------------------------- /fp8_optimization.py: -------------------------------------------------------------------------------- 1 | #based on ComfyUI's and MinusZoneAI's fp8_linear optimization 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | def fp8_linear_forward(cls, original_dtype, input): 7 | weight_dtype = cls.weight.dtype 8 | if weight_dtype in [torch.float8_e4m3fn, torch.float8_e5m2]: 9 | if len(input.shape) == 3: 10 | target_dtype = torch.float8_e5m2 if weight_dtype == torch.float8_e4m3fn else torch.float8_e4m3fn 11 | inn = input.reshape(-1, input.shape[2]).to(target_dtype) 12 | w = cls.weight.t() 13 | 14 | scale = torch.ones((1), device=input.device, dtype=torch.float32) 15 | bias = cls.bias.to(original_dtype) if cls.bias is not None else None 16 | 17 | if bias is not None: 18 | o = torch._scaled_mm(inn, w, out_dtype=original_dtype, bias=bias, scale_a=scale, scale_b=scale) 19 | else: 20 | o = torch._scaled_mm(inn, w, out_dtype=original_dtype, scale_a=scale, scale_b=scale) 21 | 22 | if isinstance(o, tuple): 23 | o = o[0] 24 | 25 | return o.reshape((-1, input.shape[1], cls.weight.shape[0])) 26 | else: 27 | return cls.original_forward(input.to(original_dtype)) 28 | else: 29 | return cls.original_forward(input) 30 | 31 | def convert_fp8_linear(module, original_dtype, params_to_keep={}): 32 | setattr(module, "fp8_matmul_enabled", True) 33 | 34 | for name, module in module.named_modules(): 35 | if not any(keyword in name for keyword in params_to_keep): 36 | if isinstance(module, nn.Linear): 37 | original_forward = module.forward 38 | setattr(module, "original_forward", original_forward) 39 | setattr(module, "forward", lambda input, m=module: fp8_linear_forward(m, original_dtype, input)) 40 | -------------------------------------------------------------------------------- /images/framepack-cascade2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nirvash/ComfyUI-FramePackWrapper/fd2fcc2f1951982d4511f41ba644c3531718fe7b/images/framepack-cascade2.mp4 -------------------------------------------------------------------------------- /images/screenshot-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nirvash/ComfyUI-FramePackWrapper/fd2fcc2f1951982d4511f41ba644c3531718fe7b/images/screenshot-01.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=1.6.0 2 | diffusers>=0.33.1 3 | transformers>=4.46.2 4 | scipy>=1.12.0 5 | torchsde>=0.2.6 6 | einops 7 | safetensors 8 | -------------------------------------------------------------------------------- /transformer_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_class_name": "HunyuanVideoTransformer3DModelPacked", 3 | "_diffusers_version": "0.33.0.dev0", 4 | "_name_or_path": "hunyuanvideo-community/HunyuanVideo", 5 | "attention_head_dim": 128, 6 | "guidance_embeds": true, 7 | "has_clean_x_embedder": true, 8 | "has_image_proj": true, 9 | "image_proj_dim": 1152, 10 | "in_channels": 16, 11 | "mlp_ratio": 4.0, 12 | "num_attention_heads": 24, 13 | "num_layers": 20, 14 | "num_refiner_layers": 2, 15 | "num_single_layers": 40, 16 | "out_channels": 16, 17 | "patch_size": 2, 18 | "patch_size_t": 1, 19 | "pooled_projection_dim": 768, 20 | "qk_norm": "rms_norm", 21 | "rope_axes_dim": [ 22 | 16, 23 | 56, 24 | 56 25 | ], 26 | "rope_theta": 256.0, 27 | "text_embed_dim": 4096 28 | } 29 | --------------------------------------------------------------------------------