├── .gitignore ├── models └── clip_fc.safetensors ├── utils ├── module_utils.py ├── ref_constants.py ├── dilate_mask.py └── sampler_utils.py ├── nodes ├── ref_bank_node.py ├── configure_ref_net_node.py ├── prepare_ref_latents.py ├── ref_model_pred_node.py ├── read_sampler_node.py ├── vision_clip_encode_node.py ├── write_sampler_node.py └── custom_ref_map_node.py ├── pyproject.toml ├── README.md ├── __init__.py ├── modules └── ref_block.py ├── LICENSE └── example_workflows └── ref_workflow_example.json /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /models/clip_fc.safetensors: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/logtd/ComfyUI-RefUNet/HEAD/models/clip_fc.safetensors -------------------------------------------------------------------------------- /utils/module_utils.py: -------------------------------------------------------------------------------- 1 | def isinstance_str(x: object, cls_name: str): 2 | for _cls in x.__class__.__mro__: 3 | if _cls.__name__ == cls_name: 4 | return True 5 | 6 | return False 7 | -------------------------------------------------------------------------------- /nodes/ref_bank_node.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class CreateRefBankNode: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": { 7 | }} 8 | RETURN_TYPES = ("REF_BANK",) 9 | FUNCTION = "create" 10 | 11 | CATEGORY = "reference" 12 | 13 | def create(self): 14 | return ({},) 15 | -------------------------------------------------------------------------------- /utils/ref_constants.py: -------------------------------------------------------------------------------- 1 | 2 | SD1_OUTPUT_MAP = set([]) 3 | for idx in [0,1,2,3,4,5,6,7,8,9]: # TODO check these 4 | SD1_OUTPUT_MAP.add(('output', idx)) 5 | 6 | SD1_MIDDLE_MAP = set([('middle', 0)]) 7 | 8 | SD1_INPUT_MAP = set() 9 | for idx in [0,1,2,3,4,5,6,7,8,9,10,11]: 10 | SD1_INPUT_MAP.add(('input', idx)) 11 | 12 | 13 | SD1_REF_MAP = SD1_INPUT_MAP | SD1_MIDDLE_MAP | SD1_OUTPUT_MAP -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "comfyui-refunet" 3 | description = "A set of nodes to use Reference UNets" 4 | version = "1.0.0" 5 | license = {file = "LICENSE"} 6 | 7 | [project.urls] 8 | Repository = "https://github.com/logtd/ComfyUI-RefUNet" 9 | # Used by Comfy Registry https://comfyregistry.org 10 | 11 | [tool.comfy] 12 | PublisherId = "logtd" 13 | DisplayName = "ComfyUI-RefUNet" 14 | Icon = "" 15 | -------------------------------------------------------------------------------- /nodes/configure_ref_net_node.py: -------------------------------------------------------------------------------- 1 | from ..modules.ref_block import inject_ref_blocks 2 | 3 | 4 | class ConfigureRefNetNode: 5 | @classmethod 6 | def INPUT_TYPES(s): 7 | return {"required": { 8 | "model": ("MODEL",), 9 | }} 10 | RETURN_TYPES = ("MODEL",) 11 | FUNCTION = "apply" 12 | 13 | CATEGORY = "reference" 14 | 15 | def apply(self, model): 16 | inject_ref_blocks(model.model.diffusion_model) 17 | return (model,) 18 | -------------------------------------------------------------------------------- /nodes/prepare_ref_latents.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class PrepareRefLatentsNode: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": { 7 | "src_latents": ("LATENT",), 8 | "ref_latents": ("LATENT",), 9 | }} 10 | RETURN_TYPES = ("LATENT",) 11 | FUNCTION = "apply" 12 | 13 | CATEGORY = "reference" 14 | 15 | def apply(self, src_latents, ref_latents): 16 | # This is mostly a trick node to ensure that comfy executes the sampling in the correct order 17 | return (src_latents,) 18 | -------------------------------------------------------------------------------- /utils/dilate_mask.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | 6 | def dilate_mask(n: torch.Tensor, transformer_options): 7 | mask = transformer_options.get('REF_MASK', None) 8 | mask_dilation = transformer_options.get('REF_MASK_DILATION', 0) 9 | 10 | if mask is None or mask_dilation <= 1: 11 | return n.clone() 12 | 13 | H, W = mask.shape[-2:] 14 | scale = 1 << int( 15 | math.ceil(math.log2((H * W) / n.shape[-2]) / 2) 16 | ) 17 | H, W = math.ceil(H / scale), math.ceil(W / scale) 18 | resized_mask = F.interpolate(mask.unsqueeze(1), (H, W)).to(n.dtype).to(n.device) 19 | dilation_kernel = torch.ones( 20 | (1, 1, mask_dilation, mask_dilation), dtype=n.dtype, device=n.device 21 | ) 22 | dilated_mask = ( 23 | F.conv2d(resized_mask, dilation_kernel, padding=1) 24 | .view(-1, H * W, 1) 25 | .clamp(0.0, 1.0) 26 | ) 27 | n = n * dilated_mask 28 | 29 | return n.clone() -------------------------------------------------------------------------------- /nodes/ref_model_pred_node.py: -------------------------------------------------------------------------------- 1 | import comfy.sd 2 | import comfy.model_sampling 3 | import comfy.latent_formats 4 | 5 | 6 | class X0Ref(comfy.model_sampling.EPS): 7 | def calculate_input(self, sigma, noise): 8 | return noise 9 | def calculate_denoised(self, sigma, model_output, model_input): 10 | return model_output 11 | 12 | 13 | class RefModelSamplingPredNode: 14 | @classmethod 15 | def INPUT_TYPES(s): 16 | return {"required": { "model": ("MODEL",), 17 | }} 18 | 19 | RETURN_TYPES = ("MODEL",) 20 | FUNCTION = "patch" 21 | 22 | CATEGORY = "reference" 23 | 24 | def patch(self, model): 25 | m = model.clone() 26 | 27 | sampling_base = comfy.model_sampling.ModelSamplingDiscrete 28 | sampling_type = X0Ref 29 | 30 | class ModelSamplingAdvanced(sampling_base, sampling_type): 31 | pass 32 | 33 | model_sampling = ModelSamplingAdvanced(model.model.model_config) 34 | 35 | m.add_object_patch("model_sampling", model_sampling) 36 | return (m, ) 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ComfyUI-RefUNet 2 | A set of nodes to use Reference UNets 3 | 4 | 5 | https://github.com/user-attachments/assets/5c921956-7bf2-4521-a8bf-1c8594b46641 6 | 7 | 8 | Should be compatible with sampling methods that use reference unets, e.g.: 9 | * [FollowYourEmoji](https://github.com/mayuelala/FollowYourEmoji) 10 | * [MusePose](https://github.com/TMElyralab/MusePose) 11 | * [AnimateAnyone](https://github.com/guoqincode/Open-AnimateAnyone) 12 | 13 | ## Examples 14 | You can find examples of FollowYourEmoji in the `example_workflows` directory using @Kijai's FYE embedding nodes 15 | 16 | https://github.com/user-attachments/assets/6b2bf9b2-8c4e-4b6b-a65d-228dc293563d 17 | 18 | ## Installation 19 | There are no specific python requirements for this repo. 20 | 21 | ### Models 22 | You can find the models for FollowYourEmoji here https://huggingface.co/Kijai/FollowYourEmoji-safetensors/tree/main 23 | 24 | | Checkpoint | Directory | 25 | |------------|-----------| 26 | |FYE_unet-fp16.safetensors | unet | 27 | |FYE_referencenet-fp16.safetensors | unet | 28 | |fye_motion_module-fp16.safetensors | animatediff_models | 29 | | sd-image-variations-encoder-fp16.safetensors | clip_vision | 30 | -------------------------------------------------------------------------------- /nodes/read_sampler_node.py: -------------------------------------------------------------------------------- 1 | import comfy.samplers 2 | from comfy.samplers import KSAMPLER 3 | 4 | from ..utils.sampler_utils import get_sampler_fn, create_sampler 5 | from ..utils.ref_constants import SD1_REF_MAP 6 | 7 | 8 | 9 | class ReadSamplerNode: 10 | @classmethod 11 | def INPUT_TYPES(s): 12 | return {"required": { 13 | "sampler_name": (comfy.samplers.SAMPLER_NAMES, ), 14 | "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), 15 | "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), 16 | "ref_bank": ("REF_BANK",) 17 | }, "optional": { 18 | "sampler": ("SAMPLER",), 19 | # "opt_attn_map": ("ATTN_MAP",), 20 | }} 21 | RETURN_TYPES = ("SAMPLER",) 22 | FUNCTION = "build" 23 | 24 | CATEGORY = "reference/sampling" 25 | 26 | def build(self, sampler_name, start_percent, end_percent, ref_bank, sampler=None, opt_attn_map=SD1_REF_MAP): 27 | sampler_fn = get_sampler_fn(sampler_name) 28 | sampler_fn = create_sampler(sampler_fn, ref_bank, opt_attn_map, 'READ', start_percent, end_percent) 29 | 30 | if sampler is None: 31 | sampler = KSAMPLER(sampler_fn) 32 | else: 33 | sampler.sampler_function = sampler_fn 34 | 35 | return (sampler, ) -------------------------------------------------------------------------------- /nodes/vision_clip_encode_node.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | import torch 5 | 6 | import comfy.model_management 7 | import comfy.utils 8 | from comfy.clip_vision import clip_preprocess 9 | 10 | from .. import REPO_DIR 11 | 12 | 13 | class VisionClipEncodeNode: 14 | @classmethod 15 | def INPUT_TYPES(s): 16 | return {"required": { 17 | "clip_vision": ("CLIP_VISION", ), 18 | "clip_image": ("IMAGE",), 19 | "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), 20 | } 21 | } 22 | 23 | RETURN_TYPES = ("CONDITIONING",) 24 | FUNCTION = "encode" 25 | CATEGORY = "clip" 26 | 27 | def encode(self, clip_vision, clip_image, strength): 28 | dtype=clip_vision.dtype 29 | device=comfy.model_management.get_torch_device() 30 | clip_image = clip_preprocess(clip_image.clone(), 224) 31 | clip_embeds = clip_vision.encode_image(clip_image.permute(0, 2, 3, 1))["last_hidden_state"].to(dtype).to(device) 32 | clip_embeds = clip_embeds * strength 33 | clip_fc_path = os.path.join(REPO_DIR, "models","clip_fc.safetensors") 34 | sd = comfy.utils.load_torch_file(clip_fc_path) 35 | self.clip_fc = torch.nn.Linear(1024, 768, bias=True).to(clip_embeds.dtype).to(clip_embeds.device) 36 | self.clip_fc.load_state_dict(sd) 37 | 38 | clip_in = clip_embeds 39 | clip_out = self.clip_fc(clip_in) * strength 40 | clip_out = clip_out.to('cpu') 41 | 42 | return ([[clip_out, {"pooled_output": clip_out}]], ) 43 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | REPO_DIR = os.path.dirname(os.path.abspath(__file__)) 3 | 4 | from .nodes.configure_ref_net_node import ConfigureRefNetNode 5 | from .nodes.prepare_ref_latents import PrepareRefLatentsNode 6 | from .nodes.read_sampler_node import ReadSamplerNode 7 | from .nodes.write_sampler_node import WriteSamplerNode 8 | from .nodes.ref_bank_node import CreateRefBankNode 9 | from .nodes.ref_model_pred_node import RefModelSamplingPredNode 10 | from .nodes.custom_ref_map_node import ConfigRefMapAdvNode, CustomRefMapSD1Node 11 | from .nodes.vision_clip_encode_node import VisionClipEncodeNode 12 | 13 | 14 | NODE_CLASS_MAPPINGS = { 15 | "ConfigureRefNet": ConfigureRefNetNode, 16 | "PrepareRefLatents": PrepareRefLatentsNode, 17 | "ReadSampler": ReadSamplerNode, 18 | "WriteSampler": WriteSamplerNode, 19 | "CreateRefBank": CreateRefBankNode, 20 | "RefModelSamplingPred": RefModelSamplingPredNode, 21 | "CustomRefMapSD1": CustomRefMapSD1Node, 22 | "ConfigRefMapAdv": ConfigRefMapAdvNode, 23 | "VisionClipEncode": VisionClipEncodeNode, 24 | } 25 | 26 | NODE_DISPLAY_NAME_MAPPINGS = { 27 | "ConfigureRefNet": "REF] Configure Model", 28 | "PrepareRefLatents": "REF] Prep Sampling Latents", 29 | "ReadSampler": "REF] Read Sampling", 30 | "WriteSampler": "REF] Write Sampling", 31 | "CreateRefBank": "REF] Create Bank", 32 | "RefModelSamplingPred": "REF] Model Sampling Pred", 33 | "CustomRefMapSD1": "REF] Ref Attn Map SD1", 34 | "ConfigRefMapAdv": "REF] Ref Attn Map Adv", 35 | "VisionClipEncode": "Clip Vision Encode Cond" 36 | } 37 | -------------------------------------------------------------------------------- /nodes/write_sampler_node.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.samplers 3 | from comfy.samplers import KSAMPLER 4 | 5 | from ..utils.sampler_utils import get_sampler_fn, create_sampler 6 | from ..utils.ref_constants import SD1_REF_MAP 7 | 8 | 9 | @torch.no_grad() 10 | def sample_write(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.): 11 | extra_args = {} if extra_args is None else extra_args 12 | s_in = x.new_ones([x.shape[0]]) 13 | return model(x, sigmas[0] * s_in, **extra_args) 14 | 15 | 16 | class WriteSamplerNode: 17 | @classmethod 18 | def INPUT_TYPES(s): 19 | return {"required": { 20 | "sampler_name": (["REFERENCE_WRITE"] + comfy.samplers.SAMPLER_NAMES, ), 21 | "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), 22 | "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), 23 | "ref_bank": ("REF_BANK",), 24 | "mask_dilation": ("INT", {"default": 3, "min": 0, "max": 10, "step": 1}), 25 | }, "optional": { 26 | "sampler": ("SAMPLER",), 27 | "opt_attn_map": ("ATTN_MAP",), 28 | "masks": ("MASK",), 29 | }} 30 | RETURN_TYPES = ("SAMPLER","SIGMAS") 31 | FUNCTION = "build" 32 | 33 | CATEGORY = "reference/sampling" 34 | 35 | def build(self, sampler_name, start_percent, end_percent, ref_bank, mask_dilation, sampler=None, opt_attn_map=SD1_REF_MAP, masks=None): 36 | if sampler_name == 'REFERENCE_WRITE': 37 | sampler_fn = sample_write 38 | else: 39 | sampler_fn = get_sampler_fn(sampler_name) 40 | sampler_fn = create_sampler(sampler_fn, ref_bank, opt_attn_map, 'WRITE', start_percent, end_percent, mask_dilation, masks) 41 | 42 | if sampler is None: 43 | sampler = KSAMPLER(sampler_fn) 44 | else: 45 | sampler.sampler_function = sampler_fn 46 | 47 | return (sampler, torch.Tensor([0])) 48 | -------------------------------------------------------------------------------- /nodes/custom_ref_map_node.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class CustomRefMapSD1Node: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | base = {"required": { 7 | }} 8 | for i in range(6): 9 | base['required'][f'input_{i}'] = ("BOOLEAN", { "default": True}) 10 | 11 | base['required'][f'middle_0'] = ("BOOLEAN", { "default": True }) 12 | 13 | for i in range(9): 14 | base['required'][f'output_{i}'] = ("BOOLEAN", { "default": True }) 15 | 16 | return base 17 | RETURN_TYPES = ("ATTN_MAP",) 18 | FUNCTION = "apply" 19 | 20 | CATEGORY = "reference/custom" 21 | 22 | def apply(self, **kwargs): 23 | 24 | attention_map = set() 25 | for key, value in kwargs.items(): 26 | if value: 27 | block, idx = key.split('_') 28 | attention_map.add((block, int(idx))) 29 | 30 | return (attention_map, ) 31 | 32 | 33 | class ConfigRefMapAdvNode: 34 | @classmethod 35 | def INPUT_TYPES(s): 36 | base = {"required": { 37 | "input_attns": ("STRING", {"multiline": True, "default": "0,1,2,3,4,5", }), 38 | "middle_attns": ("STRING", {"multiline": True, "default": "0", }), 39 | "output_attns": ("STRING", {"multiline": True, "default": "0,1,2,3,4,5,6,7,8" }), 40 | }} 41 | return base 42 | RETURN_TYPES = ("ATTN_MAP",) 43 | FUNCTION = "apply" 44 | 45 | CATEGORY = "reference/custom" 46 | 47 | def apply(self, input_attns, middle_attns, output_attns): 48 | 49 | attention_map = set() 50 | if input_attns != '' and input_attns is not None: 51 | for idx in input_attns.split(','): 52 | idx = idx.strip() 53 | if idx is '': 54 | continue 55 | attention_map.add(('input', int(idx))) 56 | 57 | if middle_attns != '' and middle_attns is not None: 58 | for idx in middle_attns.split(','): 59 | idx = idx.strip() 60 | if idx is '': 61 | continue 62 | attention_map.add(('middle', int(idx))) 63 | 64 | if output_attns != '' and output_attns is not None: 65 | for idx in output_attns.split(','): 66 | idx = idx.strip() 67 | if idx is '': 68 | continue 69 | attention_map.add(('output', int(idx))) 70 | 71 | return (attention_map, ) -------------------------------------------------------------------------------- /utils/sampler_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | import comfy.k_diffusion.sampling as k_diffusion_sampling 4 | 5 | 6 | def get_stepper(model, model_options, sigmas, start_percent, end_percent): 7 | prev_step = [0] # hack for special samplers 8 | def sample_step(x, sigma, **extra_args): 9 | step = torch.where(sigma[0] == sigmas)[0] 10 | if not len(step): 11 | step = prev_step[0] 12 | prev_step[0] = step 13 | step_percent = step.item() / model_options['transformer_options']['TOTAL_STEPS'] 14 | ref_on = start_percent <= step_percent <= end_percent 15 | model_options['transformer_options']['REF_ON'] = ref_on 16 | 17 | output = model(x, sigma, **extra_args) 18 | 19 | del model_options['transformer_options']['REF_ON'] 20 | 21 | return output 22 | 23 | return sample_step 24 | 25 | 26 | def create_sampler(sample_fn, ref_bank, ref_map, ref_type, start_percent=0, end_percent=1, mask_dilation=0, masks=None): 27 | @torch.no_grad() 28 | def sample(model, latents, sigmas, extra_args=None, callback=None, disable=None, **extra_options): 29 | model_options = extra_args.get('model_options', {}) 30 | transformer_options = model_options.get('transformer_options', {}) 31 | 32 | if ref_type == 'WRITE': 33 | ref_bank.clear() 34 | for block_idx in ref_map: 35 | ref_bank[block_idx] = {} 36 | 37 | model_options = { 38 | **model_options, 39 | 'transformer_options': { 40 | **transformer_options, 41 | 'REF_TYPE': ref_type, 42 | 'REF_BANK': ref_bank, 43 | 'TOTAL_STEPS': len(sigmas), 44 | 'REF_MASK': masks, 45 | 'REF_MASK_DILATION': mask_dilation, 46 | } 47 | } 48 | extra_args = {**extra_args, 'model_options': model_options} 49 | 50 | sampler_stepper = get_stepper(model, model_options, sigmas, start_percent, end_percent) 51 | 52 | output = sample_fn(sampler_stepper, latents, sigmas, extra_args=extra_args, callback=callback, disable=disable, **extra_options) 53 | 54 | if 'REF_BANK' in model_options['transformer_options']: 55 | del model_options['transformer_options']['REF_BANK'] 56 | 57 | if 'REF_TYPE' in model_options['transformer_options']: 58 | del model_options['transformer_options']['REF_TYPE'] 59 | 60 | if 'TOTAL_STEPS' in model_options['transformer_options']: 61 | del model_options['transformer_options']['TOTAL_STEPS'] 62 | 63 | return output 64 | 65 | return sample 66 | 67 | 68 | def get_sampler_fn(sampler_name): 69 | if sampler_name == "dpm_fast": 70 | def dpm_fast_function(model, noise, sigmas, extra_args, callback, disable): 71 | sigma_min = sigmas[-1] 72 | if sigma_min == 0: 73 | sigma_min = sigmas[-2] 74 | total_steps = len(sigmas) - 1 75 | return k_diffusion_sampling.sample_dpm_fast(model, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=callback, disable=disable) 76 | sampler_function = dpm_fast_function 77 | elif sampler_name == "dpm_adaptive": 78 | def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable): 79 | sigma_min = sigmas[-1] 80 | if sigma_min == 0: 81 | sigma_min = sigmas[-2] 82 | return k_diffusion_sampling.sample_dpm_adaptive(model, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=callback, disable=disable) 83 | sampler_function = dpm_adaptive_function 84 | else: 85 | sampler_function = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name)) 86 | return sampler_function 87 | -------------------------------------------------------------------------------- /modules/ref_block.py: -------------------------------------------------------------------------------- 1 | from einops import rearrange 2 | import torch 3 | from comfy.ldm.modules.attention import BasicTransformerBlock 4 | from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel 5 | 6 | from ..utils.module_utils import isinstance_str 7 | from ..utils.dilate_mask import dilate_mask 8 | 9 | 10 | class RefTransformerBlock(BasicTransformerBlock): 11 | def configure(self, block, idx): 12 | self.block = block 13 | self.idx = idx 14 | self.block_idx = (block, idx) 15 | 16 | def forward(self, x, context=None, transformer_options={}): 17 | extra_options = {} 18 | block = transformer_options.get("block", None) 19 | block_index = transformer_options.get("block_index", 0) 20 | transformer_patches = {} 21 | transformer_patches_replace = {} 22 | 23 | for k in transformer_options: 24 | if k == "patches": 25 | transformer_patches = transformer_options[k] 26 | elif k == "patches_replace": 27 | transformer_patches_replace = transformer_options[k] 28 | else: 29 | extra_options[k] = transformer_options[k] 30 | 31 | extra_options["n_heads"] = self.n_heads 32 | extra_options["dim_head"] = self.d_head 33 | extra_options["attn_precision"] = self.attn_precision 34 | 35 | if self.ff_in: 36 | x_skip = x 37 | x = self.ff_in(self.norm_in(x)) 38 | if self.is_res: 39 | x += x_skip 40 | 41 | orig_normal = self.norm1(x) 42 | n = orig_normal.clone() 43 | 44 | conds = transformer_options['cond_or_uncond'] 45 | len_conds = len(conds) 46 | n_frames = len(x) // len_conds 47 | 48 | ref_type = transformer_options.get('REF_TYPE', None) 49 | ref_bank = transformer_options.get('REF_BANK', None) 50 | ref_on = transformer_options.get('REF_ON', False) 51 | REF_UNCOND_READ = False 52 | 53 | context_attn1 = n 54 | 55 | if ref_type == 'WRITE' and ref_bank is not None and self.block_idx in ref_bank and ref_on: 56 | for idx, cond in enumerate(conds): 57 | ref_n = dilate_mask(n[idx*n_frames:(idx+1)*n_frames], transformer_options) 58 | ref_n = rearrange(ref_n, '(b f) d h -> b (f d) h', b=1) 59 | ref_bank[self.block_idx][cond] = ref_n.cpu() 60 | ref_bank['num_write'] = len(n) 61 | elif ref_type == 'READ' and ref_bank is not None and self.block_idx in ref_bank and ref_on: 62 | ref_n = [] 63 | for idx, cond in enumerate(conds): 64 | if cond in ref_bank[self.block_idx]: 65 | ref_n.append(ref_bank[self.block_idx][cond].to(x.device).repeat(n_frames, 1, 1)) 66 | if cond == 1: 67 | REF_UNCOND_READ = True 68 | else: 69 | 70 | ref_n.append(context_attn1[idx*n_frames:(idx+1)*n_frames].repeat(1, ref_bank['num_write'], 1)) # TODO make this faster 71 | ref_n = torch.cat(ref_n) 72 | context_attn1 = torch.cat([context_attn1, ref_n], dim=1) 73 | 74 | value_attn1 = context_attn1 75 | 76 | if "attn1_patch" in transformer_patches: 77 | patch = transformer_patches["attn1_patch"] 78 | for p in patch: 79 | n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options) 80 | 81 | if block is not None: 82 | transformer_block = (block[0], block[1], block_index) 83 | else: 84 | transformer_block = None 85 | attn1_replace_patch = transformer_patches_replace.get("attn1", {}) 86 | block_attn1 = transformer_block 87 | if block_attn1 not in attn1_replace_patch: 88 | block_attn1 = block 89 | 90 | if block_attn1 in attn1_replace_patch: 91 | q = self.attn1.to_q(n) 92 | context_attn1 = self.attn1.to_k(context_attn1) 93 | value_attn1 = self.attn1.to_v(value_attn1) 94 | hidden_states = attn1_replace_patch[block_attn1](q, context_attn1, value_attn1, extra_options) 95 | hidden_states = self.attn1.to_out(hidden_states) 96 | del q 97 | else: 98 | if hasattr(self.attn1, 'veevee'): 99 | hidden_states = self.attn1(n, context_attn1, value_attn1, extra_options=extra_options) 100 | else: 101 | hidden_states = self.attn1(n, context_attn1, value_attn1) 102 | 103 | if REF_UNCOND_READ: 104 | hidden_states_uc_c = hidden_states.clone() 105 | uc_mask = [] 106 | for cond in conds: 107 | uc_mask.append(torch.Tensor([cond] * n_frames)) 108 | uc_mask = torch.cat(uc_mask).to(hidden_states.device).bool() 109 | hidden_states_uc_c[uc_mask] = self.attn1(orig_normal[uc_mask]) 110 | hidden_states = hidden_states_uc_c.clone() 111 | 112 | n = hidden_states 113 | 114 | if "attn1_output_patch" in transformer_patches: 115 | patch = transformer_patches["attn1_output_patch"] 116 | for p in patch: 117 | n = p(n, extra_options) 118 | 119 | x += n 120 | 121 | if "middle_patch" in transformer_patches: 122 | patch = transformer_patches["middle_patch"] 123 | for p in patch: 124 | x = p(x, extra_options) 125 | 126 | if self.attn2 is not None: 127 | n = self.norm2(x) 128 | if self.switch_temporal_ca_to_sa: 129 | context_attn2 = n 130 | else: 131 | context_attn2 = context 132 | value_attn2 = None 133 | if "attn2_patch" in transformer_patches: 134 | patch = transformer_patches["attn2_patch"] 135 | value_attn2 = context_attn2 136 | for p in patch: 137 | n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options) 138 | 139 | attn2_replace_patch = transformer_patches_replace.get("attn2", {}) 140 | block_attn2 = transformer_block 141 | if block_attn2 not in attn2_replace_patch: 142 | block_attn2 = block 143 | 144 | if block_attn2 in attn2_replace_patch: 145 | if value_attn2 is None: 146 | value_attn2 = context_attn2 147 | n = self.attn2.to_q(n) 148 | context_attn2 = self.attn2.to_k(context_attn2) 149 | value_attn2 = self.attn2.to_v(value_attn2) 150 | n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options) 151 | n = self.attn2.to_out(n) 152 | else: 153 | n = self.attn2(n, context=context_attn2, value=value_attn2) 154 | 155 | if "attn2_output_patch" in transformer_patches: 156 | patch = transformer_patches["attn2_output_patch"] 157 | for p in patch: 158 | n = p(n, extra_options) 159 | 160 | x += n 161 | if self.is_res: 162 | x_skip = x 163 | x = self.ff(self.norm3(x)) 164 | if self.is_res: 165 | x += x_skip 166 | 167 | return x 168 | 169 | 170 | def _get_block_modules(module): 171 | blocks = list(filter(lambda x: isinstance_str(x[1], 'BasicTransformerBlock'), module.named_modules())) 172 | return [block for _, block in blocks] 173 | 174 | 175 | def inject_ref_blocks(diffusion_model: UNetModel): 176 | input = _get_block_modules(diffusion_model.input_blocks) 177 | middle = _get_block_modules(diffusion_model.middle_block) 178 | output = _get_block_modules(diffusion_model.output_blocks) 179 | 180 | for i, block in enumerate(input): 181 | block.__class__ = RefTransformerBlock 182 | block.configure('input', i) 183 | 184 | for i, block in enumerate(middle): 185 | block.__class__ = RefTransformerBlock 186 | block.configure('middle', i) 187 | 188 | for i, block in enumerate(output): 189 | block.__class__ = RefTransformerBlock 190 | block.configure('output', i) 191 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /example_workflows/ref_workflow_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "last_node_id": 71, 3 | "last_link_id": 123, 4 | "nodes": [ 5 | { 6 | "id": 30, 7 | "type": "FYELandmarkEncode", 8 | "pos": [ 9 | 380, 10 | -1036 11 | ], 12 | "size": { 13 | "0": 211.60000610351562, 14 | "1": 58 15 | }, 16 | "flags": {}, 17 | "order": 24, 18 | "mode": 0, 19 | "inputs": [ 20 | { 21 | "name": "motions", 22 | "type": "IMAGE", 23 | "link": 30 24 | } 25 | ], 26 | "outputs": [ 27 | { 28 | "name": "landmark_features", 29 | "type": "LMKFEAT", 30 | "links": [ 31 | 31 32 | ], 33 | "shape": 3, 34 | "slot_index": 0 35 | } 36 | ], 37 | "properties": { 38 | "Node name for S&R": "FYELandmarkEncode" 39 | }, 40 | "widgets_values": [ 41 | 1 42 | ] 43 | }, 44 | { 45 | "id": 19, 46 | "type": "ImageScale", 47 | "pos": [ 48 | -301, 49 | -946 50 | ], 51 | "size": { 52 | "0": 210, 53 | "1": 130 54 | }, 55 | "flags": {}, 56 | "order": 16, 57 | "mode": 0, 58 | "inputs": [ 59 | { 60 | "name": "image", 61 | "type": "IMAGE", 62 | "link": 23 63 | } 64 | ], 65 | "outputs": [ 66 | { 67 | "name": "IMAGE", 68 | "type": "IMAGE", 69 | "links": [ 70 | 28, 71 | 32 72 | ], 73 | "shape": 3, 74 | "slot_index": 0 75 | } 76 | ], 77 | "properties": { 78 | "Node name for S&R": "ImageScale" 79 | }, 80 | "widgets_values": [ 81 | "nearest-exact", 82 | 512, 83 | 512, 84 | "disabled" 85 | ] 86 | }, 87 | { 88 | "id": 32, 89 | "type": "PrepareRefLatents", 90 | "pos": [ 91 | 681, 92 | -388 93 | ], 94 | "size": { 95 | "0": 229.20001220703125, 96 | "1": 46 97 | }, 98 | "flags": {}, 99 | "order": 32, 100 | "mode": 0, 101 | "inputs": [ 102 | { 103 | "name": "src_latents", 104 | "type": "LATENT", 105 | "link": 34 106 | }, 107 | { 108 | "name": "ref_latents", 109 | "type": "LATENT", 110 | "link": 33 111 | } 112 | ], 113 | "outputs": [ 114 | { 115 | "name": "LATENT", 116 | "type": "LATENT", 117 | "links": [ 118 | 36 119 | ], 120 | "shape": 3, 121 | "slot_index": 0 122 | } 123 | ], 124 | "properties": { 125 | "Node name for S&R": "PrepareRefLatents" 126 | } 127 | }, 128 | { 129 | "id": 52, 130 | "type": "ADE_LoadAnimateDiffModel", 131 | "pos": [ 132 | 920, 133 | -1380 134 | ], 135 | "size": { 136 | "0": 252, 137 | "1": 58 138 | }, 139 | "flags": {}, 140 | "order": 0, 141 | "mode": 0, 142 | "inputs": [ 143 | { 144 | "name": "ad_settings", 145 | "type": "AD_SETTINGS", 146 | "link": null 147 | } 148 | ], 149 | "outputs": [ 150 | { 151 | "name": "MOTION_MODEL", 152 | "type": "MOTION_MODEL_ADE", 153 | "links": [ 154 | 59 155 | ], 156 | "shape": 3, 157 | "slot_index": 0 158 | } 159 | ], 160 | "properties": { 161 | "Node name for S&R": "ADE_LoadAnimateDiffModel" 162 | }, 163 | "widgets_values": [ 164 | "fye_motion_module-fp16.safetensors" 165 | ] 166 | }, 167 | { 168 | "id": 51, 169 | "type": "ADE_ApplyAnimateDiffModelSimple", 170 | "pos": [ 171 | 920, 172 | -1270 173 | ], 174 | "size": { 175 | "0": 260.3999938964844, 176 | "1": 114 177 | }, 178 | "flags": {}, 179 | "order": 10, 180 | "mode": 0, 181 | "inputs": [ 182 | { 183 | "name": "motion_model", 184 | "type": "MOTION_MODEL_ADE", 185 | "link": 59 186 | }, 187 | { 188 | "name": "motion_lora", 189 | "type": "MOTION_LORA", 190 | "link": null 191 | }, 192 | { 193 | "name": "scale_multival", 194 | "type": "MULTIVAL", 195 | "link": null 196 | }, 197 | { 198 | "name": "effect_multival", 199 | "type": "MULTIVAL", 200 | "link": null 201 | }, 202 | { 203 | "name": "ad_keyframes", 204 | "type": "AD_KEYFRAMES", 205 | "link": null 206 | } 207 | ], 208 | "outputs": [ 209 | { 210 | "name": "M_MODELS", 211 | "type": "M_MODELS", 212 | "links": [ 213 | 60 214 | ], 215 | "shape": 3, 216 | "slot_index": 0 217 | } 218 | ], 219 | "properties": { 220 | "Node name for S&R": "ADE_ApplyAnimateDiffModelSimple" 221 | }, 222 | "widgets_values": [ 223 | "" 224 | ] 225 | }, 226 | { 227 | "id": 39, 228 | "type": "FYECLIPEncode", 229 | "pos": [ 230 | -740, 231 | -270 232 | ], 233 | "size": { 234 | "0": 210, 235 | "1": 78 236 | }, 237 | "flags": {}, 238 | "order": 12, 239 | "mode": 0, 240 | "inputs": [ 241 | { 242 | "name": "clip_vision", 243 | "type": "CLIP_VISION", 244 | "link": 45 245 | }, 246 | { 247 | "name": "clip_image", 248 | "type": "IMAGE", 249 | "link": 46 250 | } 251 | ], 252 | "outputs": [ 253 | { 254 | "name": "clip_embeds", 255 | "type": "FYECLIPEMBED", 256 | "links": [ 257 | 48 258 | ], 259 | "shape": 3, 260 | "slot_index": 0 261 | } 262 | ], 263 | "properties": { 264 | "Node name for S&R": "FYECLIPEncode" 265 | }, 266 | "widgets_values": [ 267 | 1 268 | ] 269 | }, 270 | { 271 | "id": 31, 272 | "type": "VAEEncode", 273 | "pos": [ 274 | 240, 275 | -715 276 | ], 277 | "size": { 278 | "0": 210, 279 | "1": 46 280 | }, 281 | "flags": {}, 282 | "order": 21, 283 | "mode": 0, 284 | "inputs": [ 285 | { 286 | "name": "pixels", 287 | "type": "IMAGE", 288 | "link": 32 289 | }, 290 | { 291 | "name": "vae", 292 | "type": "VAE", 293 | "link": 108 294 | } 295 | ], 296 | "outputs": [ 297 | { 298 | "name": "LATENT", 299 | "type": "LATENT", 300 | "links": [ 301 | 34 302 | ], 303 | "shape": 3, 304 | "slot_index": 0 305 | } 306 | ], 307 | "properties": { 308 | "Node name for S&R": "VAEEncode" 309 | } 310 | }, 311 | { 312 | "id": 49, 313 | "type": "VAEDecode", 314 | "pos": [ 315 | 1477, 316 | -863 317 | ], 318 | "size": { 319 | "0": 210, 320 | "1": 46 321 | }, 322 | "flags": {}, 323 | "order": 35, 324 | "mode": 0, 325 | "inputs": [ 326 | { 327 | "name": "samples", 328 | "type": "LATENT", 329 | "link": 56 330 | }, 331 | { 332 | "name": "vae", 333 | "type": "VAE", 334 | "link": 110 335 | } 336 | ], 337 | "outputs": [ 338 | { 339 | "name": "IMAGE", 340 | "type": "IMAGE", 341 | "links": [ 342 | 58 343 | ], 344 | "shape": 3, 345 | "slot_index": 0 346 | } 347 | ], 348 | "properties": { 349 | "Node name for S&R": "VAEDecode" 350 | } 351 | }, 352 | { 353 | "id": 25, 354 | "type": "UNETLoader", 355 | "pos": [ 356 | 103, 357 | -1190 358 | ], 359 | "size": { 360 | "0": 210, 361 | "1": 82 362 | }, 363 | "flags": {}, 364 | "order": 1, 365 | "mode": 0, 366 | "outputs": [ 367 | { 368 | "name": "MODEL", 369 | "type": "MODEL", 370 | "links": [ 371 | 25 372 | ], 373 | "shape": 3, 374 | "slot_index": 0 375 | } 376 | ], 377 | "properties": { 378 | "Node name for S&R": "UNETLoader" 379 | }, 380 | "widgets_values": [ 381 | "FYE_unet-fp16.safetensors", 382 | "default" 383 | ] 384 | }, 385 | { 386 | "id": 38, 387 | "type": "CLIPVisionLoader", 388 | "pos": [ 389 | -1076, 390 | -563 391 | ], 392 | "size": { 393 | "0": 210, 394 | "1": 58 395 | }, 396 | "flags": {}, 397 | "order": 2, 398 | "mode": 0, 399 | "outputs": [ 400 | { 401 | "name": "CLIP_VISION", 402 | "type": "CLIP_VISION", 403 | "links": [ 404 | 44, 405 | 45 406 | ], 407 | "shape": 3, 408 | "slot_index": 0 409 | } 410 | ], 411 | "properties": { 412 | "Node name for S&R": "CLIPVisionLoader" 413 | }, 414 | "widgets_values": [ 415 | "sd-image-variations-encoder-fp16.safetensors" 416 | ] 417 | }, 418 | { 419 | "id": 28, 420 | "type": "FYEMediaPipe", 421 | "pos": [ 422 | 38, 423 | -1045 424 | ], 425 | "size": { 426 | "0": 287.20001220703125, 427 | "1": 102 428 | }, 429 | "flags": {}, 430 | "order": 20, 431 | "mode": 0, 432 | "inputs": [ 433 | { 434 | "name": "images", 435 | "type": "IMAGE", 436 | "link": 28 437 | }, 438 | { 439 | "name": "align_to_face_results", 440 | "type": "FACERESULTS", 441 | "link": null 442 | } 443 | ], 444 | "outputs": [ 445 | { 446 | "name": "images", 447 | "type": "IMAGE", 448 | "links": [ 449 | 30 450 | ], 451 | "shape": 3, 452 | "slot_index": 0 453 | }, 454 | { 455 | "name": "face_results", 456 | "type": "FACERESULTS", 457 | "links": null, 458 | "shape": 3 459 | } 460 | ], 461 | "properties": { 462 | "Node name for S&R": "FYEMediaPipe" 463 | }, 464 | "widgets_values": [ 465 | false, 466 | true 467 | ] 468 | }, 469 | { 470 | "id": 22, 471 | "type": "FYELandmarkToComfy", 472 | "pos": [ 473 | 687, 474 | -1132 475 | ], 476 | "size": { 477 | "0": 194.8000030517578, 478 | "1": 46 479 | }, 480 | "flags": {}, 481 | "order": 26, 482 | "mode": 0, 483 | "inputs": [ 484 | { 485 | "name": "model", 486 | "type": "MODEL", 487 | "link": 29 488 | }, 489 | { 490 | "name": "landmark_features", 491 | "type": "LMKFEAT", 492 | "link": 31 493 | } 494 | ], 495 | "outputs": [ 496 | { 497 | "name": "model", 498 | "type": "MODEL", 499 | "links": [ 500 | 106 501 | ], 502 | "shape": 3, 503 | "slot_index": 0 504 | } 505 | ], 506 | "properties": { 507 | "Node name for S&R": "FYELandmarkToComfy" 508 | } 509 | }, 510 | { 511 | "id": 40, 512 | "type": "IPAdapterNoise", 513 | "pos": [ 514 | -1100, 515 | -95 516 | ], 517 | "size": { 518 | "0": 210, 519 | "1": 106 520 | }, 521 | "flags": {}, 522 | "order": 3, 523 | "mode": 0, 524 | "inputs": [ 525 | { 526 | "name": "image_optional", 527 | "type": "IMAGE", 528 | "link": null 529 | } 530 | ], 531 | "outputs": [ 532 | { 533 | "name": "IMAGE", 534 | "type": "IMAGE", 535 | "links": [ 536 | 46 537 | ], 538 | "shape": 3, 539 | "slot_index": 0 540 | } 541 | ], 542 | "properties": { 543 | "Node name for S&R": "IPAdapterNoise" 544 | }, 545 | "widgets_values": [ 546 | "gaussian", 547 | 1, 548 | 0 549 | ] 550 | }, 551 | { 552 | "id": 45, 553 | "type": "WriteSampler", 554 | "pos": [ 555 | 184, 556 | -293 557 | ], 558 | "size": { 559 | "0": 210, 560 | "1": 146 561 | }, 562 | "flags": {}, 563 | "order": 14, 564 | "mode": 0, 565 | "inputs": [ 566 | { 567 | "name": "ref_bank", 568 | "type": "REF_BANK", 569 | "link": 98 570 | }, 571 | { 572 | "name": "sampler", 573 | "type": "SAMPLER", 574 | "link": null 575 | }, 576 | { 577 | "name": "opt_attn_map", 578 | "type": "ATTN_MAP", 579 | "link": null 580 | } 581 | ], 582 | "outputs": [ 583 | { 584 | "name": "SAMPLER", 585 | "type": "SAMPLER", 586 | "links": [ 587 | 49 588 | ], 589 | "shape": 3, 590 | "slot_index": 0 591 | }, 592 | { 593 | "name": "SIGMAS", 594 | "type": "SIGMAS", 595 | "links": [ 596 | 51 597 | ], 598 | "shape": 3, 599 | "slot_index": 1 600 | } 601 | ], 602 | "properties": { 603 | "Node name for S&R": "WriteSampler" 604 | }, 605 | "widgets_values": [ 606 | "REFERENCE_WRITE", 607 | 0, 608 | 1 609 | ] 610 | }, 611 | { 612 | "id": 18, 613 | "type": "ConfigureRefNet", 614 | "pos": [ 615 | 413, 616 | -1115 617 | ], 618 | "size": { 619 | "0": 190.90481567382812, 620 | "1": 26 621 | }, 622 | "flags": {}, 623 | "order": 11, 624 | "mode": 0, 625 | "inputs": [ 626 | { 627 | "name": "model", 628 | "type": "MODEL", 629 | "link": 25 630 | } 631 | ], 632 | "outputs": [ 633 | { 634 | "name": "MODEL", 635 | "type": "MODEL", 636 | "links": [ 637 | 29 638 | ], 639 | "shape": 3, 640 | "slot_index": 0 641 | } 642 | ], 643 | "properties": { 644 | "Node name for S&R": "ConfigureRefNet" 645 | } 646 | }, 647 | { 648 | "id": 35, 649 | "type": "FYECLIPEncode", 650 | "pos": [ 651 | -740, 652 | -410 653 | ], 654 | "size": { 655 | "0": 210, 656 | "1": 78 657 | }, 658 | "flags": {}, 659 | "order": 25, 660 | "mode": 0, 661 | "inputs": [ 662 | { 663 | "name": "clip_vision", 664 | "type": "CLIP_VISION", 665 | "link": 44 666 | }, 667 | { 668 | "name": "clip_image", 669 | "type": "IMAGE", 670 | "link": 123 671 | } 672 | ], 673 | "outputs": [ 674 | { 675 | "name": "clip_embeds", 676 | "type": "FYECLIPEMBED", 677 | "links": [ 678 | 47 679 | ], 680 | "shape": 3, 681 | "slot_index": 0 682 | } 683 | ], 684 | "properties": { 685 | "Node name for S&R": "FYECLIPEncode" 686 | }, 687 | "widgets_values": [ 688 | 1 689 | ] 690 | }, 691 | { 692 | "id": 69, 693 | "type": "VAEEncode", 694 | "pos": [ 695 | -196, 696 | -124 697 | ], 698 | "size": { 699 | "0": 210, 700 | "1": 46 701 | }, 702 | "flags": {}, 703 | "order": 23, 704 | "mode": 0, 705 | "inputs": [ 706 | { 707 | "name": "pixels", 708 | "type": "IMAGE", 709 | "link": 101 710 | }, 711 | { 712 | "name": "vae", 713 | "type": "VAE", 714 | "link": 109 715 | } 716 | ], 717 | "outputs": [ 718 | { 719 | "name": "LATENT", 720 | "type": "LATENT", 721 | "links": [ 722 | 102 723 | ], 724 | "shape": 3, 725 | "slot_index": 0 726 | } 727 | ], 728 | "properties": { 729 | "Node name for S&R": "VAEEncode" 730 | } 731 | }, 732 | { 733 | "id": 67, 734 | "type": "VAELoader", 735 | "pos": [ 736 | 247, 737 | -819 738 | ], 739 | "size": { 740 | "0": 210, 741 | "1": 58 742 | }, 743 | "flags": {}, 744 | "order": 4, 745 | "mode": 0, 746 | "outputs": [ 747 | { 748 | "name": "VAE", 749 | "type": "VAE", 750 | "links": [ 751 | 108, 752 | 109, 753 | 110 754 | ], 755 | "shape": 3, 756 | "slot_index": 0 757 | } 758 | ], 759 | "properties": { 760 | "Node name for S&R": "VAELoader" 761 | }, 762 | "widgets_values": [ 763 | "vae-ft-mse-840000-ema-pruned.safetensors" 764 | ] 765 | }, 766 | { 767 | "id": 23, 768 | "type": "UNETLoader", 769 | "pos": [ 770 | -599, 771 | -696 772 | ], 773 | "size": { 774 | "0": 210, 775 | "1": 82 776 | }, 777 | "flags": {}, 778 | "order": 5, 779 | "mode": 0, 780 | "outputs": [ 781 | { 782 | "name": "MODEL", 783 | "type": "MODEL", 784 | "links": [ 785 | 26 786 | ], 787 | "shape": 3, 788 | "slot_index": 0 789 | } 790 | ], 791 | "properties": { 792 | "Node name for S&R": "UNETLoader" 793 | }, 794 | "widgets_values": [ 795 | "FYE_referencenet-fp16.safetensors", 796 | "default" 797 | ] 798 | }, 799 | { 800 | "id": 65, 801 | "type": "ConditioningZeroOut", 802 | "pos": [ 803 | 736, 804 | -278 805 | ], 806 | "size": { 807 | "0": 211.60000610351562, 808 | "1": 26 809 | }, 810 | "flags": {}, 811 | "order": 29, 812 | "mode": 0, 813 | "inputs": [ 814 | { 815 | "name": "conditioning", 816 | "type": "CONDITIONING", 817 | "link": 82 818 | } 819 | ], 820 | "outputs": [ 821 | { 822 | "name": "CONDITIONING", 823 | "type": "CONDITIONING", 824 | "links": [ 825 | 114 826 | ], 827 | "shape": 3, 828 | "slot_index": 0 829 | } 830 | ], 831 | "properties": { 832 | "Node name for S&R": "ConditioningZeroOut" 833 | } 834 | }, 835 | { 836 | "id": 42, 837 | "type": "FYEClipEmbedToComfy", 838 | "pos": [ 839 | -411, 840 | -384 841 | ], 842 | "size": { 843 | "0": 210, 844 | "1": 63.7000617980957 845 | }, 846 | "flags": {}, 847 | "order": 27, 848 | "mode": 0, 849 | "inputs": [ 850 | { 851 | "name": "clip_embeds", 852 | "type": "FYECLIPEMBED", 853 | "link": 47 854 | } 855 | ], 856 | "outputs": [ 857 | { 858 | "name": "conditioning", 859 | "type": "CONDITIONING", 860 | "links": [ 861 | 52, 862 | 82, 863 | 115, 864 | 117 865 | ], 866 | "shape": 3, 867 | "slot_index": 0 868 | } 869 | ], 870 | "properties": { 871 | "Node name for S&R": "FYEClipEmbedToComfy" 872 | }, 873 | "widgets_values": [ 874 | 1 875 | ] 876 | }, 877 | { 878 | "id": 44, 879 | "type": "ReadSampler", 880 | "pos": [ 881 | 1180, 882 | -302 883 | ], 884 | "size": { 885 | "0": 210, 886 | "1": 146 887 | }, 888 | "flags": {}, 889 | "order": 15, 890 | "mode": 0, 891 | "inputs": [ 892 | { 893 | "name": "ref_bank", 894 | "type": "REF_BANK", 895 | "link": 99 896 | }, 897 | { 898 | "name": "sampler", 899 | "type": "SAMPLER", 900 | "link": null 901 | }, 902 | { 903 | "name": "opt_attn_map", 904 | "type": "ATTN_MAP", 905 | "link": null 906 | } 907 | ], 908 | "outputs": [ 909 | { 910 | "name": "SAMPLER", 911 | "type": "SAMPLER", 912 | "links": [ 913 | 50 914 | ], 915 | "shape": 3, 916 | "slot_index": 0 917 | } 918 | ], 919 | "properties": { 920 | "Node name for S&R": "ReadSampler" 921 | }, 922 | "widgets_values": [ 923 | "euler", 924 | 0, 925 | 1 926 | ] 927 | }, 928 | { 929 | "id": 58, 930 | "type": "ImageScale", 931 | "pos": [ 932 | -1068, 933 | -274 934 | ], 935 | "size": { 936 | "0": 210, 937 | "1": 130 938 | }, 939 | "flags": {}, 940 | "order": 17, 941 | "mode": 0, 942 | "inputs": [ 943 | { 944 | "name": "image", 945 | "type": "IMAGE", 946 | "link": 68 947 | } 948 | ], 949 | "outputs": [ 950 | { 951 | "name": "IMAGE", 952 | "type": "IMAGE", 953 | "links": [ 954 | 69, 955 | 101 956 | ], 957 | "shape": 3, 958 | "slot_index": 0 959 | } 960 | ], 961 | "properties": { 962 | "Node name for S&R": "ImageScale" 963 | }, 964 | "widgets_values": [ 965 | "nearest-exact", 966 | 512, 967 | 512, 968 | "center" 969 | ] 970 | }, 971 | { 972 | "id": 27, 973 | "type": "SamplerCustom", 974 | "pos": [ 975 | 167, 976 | -567 977 | ], 978 | "size": { 979 | "0": 236.8000030517578, 980 | "1": 230 981 | }, 982 | "flags": {}, 983 | "order": 30, 984 | "mode": 0, 985 | "inputs": [ 986 | { 987 | "name": "model", 988 | "type": "MODEL", 989 | "link": 121 990 | }, 991 | { 992 | "name": "positive", 993 | "type": "CONDITIONING", 994 | "link": 52 995 | }, 996 | { 997 | "name": "negative", 998 | "type": "CONDITIONING", 999 | "link": 117 1000 | }, 1001 | { 1002 | "name": "sampler", 1003 | "type": "SAMPLER", 1004 | "link": 49 1005 | }, 1006 | { 1007 | "name": "sigmas", 1008 | "type": "SIGMAS", 1009 | "link": 51 1010 | }, 1011 | { 1012 | "name": "latent_image", 1013 | "type": "LATENT", 1014 | "link": 102 1015 | } 1016 | ], 1017 | "outputs": [ 1018 | { 1019 | "name": "output", 1020 | "type": "LATENT", 1021 | "links": [ 1022 | 33 1023 | ], 1024 | "shape": 3, 1025 | "slot_index": 0 1026 | }, 1027 | { 1028 | "name": "denoised_output", 1029 | "type": "LATENT", 1030 | "links": null, 1031 | "shape": 3 1032 | } 1033 | ], 1034 | "properties": { 1035 | "Node name for S&R": "SamplerCustom" 1036 | }, 1037 | "widgets_values": [ 1038 | false, 1039 | 0, 1040 | "fixed", 1041 | 1 1042 | ] 1043 | }, 1044 | { 1045 | "id": 43, 1046 | "type": "FYEClipEmbedToComfy", 1047 | "pos": [ 1048 | -400, 1049 | -270 1050 | ], 1051 | "size": { 1052 | "0": 210, 1053 | "1": 58 1054 | }, 1055 | "flags": {}, 1056 | "order": 18, 1057 | "mode": 0, 1058 | "inputs": [ 1059 | { 1060 | "name": "clip_embeds", 1061 | "type": "FYECLIPEMBED", 1062 | "link": 48 1063 | } 1064 | ], 1065 | "outputs": [ 1066 | { 1067 | "name": "conditioning", 1068 | "type": "CONDITIONING", 1069 | "links": [], 1070 | "shape": 3, 1071 | "slot_index": 0 1072 | } 1073 | ], 1074 | "properties": { 1075 | "Node name for S&R": "FYEClipEmbedToComfy" 1076 | }, 1077 | "widgets_values": [ 1078 | 1 1079 | ] 1080 | }, 1081 | { 1082 | "id": 68, 1083 | "type": "CreateRefBank", 1084 | "pos": [ 1085 | 203, 1086 | -107 1087 | ], 1088 | "size": { 1089 | "0": 178.8992156982422, 1090 | "1": 26 1091 | }, 1092 | "flags": {}, 1093 | "order": 6, 1094 | "mode": 0, 1095 | "outputs": [ 1096 | { 1097 | "name": "REF_BANK", 1098 | "type": "REF_BANK", 1099 | "links": [ 1100 | 98, 1101 | 99 1102 | ], 1103 | "shape": 3, 1104 | "slot_index": 0 1105 | } 1106 | ], 1107 | "properties": { 1108 | "Node name for S&R": "CreateRefBank" 1109 | } 1110 | }, 1111 | { 1112 | "id": 33, 1113 | "type": "SamplerCustom", 1114 | "pos": [ 1115 | 1162, 1116 | -577 1117 | ], 1118 | "size": { 1119 | "0": 236.8000030517578, 1120 | "1": 230 1121 | }, 1122 | "flags": {}, 1123 | "order": 34, 1124 | "mode": 0, 1125 | "inputs": [ 1126 | { 1127 | "name": "model", 1128 | "type": "MODEL", 1129 | "link": 88 1130 | }, 1131 | { 1132 | "name": "positive", 1133 | "type": "CONDITIONING", 1134 | "link": 115 1135 | }, 1136 | { 1137 | "name": "negative", 1138 | "type": "CONDITIONING", 1139 | "link": 114 1140 | }, 1141 | { 1142 | "name": "sampler", 1143 | "type": "SAMPLER", 1144 | "link": 50 1145 | }, 1146 | { 1147 | "name": "sigmas", 1148 | "type": "SIGMAS", 1149 | "link": 73 1150 | }, 1151 | { 1152 | "name": "latent_image", 1153 | "type": "LATENT", 1154 | "link": 36 1155 | } 1156 | ], 1157 | "outputs": [ 1158 | { 1159 | "name": "output", 1160 | "type": "LATENT", 1161 | "links": [ 1162 | 56 1163 | ], 1164 | "shape": 3, 1165 | "slot_index": 0 1166 | }, 1167 | { 1168 | "name": "denoised_output", 1169 | "type": "LATENT", 1170 | "links": null, 1171 | "shape": 3 1172 | } 1173 | ], 1174 | "properties": { 1175 | "Node name for S&R": "SamplerCustom" 1176 | }, 1177 | "widgets_values": [ 1178 | true, 1179 | 0, 1180 | "fixed", 1181 | 3.5 1182 | ] 1183 | }, 1184 | { 1185 | "id": 66, 1186 | "type": "ModelSamplingDiscrete", 1187 | "pos": [ 1188 | 954, 1189 | -947 1190 | ], 1191 | "size": { 1192 | "0": 210, 1193 | "1": 82 1194 | }, 1195 | "flags": {}, 1196 | "order": 31, 1197 | "mode": 0, 1198 | "inputs": [ 1199 | { 1200 | "name": "model", 1201 | "type": "MODEL", 1202 | "link": 87 1203 | } 1204 | ], 1205 | "outputs": [ 1206 | { 1207 | "name": "MODEL", 1208 | "type": "MODEL", 1209 | "links": [ 1210 | 88, 1211 | 119 1212 | ], 1213 | "shape": 3, 1214 | "slot_index": 0 1215 | } 1216 | ], 1217 | "properties": { 1218 | "Node name for S&R": "ModelSamplingDiscrete" 1219 | }, 1220 | "widgets_values": [ 1221 | "v_prediction", 1222 | false 1223 | ] 1224 | }, 1225 | { 1226 | "id": 26, 1227 | "type": "ConfigureRefNet", 1228 | "pos": [ 1229 | -356, 1230 | -697 1231 | ], 1232 | "size": { 1233 | "0": 168, 1234 | "1": 26 1235 | }, 1236 | "flags": {}, 1237 | "order": 13, 1238 | "mode": 0, 1239 | "inputs": [ 1240 | { 1241 | "name": "model", 1242 | "type": "MODEL", 1243 | "link": 26 1244 | } 1245 | ], 1246 | "outputs": [ 1247 | { 1248 | "name": "MODEL", 1249 | "type": "MODEL", 1250 | "links": [ 1251 | 120 1252 | ], 1253 | "shape": 3, 1254 | "slot_index": 0 1255 | } 1256 | ], 1257 | "properties": { 1258 | "Node name for S&R": "ConfigureRefNet" 1259 | } 1260 | }, 1261 | { 1262 | "id": 70, 1263 | "type": "RefModelSamplingPred", 1264 | "pos": [ 1265 | -133, 1266 | -681 1267 | ], 1268 | "size": { 1269 | "0": 201.60000610351562, 1270 | "1": 26 1271 | }, 1272 | "flags": {}, 1273 | "order": 19, 1274 | "mode": 0, 1275 | "inputs": [ 1276 | { 1277 | "name": "model", 1278 | "type": "MODEL", 1279 | "link": 120 1280 | } 1281 | ], 1282 | "outputs": [ 1283 | { 1284 | "name": "MODEL", 1285 | "type": "MODEL", 1286 | "links": [ 1287 | 121 1288 | ], 1289 | "shape": 3, 1290 | "slot_index": 0 1291 | } 1292 | ], 1293 | "properties": { 1294 | "Node name for S&R": "RefModelSamplingPred" 1295 | } 1296 | }, 1297 | { 1298 | "id": 71, 1299 | "type": "ADE_StandardStaticContextOptions", 1300 | "pos": [ 1301 | 478, 1302 | -1438 1303 | ], 1304 | "size": { 1305 | "0": 319.20001220703125, 1306 | "1": 198 1307 | }, 1308 | "flags": {}, 1309 | "order": 7, 1310 | "mode": 0, 1311 | "inputs": [ 1312 | { 1313 | "name": "prev_context", 1314 | "type": "CONTEXT_OPTIONS", 1315 | "link": null 1316 | }, 1317 | { 1318 | "name": "view_opts", 1319 | "type": "VIEW_OPTS", 1320 | "link": null 1321 | } 1322 | ], 1323 | "outputs": [ 1324 | { 1325 | "name": "CONTEXT_OPTS", 1326 | "type": "CONTEXT_OPTIONS", 1327 | "links": [ 1328 | 122 1329 | ], 1330 | "shape": 3, 1331 | "slot_index": 0 1332 | } 1333 | ], 1334 | "properties": { 1335 | "Node name for S&R": "ADE_StandardStaticContextOptions" 1336 | }, 1337 | "widgets_values": [ 1338 | 16, 1339 | 4, 1340 | "pyramid", 1341 | false, 1342 | 0, 1343 | 1 1344 | ] 1345 | }, 1346 | { 1347 | "id": 57, 1348 | "type": "PrepImageForClipVision", 1349 | "pos": [ 1350 | -1049, 1351 | -446 1352 | ], 1353 | "size": { 1354 | "0": 210, 1355 | "1": 106 1356 | }, 1357 | "flags": {}, 1358 | "order": 22, 1359 | "mode": 0, 1360 | "inputs": [ 1361 | { 1362 | "name": "image", 1363 | "type": "IMAGE", 1364 | "link": 69 1365 | } 1366 | ], 1367 | "outputs": [ 1368 | { 1369 | "name": "IMAGE", 1370 | "type": "IMAGE", 1371 | "links": [ 1372 | 123 1373 | ], 1374 | "shape": 3, 1375 | "slot_index": 0 1376 | } 1377 | ], 1378 | "properties": { 1379 | "Node name for S&R": "PrepImageForClipVision" 1380 | }, 1381 | "widgets_values": [ 1382 | "LANCZOS", 1383 | "top", 1384 | 0 1385 | ] 1386 | }, 1387 | { 1388 | "id": 11, 1389 | "type": "VHS_LoadVideo", 1390 | "pos": [ 1391 | -675, 1392 | -1325 1393 | ], 1394 | "size": [ 1395 | 235.1999969482422, 1396 | 491.1999969482422 1397 | ], 1398 | "flags": {}, 1399 | "order": 8, 1400 | "mode": 0, 1401 | "inputs": [ 1402 | { 1403 | "name": "meta_batch", 1404 | "type": "VHS_BatchManager", 1405 | "link": null 1406 | }, 1407 | { 1408 | "name": "vae", 1409 | "type": "VAE", 1410 | "link": null 1411 | } 1412 | ], 1413 | "outputs": [ 1414 | { 1415 | "name": "IMAGE", 1416 | "type": "IMAGE", 1417 | "links": [ 1418 | 23 1419 | ], 1420 | "shape": 3, 1421 | "slot_index": 0 1422 | }, 1423 | { 1424 | "name": "frame_count", 1425 | "type": "INT", 1426 | "links": null, 1427 | "shape": 3 1428 | }, 1429 | { 1430 | "name": "audio", 1431 | "type": "AUDIO", 1432 | "links": null, 1433 | "shape": 3 1434 | }, 1435 | { 1436 | "name": "video_info", 1437 | "type": "VHS_VIDEOINFO", 1438 | "links": null, 1439 | "shape": 3 1440 | } 1441 | ], 1442 | "properties": { 1443 | "Node name for S&R": "VHS_LoadVideo" 1444 | }, 1445 | "widgets_values": { 1446 | "video": "d6.mp4", 1447 | "force_rate": 0, 1448 | "force_size": "Disabled", 1449 | "custom_width": 512, 1450 | "custom_height": 512, 1451 | "frame_load_cap": 64, 1452 | "skip_first_frames": 2, 1453 | "select_every_nth": 1, 1454 | "choose video to upload": "image", 1455 | "videopreview": { 1456 | "hidden": false, 1457 | "paused": false, 1458 | "params": { 1459 | "frame_load_cap": 64, 1460 | "skip_first_frames": 2, 1461 | "force_rate": 0, 1462 | "filename": "d6.mp4", 1463 | "type": "input", 1464 | "format": "video/mp4", 1465 | "select_every_nth": 1 1466 | } 1467 | } 1468 | } 1469 | }, 1470 | { 1471 | "id": 60, 1472 | "type": "BasicScheduler", 1473 | "pos": [ 1474 | 1184, 1475 | -111 1476 | ], 1477 | "size": { 1478 | "0": 210, 1479 | "1": 106 1480 | }, 1481 | "flags": {}, 1482 | "order": 33, 1483 | "mode": 0, 1484 | "inputs": [ 1485 | { 1486 | "name": "model", 1487 | "type": "MODEL", 1488 | "link": 119 1489 | } 1490 | ], 1491 | "outputs": [ 1492 | { 1493 | "name": "SIGMAS", 1494 | "type": "SIGMAS", 1495 | "links": [ 1496 | 73 1497 | ], 1498 | "shape": 3, 1499 | "slot_index": 0 1500 | } 1501 | ], 1502 | "properties": { 1503 | "Node name for S&R": "BasicScheduler" 1504 | }, 1505 | "widgets_values": [ 1506 | "normal", 1507 | 20, 1508 | 1 1509 | ] 1510 | }, 1511 | { 1512 | "id": 36, 1513 | "type": "LoadImage", 1514 | "pos": [ 1515 | 2100, 1516 | -569 1517 | ], 1518 | "size": { 1519 | "0": 315, 1520 | "1": 314 1521 | }, 1522 | "flags": {}, 1523 | "order": 9, 1524 | "mode": 0, 1525 | "outputs": [ 1526 | { 1527 | "name": "IMAGE", 1528 | "type": "IMAGE", 1529 | "links": [ 1530 | 68 1531 | ], 1532 | "shape": 3, 1533 | "slot_index": 0 1534 | }, 1535 | { 1536 | "name": "MASK", 1537 | "type": "MASK", 1538 | "links": null, 1539 | "shape": 3 1540 | } 1541 | ], 1542 | "properties": { 1543 | "Node name for S&R": "LoadImage" 1544 | }, 1545 | "widgets_values": [ 1546 | "naruto.webp", 1547 | "image" 1548 | ] 1549 | }, 1550 | { 1551 | "id": 50, 1552 | "type": "VHS_VideoCombine", 1553 | "pos": [ 1554 | 1767, 1555 | -859 1556 | ], 1557 | "size": [ 1558 | 320, 1559 | 624 1560 | ], 1561 | "flags": {}, 1562 | "order": 36, 1563 | "mode": 0, 1564 | "inputs": [ 1565 | { 1566 | "name": "images", 1567 | "type": "IMAGE", 1568 | "link": 58 1569 | }, 1570 | { 1571 | "name": "audio", 1572 | "type": "AUDIO", 1573 | "link": null 1574 | }, 1575 | { 1576 | "name": "meta_batch", 1577 | "type": "VHS_BatchManager", 1578 | "link": null 1579 | }, 1580 | { 1581 | "name": "vae", 1582 | "type": "VAE", 1583 | "link": null 1584 | } 1585 | ], 1586 | "outputs": [ 1587 | { 1588 | "name": "Filenames", 1589 | "type": "VHS_FILENAMES", 1590 | "links": null, 1591 | "shape": 3 1592 | } 1593 | ], 1594 | "properties": { 1595 | "Node name for S&R": "VHS_VideoCombine" 1596 | }, 1597 | "widgets_values": { 1598 | "frame_rate": 8, 1599 | "loop_count": 0, 1600 | "filename_prefix": "AnimateDiff", 1601 | "format": "video/h264-mp4", 1602 | "pix_fmt": "yuv420p", 1603 | "crf": 19, 1604 | "save_metadata": false, 1605 | "pingpong": false, 1606 | "save_output": false, 1607 | "videopreview": { 1608 | "hidden": false, 1609 | "paused": false, 1610 | "params": { 1611 | "filename": "AnimateDiff_00009.mp4", 1612 | "subfolder": "", 1613 | "type": "temp", 1614 | "format": "video/h264-mp4", 1615 | "frame_rate": 8 1616 | } 1617 | } 1618 | } 1619 | }, 1620 | { 1621 | "id": 53, 1622 | "type": "ADE_UseEvolvedSampling", 1623 | "pos": [ 1624 | 940, 1625 | -1110 1626 | ], 1627 | "size": { 1628 | "0": 235.1999969482422, 1629 | "1": 118 1630 | }, 1631 | "flags": {}, 1632 | "order": 28, 1633 | "mode": 0, 1634 | "inputs": [ 1635 | { 1636 | "name": "model", 1637 | "type": "MODEL", 1638 | "link": 106 1639 | }, 1640 | { 1641 | "name": "m_models", 1642 | "type": "M_MODELS", 1643 | "link": 60 1644 | }, 1645 | { 1646 | "name": "context_options", 1647 | "type": "CONTEXT_OPTIONS", 1648 | "link": 122 1649 | }, 1650 | { 1651 | "name": "sample_settings", 1652 | "type": "SAMPLE_SETTINGS", 1653 | "link": null 1654 | } 1655 | ], 1656 | "outputs": [ 1657 | { 1658 | "name": "MODEL", 1659 | "type": "MODEL", 1660 | "links": [ 1661 | 87 1662 | ], 1663 | "shape": 3, 1664 | "slot_index": 0 1665 | } 1666 | ], 1667 | "properties": { 1668 | "Node name for S&R": "ADE_UseEvolvedSampling" 1669 | }, 1670 | "widgets_values": [ 1671 | "autoselect" 1672 | ] 1673 | } 1674 | ], 1675 | "links": [ 1676 | [ 1677 | 23, 1678 | 11, 1679 | 0, 1680 | 19, 1681 | 0, 1682 | "IMAGE" 1683 | ], 1684 | [ 1685 | 25, 1686 | 25, 1687 | 0, 1688 | 18, 1689 | 0, 1690 | "MODEL" 1691 | ], 1692 | [ 1693 | 26, 1694 | 23, 1695 | 0, 1696 | 26, 1697 | 0, 1698 | "MODEL" 1699 | ], 1700 | [ 1701 | 28, 1702 | 19, 1703 | 0, 1704 | 28, 1705 | 0, 1706 | "IMAGE" 1707 | ], 1708 | [ 1709 | 29, 1710 | 18, 1711 | 0, 1712 | 22, 1713 | 0, 1714 | "MODEL" 1715 | ], 1716 | [ 1717 | 30, 1718 | 28, 1719 | 0, 1720 | 30, 1721 | 0, 1722 | "IMAGE" 1723 | ], 1724 | [ 1725 | 31, 1726 | 30, 1727 | 0, 1728 | 22, 1729 | 1, 1730 | "LMKFEAT" 1731 | ], 1732 | [ 1733 | 32, 1734 | 19, 1735 | 0, 1736 | 31, 1737 | 0, 1738 | "IMAGE" 1739 | ], 1740 | [ 1741 | 33, 1742 | 27, 1743 | 0, 1744 | 32, 1745 | 1, 1746 | "LATENT" 1747 | ], 1748 | [ 1749 | 34, 1750 | 31, 1751 | 0, 1752 | 32, 1753 | 0, 1754 | "LATENT" 1755 | ], 1756 | [ 1757 | 36, 1758 | 32, 1759 | 0, 1760 | 33, 1761 | 5, 1762 | "LATENT" 1763 | ], 1764 | [ 1765 | 44, 1766 | 38, 1767 | 0, 1768 | 35, 1769 | 0, 1770 | "CLIP_VISION" 1771 | ], 1772 | [ 1773 | 45, 1774 | 38, 1775 | 0, 1776 | 39, 1777 | 0, 1778 | "CLIP_VISION" 1779 | ], 1780 | [ 1781 | 46, 1782 | 40, 1783 | 0, 1784 | 39, 1785 | 1, 1786 | "IMAGE" 1787 | ], 1788 | [ 1789 | 47, 1790 | 35, 1791 | 0, 1792 | 42, 1793 | 0, 1794 | "FYECLIPEMBED" 1795 | ], 1796 | [ 1797 | 48, 1798 | 39, 1799 | 0, 1800 | 43, 1801 | 0, 1802 | "FYECLIPEMBED" 1803 | ], 1804 | [ 1805 | 49, 1806 | 45, 1807 | 0, 1808 | 27, 1809 | 3, 1810 | "SAMPLER" 1811 | ], 1812 | [ 1813 | 50, 1814 | 44, 1815 | 0, 1816 | 33, 1817 | 3, 1818 | "SAMPLER" 1819 | ], 1820 | [ 1821 | 51, 1822 | 45, 1823 | 1, 1824 | 27, 1825 | 4, 1826 | "SIGMAS" 1827 | ], 1828 | [ 1829 | 52, 1830 | 42, 1831 | 0, 1832 | 27, 1833 | 1, 1834 | "CONDITIONING" 1835 | ], 1836 | [ 1837 | 56, 1838 | 33, 1839 | 0, 1840 | 49, 1841 | 0, 1842 | "LATENT" 1843 | ], 1844 | [ 1845 | 58, 1846 | 49, 1847 | 0, 1848 | 50, 1849 | 0, 1850 | "IMAGE" 1851 | ], 1852 | [ 1853 | 59, 1854 | 52, 1855 | 0, 1856 | 51, 1857 | 0, 1858 | "MOTION_MODEL_ADE" 1859 | ], 1860 | [ 1861 | 60, 1862 | 51, 1863 | 0, 1864 | 53, 1865 | 1, 1866 | "M_MODELS" 1867 | ], 1868 | [ 1869 | 68, 1870 | 36, 1871 | 0, 1872 | 58, 1873 | 0, 1874 | "IMAGE" 1875 | ], 1876 | [ 1877 | 69, 1878 | 58, 1879 | 0, 1880 | 57, 1881 | 0, 1882 | "IMAGE" 1883 | ], 1884 | [ 1885 | 73, 1886 | 60, 1887 | 0, 1888 | 33, 1889 | 4, 1890 | "SIGMAS" 1891 | ], 1892 | [ 1893 | 82, 1894 | 42, 1895 | 0, 1896 | 65, 1897 | 0, 1898 | "CONDITIONING" 1899 | ], 1900 | [ 1901 | 87, 1902 | 53, 1903 | 0, 1904 | 66, 1905 | 0, 1906 | "MODEL" 1907 | ], 1908 | [ 1909 | 88, 1910 | 66, 1911 | 0, 1912 | 33, 1913 | 0, 1914 | "MODEL" 1915 | ], 1916 | [ 1917 | 98, 1918 | 68, 1919 | 0, 1920 | 45, 1921 | 0, 1922 | "REF_BANK" 1923 | ], 1924 | [ 1925 | 99, 1926 | 68, 1927 | 0, 1928 | 44, 1929 | 0, 1930 | "REF_BANK" 1931 | ], 1932 | [ 1933 | 101, 1934 | 58, 1935 | 0, 1936 | 69, 1937 | 0, 1938 | "IMAGE" 1939 | ], 1940 | [ 1941 | 102, 1942 | 69, 1943 | 0, 1944 | 27, 1945 | 5, 1946 | "LATENT" 1947 | ], 1948 | [ 1949 | 106, 1950 | 22, 1951 | 0, 1952 | 53, 1953 | 0, 1954 | "MODEL" 1955 | ], 1956 | [ 1957 | 108, 1958 | 67, 1959 | 0, 1960 | 31, 1961 | 1, 1962 | "VAE" 1963 | ], 1964 | [ 1965 | 109, 1966 | 67, 1967 | 0, 1968 | 69, 1969 | 1, 1970 | "VAE" 1971 | ], 1972 | [ 1973 | 110, 1974 | 67, 1975 | 0, 1976 | 49, 1977 | 1, 1978 | "VAE" 1979 | ], 1980 | [ 1981 | 114, 1982 | 65, 1983 | 0, 1984 | 33, 1985 | 2, 1986 | "CONDITIONING" 1987 | ], 1988 | [ 1989 | 115, 1990 | 42, 1991 | 0, 1992 | 33, 1993 | 1, 1994 | "CONDITIONING" 1995 | ], 1996 | [ 1997 | 117, 1998 | 42, 1999 | 0, 2000 | 27, 2001 | 2, 2002 | "CONDITIONING" 2003 | ], 2004 | [ 2005 | 119, 2006 | 66, 2007 | 0, 2008 | 60, 2009 | 0, 2010 | "MODEL" 2011 | ], 2012 | [ 2013 | 120, 2014 | 26, 2015 | 0, 2016 | 70, 2017 | 0, 2018 | "MODEL" 2019 | ], 2020 | [ 2021 | 121, 2022 | 70, 2023 | 0, 2024 | 27, 2025 | 0, 2026 | "MODEL" 2027 | ], 2028 | [ 2029 | 122, 2030 | 71, 2031 | 0, 2032 | 53, 2033 | 2, 2034 | "CONTEXT_OPTIONS" 2035 | ], 2036 | [ 2037 | 123, 2038 | 57, 2039 | 0, 2040 | 35, 2041 | 1, 2042 | "IMAGE" 2043 | ] 2044 | ], 2045 | "groups": [], 2046 | "config": {}, 2047 | "extra": { 2048 | "ds": { 2049 | "scale": 0.5559917313492684, 2050 | "offset": [ 2051 | -529.2273525643739, 2052 | 1186.2830152933398 2053 | ] 2054 | } 2055 | }, 2056 | "version": 0.4 2057 | } --------------------------------------------------------------------------------