├── examples
├── i5.jpg
├── jgz.mp4
├── qie.mp4
├── yao.jpg
├── amns.mp4
├── tiktok.mp4
├── toon.png
├── chillout.jpg
├── civitai2.jpg
├── helloicon.png
├── majicmix2.jpg
└── ComfyUI_Manager.jpg
├── example_workflows
├── image_generation.jpg
├── video_generation.jpg
├── image_style_transfer.jpg
├── hellomeme_image_workflow.jpg
├── hellomeme_style_workflow.jpg
├── hellomeme_video_workflow.jpg
├── image_generation.json
├── video_generation.json
└── image_style_transfer.json
├── __init__.py
├── .gitignore
├── requirements.txt
├── hellomeme
├── __init__.py
├── tools
│ ├── __init__.py
│ ├── hello_arkit.py
│ ├── hello_face_alignment.py
│ ├── hello_3dmm.py
│ ├── utils.py
│ ├── pdf.py
│ ├── hello_face_det.py
│ └── sr.py
├── pipelines
│ ├── __init__.py
│ ├── pipline_hm_image.py
│ └── pipline_hm3_image.py
├── models
│ ├── __init__.py
│ ├── hm3_denoising_motion.py
│ ├── hm_denoising_motion.py
│ ├── hm3_denoising_3d.py
│ └── hm_denoising_3d.py
└── model_config.json
├── pyproject.toml
├── .github
└── workflows
│ └── publish.yml
├── LICENSE
└── README.md
/examples/i5.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/i5.jpg
--------------------------------------------------------------------------------
/examples/jgz.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/jgz.mp4
--------------------------------------------------------------------------------
/examples/qie.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/qie.mp4
--------------------------------------------------------------------------------
/examples/yao.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/yao.jpg
--------------------------------------------------------------------------------
/examples/amns.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/amns.mp4
--------------------------------------------------------------------------------
/examples/tiktok.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/tiktok.mp4
--------------------------------------------------------------------------------
/examples/toon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/toon.png
--------------------------------------------------------------------------------
/examples/chillout.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/chillout.jpg
--------------------------------------------------------------------------------
/examples/civitai2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/civitai2.jpg
--------------------------------------------------------------------------------
/examples/helloicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/helloicon.png
--------------------------------------------------------------------------------
/examples/majicmix2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/majicmix2.jpg
--------------------------------------------------------------------------------
/examples/ComfyUI_Manager.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/examples/ComfyUI_Manager.jpg
--------------------------------------------------------------------------------
/example_workflows/image_generation.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/example_workflows/image_generation.jpg
--------------------------------------------------------------------------------
/example_workflows/video_generation.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/example_workflows/video_generation.jpg
--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
1 | from .meme import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
2 |
3 | __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']
--------------------------------------------------------------------------------
/example_workflows/image_style_transfer.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/example_workflows/image_style_transfer.jpg
--------------------------------------------------------------------------------
/example_workflows/hellomeme_image_workflow.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/example_workflows/hellomeme_image_workflow.jpg
--------------------------------------------------------------------------------
/example_workflows/hellomeme_style_workflow.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/example_workflows/hellomeme_style_workflow.jpg
--------------------------------------------------------------------------------
/example_workflows/hellomeme_video_workflow.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HelloVision/ComfyUI_HelloMeme/HEAD/example_workflows/hellomeme_video_workflow.jpg
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.pyd
3 | *.pth
4 | *.pkl
5 | *.pyc*
6 | *.pyd*
7 | *~
8 | *_fps15.mp4
9 | *__pycache__/*
10 | .idea/
11 | data/results
12 | pretrained_models/
13 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | accelerate>=1.1.1
2 | diffusers>=0.33.1
3 | einops>=0.8.0
4 | huggingface-hub>=0.30.2
5 | imageio>=2.36.0
6 | imageio-ffmpeg>=0.5.1
7 | onnxruntime>=1.14.0
8 | opencv-python>=4.10.0.84
9 | peft>=0.15.2
10 | pillow>=10.2.0
11 | safetensors>=0.4.5
12 | tqdm>=4.67.0
13 | transformers>=4.46.3
14 | scikit-image>=0.25.2
15 | modelscope>=1.21.0
16 | torchvision>=0.18.0
--------------------------------------------------------------------------------
/hellomeme/__init__.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : __init__.py.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 8/28/2024
8 | @Desc :
9 | """
10 |
11 | from .pipelines import (HMImagePipeline, HMVideoPipeline,
12 | HM3ImagePipeline, HM3VideoPipeline,
13 | HM5ImagePipeline, HM5VideoPipeline)
14 | from .tools.utils import download_file_from_cloud, creat_model_from_cloud
--------------------------------------------------------------------------------
/hellomeme/tools/__init__.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | # @File : __init__.py
4 | # @Author : Songkey
5 | # @Email : songkey@pku.edu.cn
6 | # @Date : 8/28/2024
7 | # @Desc :
8 |
9 | from .hello_arkit import HelloARKitBSPred
10 | from .hello_face_det import HelloFaceDet
11 | from .hello_camera_demo import HelloCameraDemo
12 | from .hello_3dmm import Hello3DMMPred
13 | from .hello_face_alignment import HelloFaceAlignment
14 | from .pdf import FanEncoder
15 |
--------------------------------------------------------------------------------
/hellomeme/pipelines/__init__.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : __init__.py.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 8/29/2024
8 | @Desc :
9 | """
10 |
11 | from .pipline_hm_image import HMImagePipeline
12 | from .pipline_hm_video import HMVideoPipeline
13 | from .pipline_hm3_image import HM3ImagePipeline
14 | from .pipline_hm3_video import HM3VideoPipeline
15 | from .pipline_hm5_image import HM5ImagePipeline
16 | from .pipline_hm5_video import HM5VideoPipeline
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [project]
2 | name = "comfyui_hellomeme"
3 | description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface"
4 | version = "1.3.28"
5 | license = {file = "LICENSE"}
6 |
7 | [project.urls]
8 | Repository = "https://github.com/HelloVision/ComfyUI_HelloMeme"
9 | # Used by Comfy Registry https://comfyregistry.org
10 |
11 | [tool.comfy]
12 | PublisherId = "hellomeme-api"
13 | DisplayName = "hellomeme-api"
14 | Icon = "https://github.com/HelloVision/ComfyUI_HelloMeme/blob/eee70819a114a44c4c689c865b42fa0e523a31e5/examples/helloicon.png"
15 |
--------------------------------------------------------------------------------
/.github/workflows/publish.yml:
--------------------------------------------------------------------------------
1 | name: Publish to Comfy registry
2 | on:
3 | workflow_dispatch:
4 | push:
5 | branches:
6 | - main
7 | - master
8 | paths:
9 | - "pyproject.toml"
10 |
11 | permissions:
12 | issues: write
13 |
14 | jobs:
15 | publish-node:
16 | name: Publish Custom Node to registry
17 | runs-on: ubuntu-latest
18 | if: ${{ github.repository_owner == 'HelloVision' }}
19 | steps:
20 | - name: Check out code
21 | uses: actions/checkout@v4
22 | - name: Publish Custom Node
23 | uses: Comfy-Org/publish-node-action@v1
24 | with:
25 | ## Add your own personal access token to your Github Repository secrets and reference it here.
26 | personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}
27 |
--------------------------------------------------------------------------------
/hellomeme/models/__init__.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : __init__.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 8/14/2024
8 | @Desc :
9 | """
10 |
11 | from .hm_denoising_motion import HMDenoisingMotion
12 | from .hm_control import (HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2,
13 | HMV3ControlNet, HMControlNetBase, HM5ControlNetBase,
14 | HM4SD15ControlProj, HM5SD15ControlProj)
15 | from .hm_adapters import (HMReferenceAdapter, HM3ReferenceAdapter, HM5ReferenceAdapter, HM5bReferenceAdapter,
16 | HM3MotionAdapter, HM5MotionAdapter, HMPipeline)
17 | from .hm_denoising_3d import HMDenoising3D
18 | from .hm3_denoising_3d import HM3Denoising3D
19 | from .hm3_denoising_motion import HM3DenoisingMotion
20 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024 HelloVision
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/hellomeme/tools/hello_arkit.py:
--------------------------------------------------------------------------------
1 | """
2 | @File : test.py
3 | @Author : Songkey
4 | @Email : songkey@pku.edu.cn
5 | @Date : 11/1/2024
6 | @Desc : Created by Shengjie Wu (wu.shengjie@immomo.com)
7 | """
8 |
9 | import numpy as np
10 | import cv2
11 | import os.path as osp
12 | from .utils import create_onnx_session, get_warp_mat_bbox_by_gt_pts_float, download_file_from_cloud
13 |
14 | class HelloARKitBSPred(object):
15 | def __init__(self, gpu_id=0, modelscope=False):
16 | model_path = download_file_from_cloud(model_id='songkey/hello_group_facemodel',
17 | file_name='hello_arkit_blendshape.onnx',
18 | modelscope=modelscope)
19 |
20 | self.face_rig_net = create_onnx_session(model_path, gpu_id=gpu_id)
21 | self.onnx_input_name = self.face_rig_net.get_inputs()[0].name
22 | self.onnx_output_name = [output.name for output in self.face_rig_net.get_outputs()]
23 | self.image_size = 224
24 | self.expand_ratio = 0.15
25 |
26 | def forward(self, src_image, src_pt):
27 | left_eye_corner = src_pt[74]
28 | right_eye_corner = src_pt[96]
29 | radian = np.arctan2(right_eye_corner[1] - left_eye_corner[1], right_eye_corner[0] - left_eye_corner[0] + 0.00000001)
30 | rotate_angle = np.rad2deg(radian)
31 | align_warp_mat = get_warp_mat_bbox_by_gt_pts_float(src_pt, base_angle=rotate_angle, dst_size=self.image_size,
32 | expand_ratio=self.expand_ratio)
33 | face_rig_input = cv2.warpAffine(src_image, align_warp_mat, (self.image_size, self.image_size))
34 |
35 | face_rig_onnx_input = face_rig_input.transpose((2, 0, 1)).astype(np.float32)[np.newaxis, :, :, :] / 255.0
36 | face_rig_params = self.face_rig_net.run(self.onnx_output_name,
37 | {self.onnx_input_name: face_rig_onnx_input})
38 | face_rig_params = face_rig_params[0][0]
39 | return face_rig_params
40 |
--------------------------------------------------------------------------------
/hellomeme/model_config.json:
--------------------------------------------------------------------------------
1 | {
2 | "sd15": {
3 | "checkpoints": {
4 | "SD1.5": "songkey/stable-diffusion-v1-5",
5 | "[preset]RealisticVisionV60B1": "songkey/realisticVisionV60B1_v51VAE",
6 | "[preset]DisneyPixarCartoonB": "songkey/disney-pixar-cartoon-b",
7 | "[preset]toonyou_beta6": "songkey/toonyou_beta6",
8 | "[preset]LZ_2DCartoon_V2": "songkey/LZ_2DCartoon_V2",
9 | "[preset]meinamix_v12Final": "songkey/meinamix_v12Final",
10 | "[preset]animedark_v10": "songkey/animedark_v10",
11 | "[preset]absolutereality_v181": "songkey/absolutereality_v181",
12 | "[preset]dreamshaper_8": "songkey/dreamshaper_8",
13 | "[preset]epicphotogasm_ultimateFidelity": "songkey/epicphotogasm_ultimateFidelity",
14 | "[preset]epicrealism_naturalSinRC1VAE": "songkey/epicrealism_naturalSinRC1VAE",
15 | "[preset]xxmix9realistic_v40": "songkey/xxmix9realistic_v40",
16 | "[preset]cyberrealistic_v80": "songkey/cyberrealistic_v80",
17 | "[preset]animeanything_v10": "songkey/animeanything_v10"
18 | },
19 | "loras": {
20 | "[preset]BabyFaceV1": ["songkey/loras_sd_1_5", "baby_face_v1.safetensors"],
21 | "[preset]MoreDetails": ["songkey/loras_sd_1_5", "more_details.safetensors"],
22 | "[preset]PixelPortraitV1": ["songkey/loras_sd_1_5", "pixel-portrait-v1.safetensors"],
23 | "[preset]Drawing": ["songkey/loras_sd_1_5", "Drawing.safetensors"],
24 | "[preset]anime_extract": ["songkey/loras_sd_1_5", "anime_extract.safetensors"],
25 | "[preset]animemix_v3_offset": ["songkey/loras_sd_1_5", "animemix_v3_offset.safetensors"],
26 | "[preset]animeoutlineV4_16": ["songkey/loras_sd_1_5", "animeoutlineV4_16.safetensors"],
27 | "[preset]ghibli_anime_v1": ["songkey/loras_sd_1_5", "ghibli_anime_v1.safetensors"],
28 | "[preset]zyd232_InkStyle_v1_0": ["songkey/loras_sd_1_5", "zyd232_InkStyle_v1_0.safetensors"]
29 | }
30 | },
31 | "prompt": "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
32 | "prompt_new": "best quality, highly detailed, masterpiece, person, portrait, headshot, looking at the viewer, sharp focus."
33 | }
--------------------------------------------------------------------------------
/hellomeme/tools/hello_face_alignment.py:
--------------------------------------------------------------------------------
1 | """
2 | @File : test.py
3 | @Author : Songkey
4 | @Email : songkey@pku.edu.cn
5 | @Date : 11/1/2024
6 | @Desc : Created by Shengjie Wu (wu.shengjie@immomo.com)
7 | """
8 |
9 | import cv2
10 | import os.path as osp
11 | import numpy as np
12 | from .hello_face_det import HelloFaceDet
13 | from .utils import get_warp_mat_bbox, get_warp_mat_bbox_by_gt_pts_float, transform_points
14 | from .utils import create_onnx_session, download_file_from_cloud
15 |
16 | class HelloFaceAlignment(object):
17 | def __init__(self, gpu_id=None, modelscope=False):
18 | expand_ratio = 0.15
19 |
20 | alignment_model_path = download_file_from_cloud(model_id='songkey/hello_group_facemodel',
21 | file_name='hello_face_landmark.onnx',
22 | modelscope=modelscope)
23 |
24 | det_model_path = download_file_from_cloud(model_id='songkey/hello_group_facemodel',
25 | file_name='hello_face_det.onnx',
26 | modelscope=modelscope)
27 | self.face_alignment_net_222 = (
28 | create_onnx_session(alignment_model_path, gpu_id=gpu_id))
29 | self.onnx_input_name_222 = self.face_alignment_net_222.get_inputs()[0].name
30 | self.onnx_output_name_222 = [output.name for output in self.face_alignment_net_222.get_outputs()]
31 | self.face_image_size = 128
32 |
33 | self.face_detector = HelloFaceDet(det_model_path, gpu_id=gpu_id)
34 | self.expand_ratio = expand_ratio
35 |
36 | def onnx_infer(self, input_uint8):
37 | assert input_uint8.shape[0] == input_uint8.shape[1] == self.face_image_size
38 | onnx_input = input_uint8.transpose((2, 0, 1)).astype(np.float32)[np.newaxis, :, :, :] / 255.0
39 | landmark, euler, prob = self.face_alignment_net_222.run(self.onnx_output_name_222,
40 | {self.onnx_input_name_222: onnx_input})
41 |
42 | landmark = np.reshape(landmark[0], (2, -1)).transpose((1, 0)) * self.face_image_size
43 | left_eye_corner = landmark[74]
44 | right_eye_corner = landmark[96]
45 | radian = np.arctan2(right_eye_corner[1] - left_eye_corner[1],
46 | right_eye_corner[0] - left_eye_corner[0] + 0.00000001)
47 | euler_rad = np.array([euler[0, 0], euler[0, 1], radian], dtype=np.float32)
48 | prob = prob[0]
49 |
50 | return landmark, euler_rad, prob
51 |
52 | def forward(self, src_image, face_box=None, pre_pts=None, iterations=3):
53 | if pre_pts is None:
54 | if face_box is None:
55 | # Detect max size face
56 | bounding_boxes, _, score = self.face_detector.detect(src_image)
57 | print("facedet score", score)
58 | if len(bounding_boxes) == 0:
59 | return None
60 | bbox = np.zeros(4, dtype=np.float32)
61 | if len(bounding_boxes) >= 1:
62 | max_area = 0.0
63 | for each_bbox in bounding_boxes:
64 | area = (each_bbox[2] - each_bbox[0]) * (each_bbox[3] - each_bbox[1])
65 | if area > max_area:
66 | bbox[:4] = each_bbox[:4]
67 | max_area = area
68 | else:
69 | bbox = bounding_boxes[0, :4]
70 | else:
71 | bbox = face_box.copy()
72 | M_Face = get_warp_mat_bbox(bbox, 0, self.face_image_size, expand_ratio=self.expand_ratio)
73 | else:
74 | left_eye_corner = pre_pts[74]
75 | right_eye_corner = pre_pts[96]
76 |
77 | radian = np.arctan2(right_eye_corner[1] - left_eye_corner[1],
78 | right_eye_corner[0] - left_eye_corner[0] + 0.00000001)
79 | M_Face = get_warp_mat_bbox_by_gt_pts_float(pre_pts, np.rad2deg(radian), self.face_image_size,
80 | expand_ratio=self.expand_ratio)
81 |
82 | face_input = cv2.warpAffine(src_image, M_Face, (self.face_image_size, self.face_image_size))
83 | landmarks, euler, prob = self.onnx_infer(face_input)
84 | landmarks = transform_points(landmarks, M_Face, invert=True)
85 |
86 | # Repeat
87 | for i in range(iterations - 1):
88 | M_Face = get_warp_mat_bbox_by_gt_pts_float(landmarks, np.rad2deg(euler[2]), self.face_image_size,
89 | expand_ratio=self.expand_ratio)
90 | face_input = cv2.warpAffine(src_image, M_Face, (self.face_image_size, self.face_image_size))
91 | landmarks, euler, prob = self.onnx_infer(face_input)
92 | landmarks = transform_points(landmarks, M_Face, invert=True)
93 |
94 | return_dict = {
95 | "pt222": landmarks,
96 | "euler_rad": euler,
97 | "prob": prob,
98 | "M_Face": M_Face,
99 | "face_input": face_input
100 | }
101 |
102 | return return_dict
103 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | HelloMeme: Integrating Spatial Knitting Attentions to Embed High-Level and Fidelity-Rich Conditions in Diffusion Models
2 |
3 |
6 |
7 |
8 | HelloVision | HelloGroup Inc.
9 |
10 |
11 |
12 | * Intern
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
62 | 
63 |
64 |
65 |
66 | ### Workflows
67 |
68 | | workflow file | Video Generation | Image Generation | HMControlNet | HMControlNet2 |
69 | |---------------|------------------|------------------|-----------|---------------|
70 | | image_generation.json | | ✅ | | ✅ |
71 | | image_style_transfer.json | | ✅ | | ✅ |
72 | | video_generation.json | ✅ | | | ✅ |
73 |
74 | ### Image Generation Interface
75 |
76 |
77 | 
78 |
79 |
80 | ### Style Transfer Interface
81 |
82 |
83 | 
84 |
85 |
86 | ### Video Generation Interface
87 |
88 |
89 | 
90 |
91 |
92 |
--------------------------------------------------------------------------------
/hellomeme/tools/hello_3dmm.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : test.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 11/1/2024
8 | @Desc : Created by Shengjie Wu (wu.shengjie@immomo.com)
9 | 这可能是一个很强大的模型
10 | """
11 |
12 | import numpy as np
13 | import cv2
14 | import os.path as osp
15 |
16 | from .utils import get_warp_mat_bbox_by_gt_pts_float, create_onnx_session, download_file_from_cloud
17 |
18 | def crop_transl_to_full_transl(crop_trans, crop_center, scale, full_center, focal_length):
19 | """
20 | :param crop_trans: (3), float
21 | :param crop_center: (2), float
22 | :param scale: (1), float
23 | :param full_center: (2), float
24 | :param focal_length: (1), float
25 | :return:
26 | """
27 | crop_c_x, crop_c_y = crop_center
28 | full_c_x, full_c_y = full_center
29 | bs = 2 * focal_length / scale / crop_trans[2]
30 | full_x = crop_trans[0] - 2 * (crop_c_x - full_c_x) / bs
31 | full_y = crop_trans[1] + 2 * (crop_c_y - full_c_y) / bs
32 | full_z = crop_trans[2] * scale
33 |
34 | full_trans = np.array([full_x, full_y, full_z], dtype=np.float32)
35 |
36 | return full_trans
37 |
38 | class Hello3DMMPred(object):
39 | def __init__(self, gpu_id=None, modelscope=False):
40 | model_path = download_file_from_cloud(model_id='songkey/hello_group_facemodel',
41 | file_name='hello_3dmm.onnx',
42 | modelscope=modelscope)
43 | self.deep3d_pred_net = create_onnx_session(model_path, gpu_id=gpu_id)
44 | self.deep3d_pred_net_input_name = self.deep3d_pred_net.get_inputs()[0].name
45 | self.deep3d_pred_net_output_name = [output.name for output in self.deep3d_pred_net.get_outputs()]
46 |
47 | self.image_size = 224
48 | self.camera_init_z = -0.4
49 | self.camera_init_focal_len = 386.2879122887948
50 | self.used_focal_len = -5.0 / self.camera_init_z * self.camera_init_focal_len
51 | self.id_dims = 526
52 | self.exp_dims = 203
53 | self.tex_dims = 439
54 |
55 | def forward_params(self, src_image, src_pt):
56 | align_mat_info = get_warp_mat_bbox_by_gt_pts_float(src_pt, base_angle=0, dst_size=self.image_size, expand_ratio=0.35, return_info=True)
57 | align_mat = align_mat_info["M"]
58 |
59 | align_image_rgb_uint8 = cv2.cvtColor(cv2.warpAffine(src_image, align_mat, (self.image_size, self.image_size)), cv2.COLOR_BGR2RGB)
60 |
61 | # cv2.imshow('align_image_rgb_uint8', align_image_rgb_uint8)
62 |
63 | align_image_rgb_fp32 = align_image_rgb_uint8.astype(np.float32) / 255.0
64 | align_image_rgb_fp32_onnx_input = align_image_rgb_fp32.copy().transpose((2, 0, 1))[np.newaxis, ...]
65 | pred_coeffs = self.deep3d_pred_net.run(self.deep3d_pred_net_output_name,
66 | {self.deep3d_pred_net_input_name: align_image_rgb_fp32_onnx_input})[0]
67 |
68 | angles = pred_coeffs[:, self.id_dims + self.exp_dims + self.tex_dims:self.id_dims + self.exp_dims + self.tex_dims + 3]
69 | translations = pred_coeffs[:, self.id_dims + self.exp_dims + self.tex_dims + 3 + 27:]
70 |
71 | crop_global_transl = crop_transl_to_full_transl(translations[0],
72 | crop_center=[align_mat_info["center_x"],
73 | align_mat_info["center_y"]],
74 | scale=align_mat_info["scale"],
75 | full_center=[src_image.shape[1] * 0.5, src_image.shape[0] * 0.5],
76 | focal_length=self.used_focal_len)
77 | return angles, crop_global_transl[np.newaxis, :]
78 |
79 | def compute_rotation_matrix(angles):
80 | n_b = angles.shape[0]
81 | sinx = np.sin(angles[:, 0])
82 | siny = np.sin(angles[:, 1])
83 | sinz = np.sin(angles[:, 2])
84 | cosx = np.cos(angles[:, 0])
85 | cosy = np.cos(angles[:, 1])
86 | cosz = np.cos(angles[:, 2])
87 | rotXYZ = np.eye(3).reshape(1, 3, 3).repeat(n_b*3, 0).reshape(3, n_b, 3, 3)
88 | rotXYZ[0, :, 1, 1] = cosx
89 | rotXYZ[0, :, 1, 2] = -sinx
90 | rotXYZ[0, :, 2, 1] = sinx
91 | rotXYZ[0, :, 2, 2] = cosx
92 | rotXYZ[1, :, 0, 0] = cosy
93 | rotXYZ[1, :, 0, 2] = siny
94 | rotXYZ[1, :, 2, 0] = -siny
95 | rotXYZ[1, :, 2, 2] = cosy
96 | rotXYZ[2, :, 0, 0] = cosz
97 | rotXYZ[2, :, 0, 1] = -sinz
98 | rotXYZ[2, :, 1, 0] = sinz
99 | rotXYZ[2, :, 1, 1] = cosz
100 | rotation = np.matmul(np.matmul(rotXYZ[2], rotXYZ[1]), rotXYZ[0])
101 | return rotation.transpose(0, 2, 1)
102 |
103 | def rigid_transform(vs, rot, trans):
104 | vs_r = np.matmul(vs, rot)
105 | vs_t = vs_r + trans.reshape(-1, 1, 3)
106 | return vs_t
107 |
108 | def perspective_projection_points(points, image_w, image_h, used_focal_len):
109 | batch_size = points.shape[0]
110 | K = np.zeros([batch_size, 3, 3])
111 | K[:, 0, 0] = used_focal_len
112 | K[:, 1, 1] = used_focal_len
113 | K[:, 2, 2] = 1.
114 | K[:, 0, 2] = image_w * 0.5
115 | K[:, 1, 2] = image_h * 0.5
116 |
117 | reverse_z = np.array([[1, 0, 0], [0, 1, 0], [0, 0, -1]])[np.newaxis, :, :].repeat(batch_size, 0)
118 |
119 | # Transform points
120 | aug_projection = np.matmul(points, reverse_z)
121 | aug_projection = np.matmul(aug_projection, K.transpose((0, 2, 1)))
122 |
123 | # Apply perspective distortion
124 | projected_points = aug_projection[:, :, :2] / aug_projection[:, :, 2:]
125 | return projected_points
126 |
127 | def get_project_points_rect(angle, trans, image_w, image_h, used_focal_len=4828.598903609935):
128 | vs = np.array(
129 | [[-1, -1, 0], [-1, 1, 0], [1, 1, 0], [1, -1, 0]],
130 | ) * 0.05
131 | vs = vs[np.newaxis, :, :]
132 |
133 | rotation = compute_rotation_matrix(angle)
134 | translation = trans.copy()
135 | translation[0, 2] *= 0.05
136 |
137 | vs_t = rigid_transform(vs, rotation, translation)
138 |
139 | project_points = perspective_projection_points(vs_t, image_w, image_h, used_focal_len*0.05)
140 | project_points = np.stack([project_points[:, :, 0], image_h - project_points[:, :, 1]], axis=2)
141 |
142 | return project_points[0]
143 |
144 |
--------------------------------------------------------------------------------
/hellomeme/tools/utils.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | # @File : utils.py
4 | # @Author : Songkey
5 | # @Email : songkey@pku.edu.cn
6 | # @Date : 8/18/2024
7 | # @Desc :
8 |
9 | import onnxruntime
10 | import time
11 | import cv2
12 | import numpy as np
13 | import math
14 | import os.path as osp
15 |
16 | def create_onnx_session(onnx_path, gpu_id=None)->onnxruntime.InferenceSession:
17 | start = time.perf_counter()
18 | providers = [
19 | ('CUDAExecutionProvider', {
20 | 'device_id': int(gpu_id),
21 | 'arena_extend_strategy': 'kNextPowerOfTwo',
22 | #'cuda_mem_limit': 5 * 1024 * 1024 * 1024,
23 | 'cudnn_conv_algo_search': 'EXHAUSTIVE',
24 | 'do_copy_in_default_stream': True,
25 | }),
26 | 'CPUExecutionProvider',
27 | ] if (gpu_id is not None and gpu_id >= 0) else ['CPUExecutionProvider']
28 |
29 | sess = onnxruntime.InferenceSession(onnx_path, providers=providers)
30 | print('create onnx session cost: {:.3f}s. {}'.format(time.perf_counter() - start, onnx_path))
31 | return sess
32 |
33 | def smoothing_factor(t_e, cutoff):
34 | r = 2 * math.pi * cutoff * t_e
35 | return r / (r + 1)
36 |
37 | def exponential_smoothing(a, x, x_prev):
38 | return a * x + (1 - a) * x_prev
39 |
40 | class OneEuroFilter:
41 | def __init__(self, dx0=0.0, d_cutoff=1.0):
42 | """Initialize the one euro filter."""
43 | # self.min_cutoff = float(min_cutoff)
44 | # self.beta = float(beta)
45 | self.d_cutoff = float(d_cutoff)
46 | self.dx_prev = float(dx0)
47 | # self.t_e = fcmin
48 |
49 | def __call__(self, x, x_prev, fcmin=1.0, min_cutoff=1.0, beta=0.0):
50 | if x_prev is None:
51 | return x
52 | # t_e = 1
53 | a_d = smoothing_factor(fcmin, self.d_cutoff)
54 | dx = (x - x_prev) / fcmin
55 | dx_hat = exponential_smoothing(a_d, dx, self.dx_prev)
56 | cutoff = min_cutoff + beta * abs(dx_hat)
57 | a = smoothing_factor(fcmin, cutoff)
58 | x_hat = exponential_smoothing(a, x, x_prev)
59 | self.dx_prev = dx_hat
60 | return x_hat
61 |
62 | def get_warp_mat_bbox(face_bbox, base_angle, dst_size=128, expand_ratio=0.15, aug_angle=0.0, aug_scale=1.0):
63 | face_x_min, face_y_min, face_x_max, face_y_max = face_bbox
64 | face_x_center = (face_x_min + face_x_max) / 2
65 | face_y_center = (face_y_min + face_y_max) / 2
66 | face_width = face_x_max - face_x_min
67 | face_height = face_y_max - face_y_min
68 | scale = dst_size / max(face_width, face_height) * (1 - expand_ratio) * aug_scale
69 | M = cv2.getRotationMatrix2D((face_x_center, face_y_center), angle=base_angle + aug_angle, scale=scale)
70 | offset = [dst_size / 2 - face_x_center, dst_size / 2 - face_y_center]
71 | M[:, 2] += offset
72 | return M
73 |
74 | def transform_points(points, mat, invert=False):
75 | if invert:
76 | mat = cv2.invertAffineTransform(mat)
77 | points = np.expand_dims(points, axis=1)
78 | points = cv2.transform(points, mat, points.shape)
79 | points = np.squeeze(points)
80 | return points
81 |
82 | def get_warp_mat_bbox_by_gt_pts_float(gt_pts, base_angle=0.0, dst_size=128, expand_ratio=0.15, return_info=False):
83 | # step 1
84 | face_x_min, face_x_max = np.min(gt_pts[:, 0]), np.max(gt_pts[:, 0])
85 | face_y_min, face_y_max = np.min(gt_pts[:, 1]), np.max(gt_pts[:, 1])
86 | face_x_center = (face_x_min + face_x_max) / 2
87 | face_y_center = (face_y_min + face_y_max) / 2
88 | M_step_1 = cv2.getRotationMatrix2D((face_x_center, face_y_center), angle=base_angle, scale=1.0)
89 | pts_step_1 = transform_points(gt_pts, M_step_1)
90 | face_x_min_step_1, face_x_max_step_1 = np.min(pts_step_1[:, 0]), np.max(pts_step_1[:, 0])
91 | face_y_min_step_1, face_y_max_step_1 = np.min(pts_step_1[:, 1]), np.max(pts_step_1[:, 1])
92 | # step 2
93 | face_width = face_x_max_step_1 - face_x_min_step_1
94 | face_height = face_y_max_step_1 - face_y_min_step_1
95 | scale = dst_size / max(face_width, face_height) * (1 - expand_ratio)
96 | M_step_2 = cv2.getRotationMatrix2D((face_x_center, face_y_center), angle=base_angle, scale=scale)
97 | pts_step_2 = transform_points(gt_pts, M_step_2)
98 | face_x_min_step_2, face_x_max_step_2 = np.min(pts_step_2[:, 0]), np.max(pts_step_2[:, 0])
99 | face_y_min_step_2, face_y_max_step_2 = np.min(pts_step_2[:, 1]), np.max(pts_step_2[:, 1])
100 | face_x_center_step_2 = (face_x_min_step_2 + face_x_max_step_2) / 2
101 | face_y_center_step_2 = (face_y_min_step_2 + face_y_max_step_2) / 2
102 |
103 | M = cv2.getRotationMatrix2D((face_x_center, face_y_center), angle=base_angle, scale=scale)
104 | offset = [dst_size / 2 - face_x_center_step_2, dst_size / 2 - face_y_center_step_2]
105 | M[:, 2] += offset
106 |
107 | if not return_info:
108 | return M
109 | else:
110 | transform_info = {
111 | "M": M,
112 | "center_x": face_x_center,
113 | "center_y": face_y_center,
114 | "rotate_angle": base_angle,
115 | "scale": scale
116 | }
117 | return transform_info
118 |
119 |
120 | def download_file_from_cloud(model_id,
121 | file_name,
122 | modelscope=False,
123 | cache_dir=None,
124 | hf_token=None):
125 | if modelscope:
126 | from modelscope import snapshot_download
127 | try:
128 | model_path = osp.join(snapshot_download(model_id, cache_dir=cache_dir), file_name)
129 | except Exception as e:
130 | print(e)
131 | assert False, "@@ Failed to download model from modelscope (using `hugginface`)"
132 | else:
133 | from huggingface_hub import hf_hub_download
134 | try:
135 | model_path = hf_hub_download(model_id, filename=file_name, cache_dir=cache_dir, token=hf_token)
136 | except Exception as e:
137 | print(e)
138 | assert False, "@@ `huggingface-cli login` or using `modelscope`"
139 | return model_path
140 |
141 | def creat_model_from_cloud(model_cls,
142 | model_id,
143 | modelscope=False,
144 | cache_dir=None,
145 | subfolder=None,
146 | hf_token=None):
147 | if osp.isdir(model_id):
148 | model = model_cls.from_pretrained(model_id)
149 | elif osp.isfile(model_id) and model_id.endswith('.safetensors'):
150 | model = model_cls.from_single_file(model_id)
151 | else:
152 | if modelscope:
153 | from modelscope import snapshot_download
154 | try:
155 | model_path = snapshot_download(model_id, cache_dir=cache_dir)
156 | except Exception as e:
157 | print(e)
158 | assert False, "@@ Failed to download model from modelscope (using `hugginface`)"
159 |
160 | if subfolder is None:
161 | model = model_cls.from_pretrained(model_path)
162 | else:
163 | model = model_cls.from_pretrained(model_path, subfolder=subfolder)
164 | else:
165 | try:
166 | if subfolder is None:
167 | model = model_cls.from_pretrained(model_id, cache_dir=cache_dir, token=hf_token)
168 | else:
169 | model = model_cls.from_pretrained(model_id, subfolder=subfolder, cache_dir=cache_dir, token=hf_token)
170 | except Exception as e:
171 | print(e)
172 | assert False, "@@ `huggingface-cli login` or using `modelscope`"
173 | return model
--------------------------------------------------------------------------------
/hellomeme/tools/pdf.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : pdf.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 11/7/2024
8 | @Desc : Adapted from: https://github.com/Dorniwang/PD-FGC-inference/blob/main/lib/models/networks/encoder.py
9 | """
10 |
11 | import torch
12 | import torch.nn as nn
13 | import torch.nn.functional as F
14 |
15 | from diffusers.models.modeling_utils import ModelMixin
16 | from diffusers.configuration_utils import ConfigMixin, register_to_config
17 |
18 | def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):
19 | "3x3 convolution with padding"
20 | return nn.Conv2d(in_planes, out_planes, kernel_size=3,
21 | stride=strd, padding=padding, bias=bias)
22 |
23 | class ConvBlock(nn.Module):
24 | def __init__(self, in_planes, out_planes):
25 | super(ConvBlock, self).__init__()
26 | self.bn1 = nn.BatchNorm2d(in_planes)
27 | self.conv1 = conv3x3(in_planes, int(out_planes / 2))
28 | self.bn2 = nn.BatchNorm2d(int(out_planes / 2))
29 | self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))
30 | self.bn3 = nn.BatchNorm2d(int(out_planes / 4))
31 | self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))
32 |
33 | if in_planes != out_planes:
34 | self.downsample = nn.Sequential(
35 | nn.BatchNorm2d(in_planes),
36 | nn.ReLU(True),
37 | nn.Conv2d(in_planes, out_planes,
38 | kernel_size=1, stride=1, bias=False),
39 | )
40 | else:
41 | self.downsample = None
42 |
43 | def forward(self, x):
44 | residual = x
45 |
46 | out1 = self.bn1(x)
47 | out1 = F.relu(out1, True)
48 | out1 = self.conv1(out1)
49 |
50 | out2 = self.bn2(out1)
51 | out2 = F.relu(out2, True)
52 | out2 = self.conv2(out2)
53 |
54 | out3 = self.bn3(out2)
55 | out3 = F.relu(out3, True)
56 | out3 = self.conv3(out3)
57 |
58 | out3 = torch.cat((out1, out2, out3), 1)
59 |
60 | if self.downsample is not None:
61 | residual = self.downsample(residual)
62 |
63 | out3 += residual
64 |
65 | return out3
66 |
67 |
68 | class HourGlass(nn.Module):
69 | def __init__(self, num_modules, depth, num_features):
70 | super(HourGlass, self).__init__()
71 | self.num_modules = num_modules
72 | self.depth = depth
73 | self.features = num_features
74 | self.dropout = nn.Dropout(0.5)
75 |
76 | self._generate_network(self.depth)
77 |
78 | def _generate_network(self, level):
79 | self.add_module('b1_' + str(level), ConvBlock(256, 256))
80 |
81 | self.add_module('b2_' + str(level), ConvBlock(256, 256))
82 |
83 | if level > 1:
84 | self._generate_network(level - 1)
85 | else:
86 | self.add_module('b2_plus_' + str(level), ConvBlock(256, 256))
87 |
88 | self.add_module('b3_' + str(level), ConvBlock(256, 256))
89 |
90 | def _forward(self, level, inp):
91 | # Upper branch
92 | up1 = inp
93 | up1 = self._modules['b1_' + str(level)](up1)
94 | up1 = self.dropout(up1)
95 | # Lower branch
96 | low1 = F.max_pool2d(inp, 2, stride=2)
97 | low1 = self._modules['b2_' + str(level)](low1)
98 |
99 | if level > 1:
100 | low2 = self._forward(level - 1, low1)
101 | else:
102 | low2 = low1
103 | low2 = self._modules['b2_plus_' + str(level)](low2)
104 |
105 | low3 = low2
106 | low3 = self._modules['b3_' + str(level)](low3)
107 | up1size = up1.size()
108 | rescale_size = (up1size[2], up1size[3])
109 | up2 = F.interpolate(low3, size=rescale_size, mode='bilinear')
110 |
111 | return up1 + up2
112 |
113 | def forward(self, x):
114 | return self._forward(self.depth, x)
115 |
116 | class FAN_use(nn.Module):
117 | def __init__(self):
118 | super(FAN_use, self).__init__()
119 | self.num_modules = 1
120 |
121 | # Base part
122 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)
123 | self.bn1 = nn.BatchNorm2d(64)
124 | self.conv2 = ConvBlock(64, 128)
125 | self.conv3 = ConvBlock(128, 128)
126 | self.conv4 = ConvBlock(128, 256)
127 |
128 | # Stacking part
129 | hg_module = 0
130 | self.add_module('m' + str(hg_module), HourGlass(1, 4, 256))
131 | self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256))
132 | self.add_module('conv_last' + str(hg_module),
133 | nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
134 | self.add_module('l' + str(hg_module), nn.Conv2d(256,
135 | 68, kernel_size=1, stride=1, padding=0))
136 | self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))
137 |
138 | if hg_module < self.num_modules - 1:
139 | self.add_module(
140 | 'bl' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
141 | self.add_module('al' + str(hg_module), nn.Conv2d(68,
142 | 256, kernel_size=1, stride=1, padding=0))
143 |
144 | self.avgpool = nn.MaxPool2d((2, 2), 2)
145 | self.conv6 = nn.Conv2d(68, 1, 3, 2, 1)
146 | self.fc = nn.Linear(28 * 28, 512)
147 | self.bn5 = nn.BatchNorm2d(68)
148 | self.relu = nn.ReLU(True)
149 |
150 | def forward(self, x):
151 | x = F.relu(self.bn1(self.conv1(x)), True)
152 | x = F.max_pool2d(self.conv2(x), 2)
153 | x = self.conv3(x)
154 | x = self.conv4(x)
155 |
156 | previous = x
157 |
158 | i = 0
159 | hg = self._modules['m' + str(i)](previous)
160 |
161 | ll = hg
162 | ll = self._modules['top_m_' + str(i)](ll)
163 |
164 | ll = self._modules['bn_end' + str(i)](self._modules['conv_last' + str(i)](ll))
165 | tmp_out = self._modules['l' + str(i)](F.relu(ll))
166 |
167 | net = self.relu(self.bn5(tmp_out))
168 | net = self.conv6(net)
169 | net = net.view(-1, net.shape[-2] * net.shape[-1])
170 | net = self.relu(net)
171 | net = self.fc(net)
172 | return net
173 |
174 | class FanEncoder(ModelMixin, ConfigMixin):
175 | @register_to_config
176 | def __init__(self, pose_dim=6, eye_dim=6):
177 | super().__init__()
178 |
179 | self.model = FAN_use()
180 |
181 | self.to_mouth = nn.Sequential(nn.Linear(512, 512), nn.ReLU(), nn.BatchNorm1d(512), nn.Linear(512, 512))
182 | self.mouth_embed = nn.Sequential(nn.ReLU(), nn.Linear(512, 512 - pose_dim - eye_dim))
183 |
184 | # self.to_headpose = nn.Sequential(nn.Linear(512, 512), nn.ReLU(), nn.BatchNorm1d(512), nn.Linear(512, 512))
185 | # self.headpose_embed = nn.Sequential(nn.ReLU(), nn.Linear(512, pose_dim))
186 |
187 | self.to_eye = nn.Sequential(nn.Linear(512, 512), nn.ReLU(), nn.BatchNorm1d(512), nn.Linear(512, 512))
188 | self.eye_embed = nn.Sequential(nn.ReLU(), nn.Linear(512, eye_dim))
189 |
190 | self.to_emo = nn.Sequential(nn.Linear(512, 512), nn.ReLU(), nn.BatchNorm1d(512), nn.Linear(512, 512))
191 | self.emo_embed = nn.Sequential(nn.ReLU(), nn.Linear(512, 30))
192 |
193 | def forward_feature(self, x):
194 | net = self.model(x)
195 | return net
196 |
197 | def forward(self, x):
198 | x = self.model(x)
199 | mouth_feat = self.to_mouth(x)
200 | # headpose_feat = self.to_headpose(x)
201 | # headpose_emb = self.headpose_embed(headpose_feat)
202 | eye_feat = self.to_eye(x)
203 | eye_embed = self.eye_embed(eye_feat)
204 | emo_feat = self.to_emo(x)
205 | emo_embed = self.emo_embed(emo_feat)
206 |
207 | return torch.cat([eye_embed, emo_embed, mouth_feat], dim=1)
208 | # return headpose_emb, eye_embed, emo_embed, mouth_feat
209 |
--------------------------------------------------------------------------------
/hellomeme/models/hm3_denoising_motion.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : models6/hm_denoising_motion.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 1/3/2025
8 | @Desc :
9 | adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unets/unet_motion_model.py
10 | """
11 |
12 | import torch
13 | import torch.utils.checkpoint
14 | from typing import Any, Dict, Optional, Tuple, Union
15 |
16 | from einops import rearrange
17 |
18 | from diffusers.utils import logging
19 | from diffusers.models.unets.unet_motion_model import UNetMotionModel, UNetMotionOutput
20 | from .hm_adapters import InsertReferenceAdapter
21 |
22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name
23 |
24 |
25 | class HM3DenoisingMotion(UNetMotionModel, InsertReferenceAdapter):
26 | def forward(
27 | self,
28 | sample: torch.Tensor,
29 | timestep: Union[torch.Tensor, float, int],
30 | encoder_hidden_states: torch.Tensor,
31 | reference_hidden_states: Optional[dict] = None,
32 | control_hidden_states: Optional[torch.Tensor] = None,
33 | motion_pad_hidden_states: Optional[dict] = None,
34 | use_motion: bool = False,
35 | timestep_cond: Optional[torch.Tensor] = None,
36 | attention_mask: Optional[torch.Tensor] = None,
37 | cross_attention_kwargs: Optional[Dict[str, Any]] = None,
38 | added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
39 | down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
40 | mid_block_additional_residual: Optional[torch.Tensor] = None,
41 | return_dict: bool = True,
42 | ) -> Union[UNetMotionOutput, Tuple[torch.Tensor]]:
43 |
44 | # By default samples have to be AT least a multiple of the overall upsampling factor.
45 | # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
46 | # However, the upsampling interpolation output size can be forced to fit any upsampling size
47 | # on the fly if necessary.
48 | default_overall_up_factor = 2 ** self.num_upsamplers
49 |
50 | # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
51 | forward_upsample_size = False
52 | upsample_size = None
53 |
54 | if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
55 | logger.info("Forward upsample size to force interpolation output size.")
56 | forward_upsample_size = True
57 |
58 | # prepare attention_mask
59 | if attention_mask is not None:
60 | attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
61 | attention_mask = attention_mask.unsqueeze(1)
62 |
63 | # 1. time
64 | timesteps = timestep
65 | if not torch.is_tensor(timesteps):
66 | # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
67 | # This would be a good case for the `match` statement (Python 3.10+)
68 | is_mps = sample.device.type == "mps"
69 | if isinstance(timestep, float):
70 | dtype = torch.float32 if is_mps else torch.float64
71 | else:
72 | dtype = torch.int32 if is_mps else torch.int64
73 | timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
74 | elif len(timesteps.shape) == 0:
75 | timesteps = timesteps[None].to(sample.device)
76 |
77 | # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
78 | num_frames = sample.shape[2]
79 | timesteps = timesteps.expand(sample.shape[0])
80 |
81 | t_emb = self.time_proj(timesteps)
82 |
83 | # timesteps does not contain any weights and will always return f32 tensors
84 | # but time_embedding might actually be running in fp16. so we need to cast here.
85 | # there might be better ways to encapsulate this.
86 | t_emb = t_emb.to(dtype=self.dtype)
87 |
88 | emb = self.time_embedding(t_emb, timestep_cond)
89 | aug_emb = None
90 |
91 | if self.config.addition_embed_type == "text_time":
92 | if "text_embeds" not in added_cond_kwargs:
93 | raise ValueError(
94 | f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"
95 | )
96 |
97 | text_embeds = added_cond_kwargs.get("text_embeds")
98 | if "time_ids" not in added_cond_kwargs:
99 | raise ValueError(
100 | f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"
101 | )
102 | time_ids = added_cond_kwargs.get("time_ids")
103 | time_embeds = self.add_time_proj(time_ids.flatten())
104 | time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))
105 |
106 | add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)
107 | add_embeds = add_embeds.to(emb.dtype)
108 | aug_emb = self.add_embedding(add_embeds)
109 |
110 | emb = emb if aug_emb is None else emb + aug_emb
111 | emb = emb.repeat_interleave(repeats=num_frames, dim=0)
112 |
113 | if len(encoder_hidden_states.shape) == 3:
114 | encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0)
115 | elif len(encoder_hidden_states.shape) == 4:
116 | encoder_hidden_states = rearrange(encoder_hidden_states, "b f l d -> (b f) l d")
117 |
118 | if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj":
119 | if "image_embeds" not in added_cond_kwargs:
120 | raise ValueError(
121 | f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
122 | )
123 | image_embeds = added_cond_kwargs.get("image_embeds")
124 | image_embeds = self.encoder_hid_proj(image_embeds)
125 | image_embeds = [image_embed.repeat_interleave(repeats=num_frames, dim=0) for image_embed in image_embeds]
126 | encoder_hidden_states = (encoder_hidden_states, image_embeds)
127 |
128 | # 2. pre-process
129 | sample = sample.permute(0, 2, 1, 3, 4).reshape((sample.shape[0] * num_frames, -1) + sample.shape[3:])
130 | sample = self.conv_in(sample)
131 |
132 | # 3. down
133 | down_block_res_samples = (sample,)
134 | for idx, downsample_block in enumerate(self.down_blocks):
135 | if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
136 | sample, res_samples = downsample_block(
137 | hidden_states=sample,
138 | temb=emb,
139 | encoder_hidden_states=encoder_hidden_states,
140 | attention_mask=attention_mask,
141 | num_frames=num_frames,
142 | cross_attention_kwargs=cross_attention_kwargs,
143 | )
144 | else:
145 | sample, res_samples = downsample_block(hidden_states=sample, temb=emb, num_frames=num_frames)
146 |
147 | if not control_hidden_states is None and f'down3_{idx}' in control_hidden_states:
148 | sample += rearrange(control_hidden_states[f'down3_{idx}'], "b c f h w -> (b f) c h w")
149 | if hasattr(self, 'motion_down') and use_motion:
150 | sample = self.motion_down[idx](sample, motion_pad_hidden_states[f'down_{idx}'], emb, num_frames)
151 |
152 | down_block_res_samples += res_samples
153 |
154 | if down_block_additional_residuals is not None:
155 | new_down_block_res_samples = ()
156 |
157 | for down_block_res_sample, down_block_additional_residual in zip(
158 | down_block_res_samples, down_block_additional_residuals
159 | ):
160 | down_block_res_sample = down_block_res_sample + down_block_additional_residual
161 | new_down_block_res_samples += (down_block_res_sample,)
162 |
163 | down_block_res_samples = new_down_block_res_samples
164 |
165 | # 4. mid
166 | if self.mid_block is not None:
167 | # To support older versions of motion modules that don't have a mid_block
168 | if hasattr(self.mid_block, "motion_modules"):
169 | sample = self.mid_block(
170 | sample,
171 | emb,
172 | encoder_hidden_states=encoder_hidden_states,
173 | attention_mask=attention_mask,
174 | num_frames=num_frames,
175 | cross_attention_kwargs=cross_attention_kwargs,
176 | )
177 | else:
178 | sample = self.mid_block(
179 | sample,
180 | emb,
181 | encoder_hidden_states=encoder_hidden_states,
182 | attention_mask=attention_mask,
183 | cross_attention_kwargs=cross_attention_kwargs,
184 | )
185 |
186 | if mid_block_additional_residual is not None:
187 | sample = sample + mid_block_additional_residual
188 |
189 | # 5. up
190 | for i, upsample_block in enumerate(self.up_blocks):
191 | is_final_block = i == len(self.up_blocks) - 1
192 |
193 | res_samples = down_block_res_samples[-len(upsample_block.resnets):]
194 | down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
195 |
196 | # if we have not reached the final block and need to forward the
197 | # upsample size, we do it here
198 | if not is_final_block and forward_upsample_size:
199 | upsample_size = down_block_res_samples[-1].shape[2:]
200 |
201 | if not control_hidden_states is None and f'up3_{i}' in control_hidden_states:
202 | sample += rearrange(control_hidden_states[f'up3_{i}'], "b c f h w -> (b f) c h w")
203 | if hasattr(self, "reference_modules_up") and not reference_hidden_states is None and f'up_{i}' in reference_hidden_states:
204 | sample = self.reference_modules_up[i](sample, reference_hidden_states[f'up_{i}'], num_frames)
205 | if hasattr(self, 'motion_up') and use_motion:
206 | sample = self.motion_up[i](sample, motion_pad_hidden_states[f'up_{i}'], emb, num_frames)
207 |
208 | if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
209 | sample = upsample_block(
210 | hidden_states=sample,
211 | temb=emb,
212 | res_hidden_states_tuple=res_samples,
213 | encoder_hidden_states=encoder_hidden_states,
214 | upsample_size=upsample_size,
215 | attention_mask=attention_mask,
216 | num_frames=num_frames,
217 | cross_attention_kwargs=cross_attention_kwargs,
218 | )
219 | else:
220 | sample = upsample_block(
221 | hidden_states=sample,
222 | temb=emb,
223 | res_hidden_states_tuple=res_samples,
224 | upsample_size=upsample_size,
225 | num_frames=num_frames,
226 | )
227 |
228 | # 6. post-process
229 | if self.conv_norm_out:
230 | sample = self.conv_norm_out(sample)
231 | sample = self.conv_act(sample)
232 |
233 | sample = self.conv_out(sample)
234 |
235 | # reshape to (batch, channel, framerate, width, height)
236 | sample = sample[None, :].reshape((-1, num_frames) + sample.shape[1:]).permute(0, 2, 1, 3, 4)
237 |
238 | if not return_dict:
239 | return (sample,)
240 |
241 | return UNetMotionOutput(sample=sample)
--------------------------------------------------------------------------------
/hellomeme/tools/hello_face_det.py:
--------------------------------------------------------------------------------
1 | """
2 | @File : test.py
3 | @Author : Songkey
4 | @Email : songkey@pku.edu.cn
5 | @Date : 11/1/2024
6 | @Desc : Created by Zemin An (an.zemin@hellogroup.com)
7 | """
8 |
9 | from abc import ABCMeta, abstractmethod
10 | import cv2
11 | import numpy as np
12 | from scipy.special import softmax
13 | import os.path as osp
14 | from .utils import create_onnx_session
15 |
16 | songkey_weights_dir = 'pretrained_models'
17 |
18 | _COLORS = (
19 | np.array(
20 | [
21 | 0.000,
22 | 0.447,
23 | 0.741,
24 | ]
25 | )
26 | .astype(np.float32)
27 | .reshape(-1, 3)
28 | )
29 |
30 | def get_resize_matrix(raw_shape, dst_shape, keep_ratio):
31 | """
32 | Get resize matrix for resizing raw img to input size
33 | :param raw_shape: (width, height) of raw image
34 | :param dst_shape: (width, height) of input image
35 | :param keep_ratio: whether keep original ratio
36 | :return: 3x3 Matrix
37 | """
38 | r_w, r_h = raw_shape
39 | d_w, d_h = dst_shape
40 | Rs = np.eye(3)
41 | if keep_ratio:
42 | C = np.eye(3)
43 | C[0, 2] = -r_w / 2
44 | C[1, 2] = -r_h / 2
45 |
46 | if r_w / r_h < d_w / d_h:
47 | ratio = d_h / r_h
48 | else:
49 | ratio = d_w / r_w
50 | Rs[0, 0] *= ratio
51 | Rs[1, 1] *= ratio
52 |
53 | T = np.eye(3)
54 | T[0, 2] = 0.5 * d_w
55 | T[1, 2] = 0.5 * d_h
56 | return T @ Rs @ C
57 | else:
58 | Rs[0, 0] *= d_w / r_w
59 | Rs[1, 1] *= d_h / r_h
60 | return Rs
61 |
62 | def warp_boxes(boxes, M, width, height):
63 | """Apply transform to boxes
64 | Copy from nanodet/data/transform/warp.py
65 | """
66 | n = len(boxes)
67 | if n:
68 | # warp points
69 | xy = np.ones((n * 4, 3))
70 | xy[:, :2] = boxes[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(
71 | n * 4, 2
72 | ) # x1y1, x2y2, x1y2, x2y1
73 | xy = xy @ M.T # transform
74 | xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) # rescale
75 | # create new boxes
76 | x = xy[:, [0, 2, 4, 6]]
77 | y = xy[:, [1, 3, 5, 7]]
78 | xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T
79 | # clip boxes
80 | xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)
81 | xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)
82 | return xy.astype(np.float32)
83 | else:
84 | return boxes
85 |
86 | def overlay_bbox_cv(img, all_box, class_names):
87 | """Draw result boxes
88 | Copy from nanodet/util/visualization.py
89 | """
90 | # all_box array of [label, x0, y0, x1, y1, score]
91 | all_box.sort(key=lambda v: v[5])
92 | for box in all_box:
93 | label, x0, y0, x1, y1, score = box
94 | # color = self.cmap(i)[:3]
95 | color = (_COLORS[label] * 255).astype(np.uint8).tolist()
96 | text = "{}:{:.1f}%".format(class_names[label], score * 100)
97 | txt_color = (0, 0, 0) if np.mean(_COLORS[label]) > 0.5 else (255, 255, 255)
98 | font = cv2.FONT_HERSHEY_SIMPLEX
99 | txt_size = cv2.getTextSize(text, font, 0.5, 2)[0]
100 | cv2.rectangle(img, (x0, y0), (x1, y1), color, 2)
101 |
102 | cv2.rectangle(
103 | img,
104 | (x0, y0 - txt_size[1] - 1),
105 | (x0 + txt_size[0] + txt_size[1], y0 - 1),
106 | color,
107 | -1,
108 | )
109 | cv2.putText(img, text, (x0, y0 - 1), font, 0.5, txt_color, thickness=1)
110 | return img
111 |
112 | def hard_nms(box_scores, iou_threshold, top_k=-1, candidate_size=200):
113 | """
114 |
115 | Args:
116 | box_scores (N, 5): boxes in corner-form and probabilities.
117 | iou_threshold: intersection over union threshold.
118 | top_k: keep top_k results. If k <= 0, keep all the results.
119 | candidate_size: only consider the candidates with the highest scores.
120 | Returns:
121 | picked: a list of indexes of the kept boxes
122 | """
123 | scores = box_scores[:, -1]
124 | boxes = box_scores[:, :-1]
125 | picked = []
126 | # _, indexes = scores.sort(descending=True)
127 | indexes = np.argsort(scores)
128 | # indexes = indexes[:candidate_size]
129 | indexes = indexes[-candidate_size:]
130 | while len(indexes) > 0:
131 | # current = indexes[0]
132 | current = indexes[-1]
133 | picked.append(current)
134 | if 0 < top_k == len(picked) or len(indexes) == 1:
135 | break
136 | current_box = boxes[current, :]
137 | # indexes = indexes[1:]
138 | indexes = indexes[:-1]
139 | rest_boxes = boxes[indexes, :]
140 | iou = iou_of(
141 | rest_boxes,
142 | np.expand_dims(current_box, axis=0),
143 | )
144 | indexes = indexes[iou <= iou_threshold]
145 |
146 | return box_scores[picked, :]
147 |
148 |
149 | def iou_of(boxes0, boxes1, eps=1e-5):
150 | """Return intersection-over-union (Jaccard index) of boxes.
151 |
152 | Args:
153 | boxes0 (N, 4): ground truth boxes.
154 | boxes1 (N or 1, 4): predicted boxes.
155 | eps: a small number to avoid 0 as denominator.
156 | Returns:
157 | iou (N): IoU values.
158 | """
159 | overlap_left_top = np.maximum(boxes0[..., :2], boxes1[..., :2])
160 | overlap_right_bottom = np.minimum(boxes0[..., 2:], boxes1[..., 2:])
161 |
162 | overlap_area = area_of(overlap_left_top, overlap_right_bottom)
163 | area0 = area_of(boxes0[..., :2], boxes0[..., 2:])
164 | area1 = area_of(boxes1[..., :2], boxes1[..., 2:])
165 | return overlap_area / (area0 + area1 - overlap_area + eps)
166 |
167 |
168 | def area_of(left_top, right_bottom):
169 | """Compute the areas of rectangles given two corners.
170 |
171 | Args:
172 | left_top (N, 2): left top corner.
173 | right_bottom (N, 2): right bottom corner.
174 |
175 | Returns:
176 | area (N): return the area.
177 | """
178 | hw = np.clip(right_bottom - left_top, 0.0, None)
179 | return hw[..., 0] * hw[..., 1]
180 |
181 |
182 | class NanoDetABC(metaclass=ABCMeta):
183 | def __init__(
184 | self,
185 | input_shape=[272, 160],
186 | reg_max=7,
187 | strides=[8, 16, 32],
188 | prob_threshold=0.4,
189 | iou_threshold=0.3,
190 | num_candidate=1000,
191 | top_k=-1,
192 | class_names=["face"]
193 | ):
194 | self.strides = strides
195 | self.input_shape = input_shape
196 | self.reg_max = reg_max
197 | self.prob_threshold = prob_threshold
198 | self.iou_threshold = iou_threshold
199 | self.num_candidate = num_candidate
200 | self.top_k = top_k
201 | self.img_mean = [103.53, 116.28, 123.675]
202 | self.img_std = [57.375, 57.12, 58.395]
203 | self.input_size = (self.input_shape[1], self.input_shape[0])
204 | self.class_names = class_names
205 | self.num_classes = len(self.class_names)
206 |
207 | def preprocess(self, img):
208 | # resize image
209 | ResizeM = get_resize_matrix((img.shape[1], img.shape[0]), self.input_size, True)
210 | img_resize = cv2.warpPerspective(img, ResizeM, dsize=self.input_size)
211 |
212 | # normalize image
213 | img_input = img_resize.astype(np.float32) / 255
214 | img_mean = np.array(self.img_mean, dtype=np.float32).reshape(1, 1, 3) / 255
215 | img_std = np.array(self.img_std, dtype=np.float32).reshape(1, 1, 3) / 255
216 | img_input = (img_input - img_mean) / img_std
217 |
218 | # expand dims
219 | img_input = np.transpose(img_input, [2, 0, 1])
220 | img_input = np.expand_dims(img_input, axis=0)
221 | return img_input, ResizeM
222 |
223 | def postprocess(self, scores, raw_boxes, ResizeM, raw_shape):
224 | # generate centers
225 | decode_boxes = []
226 | select_scores = []
227 | for stride, box_distribute, score in zip(self.strides, raw_boxes, scores):
228 | # centers
229 | fm_h = self.input_shape[0] / stride
230 | fm_w = self.input_shape[1] / stride
231 |
232 | h_range = np.arange(fm_h)
233 | w_range = np.arange(fm_w)
234 | ww, hh = np.meshgrid(w_range, h_range)
235 |
236 | ct_row = hh.flatten() * stride
237 | ct_col = ww.flatten() * stride
238 |
239 | center = np.stack((ct_col, ct_row, ct_col, ct_row), axis=1)
240 |
241 | # box distribution to distance
242 | reg_range = np.arange(self.reg_max + 1)
243 | box_distance = box_distribute.reshape((-1, self.reg_max + 1))
244 | box_distance = softmax(box_distance, axis=1)
245 | box_distance = box_distance * np.expand_dims(reg_range, axis=0)
246 | box_distance = np.sum(box_distance, axis=1).reshape((-1, 4))
247 | box_distance = box_distance * stride
248 |
249 | # top K candidate
250 | topk_idx = np.argsort(score.max(axis=1))[::-1]
251 | topk_idx = topk_idx[: self.num_candidate]
252 | center = center[topk_idx]
253 | score = score[topk_idx]
254 | box_distance = box_distance[topk_idx]
255 |
256 | # decode box
257 | decode_box = center + [-1, -1, 1, 1] * box_distance
258 |
259 | select_scores.append(score)
260 | decode_boxes.append(decode_box)
261 |
262 | # nms
263 | bboxes = np.concatenate(decode_boxes, axis=0)
264 | confidences = np.concatenate(select_scores, axis=0)
265 | picked_box_probs = []
266 | picked_labels = []
267 | for class_index in range(0, confidences.shape[1]):
268 | probs = confidences[:, class_index]
269 | mask = probs > self.prob_threshold
270 | probs = probs[mask]
271 | if probs.shape[0] == 0:
272 | continue
273 | subset_boxes = bboxes[mask, :]
274 | box_probs = np.concatenate([subset_boxes, probs.reshape(-1, 1)], axis=1)
275 | box_probs = hard_nms(
276 | box_probs,
277 | iou_threshold=self.iou_threshold,
278 | top_k=self.top_k,
279 | )
280 | picked_box_probs.append(box_probs)
281 | picked_labels.extend([class_index] * box_probs.shape[0])
282 | if not picked_box_probs:
283 | return np.array([]), np.array([]), np.array([])
284 | picked_box_probs = np.concatenate(picked_box_probs)
285 |
286 | # resize output boxes
287 | picked_box_probs[:, :4] = warp_boxes(
288 | picked_box_probs[:, :4], np.linalg.inv(ResizeM), raw_shape[1], raw_shape[0]
289 | )
290 | return (
291 | picked_box_probs[:, :4].astype(np.int32),
292 | np.array(picked_labels),
293 | picked_box_probs[:, 4],
294 | )
295 |
296 | @abstractmethod
297 | def infer_image(self, img_input):
298 | pass
299 |
300 | def detect(self, img):
301 | raw_shape = img.shape
302 | img_input, ResizeM = self.preprocess(img)
303 | scores, raw_boxes = self.infer_image(img_input)
304 | if scores[0].ndim == 1: # handling num_classes=1 case
305 | scores = [x[:, None] for x in scores]
306 | bbox, label, score = self.postprocess(scores, raw_boxes, ResizeM, raw_shape)
307 |
308 | return bbox, label, score
309 |
310 | class HelloFaceDet(NanoDetABC):
311 | def __init__(self, model_path=osp.join(songkey_weights_dir, 'face/nanodet_humandet_320-192_220302_model_20220315_test3.onnx'), gpu_id=None, *args, **kwargs):
312 | super(HelloFaceDet, self).__init__(*args, **kwargs)
313 | # print("Using ONNX as inference backend")
314 | # print(f"Using weight: {model_path}")
315 |
316 | # load model
317 | self.model_path = model_path
318 | self.ort_session = create_onnx_session(model_path, gpu_id=gpu_id)
319 | self.input_name = self.ort_session.get_inputs()[0].name
320 |
321 | def infer_image(self, img_input):
322 | inference_results = self.ort_session.run(None, {self.input_name: img_input})
323 |
324 | scores = [np.squeeze(x) for x in inference_results[:3]]
325 | raw_boxes = [np.squeeze(x) for x in inference_results[3:]]
326 | return scores, raw_boxes
327 |
--------------------------------------------------------------------------------
/hellomeme/models/hm_denoising_motion.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : models6/hm_denoising_motion.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 9/9/2024
8 | @Desc :
9 | adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unets/unet_motion_model.py
10 | """
11 |
12 | import torch
13 | import torch.utils.checkpoint
14 | from typing import Any, Dict, Optional, Tuple, Union
15 |
16 | from einops import rearrange
17 |
18 | from diffusers.utils import logging
19 | from diffusers.models.unets.unet_motion_model import UNetMotionModel, UNetMotionOutput
20 | from .hm_adapters import InsertReferenceAdapter
21 |
22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name
23 |
24 |
25 | class HMDenoisingMotion(UNetMotionModel, InsertReferenceAdapter):
26 | def forward(
27 | self,
28 | sample: torch.Tensor,
29 | timestep: Union[torch.Tensor, float, int],
30 | encoder_hidden_states: torch.Tensor,
31 | reference_hidden_states: Optional[dict] = None,
32 | control_hidden_states: Optional[torch.Tensor] = None,
33 | timestep_cond: Optional[torch.Tensor] = None,
34 | attention_mask: Optional[torch.Tensor] = None,
35 | cross_attention_kwargs: Optional[Dict[str, Any]] = None,
36 | added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
37 | down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
38 | mid_block_additional_residual: Optional[torch.Tensor] = None,
39 | return_dict: bool = True,
40 | ) -> Union[UNetMotionOutput, Tuple[torch.Tensor]]:
41 |
42 | # By default samples have to be AT least a multiple of the overall upsampling factor.
43 | # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
44 | # However, the upsampling interpolation output size can be forced to fit any upsampling size
45 | # on the fly if necessary.
46 | default_overall_up_factor = 2 ** self.num_upsamplers
47 |
48 | # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
49 | forward_upsample_size = False
50 | upsample_size = None
51 |
52 | if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
53 | logger.info("Forward upsample size to force interpolation output size.")
54 | forward_upsample_size = True
55 |
56 | # prepare attention_mask
57 | if attention_mask is not None:
58 | attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
59 | attention_mask = attention_mask.unsqueeze(1)
60 |
61 | # 1. time
62 | timesteps = timestep
63 | if not torch.is_tensor(timesteps):
64 | # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
65 | # This would be a good case for the `match` statement (Python 3.10+)
66 | is_mps = sample.device.type == "mps"
67 | if isinstance(timestep, float):
68 | dtype = torch.float32 if is_mps else torch.float64
69 | else:
70 | dtype = torch.int32 if is_mps else torch.int64
71 | timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
72 | elif len(timesteps.shape) == 0:
73 | timesteps = timesteps[None].to(sample.device)
74 |
75 | # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
76 | num_frames = sample.shape[2]
77 | timesteps = timesteps.expand(sample.shape[0])
78 |
79 | t_emb = self.time_proj(timesteps)
80 |
81 | # timesteps does not contain any weights and will always return f32 tensors
82 | # but time_embedding might actually be running in fp16. so we need to cast here.
83 | # there might be better ways to encapsulate this.
84 | t_emb = t_emb.to(dtype=self.dtype)
85 |
86 | emb = self.time_embedding(t_emb, timestep_cond)
87 | aug_emb = None
88 |
89 | if self.config.addition_embed_type == "text_time":
90 | if "text_embeds" not in added_cond_kwargs:
91 | raise ValueError(
92 | f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"
93 | )
94 |
95 | text_embeds = added_cond_kwargs.get("text_embeds")
96 | if "time_ids" not in added_cond_kwargs:
97 | raise ValueError(
98 | f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"
99 | )
100 | time_ids = added_cond_kwargs.get("time_ids")
101 | time_embeds = self.add_time_proj(time_ids.flatten())
102 | time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))
103 |
104 | add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)
105 | add_embeds = add_embeds.to(emb.dtype)
106 | aug_emb = self.add_embedding(add_embeds)
107 |
108 | emb = emb if aug_emb is None else emb + aug_emb
109 | emb = emb.repeat_interleave(repeats=num_frames, dim=0)
110 | if len(encoder_hidden_states.shape) == 3:
111 | encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0)
112 | elif len(encoder_hidden_states.shape) == 4:
113 | encoder_hidden_states = rearrange(encoder_hidden_states, "b f l d -> (b f) l d")
114 |
115 | if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj":
116 | if "image_embeds" not in added_cond_kwargs:
117 | raise ValueError(
118 | f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
119 | )
120 | image_embeds = added_cond_kwargs.get("image_embeds")
121 | image_embeds = self.encoder_hid_proj(image_embeds)
122 | image_embeds = [image_embed.repeat_interleave(repeats=num_frames, dim=0) for image_embed in image_embeds]
123 | encoder_hidden_states = (encoder_hidden_states, image_embeds)
124 |
125 | # 2. pre-process
126 | sample = sample.permute(0, 2, 1, 3, 4).reshape((sample.shape[0] * num_frames, -1) + sample.shape[3:])
127 | sample = self.conv_in(sample)
128 |
129 | # 3. down
130 | down_block_res_samples = (sample,)
131 | for idx, downsample_block in enumerate(self.down_blocks):
132 | if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
133 | sample, res_samples = downsample_block(
134 | hidden_states=sample,
135 | temb=emb,
136 | encoder_hidden_states=encoder_hidden_states,
137 | attention_mask=attention_mask,
138 | num_frames=num_frames,
139 | cross_attention_kwargs=cross_attention_kwargs,
140 | )
141 | else:
142 | sample, res_samples = downsample_block(hidden_states=sample, temb=emb, num_frames=num_frames)
143 |
144 | if not control_hidden_states is None and f'down_{idx}' in control_hidden_states:
145 | sample += rearrange(control_hidden_states[f'down_{idx}'], "b c f h w -> (b f) c h w")
146 | if not control_hidden_states is None and f'down2_{idx}' in control_hidden_states:
147 | sample += rearrange(control_hidden_states[f'down2_{idx}'], "b c f h w -> (b f) c h w")
148 |
149 | if hasattr(self, 'reference_modules_down') and not reference_hidden_states is None and f'down_{idx}' in reference_hidden_states:
150 | sample = self.reference_modules_down[idx](sample, reference_hidden_states[f'down_{idx}'], num_frames)
151 |
152 | down_block_res_samples += res_samples
153 |
154 | if down_block_additional_residuals is not None:
155 | new_down_block_res_samples = ()
156 |
157 | for down_block_res_sample, down_block_additional_residual in zip(
158 | down_block_res_samples, down_block_additional_residuals
159 | ):
160 | down_block_res_sample = down_block_res_sample + down_block_additional_residual
161 | new_down_block_res_samples += (down_block_res_sample,)
162 |
163 | down_block_res_samples = new_down_block_res_samples
164 |
165 | # 4. mid
166 | if self.mid_block is not None:
167 | # To support older versions of motion modules that don't have a mid_block
168 | if hasattr(self.mid_block, "motion_modules"):
169 | sample = self.mid_block(
170 | sample,
171 | emb,
172 | encoder_hidden_states=encoder_hidden_states,
173 | attention_mask=attention_mask,
174 | num_frames=num_frames,
175 | cross_attention_kwargs=cross_attention_kwargs,
176 | )
177 | else:
178 | sample = self.mid_block(
179 | sample,
180 | emb,
181 | encoder_hidden_states=encoder_hidden_states,
182 | attention_mask=attention_mask,
183 | cross_attention_kwargs=cross_attention_kwargs,
184 | )
185 | if hasattr(self, 'reference_modules_mid') and not reference_hidden_states is None and f'mid' in reference_hidden_states:
186 | sample = self.reference_modules_mid(sample, reference_hidden_states[f'mid'], num_frames)
187 |
188 | if mid_block_additional_residual is not None:
189 | sample = sample + mid_block_additional_residual
190 |
191 | # 5. up
192 | for i, upsample_block in enumerate(self.up_blocks):
193 | is_final_block = i == len(self.up_blocks) - 1
194 |
195 | res_samples = down_block_res_samples[-len(upsample_block.resnets):]
196 | down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
197 |
198 | # if we have not reached the final block and need to forward the
199 | # upsample size, we do it here
200 | if not is_final_block and forward_upsample_size:
201 | upsample_size = down_block_res_samples[-1].shape[2:]
202 |
203 | if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
204 | if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
205 | if not control_hidden_states is None and f'up_v2_{i}' in control_hidden_states:
206 | sample += rearrange(control_hidden_states[f'up_v2_{i}'], "b c f h w -> (b f) c h w")
207 | if not control_hidden_states is None and f'up2_v2_{i}' in control_hidden_states:
208 | sample += rearrange(control_hidden_states[f'up2_v2_{i}'], "b c f h w -> (b f) c h w")
209 | if hasattr(self,
210 | "reference_modules_up") and not reference_hidden_states is None and f'up_{i}' in reference_hidden_states:
211 | sample = self.reference_modules_up[i - 1](sample, reference_hidden_states[f'up_{i}'],
212 | num_frames)
213 |
214 | sample = upsample_block(
215 | hidden_states=sample,
216 | temb=emb,
217 | res_hidden_states_tuple=res_samples,
218 | encoder_hidden_states=encoder_hidden_states,
219 | upsample_size=upsample_size,
220 | attention_mask=attention_mask,
221 | num_frames=num_frames,
222 | cross_attention_kwargs=cross_attention_kwargs,
223 | )
224 | else:
225 | if not control_hidden_states is None and f'up_v2_{i}' in control_hidden_states:
226 | sample += rearrange(control_hidden_states[f'up_v2_{i}'], "b c f h w -> (b f) c h w")
227 | if not control_hidden_states is None and f'up2_v2_{i}' in control_hidden_states:
228 | sample += rearrange(control_hidden_states[f'up2_v2_{i}'], "b c f h w -> (b f) c h w")
229 | sample = upsample_block(
230 | hidden_states=sample,
231 | temb=emb,
232 | res_hidden_states_tuple=res_samples,
233 | upsample_size=upsample_size,
234 | num_frames=num_frames,
235 | )
236 |
237 | # 6. post-process
238 | if self.conv_norm_out:
239 | sample = self.conv_norm_out(sample)
240 | sample = self.conv_act(sample)
241 |
242 | sample = self.conv_out(sample)
243 |
244 | # reshape to (batch, channel, framerate, width, height)
245 | sample = sample[None, :].reshape((-1, num_frames) + sample.shape[1:]).permute(0, 2, 1, 3, 4)
246 |
247 | if not return_dict:
248 | return (sample,)
249 |
250 | return UNetMotionOutput(sample=sample)
--------------------------------------------------------------------------------
/hellomeme/models/hm3_denoising_3d.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : models6/hm_denoising_3d.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 1/3/2025
8 | @Desc :
9 | adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unets/unet_2d_condition.py
10 | """
11 |
12 | import torch
13 | import torch.utils.checkpoint
14 | from typing import Any, Dict, Optional, Tuple, Union
15 |
16 | from einops import rearrange
17 |
18 | from diffusers.utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers
19 | from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel, UNet2DConditionOutput
20 | from .hm_adapters import CopyWeights, InsertReferenceAdapter
21 |
22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name
23 |
24 | class HM3Denoising3D(UNet2DConditionModel, CopyWeights, InsertReferenceAdapter):
25 | def forward(
26 | self,
27 | sample: torch.Tensor,
28 | timestep: Union[torch.Tensor, float, int],
29 | encoder_hidden_states: torch.Tensor,
30 | reference_hidden_states: Optional[dict] = None,
31 | control_hidden_states: Optional[dict] = None,
32 | motion_pad_hidden_states: Optional[dict] = None,
33 | use_motion: bool = False,
34 | class_labels: Optional[torch.Tensor] = None,
35 | timestep_cond: Optional[torch.Tensor] = None,
36 | attention_mask: Optional[torch.Tensor] = None,
37 | cross_attention_kwargs: Optional[Dict[str, Any]] = None,
38 | added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
39 | down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
40 | mid_block_additional_residual: Optional[torch.Tensor] = None,
41 | down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
42 | encoder_attention_mask: Optional[torch.Tensor] = None,
43 | return_dict: bool = True,
44 | ) -> Union[UNet2DConditionOutput, Tuple]:
45 | # By default samples have to be AT least a multiple of the overall upsampling factor.
46 | # The overall upsampling factor is equal to 2 ** (# num of upsampling layers).
47 | # However, the upsampling interpolation output size can be forced to fit any upsampling size
48 | # on the fly if necessary.
49 | default_overall_up_factor = 2**self.num_upsamplers
50 |
51 | # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
52 | forward_upsample_size = False
53 | upsample_size = None
54 |
55 | for dim in sample.shape[-2:]:
56 | if dim % default_overall_up_factor != 0:
57 | # Forward upsample size to force interpolation output size.
58 | forward_upsample_size = True
59 | break
60 |
61 | # ensure attention_mask is a bias, and give it a singleton query_tokens dimension
62 | # expects mask of shape:
63 | # [batch, key_tokens]
64 | # adds singleton query_tokens dimension:
65 | # [batch, 1, key_tokens]
66 | # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
67 | # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)
68 | # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
69 | if attention_mask is not None:
70 | # assume that mask is expressed as:
71 | # (1 = keep, 0 = discard)
72 | # convert mask into a bias that can be added to attention scores:
73 | # (keep = +0, discard = -10000.0)
74 | attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
75 | attention_mask = attention_mask.unsqueeze(1)
76 |
77 | # convert encoder_attention_mask to a bias the same way we do for attention_mask
78 | if encoder_attention_mask is not None:
79 | encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0
80 | encoder_attention_mask = encoder_attention_mask.unsqueeze(1)
81 |
82 | # 0. center input if necessary
83 | if self.config.center_input_sample:
84 | sample = 2 * sample - 1.0
85 |
86 | # 1. time
87 | t_emb = self.get_time_embed(sample=sample, timestep=timestep)
88 | emb = self.time_embedding(t_emb, timestep_cond)
89 |
90 | class_emb = self.get_class_embed(sample=sample, class_labels=class_labels)
91 | if class_emb is not None:
92 | if self.config.class_embeddings_concat:
93 | emb = torch.cat([emb, class_emb], dim=-1)
94 | else:
95 | emb = emb + class_emb
96 |
97 | aug_emb = self.get_aug_embed(
98 | emb=emb, encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs
99 | )
100 | if self.config.addition_embed_type == "image_hint":
101 | aug_emb, hint = aug_emb
102 | sample = torch.cat([sample, hint], dim=1)
103 |
104 | emb = emb + aug_emb if aug_emb is not None else emb
105 |
106 | if self.time_embed_act is not None:
107 | emb = self.time_embed_act(emb)
108 |
109 | num_frames = sample.shape[2]
110 | emb = emb.repeat_interleave(repeats=num_frames, dim=0)
111 |
112 | if not added_cond_kwargs is None:
113 | if 'image_embeds' in added_cond_kwargs:
114 | if isinstance(added_cond_kwargs['image_embeds'], torch.Tensor):
115 | added_cond_kwargs['image_embeds'] = added_cond_kwargs['image_embeds'].repeat_interleave(repeats=num_frames, dim=0)
116 | else:
117 | added_cond_kwargs['image_embeds'] = [x.repeat_interleave(repeats=num_frames, dim=0) for x in added_cond_kwargs['image_embeds']]
118 |
119 | if len(encoder_hidden_states.shape) == 3:
120 | encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0)
121 | elif len(encoder_hidden_states.shape) == 4:
122 | encoder_hidden_states = rearrange(encoder_hidden_states, "b f l d -> (b f) l d")
123 |
124 | encoder_hidden_states = self.process_encoder_hidden_states(
125 | encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs
126 | )
127 |
128 | # 2. pre-process
129 | sample = rearrange(sample, "b c f h w -> (b f) c h w")
130 | sample = self.conv_in(sample)
131 |
132 | # 2.5 GLIGEN position net
133 | if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None:
134 | cross_attention_kwargs = cross_attention_kwargs.copy()
135 | gligen_args = cross_attention_kwargs.pop("gligen")
136 | cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)}
137 |
138 | # 3. down
139 | # we're popping the `scale` instead of getting it because otherwise `scale` will be propagated
140 | # to the internal blocks and will raise deprecation warnings. this will be confusing for our users.
141 | if cross_attention_kwargs is not None:
142 | cross_attention_kwargs = cross_attention_kwargs.copy()
143 | lora_scale = cross_attention_kwargs.pop("scale", 1.0)
144 | else:
145 | lora_scale = 1.0
146 |
147 | if USE_PEFT_BACKEND:
148 | # weight the lora layers by setting `lora_scale` for each PEFT layer
149 | scale_lora_layers(self, lora_scale)
150 |
151 | is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None
152 | # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets
153 | is_adapter = down_intrablock_additional_residuals is not None
154 | # maintain backward compatibility for legacy usage, where
155 | # T2I-Adapter and ControlNet both use down_block_additional_residuals arg
156 | # but can only use one or the other
157 | if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None:
158 | deprecate(
159 | "T2I should not use down_block_additional_residuals",
160 | "1.3.0",
161 | "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \
162 | and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \
163 | for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ",
164 | standard_warn=False,
165 | )
166 | down_intrablock_additional_residuals = down_block_additional_residuals
167 | is_adapter = True
168 |
169 | res_cache = dict()
170 | down_block_res_samples = (sample,)
171 | for idx, downsample_block in enumerate(self.down_blocks):
172 | if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
173 | # For t2i-adapter CrossAttnDownBlock2D
174 | additional_residuals = {}
175 | if is_adapter and len(down_intrablock_additional_residuals) > 0:
176 | additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0)
177 |
178 | sample, res_samples = downsample_block(
179 | hidden_states=sample,
180 | temb=emb,
181 | encoder_hidden_states=encoder_hidden_states,
182 | attention_mask=attention_mask,
183 | cross_attention_kwargs=cross_attention_kwargs,
184 | encoder_attention_mask=encoder_attention_mask,
185 | **additional_residuals,
186 | )
187 | else:
188 | sample, res_samples = downsample_block(hidden_states=sample, temb=emb)
189 | if is_adapter and len(down_intrablock_additional_residuals) > 0:
190 | sample += down_intrablock_additional_residuals.pop(0)
191 |
192 | res_cache[f"down_{idx}"] = sample.clone()
193 | if not control_hidden_states is None and f'down3_{idx}' in control_hidden_states:
194 | sample += rearrange(control_hidden_states[f'down3_{idx}'], "b c f h w -> (b f) c h w")
195 | if hasattr(self, 'motion_down') and use_motion:
196 | sample = self.motion_down[idx](sample,
197 | None if motion_pad_hidden_states is None else motion_pad_hidden_states[f'down_{idx}'],
198 | emb, num_frames)
199 |
200 | down_block_res_samples += res_samples
201 |
202 | if is_controlnet:
203 | new_down_block_res_samples = ()
204 |
205 | for down_block_res_sample, down_block_additional_residual in zip(
206 | down_block_res_samples, down_block_additional_residuals
207 | ):
208 | down_block_res_sample = down_block_res_sample + down_block_additional_residual
209 | new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,)
210 |
211 | down_block_res_samples = new_down_block_res_samples
212 |
213 | # 4. mid
214 | if self.mid_block is not None:
215 | if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention:
216 | sample = self.mid_block(
217 | sample,
218 | emb,
219 | encoder_hidden_states=encoder_hidden_states,
220 | attention_mask=attention_mask,
221 | cross_attention_kwargs=cross_attention_kwargs,
222 | encoder_attention_mask=encoder_attention_mask,
223 | )
224 | else:
225 | sample = self.mid_block(sample, emb)
226 |
227 | # To support T2I-Adapter-XL
228 | if (
229 | is_adapter
230 | and len(down_intrablock_additional_residuals) > 0
231 | and sample.shape == down_intrablock_additional_residuals[0].shape
232 | ):
233 | sample += down_intrablock_additional_residuals.pop(0)
234 |
235 | if is_controlnet:
236 | sample = sample + mid_block_additional_residual
237 |
238 | # 5. up
239 | for i, upsample_block in enumerate(self.up_blocks):
240 | is_final_block = i == len(self.up_blocks) - 1
241 |
242 | res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
243 | down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
244 |
245 | # if we have not reached the final block and need to forward the
246 | # upsample size, we do it here
247 | if not is_final_block and forward_upsample_size:
248 | upsample_size = down_block_res_samples[-1].shape[2:]
249 |
250 | res_cache[f"up_{i}"] = sample.clone()
251 | if not control_hidden_states is None and f'up3_{i}' in control_hidden_states:
252 | sample += rearrange(control_hidden_states[f'up3_{i}'], "b c f h w -> (b f) c h w")
253 | if hasattr(self, "reference_modules_up") and not reference_hidden_states is None and f'up_{i}' in reference_hidden_states:
254 | sample = self.reference_modules_up[i](sample, reference_hidden_states[f'up_{i}'], num_frames)
255 | if hasattr(self, 'motion_up') and use_motion:
256 | sample = self.motion_up[i](sample,
257 | None if motion_pad_hidden_states is None else motion_pad_hidden_states[f'up_{i}'],
258 | emb, num_frames)
259 |
260 | if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
261 | sample = upsample_block(
262 | hidden_states=sample,
263 | temb=emb,
264 | res_hidden_states_tuple=res_samples,
265 | encoder_hidden_states=encoder_hidden_states,
266 | cross_attention_kwargs=cross_attention_kwargs,
267 | upsample_size=upsample_size,
268 | attention_mask=attention_mask,
269 | encoder_attention_mask=encoder_attention_mask,
270 | )
271 | else:
272 | sample = upsample_block(
273 | hidden_states=sample,
274 | temb=emb,
275 | res_hidden_states_tuple=res_samples,
276 | upsample_size=upsample_size,
277 | )
278 |
279 | # 6. post-process
280 | if self.conv_norm_out:
281 | sample = self.conv_norm_out(sample)
282 | sample = self.conv_act(sample)
283 | sample = self.conv_out(sample)
284 |
285 | if USE_PEFT_BACKEND:
286 | # remove `lora_scale` from each PEFT layer
287 | unscale_lora_layers(self, lora_scale)
288 |
289 | # reshape to (batch, channel, framerate, width, height)
290 | sample = rearrange(sample, "(b f) c h w -> b c f h w", f=num_frames)
291 |
292 | if not return_dict:
293 | return (sample, res_cache)
294 |
295 | return (UNet2DConditionOutput(sample=sample), res_cache)
296 |
--------------------------------------------------------------------------------
/example_workflows/image_generation.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "a8e8ebf3-8dc8-4d76-ad73-dfe0b945cc77",
3 | "revision": 0,
4 | "last_node_id": 80,
5 | "last_link_id": 187,
6 | "nodes": [
7 | {
8 | "id": 2,
9 | "type": "HMFaceToolkitsLoader",
10 | "pos": [
11 | 45.41996383666992,
12 | 769.9866333007812
13 | ],
14 | "size": [
15 | 315,
16 | 82
17 | ],
18 | "flags": {},
19 | "order": 0,
20 | "mode": 0,
21 | "inputs": [],
22 | "outputs": [
23 | {
24 | "name": "face_toolkits",
25 | "type": "FACE_TOOLKITS",
26 | "slot_index": 0,
27 | "links": [
28 | 122,
29 | 129,
30 | 147,
31 | 149,
32 | 173,
33 | 174
34 | ]
35 | }
36 | ],
37 | "properties": {
38 | "Node name for S&R": "HMFaceToolkitsLoader"
39 | },
40 | "widgets_values": [
41 | 0,
42 | "huggingface"
43 | ]
44 | },
45 | {
46 | "id": 61,
47 | "type": "GetFaceLandmarks",
48 | "pos": [
49 | 519.7196655273438,
50 | 938.032958984375
51 | ],
52 | "size": [
53 | 170.62362670898438,
54 | 46
55 | ],
56 | "flags": {},
57 | "order": 4,
58 | "mode": 0,
59 | "inputs": [
60 | {
61 | "name": "face_toolkits",
62 | "type": "FACE_TOOLKITS",
63 | "link": 129
64 | },
65 | {
66 | "name": "images",
67 | "type": "IMAGE",
68 | "link": 179
69 | }
70 | ],
71 | "outputs": [
72 | {
73 | "name": "landmarks",
74 | "type": "FACELANDMARKS222",
75 | "links": [
76 | 145,
77 | 177
78 | ]
79 | }
80 | ],
81 | "properties": {
82 | "Node name for S&R": "GetFaceLandmarks"
83 | },
84 | "widgets_values": []
85 | },
86 | {
87 | "id": 60,
88 | "type": "GetFaceLandmarks",
89 | "pos": [
90 | 518.9132690429688,
91 | 612.9461669921875
92 | ],
93 | "size": [
94 | 170.62362670898438,
95 | 46
96 | ],
97 | "flags": {},
98 | "order": 5,
99 | "mode": 0,
100 | "inputs": [
101 | {
102 | "name": "face_toolkits",
103 | "type": "FACE_TOOLKITS",
104 | "link": 122
105 | },
106 | {
107 | "name": "images",
108 | "type": "IMAGE",
109 | "link": 121
110 | }
111 | ],
112 | "outputs": [
113 | {
114 | "name": "landmarks",
115 | "type": "FACELANDMARKS222",
116 | "links": [
117 | 143,
118 | 178
119 | ]
120 | }
121 | ],
122 | "properties": {
123 | "Node name for S&R": "GetFaceLandmarks"
124 | },
125 | "widgets_values": []
126 | },
127 | {
128 | "id": 77,
129 | "type": "LoadImage",
130 | "pos": [
131 | 61.306365966796875,
132 | 1023.740478515625
133 | ],
134 | "size": [
135 | 270,
136 | 314
137 | ],
138 | "flags": {},
139 | "order": 1,
140 | "mode": 0,
141 | "inputs": [],
142 | "outputs": [
143 | {
144 | "name": "IMAGE",
145 | "type": "IMAGE",
146 | "links": [
147 | 179,
148 | 180,
149 | 181
150 | ]
151 | },
152 | {
153 | "name": "MASK",
154 | "type": "MASK",
155 | "links": null
156 | }
157 | ],
158 | "properties": {
159 | "Node name for S&R": "LoadImage"
160 | },
161 | "widgets_values": [
162 | "yao.jpg",
163 | "image"
164 | ]
165 | },
166 | {
167 | "id": 67,
168 | "type": "GetHeadPose",
169 | "pos": [
170 | 814.1528930664062,
171 | 495.9793701171875
172 | ],
173 | "size": [
174 | 270,
175 | 98
176 | ],
177 | "flags": {},
178 | "order": 8,
179 | "mode": 0,
180 | "inputs": [
181 | {
182 | "name": "face_toolkits",
183 | "type": "FACE_TOOLKITS",
184 | "link": 147
185 | },
186 | {
187 | "name": "images",
188 | "type": "IMAGE",
189 | "link": 151
190 | },
191 | {
192 | "name": "landmarks",
193 | "type": "FACELANDMARKS222",
194 | "link": 143
195 | }
196 | ],
197 | "outputs": [
198 | {
199 | "name": "head_pose",
200 | "type": "HEAD_POSE",
201 | "links": [
202 | 183
203 | ]
204 | }
205 | ],
206 | "properties": {
207 | "Node name for S&R": "GetHeadPose"
208 | },
209 | "widgets_values": [
210 | true
211 | ]
212 | },
213 | {
214 | "id": 75,
215 | "type": "GetExpression2",
216 | "pos": [
217 | 814.1359252929688,
218 | 689.6795654296875
219 | ],
220 | "size": [
221 | 191.63729858398438,
222 | 66
223 | ],
224 | "flags": {},
225 | "order": 9,
226 | "mode": 0,
227 | "inputs": [
228 | {
229 | "name": "face_toolkits",
230 | "type": "FACE_TOOLKITS",
231 | "link": 173
232 | },
233 | {
234 | "name": "images",
235 | "type": "IMAGE",
236 | "link": 175
237 | },
238 | {
239 | "name": "landmarks",
240 | "type": "FACELANDMARKS222",
241 | "link": 178
242 | }
243 | ],
244 | "outputs": [
245 | {
246 | "name": "expression",
247 | "type": "EXPRESSION",
248 | "links": [
249 | 184
250 | ]
251 | }
252 | ],
253 | "properties": {
254 | "Node name for S&R": "GetExpression2"
255 | },
256 | "widgets_values": []
257 | },
258 | {
259 | "id": 69,
260 | "type": "GetHeadPose",
261 | "pos": [
262 | 819.799560546875,
263 | 860.5927734375
264 | ],
265 | "size": [
266 | 270,
267 | 98
268 | ],
269 | "flags": {},
270 | "order": 6,
271 | "mode": 0,
272 | "inputs": [
273 | {
274 | "name": "face_toolkits",
275 | "type": "FACE_TOOLKITS",
276 | "link": 149
277 | },
278 | {
279 | "name": "images",
280 | "type": "IMAGE",
281 | "link": 180
282 | },
283 | {
284 | "name": "landmarks",
285 | "type": "FACELANDMARKS222",
286 | "link": 145
287 | }
288 | ],
289 | "outputs": [
290 | {
291 | "name": "head_pose",
292 | "type": "HEAD_POSE",
293 | "links": [
294 | 185
295 | ]
296 | }
297 | ],
298 | "properties": {
299 | "Node name for S&R": "GetHeadPose"
300 | },
301 | "widgets_values": [
302 | true
303 | ]
304 | },
305 | {
306 | "id": 76,
307 | "type": "GetExpression2",
308 | "pos": [
309 | 833.6573486328125,
310 | 1042.0396728515625
311 | ],
312 | "size": [
313 | 191.63729858398438,
314 | 66
315 | ],
316 | "flags": {},
317 | "order": 7,
318 | "mode": 0,
319 | "inputs": [
320 | {
321 | "name": "face_toolkits",
322 | "type": "FACE_TOOLKITS",
323 | "link": 174
324 | },
325 | {
326 | "name": "images",
327 | "type": "IMAGE",
328 | "link": 181
329 | },
330 | {
331 | "name": "landmarks",
332 | "type": "FACELANDMARKS222",
333 | "link": 177
334 | }
335 | ],
336 | "outputs": [
337 | {
338 | "name": "expression",
339 | "type": "EXPRESSION",
340 | "links": [
341 | 186
342 | ]
343 | }
344 | ],
345 | "properties": {
346 | "Node name for S&R": "GetExpression2"
347 | },
348 | "widgets_values": []
349 | },
350 | {
351 | "id": 79,
352 | "type": "HMPipelineImage",
353 | "pos": [
354 | 1334.226318359375,
355 | 613.9535522460938
356 | ],
357 | "size": [
358 | 270,
359 | 306
360 | ],
361 | "flags": {},
362 | "order": 10,
363 | "mode": 0,
364 | "inputs": [
365 | {
366 | "name": "hm_image_pipeline",
367 | "type": "HMIMAGEPIPELINE",
368 | "link": 182
369 | },
370 | {
371 | "name": "ref_head_pose",
372 | "type": "HEAD_POSE",
373 | "link": 183
374 | },
375 | {
376 | "name": "ref_expression",
377 | "type": "EXPRESSION",
378 | "link": 184
379 | },
380 | {
381 | "name": "drive_head_pose",
382 | "type": "HEAD_POSE",
383 | "link": 185
384 | },
385 | {
386 | "name": "drive_expression",
387 | "type": "EXPRESSION",
388 | "link": 186
389 | }
390 | ],
391 | "outputs": [
392 | {
393 | "name": "IMAGE",
394 | "type": "IMAGE",
395 | "links": [
396 | 187
397 | ]
398 | },
399 | {
400 | "name": "LATENT",
401 | "type": "LATENT",
402 | "links": null
403 | }
404 | ],
405 | "properties": {
406 | "Node name for S&R": "HMPipelineImage"
407 | },
408 | "widgets_values": [
409 | 0,
410 | "",
411 | "",
412 | 25,
413 | 815482659530374,
414 | "randomize",
415 | 1.5,
416 | 0
417 | ]
418 | },
419 | {
420 | "id": 58,
421 | "type": "LoadImage",
422 | "pos": [
423 | 37.4466552734375,
424 | 260.9000244140625
425 | ],
426 | "size": [
427 | 315,
428 | 314
429 | ],
430 | "flags": {},
431 | "order": 2,
432 | "mode": 0,
433 | "inputs": [],
434 | "outputs": [
435 | {
436 | "name": "IMAGE",
437 | "type": "IMAGE",
438 | "slot_index": 0,
439 | "links": [
440 | 121,
441 | 151,
442 | 175
443 | ]
444 | },
445 | {
446 | "name": "MASK",
447 | "type": "MASK",
448 | "links": null
449 | }
450 | ],
451 | "properties": {
452 | "Node name for S&R": "LoadImage"
453 | },
454 | "widgets_values": [
455 | "chillout.jpg",
456 | "image"
457 | ]
458 | },
459 | {
460 | "id": 80,
461 | "type": "PreviewImage",
462 | "pos": [
463 | 1703.6793212890625,
464 | 613.9529418945312
465 | ],
466 | "size": [
467 | 210,
468 | 258
469 | ],
470 | "flags": {},
471 | "order": 11,
472 | "mode": 0,
473 | "inputs": [
474 | {
475 | "name": "images",
476 | "type": "IMAGE",
477 | "link": 187
478 | }
479 | ],
480 | "outputs": [],
481 | "properties": {
482 | "Node name for S&R": "PreviewImage"
483 | },
484 | "widgets_values": []
485 | },
486 | {
487 | "id": 78,
488 | "type": "HMImagePipelineLoader",
489 | "pos": [
490 | 797.7930297851562,
491 | 188.84007263183594
492 | ],
493 | "size": [
494 | 298.3472595214844,
495 | 226
496 | ],
497 | "flags": {},
498 | "order": 3,
499 | "mode": 0,
500 | "inputs": [],
501 | "outputs": [
502 | {
503 | "name": "hm_image_pipeline",
504 | "type": "HMIMAGEPIPELINE",
505 | "links": [
506 | 182
507 | ]
508 | }
509 | ],
510 | "properties": {
511 | "Node name for S&R": "HMImagePipelineLoader"
512 | },
513 | "widgets_values": [
514 | "[preset]DisneyPixarCartoonB",
515 | "None",
516 | "same as checkpoint",
517 | "v5c",
518 | "x1",
519 | "huggingface",
520 | 1,
521 | "fp32"
522 | ]
523 | }
524 | ],
525 | "links": [
526 | [
527 | 121,
528 | 58,
529 | 0,
530 | 60,
531 | 1,
532 | "IMAGE"
533 | ],
534 | [
535 | 122,
536 | 2,
537 | 0,
538 | 60,
539 | 0,
540 | "FACE_TOOLKITS"
541 | ],
542 | [
543 | 129,
544 | 2,
545 | 0,
546 | 61,
547 | 0,
548 | "FACE_TOOLKITS"
549 | ],
550 | [
551 | 143,
552 | 60,
553 | 0,
554 | 67,
555 | 2,
556 | "FACELANDMARKS222"
557 | ],
558 | [
559 | 145,
560 | 61,
561 | 0,
562 | 69,
563 | 2,
564 | "FACELANDMARKS222"
565 | ],
566 | [
567 | 147,
568 | 2,
569 | 0,
570 | 67,
571 | 0,
572 | "FACE_TOOLKITS"
573 | ],
574 | [
575 | 149,
576 | 2,
577 | 0,
578 | 69,
579 | 0,
580 | "FACE_TOOLKITS"
581 | ],
582 | [
583 | 151,
584 | 58,
585 | 0,
586 | 67,
587 | 1,
588 | "IMAGE"
589 | ],
590 | [
591 | 173,
592 | 2,
593 | 0,
594 | 75,
595 | 0,
596 | "FACE_TOOLKITS"
597 | ],
598 | [
599 | 174,
600 | 2,
601 | 0,
602 | 76,
603 | 0,
604 | "FACE_TOOLKITS"
605 | ],
606 | [
607 | 175,
608 | 58,
609 | 0,
610 | 75,
611 | 1,
612 | "IMAGE"
613 | ],
614 | [
615 | 177,
616 | 61,
617 | 0,
618 | 76,
619 | 2,
620 | "FACELANDMARKS222"
621 | ],
622 | [
623 | 178,
624 | 60,
625 | 0,
626 | 75,
627 | 2,
628 | "FACELANDMARKS222"
629 | ],
630 | [
631 | 179,
632 | 77,
633 | 0,
634 | 61,
635 | 1,
636 | "IMAGE"
637 | ],
638 | [
639 | 180,
640 | 77,
641 | 0,
642 | 69,
643 | 1,
644 | "IMAGE"
645 | ],
646 | [
647 | 181,
648 | 77,
649 | 0,
650 | 76,
651 | 1,
652 | "IMAGE"
653 | ],
654 | [
655 | 182,
656 | 78,
657 | 0,
658 | 79,
659 | 0,
660 | "HMIMAGEPIPELINE"
661 | ],
662 | [
663 | 183,
664 | 67,
665 | 0,
666 | 79,
667 | 1,
668 | "HEAD_POSE"
669 | ],
670 | [
671 | 184,
672 | 75,
673 | 0,
674 | 79,
675 | 2,
676 | "EXPRESSION"
677 | ],
678 | [
679 | 185,
680 | 69,
681 | 0,
682 | 79,
683 | 3,
684 | "HEAD_POSE"
685 | ],
686 | [
687 | 186,
688 | 76,
689 | 0,
690 | 79,
691 | 4,
692 | "EXPRESSION"
693 | ],
694 | [
695 | 187,
696 | 79,
697 | 0,
698 | 80,
699 | 0,
700 | "IMAGE"
701 | ]
702 | ],
703 | "groups": [],
704 | "config": {},
705 | "extra": {
706 | "ds": {
707 | "scale": 0.8264462809917358,
708 | "offset": [
709 | 493.27712102477454,
710 | -69.45337656976547
711 | ]
712 | },
713 | "frontendVersion": "1.19.2"
714 | },
715 | "version": 0.4
716 | }
--------------------------------------------------------------------------------
/hellomeme/models/hm_denoising_3d.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : models6/hm_denoising_3d.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 8/14/2024
8 | @Desc : 删除实验代码，精简结构
9 | adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unets/unet_2d_condition.py
10 | """
11 |
12 | import torch
13 | import torch.utils.checkpoint
14 | from typing import Any, Dict, Optional, Tuple, Union
15 |
16 | from einops import rearrange
17 |
18 | from diffusers.utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers
19 | from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel, UNet2DConditionOutput
20 | from .hm_adapters import CopyWeights, InsertReferenceAdapter
21 |
22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name
23 |
24 |
25 | class HMDenoising3D(UNet2DConditionModel, CopyWeights, InsertReferenceAdapter):
26 | def forward(
27 | self,
28 | sample: torch.Tensor,
29 | timestep: Union[torch.Tensor, float, int],
30 | encoder_hidden_states: torch.Tensor,
31 | reference_hidden_states: Optional[dict] = None,
32 | control_hidden_states: Optional[torch.Tensor] = None,
33 | class_labels: Optional[torch.Tensor] = None,
34 | timestep_cond: Optional[torch.Tensor] = None,
35 | attention_mask: Optional[torch.Tensor] = None,
36 | cross_attention_kwargs: Optional[Dict[str, Any]] = None,
37 | added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
38 | down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
39 | mid_block_additional_residual: Optional[torch.Tensor] = None,
40 | down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
41 | encoder_attention_mask: Optional[torch.Tensor] = None,
42 | return_dict: bool = True,
43 | ) -> Union[UNet2DConditionOutput, Tuple]:
44 | # By default samples have to be AT least a multiple of the overall upsampling factor.
45 | # The overall upsampling factor is equal to 2 ** (# num of upsampling layers).
46 | # However, the upsampling interpolation output size can be forced to fit any upsampling size
47 | # on the fly if necessary.
48 | default_overall_up_factor = 2**self.num_upsamplers
49 |
50 | # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
51 | forward_upsample_size = False
52 | upsample_size = None
53 |
54 | for dim in sample.shape[-2:]:
55 | if dim % default_overall_up_factor != 0:
56 | # Forward upsample size to force interpolation output size.
57 | forward_upsample_size = True
58 | break
59 |
60 | # ensure attention_mask is a bias, and give it a singleton query_tokens dimension
61 | # expects mask of shape:
62 | # [batch, key_tokens]
63 | # adds singleton query_tokens dimension:
64 | # [batch, 1, key_tokens]
65 | # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
66 | # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)
67 | # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
68 | if attention_mask is not None:
69 | # assume that mask is expressed as:
70 | # (1 = keep, 0 = discard)
71 | # convert mask into a bias that can be added to attention scores:
72 | # (keep = +0, discard = -10000.0)
73 | attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
74 | attention_mask = attention_mask.unsqueeze(1)
75 |
76 | # convert encoder_attention_mask to a bias the same way we do for attention_mask
77 | if encoder_attention_mask is not None:
78 | encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0
79 | encoder_attention_mask = encoder_attention_mask.unsqueeze(1)
80 |
81 | # 0. center input if necessary
82 | if self.config.center_input_sample:
83 | sample = 2 * sample - 1.0
84 |
85 | # 1. time
86 | t_emb = self.get_time_embed(sample=sample, timestep=timestep)
87 | emb = self.time_embedding(t_emb, timestep_cond)
88 | aug_emb = None
89 |
90 | class_emb = self.get_class_embed(sample=sample, class_labels=class_labels)
91 | if class_emb is not None:
92 | if self.config.class_embeddings_concat:
93 | emb = torch.cat([emb, class_emb], dim=-1)
94 | else:
95 | emb = emb + class_emb
96 |
97 | aug_emb = self.get_aug_embed(
98 | emb=emb, encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs
99 | )
100 | if self.config.addition_embed_type == "image_hint":
101 | aug_emb, hint = aug_emb
102 | sample = torch.cat([sample, hint], dim=1)
103 |
104 | emb = emb + aug_emb if aug_emb is not None else emb
105 |
106 | if self.time_embed_act is not None:
107 | emb = self.time_embed_act(emb)
108 |
109 | num_frames = sample.shape[2]
110 | emb = emb.repeat_interleave(repeats=num_frames, dim=0)
111 |
112 | if len(encoder_hidden_states.shape) == 3:
113 | encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0)
114 | elif len(encoder_hidden_states.shape) == 4:
115 | encoder_hidden_states = rearrange(encoder_hidden_states, "b f l d -> (b f) l d")
116 |
117 | if not added_cond_kwargs is None and 'image_embeds' in added_cond_kwargs:
118 | if isinstance(added_cond_kwargs['image_embeds'], torch.Tensor):
119 | added_cond_kwargs['image_embeds'] = added_cond_kwargs['image_embeds'].repeat_interleave(repeats=num_frames, dim=0)
120 | else:
121 | added_cond_kwargs['image_embeds'] = [x.repeat_interleave(repeats=num_frames, dim=0) for x in added_cond_kwargs['image_embeds']]
122 |
123 | encoder_hidden_states = self.process_encoder_hidden_states(
124 | encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs
125 | )
126 |
127 | # 2. pre-process
128 | sample = rearrange(sample, "b c f h w -> (b f) c h w")
129 | sample = self.conv_in(sample)
130 |
131 | # 2.5 GLIGEN position net
132 | if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None:
133 | cross_attention_kwargs = cross_attention_kwargs.copy()
134 | gligen_args = cross_attention_kwargs.pop("gligen")
135 | cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)}
136 |
137 | # 3. down
138 | # we're popping the `scale` instead of getting it because otherwise `scale` will be propagated
139 | # to the internal blocks and will raise deprecation warnings. this will be confusing for our users.
140 | if cross_attention_kwargs is not None:
141 | cross_attention_kwargs = cross_attention_kwargs.copy()
142 | lora_scale = cross_attention_kwargs.pop("scale", 1.0)
143 | else:
144 | lora_scale = 1.0
145 |
146 | if USE_PEFT_BACKEND:
147 | # weight the lora layers by setting `lora_scale` for each PEFT layer
148 | scale_lora_layers(self, lora_scale)
149 |
150 | is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None
151 | # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets
152 | is_adapter = down_intrablock_additional_residuals is not None
153 | # maintain backward compatibility for legacy usage, where
154 | # T2I-Adapter and ControlNet both use down_block_additional_residuals arg
155 | # but can only use one or the other
156 | if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None:
157 | deprecate(
158 | "T2I should not use down_block_additional_residuals",
159 | "1.3.0",
160 | "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \
161 | and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \
162 | for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ",
163 | standard_warn=False,
164 | )
165 | down_intrablock_additional_residuals = down_block_additional_residuals
166 | is_adapter = True
167 |
168 | res_cache = dict()
169 | down_block_res_samples = (sample,)
170 | for idx, downsample_block in enumerate(self.down_blocks):
171 | if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
172 | # For t2i-adapter CrossAttnDownBlock2D
173 | additional_residuals = {}
174 | if is_adapter and len(down_intrablock_additional_residuals) > 0:
175 | additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0)
176 |
177 | sample, res_samples = downsample_block(
178 | hidden_states=sample,
179 | temb=emb,
180 | encoder_hidden_states=encoder_hidden_states,
181 | attention_mask=attention_mask,
182 | cross_attention_kwargs=cross_attention_kwargs,
183 | encoder_attention_mask=encoder_attention_mask,
184 | **additional_residuals,
185 | )
186 | res_cache[f"down_{idx}"] = sample.clone()
187 | else:
188 | sample, res_samples = downsample_block(hidden_states=sample, temb=emb)
189 | if is_adapter and len(down_intrablock_additional_residuals) > 0:
190 | sample += down_intrablock_additional_residuals.pop(0)
191 |
192 | if not control_hidden_states is None and f'down_{idx}' in control_hidden_states:
193 | sample += rearrange(control_hidden_states[f'down_{idx}'], "b c f h w -> (b f) c h w")
194 | if not control_hidden_states is None and f'down2_{idx}' in control_hidden_states:
195 | sample += rearrange(control_hidden_states[f'down2_{idx}'], "b c f h w -> (b f) c h w")
196 | if hasattr(self, 'reference_modules_down') and not reference_hidden_states is None and f'down_{idx}' in reference_hidden_states:
197 | sample = self.reference_modules_down[idx](sample, reference_hidden_states[f'down_{idx}'], num_frames)
198 |
199 | down_block_res_samples += res_samples
200 |
201 | if is_controlnet:
202 | new_down_block_res_samples = ()
203 |
204 | for down_block_res_sample, down_block_additional_residual in zip(
205 | down_block_res_samples, down_block_additional_residuals
206 | ):
207 | down_block_res_sample = down_block_res_sample + down_block_additional_residual
208 | new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,)
209 |
210 | down_block_res_samples = new_down_block_res_samples
211 |
212 | # 4. mid
213 | if self.mid_block is not None:
214 | if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention:
215 | sample = self.mid_block(
216 | sample,
217 | emb,
218 | encoder_hidden_states=encoder_hidden_states,
219 | attention_mask=attention_mask,
220 | cross_attention_kwargs=cross_attention_kwargs,
221 | encoder_attention_mask=encoder_attention_mask,
222 | )
223 | else:
224 | sample = self.mid_block(sample, emb)
225 | if hasattr(self, 'reference_modules_mid') and not reference_hidden_states is None and f'mid' in reference_hidden_states:
226 | sample = self.reference_modules_mid(sample, reference_hidden_states[f'mid'], num_frames)
227 |
228 | # To support T2I-Adapter-XL
229 | if (
230 | is_adapter
231 | and len(down_intrablock_additional_residuals) > 0
232 | and sample.shape == down_intrablock_additional_residuals[0].shape
233 | ):
234 | sample += down_intrablock_additional_residuals.pop(0)
235 | res_cache[f"mid"] = sample.clone()
236 |
237 | if is_controlnet:
238 | sample = sample + mid_block_additional_residual
239 |
240 | # 5. up
241 | for i, upsample_block in enumerate(self.up_blocks):
242 | is_final_block = i == len(self.up_blocks) - 1
243 |
244 | res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
245 | down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
246 |
247 | # if we have not reached the final block and need to forward the
248 | # upsample size, we do it here
249 | if not is_final_block and forward_upsample_size:
250 | upsample_size = down_block_res_samples[-1].shape[2:]
251 |
252 | if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
253 | res_cache[f"up_{i}"] = sample.clone()
254 | if not control_hidden_states is None and f'up_v2_{i}' in control_hidden_states:
255 | sample += rearrange(control_hidden_states[f'up_v2_{i}'], "b c f h w -> (b f) c h w")
256 | if not control_hidden_states is None and f'up2_v2_{i}' in control_hidden_states:
257 | sample += rearrange(control_hidden_states[f'up2_v2_{i}'], "b c f h w -> (b f) c h w")
258 | if hasattr(self, "reference_modules_up") and not reference_hidden_states is None and f'up_{i}' in reference_hidden_states:
259 | sample = self.reference_modules_up[i-1](sample, reference_hidden_states[f'up_{i}'], num_frames)
260 |
261 | sample = upsample_block(
262 | hidden_states=sample,
263 | temb=emb,
264 | res_hidden_states_tuple=res_samples,
265 | encoder_hidden_states=encoder_hidden_states,
266 | cross_attention_kwargs=cross_attention_kwargs,
267 | upsample_size=upsample_size,
268 | attention_mask=attention_mask,
269 | encoder_attention_mask=encoder_attention_mask,
270 | )
271 | else:
272 | if not control_hidden_states is None and f'up_v2_{i}' in control_hidden_states:
273 | sample += rearrange(control_hidden_states[f'up_v2_{i}'], "b c f h w -> (b f) c h w")
274 | if not control_hidden_states is None and f'up2_v2_{i}' in control_hidden_states:
275 | sample += rearrange(control_hidden_states[f'up2_v2_{i}'], "b c f h w -> (b f) c h w")
276 | sample = upsample_block(
277 | hidden_states=sample,
278 | temb=emb,
279 | res_hidden_states_tuple=res_samples,
280 | upsample_size=upsample_size,
281 | )
282 |
283 | # 6. post-process
284 | if self.conv_norm_out:
285 | sample = self.conv_norm_out(sample)
286 | sample = self.conv_act(sample)
287 | sample = self.conv_out(sample)
288 |
289 | if USE_PEFT_BACKEND:
290 | # remove `lora_scale` from each PEFT layer
291 | unscale_lora_layers(self, lora_scale)
292 |
293 | # reshape to (batch, channel, framerate, width, height)
294 | sample = rearrange(sample, "(b f) c h w -> b c f h w", f=num_frames)
295 |
296 | if not return_dict:
297 | return (sample, res_cache)
298 |
299 | return (UNet2DConditionOutput(sample=sample), res_cache)
300 |
--------------------------------------------------------------------------------
/hellomeme/tools/sr.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : sr.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 5/30/2025
8 | @Desc : adapted from: https://github.com/xinntao/Real-ESRGAN
9 | """
10 |
11 | import torch
12 | from torch import nn as nn
13 | from torch.nn import functional as F
14 | import cv2
15 | import numpy as np
16 | import math
17 | import os.path as osp
18 | from .utils import download_file_from_cloud
19 |
20 | def pixel_unshuffle(x, scale):
21 | """ Pixel unshuffle.
22 |
23 | Args:
24 | x (Tensor): Input feature with shape (b, c, hh, hw).
25 | scale (int): Downsample ratio.
26 |
27 | Returns:
28 | Tensor: the pixel unshuffled feature.
29 | """
30 | b, c, hh, hw = x.size()
31 | out_channel = c * (scale**2)
32 | assert hh % scale == 0 and hw % scale == 0
33 | h = hh // scale
34 | w = hw // scale
35 | x_view = x.view(b, c, h, scale, w, scale)
36 | return x_view.permute(0, 1, 3, 5, 2, 4).reshape(b, out_channel, h, w)
37 |
38 | def make_layer(basic_block, num_basic_block, **kwarg):
39 | """Make layers by stacking the same blocks.
40 |
41 | Args:
42 | basic_block (nn.module): nn.module class for basic block.
43 | num_basic_block (int): number of blocks.
44 |
45 | Returns:
46 | nn.Sequential: Stacked blocks in nn.Sequential.
47 | """
48 | layers = []
49 | for _ in range(num_basic_block):
50 | layers.append(basic_block(**kwarg))
51 | return nn.Sequential(*layers)
52 |
53 | class ResidualDenseBlock(nn.Module):
54 | """Residual Dense Block.
55 |
56 | Used in RRDB block in ESRGAN.
57 |
58 | Args:
59 | num_feat (int): Channel number of intermediate features.
60 | num_grow_ch (int): Channels for each growth.
61 | """
62 |
63 | def __init__(self, num_feat=64, num_grow_ch=32):
64 | super(ResidualDenseBlock, self).__init__()
65 | self.conv1 = nn.Conv2d(num_feat, num_grow_ch, 3, 1, 1)
66 | self.conv2 = nn.Conv2d(num_feat + num_grow_ch, num_grow_ch, 3, 1, 1)
67 | self.conv3 = nn.Conv2d(num_feat + 2 * num_grow_ch, num_grow_ch, 3, 1, 1)
68 | self.conv4 = nn.Conv2d(num_feat + 3 * num_grow_ch, num_grow_ch, 3, 1, 1)
69 | self.conv5 = nn.Conv2d(num_feat + 4 * num_grow_ch, num_feat, 3, 1, 1)
70 |
71 | self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True)
72 |
73 | # initialization
74 | # default_init_weights([self.conv1, self.conv2, self.conv3, self.conv4, self.conv5], 0.1)
75 |
76 | def forward(self, x):
77 | x1 = self.lrelu(self.conv1(x))
78 | x2 = self.lrelu(self.conv2(torch.cat((x, x1), 1)))
79 | x3 = self.lrelu(self.conv3(torch.cat((x, x1, x2), 1)))
80 | x4 = self.lrelu(self.conv4(torch.cat((x, x1, x2, x3), 1)))
81 | x5 = self.conv5(torch.cat((x, x1, x2, x3, x4), 1))
82 | # Empirically, we use 0.2 to scale the residual for better performance
83 | return x5 * 0.2 + x
84 |
85 |
86 | class RRDB(nn.Module):
87 | """Residual in Residual Dense Block.
88 |
89 | Used in RRDB-Net in ESRGAN.
90 |
91 | Args:
92 | num_feat (int): Channel number of intermediate features.
93 | num_grow_ch (int): Channels for each growth.
94 | """
95 |
96 | def __init__(self, num_feat, num_grow_ch=32):
97 | super(RRDB, self).__init__()
98 | self.rdb1 = ResidualDenseBlock(num_feat, num_grow_ch)
99 | self.rdb2 = ResidualDenseBlock(num_feat, num_grow_ch)
100 | self.rdb3 = ResidualDenseBlock(num_feat, num_grow_ch)
101 |
102 | def forward(self, x):
103 | out = self.rdb1(x)
104 | out = self.rdb2(out)
105 | out = self.rdb3(out)
106 | # Empirically, we use 0.2 to scale the residual for better performance
107 | return out * 0.2 + x
108 |
109 | class RRDBNet(nn.Module):
110 | """Networks consisting of Residual in Residual Dense Block, which is used
111 | in ESRGAN.
112 |
113 | ESRGAN: Enhanced Super-Resolution Generative Adversarial Networks.
114 |
115 | We extend ESRGAN for scale x2 and scale x1.
116 | Note: This is one option for scale 1, scale 2 in RRDBNet.
117 | We first employ the pixel-unshuffle (an inverse operation of pixelshuffle to reduce the spatial size
118 | and enlarge the channel size before feeding inputs into the main ESRGAN architecture.
119 |
120 | Args:
121 | num_in_ch (int): Channel number of inputs.
122 | num_out_ch (int): Channel number of outputs.
123 | num_feat (int): Channel number of intermediate features.
124 | Default: 64
125 | num_block (int): Block number in the trunk network. Defaults: 23
126 | num_grow_ch (int): Channels for each growth. Default: 32.
127 | """
128 |
129 | def __init__(self, num_in_ch, num_out_ch, scale=4, num_feat=64, num_block=23, num_grow_ch=32):
130 | super(RRDBNet, self).__init__()
131 | self.scale = scale
132 | if scale == 2:
133 | num_in_ch = num_in_ch * 4
134 | elif scale == 1:
135 | num_in_ch = num_in_ch * 16
136 | self.conv_first = nn.Conv2d(num_in_ch, num_feat, 3, 1, 1)
137 | self.body = make_layer(RRDB, num_block, num_feat=num_feat, num_grow_ch=num_grow_ch)
138 | self.conv_body = nn.Conv2d(num_feat, num_feat, 3, 1, 1)
139 | # upsample
140 | self.conv_up1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1)
141 | self.conv_up2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1)
142 | self.conv_hr = nn.Conv2d(num_feat, num_feat, 3, 1, 1)
143 | self.conv_last = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1)
144 |
145 | self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True)
146 |
147 | def forward(self, x):
148 | if self.scale == 2:
149 | feat = pixel_unshuffle(x, scale=2)
150 | elif self.scale == 1:
151 | feat = pixel_unshuffle(x, scale=4)
152 | else:
153 | feat = x
154 | feat = self.conv_first(feat)
155 | body_feat = self.conv_body(self.body(feat))
156 | feat = feat + body_feat
157 | # upsample
158 | feat = self.lrelu(self.conv_up1(F.interpolate(feat, scale_factor=2, mode='nearest')))
159 | feat = self.lrelu(self.conv_up2(F.interpolate(feat, scale_factor=2, mode='nearest')))
160 | out = self.conv_last(self.lrelu(self.conv_hr(feat)))
161 | return out
162 |
163 | class RealESRGANer():
164 | def __init__(self,
165 | scale,
166 | tile=0,
167 | tile_pad=10,
168 | pre_pad=10,
169 | half=True,
170 | device=None,
171 | gpu_id=None,
172 | modelscope=False):
173 | self.scale = scale
174 | self.tile_size = tile
175 | self.tile_pad = tile_pad
176 | self.pre_pad = pre_pad
177 | self.mod_scale = None
178 | self.half = half
179 |
180 | # initialize model
181 | if gpu_id:
182 | self.device = torch.device(
183 | f'cuda:{gpu_id}' if torch.cuda.is_available() else 'cpu') if device is None else device
184 | else:
185 | self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device
186 |
187 | model_path = download_file_from_cloud(model_id='songkey/ESRGAN', file_name='RealESRGAN_x2plus.pth', modelscope=modelscope)
188 | loadnet = torch.load(model_path, map_location=torch.device('cpu'))
189 |
190 | # prefer to use params_ema
191 | if 'params_ema' in loadnet:
192 | keyname = 'params_ema'
193 | else:
194 | keyname = 'params'
195 | self.model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=2)
196 | self.model.load_state_dict(loadnet[keyname], strict=True)
197 |
198 | self.model.eval()
199 | self.model = self.model.to(self.device)
200 | if self.half:
201 | self.model = self.model.half()
202 |
203 | def dni(self, net_a, net_b, dni_weight, key='params', loc='cpu'):
204 | """Deep network interpolation.
205 |
206 | ``Paper: Deep Network Interpolation for Continuous Imagery Effect Transition``
207 | """
208 | net_a = torch.load(net_a, map_location=torch.device(loc))
209 | net_b = torch.load(net_b, map_location=torch.device(loc))
210 | for k, v_a in net_a[key].items():
211 | net_a[key][k] = dni_weight[0] * v_a + dni_weight[1] * net_b[key][k]
212 | return net_a
213 |
214 | def pre_process(self, img):
215 | """Pre-process, such as pre-pad and mod pad, so that the images can be divisible
216 | """
217 | img = torch.from_numpy(np.transpose(img, (2, 0, 1))).float()
218 | self.img = img.unsqueeze(0).to(self.device)
219 | if self.half:
220 | self.img = self.img.half()
221 |
222 | # pre_pad
223 | if self.pre_pad != 0:
224 | self.img = F.pad(self.img, (0, self.pre_pad, 0, self.pre_pad), 'reflect')
225 | # mod pad for divisible borders
226 | if self.scale == 2:
227 | self.mod_scale = 2
228 | elif self.scale == 1:
229 | self.mod_scale = 4
230 | if self.mod_scale is not None:
231 | self.mod_pad_h, self.mod_pad_w = 0, 0
232 | _, _, h, w = self.img.size()
233 | if (h % self.mod_scale != 0):
234 | self.mod_pad_h = (self.mod_scale - h % self.mod_scale)
235 | if (w % self.mod_scale != 0):
236 | self.mod_pad_w = (self.mod_scale - w % self.mod_scale)
237 | self.img = F.pad(self.img, (0, self.mod_pad_w, 0, self.mod_pad_h), 'reflect')
238 |
239 | def process(self):
240 | # model inference
241 | self.output = self.model(self.img)
242 |
243 | def tile_process(self):
244 | """It will first crop input images to tiles, and then process each tile.
245 | Finally, all the processed tiles are merged into one images.
246 |
247 | Modified from: https://github.com/ata4/esrgan-launcher
248 | """
249 | batch, channel, height, width = self.img.shape
250 | output_height = height * self.scale
251 | output_width = width * self.scale
252 | output_shape = (batch, channel, output_height, output_width)
253 |
254 | # start with black image
255 | self.output = self.img.new_zeros(output_shape)
256 | tiles_x = math.ceil(width / self.tile_size)
257 | tiles_y = math.ceil(height / self.tile_size)
258 |
259 | # loop over all tiles
260 | for y in range(tiles_y):
261 | for x in range(tiles_x):
262 | # extract tile from input image
263 | ofs_x = x * self.tile_size
264 | ofs_y = y * self.tile_size
265 | # input tile area on total image
266 | input_start_x = ofs_x
267 | input_end_x = min(ofs_x + self.tile_size, width)
268 | input_start_y = ofs_y
269 | input_end_y = min(ofs_y + self.tile_size, height)
270 |
271 | # input tile area on total image with padding
272 | input_start_x_pad = max(input_start_x - self.tile_pad, 0)
273 | input_end_x_pad = min(input_end_x + self.tile_pad, width)
274 | input_start_y_pad = max(input_start_y - self.tile_pad, 0)
275 | input_end_y_pad = min(input_end_y + self.tile_pad, height)
276 |
277 | # input tile dimensions
278 | input_tile_width = input_end_x - input_start_x
279 | input_tile_height = input_end_y - input_start_y
280 | tile_idx = y * tiles_x + x + 1
281 | input_tile = self.img[:, :, input_start_y_pad:input_end_y_pad, input_start_x_pad:input_end_x_pad]
282 |
283 | # upscale tile
284 | try:
285 | with torch.no_grad():
286 | output_tile = self.model(input_tile)
287 | except RuntimeError as error:
288 | print('Error', error)
289 | print(f'\tTile {tile_idx}/{tiles_x * tiles_y}')
290 |
291 | # output tile area on total image
292 | output_start_x = input_start_x * self.scale
293 | output_end_x = input_end_x * self.scale
294 | output_start_y = input_start_y * self.scale
295 | output_end_y = input_end_y * self.scale
296 |
297 | # output tile area without padding
298 | output_start_x_tile = (input_start_x - input_start_x_pad) * self.scale
299 | output_end_x_tile = output_start_x_tile + input_tile_width * self.scale
300 | output_start_y_tile = (input_start_y - input_start_y_pad) * self.scale
301 | output_end_y_tile = output_start_y_tile + input_tile_height * self.scale
302 |
303 | # put tile into output image
304 | self.output[:, :, output_start_y:output_end_y,
305 | output_start_x:output_end_x] = output_tile[:, :, output_start_y_tile:output_end_y_tile,
306 | output_start_x_tile:output_end_x_tile]
307 |
308 | def post_process(self):
309 | # remove extra pad
310 | if self.mod_scale is not None:
311 | _, _, h, w = self.output.size()
312 | self.output = self.output[:, :, 0:h - self.mod_pad_h * self.scale, 0:w - self.mod_pad_w * self.scale]
313 | # remove prepad
314 | if self.pre_pad != 0:
315 | _, _, h, w = self.output.size()
316 | self.output = self.output[:, :, 0:h - self.pre_pad * self.scale, 0:w - self.pre_pad * self.scale]
317 | return self.output
318 |
319 | @torch.no_grad()
320 | def enhance(self, img, outscale=None, alpha_upsampler='realesrgan'):
321 | h_input, w_input = img.shape[0:2]
322 | # img: numpy
323 | img = img.astype(np.float32)
324 | if np.max(img) > 256: # 16-bit image
325 | max_range = 65535
326 | print('\tInput is a 16-bit image')
327 | else:
328 | max_range = 255
329 | img = img / max_range
330 | if len(img.shape) == 2: # gray image
331 | img_mode = 'L'
332 | img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
333 | elif img.shape[2] == 4: # RGBA image with alpha channel
334 | img_mode = 'RGBA'
335 | alpha = img[:, :, 3]
336 | img = img[:, :, 0:3]
337 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
338 | if alpha_upsampler == 'realesrgan':
339 | alpha = cv2.cvtColor(alpha, cv2.COLOR_GRAY2RGB)
340 | else:
341 | img_mode = 'RGB'
342 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
343 |
344 | # ------------------- process image (without the alpha channel) ------------------- #
345 | self.pre_process(img)
346 | if self.tile_size > 0:
347 | self.tile_process()
348 | else:
349 | self.process()
350 | output_img = self.post_process()
351 | output_img = output_img.data.squeeze().float().cpu().clamp_(0, 1).numpy()
352 | output_img = np.transpose(output_img[[2, 1, 0], :, :], (1, 2, 0))
353 | if img_mode == 'L':
354 | output_img = cv2.cvtColor(output_img, cv2.COLOR_BGR2GRAY)
355 |
356 | # ------------------- process the alpha channel if necessary ------------------- #
357 | if img_mode == 'RGBA':
358 | if alpha_upsampler == 'realesrgan':
359 | self.pre_process(alpha)
360 | if self.tile_size > 0:
361 | self.tile_process()
362 | else:
363 | self.process()
364 | output_alpha = self.post_process()
365 | output_alpha = output_alpha.data.squeeze().float().cpu().clamp_(0, 1).numpy()
366 | output_alpha = np.transpose(output_alpha[[2, 1, 0], :, :], (1, 2, 0))
367 | output_alpha = cv2.cvtColor(output_alpha, cv2.COLOR_BGR2GRAY)
368 | else: # use the cv2 resize for alpha channel
369 | h, w = alpha.shape[0:2]
370 | output_alpha = cv2.resize(alpha, (w * self.scale, h * self.scale), interpolation=cv2.INTER_LINEAR)
371 |
372 | # merge the alpha channel
373 | output_img = cv2.cvtColor(output_img, cv2.COLOR_BGR2BGRA)
374 | output_img[:, :, 3] = output_alpha
375 |
376 | # ------------------------------ return ------------------------------ #
377 | if max_range == 65535: # 16-bit image
378 | output = (output_img * 65535.0).round().astype(np.uint16)
379 | else:
380 | output = (output_img * 255.0).round().astype(np.uint8)
381 |
382 | if outscale is not None and outscale != float(self.scale):
383 | output = cv2.resize(
384 | output, (
385 | int(w_input * outscale),
386 | int(h_input * outscale),
387 | ), interpolation=cv2.INTER_LANCZOS4)
388 |
389 | return output, img_mode
--------------------------------------------------------------------------------
/hellomeme/pipelines/pipline_hm_image.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : hm_pipline_image.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 8/29/2024
8 | @Desc :
9 | adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py
10 | """
11 |
12 | import copy
13 | from typing import Any, Callable, Dict, List, Optional, Union
14 | import torch
15 |
16 | from diffusers import EulerDiscreteScheduler
17 | from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback
18 | from diffusers.image_processor import PipelineImageInput
19 | from diffusers.utils import deprecate
20 | from diffusers.utils.torch_utils import randn_tensor
21 | from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput
22 | from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents
23 |
24 | from ..models import HMDenoising3D, HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2, HMPipeline
25 | from ..models import HMReferenceAdapter
26 | from ..tools.utils import creat_model_from_cloud
27 |
28 |
29 | class HMImagePipeline(HMPipeline):
30 | def caryomitosis(self, **kwargs):
31 | if hasattr(self, "unet_ref"):
32 | del self.unet_ref
33 | self.unet_ref = HMDenoising3D.from_unet2d(self.unet)
34 | self.unet_ref.cpu()
35 |
36 | if not isinstance(self.unet, HMDenoising3D):
37 | unet = HMDenoising3D.from_unet2d(unet=self.unet)
38 | # todo: 不够优雅
39 | del self.unet
40 | self.unet = unet
41 | self.unet.cpu()
42 |
43 | self.vae.cpu()
44 | self.vae_decode = copy.deepcopy(self.vae)
45 | self.text_encoder.cpu()
46 | self.text_encoder_ref = copy.deepcopy(self.text_encoder)
47 | if hasattr(self, 'safety_checker'):
48 | del self.safety_checker
49 |
50 | def insert_hm_modules(self, version, dtype, modelscope=False):
51 | self.version = version
52 |
53 | if version == 'v1':
54 | hm_reference_dir = 'songkey/hm_reference'
55 | hm_control_dir = 'songkey/hm_control'
56 | hm_control2_dir = 'songkey/hm_control2'
57 | else:
58 | hm_reference_dir = 'songkey/hm2_reference'
59 | hm_control_dir = 'songkey/hm2_control'
60 | hm_control2_dir = 'songkey/hm2_control2'
61 |
62 | if isinstance(self.unet, HMDenoising3D):
63 | hm_adapter = creat_model_from_cloud(HMReferenceAdapter, hm_reference_dir, modelscope=modelscope)
64 | self.unet.insert_reference_adapter(hm_adapter)
65 | self.unet.to(device='cpu', dtype=dtype).eval()
66 |
67 | if hasattr(self, "unet_ref"):
68 | self.unet_ref.to(device='cpu', dtype=dtype).eval()
69 |
70 | if hasattr(self, "mp_control"):
71 | del self.mp_control
72 | if version == 'v1':
73 | self.mp_control = creat_model_from_cloud(HMControlNet, hm_control_dir, modelscope=modelscope)
74 | else:
75 | self.mp_control = creat_model_from_cloud(HMV2ControlNet, hm_control_dir, modelscope=modelscope)
76 | self.mp_control.to(device='cpu', dtype=dtype).eval()
77 |
78 | if hasattr(self, "mp_control2"):
79 | del self.mp_control2
80 | if version == 'v1':
81 | self.mp_control2 = creat_model_from_cloud(HMControlNet2, hm_control2_dir, modelscope=modelscope)
82 | else:
83 | self.mp_control2 = creat_model_from_cloud(HMV2ControlNet2, hm_control2_dir, modelscope=modelscope)
84 | self.mp_control2.to(device='cpu', dtype=dtype).eval()
85 |
86 | self.vae.to(device='cpu', dtype=dtype).eval()
87 | self.vae_decode.to(device='cpu', dtype=dtype).eval()
88 | self.text_encoder.to(device='cpu', dtype=dtype).eval()
89 | self.text_encoder_ref.to(device='cpu', dtype=dtype).eval()
90 |
91 | @torch.no_grad()
92 | def __call__(
93 | self,
94 | prompt: Union[str, List[str]] = None,
95 | image: PipelineImageInput = None,
96 | drive_params: Dict[str, Any] = None,
97 | strength: float = 0.8,
98 | num_inference_steps: Optional[int] = 50,
99 | timesteps: List[int] = None,
100 | sigmas: List[float] = None,
101 | guidance_scale: Optional[float] = 7.5,
102 | negative_prompt: Optional[Union[str, List[str]]] = None,
103 | eta: Optional[float] = 0.0,
104 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
105 | prompt_embeds: Optional[torch.Tensor] = None,
106 | negative_prompt_embeds: Optional[torch.Tensor] = None,
107 | ip_adapter_image: Optional[PipelineImageInput] = None,
108 | ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
109 | output_type: Optional[str] = "pil",
110 | device: Optional[str] = "cpu",
111 | return_dict: bool = True,
112 | cross_attention_kwargs: Optional[Dict[str, Any]] = None,
113 | clip_skip: int = None,
114 | callback_on_step_end: Optional[
115 | Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
116 | ] = None,
117 | callback_on_step_end_tensor_inputs: List[str] = ["latents"],
118 | **kwargs,
119 | ):
120 | callback = kwargs.pop("callback", None)
121 | callback_steps = kwargs.pop("callback_steps", None)
122 | num_images_per_prompt = 1
123 |
124 | if callback is not None:
125 | deprecate(
126 | "callback",
127 | "1.0.0",
128 | "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
129 | )
130 | if callback_steps is not None:
131 | deprecate(
132 | "callback_steps",
133 | "1.0.0",
134 | "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
135 | )
136 |
137 | if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
138 | callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
139 |
140 | # 1. Check inputs. Raise error if not correct
141 | self.check_inputs(
142 | prompt,
143 | strength,
144 | callback_steps,
145 | negative_prompt,
146 | prompt_embeds,
147 | negative_prompt_embeds,
148 | ip_adapter_image,
149 | ip_adapter_image_embeds,
150 | callback_on_step_end_tensor_inputs,
151 | )
152 |
153 | self._guidance_scale = guidance_scale
154 | self._clip_skip = clip_skip
155 | self._cross_attention_kwargs = cross_attention_kwargs
156 | self._interrupt = False
157 |
158 | # 2. Define call parameters
159 | if prompt is not None and isinstance(prompt, str):
160 | batch_size = 1
161 | elif prompt is not None and isinstance(prompt, list):
162 | batch_size = len(prompt)
163 | else:
164 | batch_size = prompt_embeds.shape[0]
165 |
166 | # device = self.device
167 |
168 | # 3. Encode input prompt
169 | text_encoder_lora_scale = (
170 | self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
171 | )
172 |
173 | self.text_encoder_ref.to(device=device)
174 | prompt_embeds_ref, negative_prompt_embeds_ref = self.encode_prompt_sk(
175 | self.text_encoder_ref,
176 | prompt,
177 | device,
178 | num_images_per_prompt,
179 | self.do_classifier_free_guidance,
180 | negative_prompt,
181 | prompt_embeds=prompt_embeds,
182 | negative_prompt_embeds=negative_prompt_embeds,
183 | lora_scale=text_encoder_lora_scale,
184 | clip_skip=self.clip_skip,
185 | )
186 | self.text_encoder_ref.cpu()
187 |
188 | self.text_encoder.to(device=device)
189 | prompt_embeds, negative_prompt_embeds = self.encode_prompt_sk(
190 | self.text_encoder,
191 | prompt,
192 | device,
193 | num_images_per_prompt,
194 | self.do_classifier_free_guidance,
195 | negative_prompt,
196 | prompt_embeds=prompt_embeds,
197 | negative_prompt_embeds=negative_prompt_embeds,
198 | lora_scale=text_encoder_lora_scale,
199 | clip_skip=self.clip_skip,
200 | )
201 | self.text_encoder.cpu()
202 |
203 | # For classifier free guidance, we need to do two forward passes.
204 | # Here we concatenate the unconditional and text embeddings into a single batch
205 | # to avoid doing two forward passes
206 | if self.do_classifier_free_guidance:
207 | prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
208 | prompt_embeds_ref = torch.cat([negative_prompt_embeds_ref, prompt_embeds_ref])
209 |
210 | if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
211 | image_embeds = self.prepare_ip_adapter_image_embeds(
212 | ip_adapter_image,
213 | ip_adapter_image_embeds,
214 | device,
215 | batch_size * num_images_per_prompt,
216 | self.do_classifier_free_guidance,
217 | )
218 |
219 | # 4. Preprocess
220 | image = self.image_processor.preprocess(image).to(device=device, dtype=prompt_embeds.dtype)
221 |
222 | scheduler = EulerDiscreteScheduler(
223 | num_train_timesteps=1000,
224 | beta_start=0.00085,
225 | beta_end=0.012,
226 | beta_schedule="scaled_linear",
227 | )
228 |
229 | # 5. set timesteps
230 | timesteps, num_inference_steps = retrieve_timesteps(
231 | scheduler, num_inference_steps, device, timesteps, sigmas
232 | )
233 | # timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
234 | latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
235 |
236 | # 6. Prepare reference latents
237 | self.vae.to(device=device)
238 | ref_latents = [
239 | retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator)
240 | for i in range(batch_size)
241 | ]
242 | self.vae.cpu()
243 |
244 | ref_latents = torch.cat(ref_latents, dim=0)
245 | ref_latents = self.vae.config.scaling_factor * ref_latents
246 | c, h, w = ref_latents.shape[1:]
247 |
248 | condition = drive_params['condition'].clone().to(device=device)
249 | if self.do_classifier_free_guidance:
250 | condition = torch.cat([torch.ones_like(condition) * -1, condition], dim=0)
251 |
252 | control_latents = {}
253 | if 'drive_coeff' in drive_params:
254 | self.mp_control.to(device=device)
255 | drive_coeff = drive_params['drive_coeff'].clone().to(device=device)
256 | face_parts = drive_params['face_parts'].clone().to(device=device)
257 | if self.do_classifier_free_guidance:
258 | drive_coeff = torch.cat([torch.zeros_like(drive_coeff), drive_coeff], dim=0)
259 | face_parts = torch.cat([torch.zeros_like(face_parts), face_parts], dim=0)
260 | control_latents1 = self.mp_control(condition=condition, drive_coeff=drive_coeff, face_parts=face_parts)
261 | control_latents.update(control_latents1)
262 | self.mp_control.cpu()
263 |
264 | if 'pd_fpg' in drive_params:
265 | self.mp_control2.to(device=device)
266 | pd_fpg = drive_params['pd_fpg'].clone().to(device=device)
267 | if self.do_classifier_free_guidance:
268 | neg_pd_fpg = drive_params['neg_pd_fpg'].clone().to(device=device)
269 | neg_pd_fpg.repeat_interleave(pd_fpg.size(1), dim=1)
270 | pd_fpg = torch.cat([neg_pd_fpg, pd_fpg], dim=0)
271 | control_latents2 = self.mp_control2(condition=condition, emo_embedding=pd_fpg)
272 | control_latents.update(control_latents2)
273 | self.mp_control2.cpu()
274 |
275 | # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
276 | extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
277 |
278 | # 7.1 Add image embeds for IP-Adapter
279 | added_cond_kwargs = (
280 | {"image_embeds": image_embeds}
281 | if ip_adapter_image is not None or ip_adapter_image_embeds is not None
282 | else None
283 | )
284 |
285 | latent_model_input = torch.cat([torch.zeros_like(ref_latents), ref_latents]) if self.do_classifier_free_guidance else ref_latents
286 | self.unet_ref.to(device=device)
287 | cached_res = self.unet_ref(
288 | latent_model_input.unsqueeze(2),
289 | 0,
290 | encoder_hidden_states=prompt_embeds_ref,
291 | return_dict=False,
292 | )[1]
293 | self.unet_ref.cpu()
294 |
295 | # 7.2 Optionally get Guidance Scale Embedding
296 | timestep_cond = None
297 | if self.unet.config.time_cond_proj_dim is not None:
298 | guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
299 | timestep_cond = self.get_guidance_scale_embedding(
300 | guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
301 | ).to(device=device, dtype=prompt_embeds.dtype)
302 |
303 | latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
304 | base_noise = randn_tensor([batch_size, c, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device)
305 | latents = base_noise * scheduler.init_noise_sigma
306 | # 8. Denoising loop
307 | num_warmup_steps = len(timesteps) - num_inference_steps * scheduler.order
308 | self._num_timesteps = len(timesteps)
309 | self.unet.to(device=device)
310 | with self.progress_bar(total=num_inference_steps) as progress_bar:
311 | for i, t in enumerate(timesteps):
312 | if self.interrupt:
313 | continue
314 |
315 | # expand the latents if we are doing classifier free guidance
316 | latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
317 | latent_model_input = scheduler.scale_model_input(latent_model_input, t)
318 |
319 | # predict the noise residual
320 | noise_pred = self.unet(
321 | latent_model_input.unsqueeze(2),
322 | t,
323 | encoder_hidden_states=prompt_embeds,
324 | reference_hidden_states=cached_res,
325 | control_hidden_states=control_latents,
326 | timestep_cond=timestep_cond,
327 | cross_attention_kwargs=self.cross_attention_kwargs,
328 | added_cond_kwargs=added_cond_kwargs,
329 | return_dict=False,
330 | )[0][:,:,0,:,:]
331 |
332 | # perform guidance
333 | if self.do_classifier_free_guidance:
334 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
335 | noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
336 |
337 | # compute the previous noisy sample x_t -> x_t-1
338 | latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
339 |
340 | if callback_on_step_end is not None:
341 | callback_kwargs = {}
342 | for k in callback_on_step_end_tensor_inputs:
343 | callback_kwargs[k] = locals()[k]
344 | callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
345 |
346 | latents = callback_outputs.pop("latents", latents)
347 | prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
348 | negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
349 |
350 | # call the callback, if provided
351 | if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0):
352 | progress_bar.update()
353 | if callback is not None and i % callback_steps == 0:
354 | step_idx = i // getattr(scheduler, "order", 1)
355 | callback(step_idx, t, latents)
356 |
357 | self.unet.cpu()
358 |
359 | self.vae_decode.to(device=device)
360 | if not output_type == "latent":
361 | image = self.vae_decode.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[
362 | 0
363 | ]
364 | else:
365 | image = latents
366 | self.vae_decode.cpu()
367 |
368 | do_denormalize = [True] * image.shape[0]
369 |
370 | image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
371 |
372 | # Offload all models
373 | self.maybe_free_model_hooks()
374 |
375 | return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=None), latents.detach().cpu() / self.vae.config.scaling_factor
376 |
--------------------------------------------------------------------------------
/hellomeme/pipelines/pipline_hm3_image.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | """
4 | @File : hm_pipline_image.py
5 | @Author : Songkey
6 | @Email : songkey@pku.edu.cn
7 | @Date : 1/3/2025
8 | @Desc :
9 | adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py
10 | """
11 |
12 | import copy
13 | from typing import Any, Callable, Dict, List, Optional, Union
14 | import torch
15 |
16 | from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback
17 | from diffusers.image_processor import PipelineImageInput
18 | from diffusers.utils import deprecate
19 | from diffusers.utils.torch_utils import randn_tensor
20 | from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput
21 | from diffusers import DPMSolverMultistepScheduler
22 | from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents
23 | from ..models import HM3Denoising3D, HMV3ControlNet, HMPipeline, HM3ReferenceAdapter, HMControlNetBase, HM4SD15ControlProj
24 | from ..tools.utils import creat_model_from_cloud
25 |
26 |
27 | class HM3ImagePipeline(HMPipeline):
28 | def caryomitosis(self, **kwargs):
29 | if hasattr(self, "unet_ref"):
30 | del self.unet_ref
31 | self.unet_ref = HM3Denoising3D.from_unet2d(self.unet)
32 | self.unet_ref.cpu()
33 |
34 | if not isinstance(self.unet, HM3Denoising3D):
35 | unet = HM3Denoising3D.from_unet2d(unet=self.unet)
36 | # todo: 不够优雅
37 | del self.unet
38 | self.unet = unet
39 | self.unet.cpu()
40 |
41 | self.vae.cpu()
42 | self.vae_decode = copy.deepcopy(self.vae)
43 | self.text_encoder.cpu()
44 | self.text_encoder_ref = copy.deepcopy(self.text_encoder)
45 | if hasattr(self, 'safety_checker'):
46 | del self.safety_checker
47 |
48 | def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False):
49 | self.version = version
50 |
51 | if version == 'v3':
52 | hm_reference_dir = 'songkey/hm3_reference'
53 | hm_control_dir = 'songkey/hm3_control_mix'
54 | else:
55 | hm_reference_dir = 'songkey/hm4_reference'
56 | hm_control_dir = 'songkey/hm_control_base'
57 | hm_control_proj_dir = 'songkey/hm4_control_proj'
58 |
59 | if isinstance(self.unet, HM3Denoising3D):
60 | hm_adapter = creat_model_from_cloud(HM3ReferenceAdapter, hm_reference_dir, modelscope=modelscope)
61 | self.unet.insert_reference_adapter(hm_adapter)
62 | self.unet.to(device='cpu', dtype=dtype).eval()
63 |
64 | if hasattr(self, "unet_ref"):
65 | self.unet_ref.to(device='cpu', dtype=dtype).eval()
66 |
67 | if hasattr(self, "mp_control"):
68 | del self.mp_control
69 |
70 | if hasattr(self, "mp_control_proj"):
71 | del self.mp_control_proj
72 |
73 | if version == 'v3':
74 | self.mp_control = creat_model_from_cloud(HMV3ControlNet, hm_control_dir, modelscope=modelscope)
75 | else:
76 | self.mp_control = creat_model_from_cloud(HMControlNetBase, hm_control_dir, modelscope=modelscope)
77 | self.mp_control_proj = creat_model_from_cloud(HM4SD15ControlProj, hm_control_proj_dir, modelscope=modelscope)
78 |
79 | self.mp_control_proj.to(device='cpu', dtype=dtype).eval()
80 |
81 | self.mp_control.to(device='cpu', dtype=dtype).eval()
82 |
83 | self.vae.to(device='cpu', dtype=dtype).eval()
84 | self.vae_decode.to(device='cpu', dtype=dtype).eval()
85 | self.text_encoder.to(device='cpu', dtype=dtype).eval()
86 | self.text_encoder_ref.to(device='cpu', dtype=dtype).eval()
87 |
88 | @torch.no_grad()
89 | def __call__(
90 | self,
91 | prompt: Union[str, List[str]] = None,
92 | image: PipelineImageInput = None,
93 | drive_params: Dict[str, Any] = None,
94 | strength: float = 0.8,
95 | num_inference_steps: Optional[int] = 50,
96 | timesteps: List[int] = None,
97 | sigmas: List[float] = None,
98 | guidance_scale: Optional[float] = 7.5,
99 | negative_prompt: Optional[Union[str, List[str]]] = None,
100 | eta: Optional[float] = 0.0,
101 | generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
102 | prompt_embeds: Optional[torch.Tensor] = None,
103 | negative_prompt_embeds: Optional[torch.Tensor] = None,
104 | ip_adapter_image: Optional[PipelineImageInput] = None,
105 | ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None,
106 | output_type: Optional[str] = "pil",
107 | device: Optional[str] = "cpu",
108 | return_dict: bool = True,
109 | cross_attention_kwargs: Optional[Dict[str, Any]] = None,
110 | clip_skip: int = None,
111 | callback_on_step_end: Optional[
112 | Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
113 | ] = None,
114 | callback_on_step_end_tensor_inputs: List[str] = ["latents"],
115 | **kwargs,
116 | ):
117 | callback = kwargs.pop("callback", None)
118 | callback_steps = kwargs.pop("callback_steps", None)
119 | num_images_per_prompt = 1
120 |
121 | if callback is not None:
122 | deprecate(
123 | "callback",
124 | "1.0.0",
125 | "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
126 | )
127 | if callback_steps is not None:
128 | deprecate(
129 | "callback_steps",
130 | "1.0.0",
131 | "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
132 | )
133 |
134 | if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
135 | callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
136 |
137 | # 1. Check inputs. Raise error if not correct
138 | self.check_inputs(
139 | prompt,
140 | strength,
141 | callback_steps,
142 | negative_prompt,
143 | prompt_embeds,
144 | negative_prompt_embeds,
145 | ip_adapter_image,
146 | ip_adapter_image_embeds,
147 | callback_on_step_end_tensor_inputs,
148 | )
149 |
150 | self._guidance_scale = guidance_scale
151 | self._clip_skip = clip_skip
152 | self._cross_attention_kwargs = cross_attention_kwargs
153 | self._interrupt = False
154 |
155 | # 2. Define call parameters
156 | if prompt is not None and isinstance(prompt, str):
157 | batch_size = 1
158 | elif prompt is not None and isinstance(prompt, list):
159 | batch_size = len(prompt)
160 | else:
161 | batch_size = prompt_embeds.shape[0]
162 |
163 | # device = self.device
164 |
165 | # 3. Encode input prompt
166 | text_encoder_lora_scale = (
167 | self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
168 | )
169 |
170 | self.text_encoder_ref.to(device=device)
171 | prompt_embeds_ref, negative_prompt_embeds_ref = self.encode_prompt_sk(
172 | self.text_encoder_ref,
173 | prompt,
174 | device,
175 | num_images_per_prompt,
176 | self.do_classifier_free_guidance,
177 | negative_prompt,
178 | prompt_embeds=prompt_embeds,
179 | negative_prompt_embeds=negative_prompt_embeds,
180 | lora_scale=text_encoder_lora_scale,
181 | clip_skip=self.clip_skip,
182 | )
183 | self.text_encoder_ref.cpu()
184 |
185 | self.text_encoder.to(device=device)
186 | prompt_embeds, negative_prompt_embeds = self.encode_prompt_sk(
187 | self.text_encoder,
188 | prompt,
189 | device,
190 | num_images_per_prompt,
191 | self.do_classifier_free_guidance,
192 | negative_prompt,
193 | prompt_embeds=prompt_embeds,
194 | negative_prompt_embeds=negative_prompt_embeds,
195 | lora_scale=text_encoder_lora_scale,
196 | clip_skip=self.clip_skip,
197 | )
198 | self.text_encoder.cpu()
199 |
200 | # For classifier free guidance, we need to do two forward passes.
201 | # Here we concatenate the unconditional and text embeddings into a single batch
202 | # to avoid doing two forward passes
203 | if self.do_classifier_free_guidance:
204 | prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
205 | prompt_embeds_ref = torch.cat([negative_prompt_embeds_ref, prompt_embeds_ref])
206 |
207 | if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
208 | image_embeds = self.prepare_ip_adapter_image_embeds(
209 | ip_adapter_image,
210 | ip_adapter_image_embeds,
211 | device,
212 | batch_size * num_images_per_prompt,
213 | self.do_classifier_free_guidance,
214 | )
215 |
216 | # 4. Preprocess
217 | image = self.image_processor.preprocess(image).to(device=device, dtype=prompt_embeds.dtype)
218 |
219 | scheduler = DPMSolverMultistepScheduler(
220 | num_train_timesteps=1000,
221 | beta_start=0.00085,
222 | beta_end=0.012,
223 | beta_schedule="scaled_linear",
224 | # use_karras_sigmas=True,
225 | algorithm_type="sde-dpmsolver++",
226 | )
227 |
228 | # 5. set timesteps
229 | timesteps, num_inference_steps = retrieve_timesteps(scheduler, num_inference_steps, device, timesteps, sigmas)
230 |
231 | # 6. Prepare reference latents
232 | self.vae.to(device=device)
233 | ref_latents = [
234 | retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator)
235 | for i in range(batch_size)
236 | ]
237 | self.vae.cpu()
238 |
239 | ref_latents = torch.cat(ref_latents, dim=0)
240 | ref_latents = self.vae.config.scaling_factor * ref_latents
241 | c, h, w = ref_latents.shape[1:]
242 |
243 | condition = drive_params['condition'].clone().to(device=device)
244 | if self.do_classifier_free_guidance:
245 | condition = torch.cat([torch.ones_like(condition) * -1, condition], dim=0)
246 |
247 | control_latents = {}
248 | self.mp_control.to(device=device)
249 | if hasattr(self, 'mp_control_proj') and self.version == 'v4':
250 | self.mp_control_proj.to(device=device)
251 | if 'drive_coeff' in drive_params:
252 | drive_coeff = drive_params['drive_coeff'].clone().to(device=device)
253 | face_parts = drive_params['face_parts'].clone().to(device=device)
254 | if self.do_classifier_free_guidance:
255 | drive_coeff = torch.cat([torch.zeros_like(drive_coeff), drive_coeff], dim=0)
256 | face_parts = torch.cat([torch.zeros_like(face_parts), face_parts], dim=0)
257 | control_latents1 = self.mp_control(condition=condition, drive_coeff=drive_coeff, face_parts=face_parts)
258 | if self.version == 'v4':
259 | control_latents1 = self.mp_control_proj(control_latents1)
260 | control_latents.update(control_latents1)
261 | elif 'pd_fpg' in drive_params:
262 | pd_fpg = drive_params['pd_fpg'].clone().to(device=device)
263 | if self.do_classifier_free_guidance:
264 | pd_fpg = torch.cat([torch.zeros_like(pd_fpg), pd_fpg], dim=0)
265 | control_latents2 = self.mp_control(condition=condition, emo_embedding=pd_fpg)
266 | if self.version == 'v4':
267 | control_latents2 = self.mp_control_proj(control_latents2)
268 | control_latents.update(control_latents2)
269 | self.mp_control.cpu()
270 | if self.version == 'v4':
271 | self.mp_control_proj.cpu()
272 |
273 | # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
274 | extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
275 |
276 | # 7.1 Add image embeds for IP-Adapter
277 | added_cond_kwargs = (
278 | {"image_embeds": image_embeds}
279 | if ip_adapter_image is not None or ip_adapter_image_embeds is not None
280 | else None
281 | )
282 | base_noise = randn_tensor([batch_size, c, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device)
283 |
284 | latent_model_input = torch.cat([torch.zeros_like(ref_latents), ref_latents]) if (
285 | self.do_classifier_free_guidance) else ref_latents
286 | # latent_model_input = torch.cat([ref_latents_neg, ref_latents], dim=0)
287 | self.unet_ref.to(device=device)
288 | cached_res = self.unet_ref(
289 | latent_model_input.unsqueeze(2),
290 | 0,
291 | encoder_hidden_states=prompt_embeds_ref,
292 | return_dict=False,
293 | )[1]
294 | self.unet_ref.cpu()
295 |
296 | # 7.2 Optionally get Guidance Scale Embedding
297 | timestep_cond = None
298 | if self.unet.config.time_cond_proj_dim is not None:
299 | guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
300 | timestep_cond = self.get_guidance_scale_embedding(
301 | guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
302 | ).to(device=device, dtype=prompt_embeds.dtype)
303 |
304 | latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
305 | # base_noise = randn_tensor([batch_size, c, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device)
306 | latents = base_noise * scheduler.init_noise_sigma
307 | # 8. Denoising loop
308 | num_warmup_steps = len(timesteps) - num_inference_steps * scheduler.order
309 | self._num_timesteps = len(timesteps)
310 | self.unet.to(device=device)
311 | with self.progress_bar(total=num_inference_steps) as progress_bar:
312 | for i, t in enumerate(timesteps):
313 | if self.interrupt:
314 | continue
315 |
316 | # expand the latents if we are doing classifier free guidance
317 | latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
318 | latent_model_input = scheduler.scale_model_input(latent_model_input, t)
319 |
320 | # predict the noise residual
321 | noise_pred = self.unet(
322 | latent_model_input.unsqueeze(2),
323 | t,
324 | encoder_hidden_states=prompt_embeds,
325 | reference_hidden_states=cached_res,
326 | control_hidden_states=control_latents,
327 | timestep_cond=timestep_cond,
328 | cross_attention_kwargs=self.cross_attention_kwargs,
329 | added_cond_kwargs=added_cond_kwargs,
330 | return_dict=False,
331 | )[0][:,:,0,:,:]
332 |
333 | # perform guidance
334 | if self.do_classifier_free_guidance:
335 | noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
336 | noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
337 |
338 | # compute the previous noisy sample x_t -> x_t-1
339 | latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
340 |
341 | if callback_on_step_end is not None:
342 | callback_kwargs = {}
343 | for k in callback_on_step_end_tensor_inputs:
344 | callback_kwargs[k] = locals()[k]
345 | callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
346 |
347 | latents = callback_outputs.pop("latents", latents)
348 | prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
349 | negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
350 |
351 | # call the callback, if provided
352 | if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0):
353 | progress_bar.update()
354 | if callback is not None and i % callback_steps == 0:
355 | step_idx = i // getattr(scheduler, "order", 1)
356 | callback(step_idx, t, latents)
357 |
358 | self.unet.cpu()
359 |
360 | self.vae_decode.to(device=device)
361 | if not output_type == "latent":
362 | image = self.vae_decode.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[
363 | 0
364 | ]
365 | else:
366 | image = latents
367 | self.vae_decode.cpu()
368 |
369 | do_denormalize = [True] * image.shape[0]
370 |
371 | image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
372 |
373 | # Offload all models
374 | self.maybe_free_model_hooks()
375 |
376 | return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=None), latents.detach().cpu() / self.vae.config.scaling_factor
377 |
--------------------------------------------------------------------------------
/example_workflows/video_generation.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "4c8b7397-a69c-46cc-b553-00e600557365",
3 | "revision": 0,
4 | "last_node_id": 76,
5 | "last_link_id": 178,
6 | "nodes": [
7 | {
8 | "id": 75,
9 | "type": "GetExpression2",
10 | "pos": [
11 | 814.1359252929688,
12 | 689.6795654296875
13 | ],
14 | "size": [
15 | 191.63729858398438,
16 | 66
17 | ],
18 | "flags": {},
19 | "order": 9,
20 | "mode": 0,
21 | "inputs": [
22 | {
23 | "name": "face_toolkits",
24 | "type": "FACE_TOOLKITS",
25 | "link": 173
26 | },
27 | {
28 | "name": "images",
29 | "type": "IMAGE",
30 | "link": 175
31 | },
32 | {
33 | "name": "landmarks",
34 | "type": "FACELANDMARKS222",
35 | "link": 178
36 | }
37 | ],
38 | "outputs": [
39 | {
40 | "name": "expression",
41 | "type": "EXPRESSION",
42 | "links": [
43 | 171
44 | ]
45 | }
46 | ],
47 | "properties": {
48 | "Node name for S&R": "GetExpression2"
49 | },
50 | "widgets_values": []
51 | },
52 | {
53 | "id": 76,
54 | "type": "GetExpression2",
55 | "pos": [
56 | 833.6573486328125,
57 | 1042.0396728515625
58 | ],
59 | "size": [
60 | 191.63729858398438,
61 | 66
62 | ],
63 | "flags": {},
64 | "order": 7,
65 | "mode": 0,
66 | "inputs": [
67 | {
68 | "name": "face_toolkits",
69 | "type": "FACE_TOOLKITS",
70 | "link": 174
71 | },
72 | {
73 | "name": "images",
74 | "type": "IMAGE",
75 | "link": 176
76 | },
77 | {
78 | "name": "landmarks",
79 | "type": "FACELANDMARKS222",
80 | "link": 177
81 | }
82 | ],
83 | "outputs": [
84 | {
85 | "name": "expression",
86 | "type": "EXPRESSION",
87 | "links": [
88 | 172
89 | ]
90 | }
91 | ],
92 | "properties": {
93 | "Node name for S&R": "GetExpression2"
94 | },
95 | "widgets_values": []
96 | },
97 | {
98 | "id": 2,
99 | "type": "HMFaceToolkitsLoader",
100 | "pos": [
101 | 45.41996383666992,
102 | 769.9866333007812
103 | ],
104 | "size": [
105 | 315,
106 | 82
107 | ],
108 | "flags": {},
109 | "order": 0,
110 | "mode": 0,
111 | "inputs": [],
112 | "outputs": [
113 | {
114 | "name": "face_toolkits",
115 | "type": "FACE_TOOLKITS",
116 | "slot_index": 0,
117 | "links": [
118 | 122,
119 | 129,
120 | 147,
121 | 149,
122 | 173,
123 | 174
124 | ]
125 | }
126 | ],
127 | "properties": {
128 | "Node name for S&R": "HMFaceToolkitsLoader"
129 | },
130 | "widgets_values": [
131 | 0,
132 | "huggingface"
133 | ]
134 | },
135 | {
136 | "id": 61,
137 | "type": "GetFaceLandmarks",
138 | "pos": [
139 | 519.7196655273438,
140 | 938.032958984375
141 | ],
142 | "size": [
143 | 170.62362670898438,
144 | 46
145 | ],
146 | "flags": {},
147 | "order": 4,
148 | "mode": 0,
149 | "inputs": [
150 | {
151 | "name": "face_toolkits",
152 | "type": "FACE_TOOLKITS",
153 | "link": 129
154 | },
155 | {
156 | "name": "images",
157 | "type": "IMAGE",
158 | "link": 131
159 | }
160 | ],
161 | "outputs": [
162 | {
163 | "name": "landmarks",
164 | "type": "FACELANDMARKS222",
165 | "links": [
166 | 145,
167 | 177
168 | ]
169 | }
170 | ],
171 | "properties": {
172 | "Node name for S&R": "GetFaceLandmarks"
173 | },
174 | "widgets_values": []
175 | },
176 | {
177 | "id": 60,
178 | "type": "GetFaceLandmarks",
179 | "pos": [
180 | 518.9132690429688,
181 | 612.9461669921875
182 | ],
183 | "size": [
184 | 170.62362670898438,
185 | 46
186 | ],
187 | "flags": {},
188 | "order": 5,
189 | "mode": 0,
190 | "inputs": [
191 | {
192 | "name": "face_toolkits",
193 | "type": "FACE_TOOLKITS",
194 | "link": 122
195 | },
196 | {
197 | "name": "images",
198 | "type": "IMAGE",
199 | "link": 121
200 | }
201 | ],
202 | "outputs": [
203 | {
204 | "name": "landmarks",
205 | "type": "FACELANDMARKS222",
206 | "links": [
207 | 143,
208 | 178
209 | ]
210 | }
211 | ],
212 | "properties": {
213 | "Node name for S&R": "GetFaceLandmarks"
214 | },
215 | "widgets_values": []
216 | },
217 | {
218 | "id": 59,
219 | "type": "VHS_VideoCombine",
220 | "pos": [
221 | 1658.52490234375,
222 | 685.6950073242188
223 | ],
224 | "size": [
225 | 329.22479248046875,
226 | 633.2247924804688
227 | ],
228 | "flags": {},
229 | "order": 11,
230 | "mode": 0,
231 | "inputs": [
232 | {
233 | "name": "images",
234 | "type": "IMAGE",
235 | "link": 170
236 | },
237 | {
238 | "name": "audio",
239 | "shape": 7,
240 | "type": "AUDIO",
241 | "link": 118
242 | },
243 | {
244 | "name": "meta_batch",
245 | "shape": 7,
246 | "type": "VHS_BatchManager",
247 | "link": null
248 | },
249 | {
250 | "name": "vae",
251 | "shape": 7,
252 | "type": "VAE",
253 | "link": null
254 | }
255 | ],
256 | "outputs": [
257 | {
258 | "name": "Filenames",
259 | "type": "VHS_FILENAMES",
260 | "links": null
261 | }
262 | ],
263 | "properties": {
264 | "Node name for S&R": "VHS_VideoCombine"
265 | },
266 | "widgets_values": {
267 | "frame_rate": 8,
268 | "loop_count": 0,
269 | "filename_prefix": "AnimateDiff",
270 | "format": "video/h264-mp4",
271 | "pix_fmt": "yuv420p",
272 | "crf": 18,
273 | "save_metadata": true,
274 | "pingpong": false,
275 | "save_output": true,
276 | "videopreview": {
277 | "hidden": false,
278 | "paused": false,
279 | "params": {
280 | "filename": "AnimateDiff_00703-audio.mp4",
281 | "subfolder": "",
282 | "type": "output",
283 | "format": "video/h264-mp4",
284 | "frame_rate": 8
285 | },
286 | "muted": false
287 | }
288 | }
289 | },
290 | {
291 | "id": 74,
292 | "type": "HMPipelineVideo",
293 | "pos": [
294 | 1296.2967529296875,
295 | 659.4104614257812
296 | ],
297 | "size": [
298 | 270,
299 | 330
300 | ],
301 | "flags": {},
302 | "order": 10,
303 | "mode": 0,
304 | "inputs": [
305 | {
306 | "name": "hm_video_pipeline",
307 | "type": "HMVIDEOPIPELINE",
308 | "link": 165
309 | },
310 | {
311 | "name": "ref_head_pose",
312 | "type": "HEAD_POSE",
313 | "link": 166
314 | },
315 | {
316 | "name": "ref_expression",
317 | "type": "EXPRESSION",
318 | "link": 171
319 | },
320 | {
321 | "name": "drive_head_pose",
322 | "type": "HEAD_POSE",
323 | "link": 168
324 | },
325 | {
326 | "name": "drive_expression",
327 | "type": "EXPRESSION",
328 | "link": 172
329 | }
330 | ],
331 | "outputs": [
332 | {
333 | "name": "IMAGE",
334 | "type": "IMAGE",
335 | "links": [
336 | 170
337 | ]
338 | },
339 | {
340 | "name": "LATENT",
341 | "type": "LATENT",
342 | "links": null
343 | }
344 | ],
345 | "properties": {
346 | "Node name for S&R": "HMPipelineVideo"
347 | },
348 | "widgets_values": [
349 | 0,
350 | 4,
351 | "",
352 | "",
353 | 25,
354 | 1075819929118817,
355 | "randomize",
356 | 1.5,
357 | 0
358 | ]
359 | },
360 | {
361 | "id": 69,
362 | "type": "GetHeadPose",
363 | "pos": [
364 | 819.799560546875,
365 | 860.5927734375
366 | ],
367 | "size": [
368 | 270,
369 | 98
370 | ],
371 | "flags": {},
372 | "order": 6,
373 | "mode": 0,
374 | "inputs": [
375 | {
376 | "name": "face_toolkits",
377 | "type": "FACE_TOOLKITS",
378 | "link": 149
379 | },
380 | {
381 | "name": "images",
382 | "type": "IMAGE",
383 | "link": 153
384 | },
385 | {
386 | "name": "landmarks",
387 | "type": "FACELANDMARKS222",
388 | "link": 145
389 | }
390 | ],
391 | "outputs": [
392 | {
393 | "name": "head_pose",
394 | "type": "HEAD_POSE",
395 | "links": [
396 | 168
397 | ]
398 | }
399 | ],
400 | "properties": {
401 | "Node name for S&R": "GetHeadPose"
402 | },
403 | "widgets_values": [
404 | true
405 | ]
406 | },
407 | {
408 | "id": 48,
409 | "type": "VHS_LoadVideo",
410 | "pos": [
411 | 74.54000091552734,
412 | 996.8733520507812
413 | ],
414 | "size": [
415 | 247.455078125,
416 | 262
417 | ],
418 | "flags": {},
419 | "order": 1,
420 | "mode": 0,
421 | "inputs": [
422 | {
423 | "name": "meta_batch",
424 | "shape": 7,
425 | "type": "VHS_BatchManager",
426 | "link": null
427 | },
428 | {
429 | "name": "vae",
430 | "shape": 7,
431 | "type": "VAE",
432 | "link": null
433 | }
434 | ],
435 | "outputs": [
436 | {
437 | "name": "IMAGE",
438 | "type": "IMAGE",
439 | "slot_index": 0,
440 | "links": [
441 | 131,
442 | 153,
443 | 176
444 | ]
445 | },
446 | {
447 | "name": "frame_count",
448 | "type": "INT",
449 | "links": null
450 | },
451 | {
452 | "name": "audio",
453 | "type": "AUDIO",
454 | "slot_index": 2,
455 | "links": [
456 | 118
457 | ]
458 | },
459 | {
460 | "name": "video_info",
461 | "type": "VHS_VIDEOINFO",
462 | "links": null
463 | }
464 | ],
465 | "properties": {
466 | "Node name for S&R": "VHS_LoadVideo"
467 | },
468 | "widgets_values": {
469 | "video": "jgz.mp4",
470 | "force_rate": 8,
471 | "force_size": "Disabled",
472 | "custom_width": 512,
473 | "custom_height": 512,
474 | "frame_load_cap": 0,
475 | "skip_first_frames": 0,
476 | "select_every_nth": 1,
477 | "choose video to upload": "image",
478 | "videopreview": {
479 | "hidden": false,
480 | "paused": false,
481 | "params": {
482 | "force_rate": 8,
483 | "frame_load_cap": 0,
484 | "skip_first_frames": 0,
485 | "select_every_nth": 1,
486 | "filename": "jgz.mp4",
487 | "type": "input",
488 | "format": "video/mp4"
489 | },
490 | "muted": false
491 | }
492 | }
493 | },
494 | {
495 | "id": 58,
496 | "type": "LoadImage",
497 | "pos": [
498 | 37.4466552734375,
499 | 260.9000244140625
500 | ],
501 | "size": [
502 | 315,
503 | 314
504 | ],
505 | "flags": {},
506 | "order": 2,
507 | "mode": 0,
508 | "inputs": [],
509 | "outputs": [
510 | {
511 | "name": "IMAGE",
512 | "type": "IMAGE",
513 | "slot_index": 0,
514 | "links": [
515 | 121,
516 | 151,
517 | 175
518 | ]
519 | },
520 | {
521 | "name": "MASK",
522 | "type": "MASK",
523 | "links": null
524 | }
525 | ],
526 | "properties": {
527 | "Node name for S&R": "LoadImage"
528 | },
529 | "widgets_values": [
530 | "i5.jpg",
531 | "image"
532 | ]
533 | },
534 | {
535 | "id": 67,
536 | "type": "GetHeadPose",
537 | "pos": [
538 | 814.1528930664062,
539 | 495.9793701171875
540 | ],
541 | "size": [
542 | 270,
543 | 98
544 | ],
545 | "flags": {},
546 | "order": 8,
547 | "mode": 0,
548 | "inputs": [
549 | {
550 | "name": "face_toolkits",
551 | "type": "FACE_TOOLKITS",
552 | "link": 147
553 | },
554 | {
555 | "name": "images",
556 | "type": "IMAGE",
557 | "link": 151
558 | },
559 | {
560 | "name": "landmarks",
561 | "type": "FACELANDMARKS222",
562 | "link": 143
563 | }
564 | ],
565 | "outputs": [
566 | {
567 | "name": "head_pose",
568 | "type": "HEAD_POSE",
569 | "links": [
570 | 166
571 | ]
572 | }
573 | ],
574 | "properties": {
575 | "Node name for S&R": "GetHeadPose"
576 | },
577 | "widgets_values": [
578 | true
579 | ]
580 | },
581 | {
582 | "id": 57,
583 | "type": "HMVideoPipelineLoader",
584 | "pos": [
585 | 798.4796752929688,
586 | 193.7666015625
587 | ],
588 | "size": [
589 | 352.79998779296875,
590 | 226
591 | ],
592 | "flags": {},
593 | "order": 3,
594 | "mode": 0,
595 | "inputs": [],
596 | "outputs": [
597 | {
598 | "name": "hm_video_pipeline",
599 | "type": "HMVIDEOPIPELINE",
600 | "slot_index": 0,
601 | "links": [
602 | 165
603 | ]
604 | }
605 | ],
606 | "properties": {
607 | "Node name for S&R": "HMVideoPipelineLoader"
608 | },
609 | "widgets_values": [
610 | "[preset]RealisticVisionV60B1",
611 | "None",
612 | "same as checkpoint",
613 | "v5",
614 | "x1",
615 | "huggingface",
616 | 1,
617 | "fp32"
618 | ]
619 | }
620 | ],
621 | "links": [
622 | [
623 | 118,
624 | 48,
625 | 2,
626 | 59,
627 | 1,
628 | "AUDIO"
629 | ],
630 | [
631 | 121,
632 | 58,
633 | 0,
634 | 60,
635 | 1,
636 | "IMAGE"
637 | ],
638 | [
639 | 122,
640 | 2,
641 | 0,
642 | 60,
643 | 0,
644 | "FACE_TOOLKITS"
645 | ],
646 | [
647 | 129,
648 | 2,
649 | 0,
650 | 61,
651 | 0,
652 | "FACE_TOOLKITS"
653 | ],
654 | [
655 | 131,
656 | 48,
657 | 0,
658 | 61,
659 | 1,
660 | "IMAGE"
661 | ],
662 | [
663 | 143,
664 | 60,
665 | 0,
666 | 67,
667 | 2,
668 | "FACELANDMARKS222"
669 | ],
670 | [
671 | 145,
672 | 61,
673 | 0,
674 | 69,
675 | 2,
676 | "FACELANDMARKS222"
677 | ],
678 | [
679 | 147,
680 | 2,
681 | 0,
682 | 67,
683 | 0,
684 | "FACE_TOOLKITS"
685 | ],
686 | [
687 | 149,
688 | 2,
689 | 0,
690 | 69,
691 | 0,
692 | "FACE_TOOLKITS"
693 | ],
694 | [
695 | 151,
696 | 58,
697 | 0,
698 | 67,
699 | 1,
700 | "IMAGE"
701 | ],
702 | [
703 | 153,
704 | 48,
705 | 0,
706 | 69,
707 | 1,
708 | "IMAGE"
709 | ],
710 | [
711 | 165,
712 | 57,
713 | 0,
714 | 74,
715 | 0,
716 | "HMVIDEOPIPELINE"
717 | ],
718 | [
719 | 166,
720 | 67,
721 | 0,
722 | 74,
723 | 1,
724 | "HEAD_POSE"
725 | ],
726 | [
727 | 168,
728 | 69,
729 | 0,
730 | 74,
731 | 3,
732 | "HEAD_POSE"
733 | ],
734 | [
735 | 170,
736 | 74,
737 | 0,
738 | 59,
739 | 0,
740 | "IMAGE"
741 | ],
742 | [
743 | 171,
744 | 75,
745 | 0,
746 | 74,
747 | 2,
748 | "EXPRESSION"
749 | ],
750 | [
751 | 172,
752 | 76,
753 | 0,
754 | 74,
755 | 4,
756 | "EXPRESSION"
757 | ],
758 | [
759 | 173,
760 | 2,
761 | 0,
762 | 75,
763 | 0,
764 | "FACE_TOOLKITS"
765 | ],
766 | [
767 | 174,
768 | 2,
769 | 0,
770 | 76,
771 | 0,
772 | "FACE_TOOLKITS"
773 | ],
774 | [
775 | 175,
776 | 58,
777 | 0,
778 | 75,
779 | 1,
780 | "IMAGE"
781 | ],
782 | [
783 | 176,
784 | 48,
785 | 0,
786 | 76,
787 | 1,
788 | "IMAGE"
789 | ],
790 | [
791 | 177,
792 | 61,
793 | 0,
794 | 76,
795 | 2,
796 | "FACELANDMARKS222"
797 | ],
798 | [
799 | 178,
800 | 60,
801 | 0,
802 | 75,
803 | 2,
804 | "FACELANDMARKS222"
805 | ]
806 | ],
807 | "groups": [],
808 | "config": {},
809 | "extra": {
810 | "ds": {
811 | "scale": 0.8264462809917358,
812 | "offset": [
813 | 493.27712102477454,
814 | -69.45337656976547
815 | ]
816 | },
817 | "frontendVersion": "1.19.2"
818 | },
819 | "version": 0.4
820 | }
--------------------------------------------------------------------------------
/example_workflows/image_style_transfer.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "a8e8ebf3-8dc8-4d76-ad73-dfe0b945cc77",
3 | "revision": 0,
4 | "last_node_id": 87,
5 | "last_link_id": 213,
6 | "nodes": [
7 | {
8 | "id": 58,
9 | "type": "LoadImage",
10 | "pos": [
11 | 41.47996520996094,
12 | 358.5067138671875
13 | ],
14 | "size": [
15 | 315,
16 | 314
17 | ],
18 | "flags": {},
19 | "order": 0,
20 | "mode": 0,
21 | "inputs": [],
22 | "outputs": [
23 | {
24 | "name": "IMAGE",
25 | "type": "IMAGE",
26 | "slot_index": 0,
27 | "links": [
28 | 121,
29 | 151,
30 | 175
31 | ]
32 | },
33 | {
34 | "name": "MASK",
35 | "type": "MASK",
36 | "links": null
37 | }
38 | ],
39 | "properties": {
40 | "cnr_id": "comfy-core",
41 | "ver": "0.3.39",
42 | "Node name for S&R": "LoadImage"
43 | },
44 | "widgets_values": [
45 | "chillout.jpg",
46 | "image"
47 | ]
48 | },
49 | {
50 | "id": 2,
51 | "type": "HMFaceToolkitsLoader",
52 | "pos": [
53 | 45.41996383666992,
54 | 769.9866333007812
55 | ],
56 | "size": [
57 | 315,
58 | 82
59 | ],
60 | "flags": {},
61 | "order": 1,
62 | "mode": 0,
63 | "inputs": [],
64 | "outputs": [
65 | {
66 | "name": "face_toolkits",
67 | "type": "FACE_TOOLKITS",
68 | "slot_index": 0,
69 | "links": [
70 | 122,
71 | 147,
72 | 173,
73 | 193,
74 | 194,
75 | 210
76 | ]
77 | }
78 | ],
79 | "properties": {
80 | "cnr_id": "comfyui_hellomeme",
81 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
82 | "Node name for S&R": "HMFaceToolkitsLoader"
83 | },
84 | "widgets_values": [
85 | 0,
86 | "huggingface"
87 | ]
88 | },
89 | {
90 | "id": 60,
91 | "type": "GetFaceLandmarks",
92 | "pos": [
93 | 438.875732421875,
94 | 586.59228515625
95 | ],
96 | "size": [
97 | 170.62362670898438,
98 | 46
99 | ],
100 | "flags": {},
101 | "order": 3,
102 | "mode": 0,
103 | "inputs": [
104 | {
105 | "name": "face_toolkits",
106 | "type": "FACE_TOOLKITS",
107 | "link": 122
108 | },
109 | {
110 | "name": "images",
111 | "type": "IMAGE",
112 | "link": 121
113 | }
114 | ],
115 | "outputs": [
116 | {
117 | "name": "landmarks",
118 | "type": "FACELANDMARKS222",
119 | "links": [
120 | 143,
121 | 178
122 | ]
123 | }
124 | ],
125 | "properties": {
126 | "cnr_id": "comfyui_hellomeme",
127 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
128 | "Node name for S&R": "GetFaceLandmarks"
129 | },
130 | "widgets_values": []
131 | },
132 | {
133 | "id": 75,
134 | "type": "GetExpression2",
135 | "pos": [
136 | 689.199462890625,
137 | 676.9906616210938
138 | ],
139 | "size": [
140 | 191.63729858398438,
141 | 66
142 | ],
143 | "flags": {},
144 | "order": 5,
145 | "mode": 0,
146 | "inputs": [
147 | {
148 | "name": "face_toolkits",
149 | "type": "FACE_TOOLKITS",
150 | "link": 173
151 | },
152 | {
153 | "name": "images",
154 | "type": "IMAGE",
155 | "link": 175
156 | },
157 | {
158 | "name": "landmarks",
159 | "type": "FACELANDMARKS222",
160 | "link": 178
161 | }
162 | ],
163 | "outputs": [
164 | {
165 | "name": "expression",
166 | "type": "EXPRESSION",
167 | "links": [
168 | 184,
169 | 207
170 | ]
171 | }
172 | ],
173 | "properties": {
174 | "cnr_id": "comfyui_hellomeme",
175 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
176 | "Node name for S&R": "GetExpression2"
177 | },
178 | "widgets_values": []
179 | },
180 | {
181 | "id": 78,
182 | "type": "HMImagePipelineLoader",
183 | "pos": [
184 | 689.4496459960938,
185 | 203.48110961914062
186 | ],
187 | "size": [
188 | 298.3472595214844,
189 | 226
190 | ],
191 | "flags": {},
192 | "order": 2,
193 | "mode": 0,
194 | "inputs": [],
195 | "outputs": [
196 | {
197 | "name": "hm_image_pipeline",
198 | "type": "HMIMAGEPIPELINE",
199 | "links": [
200 | 182,
201 | 199
202 | ]
203 | }
204 | ],
205 | "properties": {
206 | "cnr_id": "comfyui_hellomeme",
207 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
208 | "Node name for S&R": "HMImagePipelineLoader"
209 | },
210 | "widgets_values": [
211 | "[preset]animeanything_v10",
212 | "[preset]ghibli_anime_v1",
213 | "same as checkpoint",
214 | "v5c",
215 | "x1",
216 | "huggingface",
217 | 1.2,
218 | "fp32"
219 | ]
220 | },
221 | {
222 | "id": 67,
223 | "type": "GetHeadPose",
224 | "pos": [
225 | 688.240234375,
226 | 516.476806640625
227 | ],
228 | "size": [
229 | 270,
230 | 98
231 | ],
232 | "flags": {},
233 | "order": 4,
234 | "mode": 0,
235 | "inputs": [
236 | {
237 | "name": "face_toolkits",
238 | "type": "FACE_TOOLKITS",
239 | "link": 147
240 | },
241 | {
242 | "name": "images",
243 | "type": "IMAGE",
244 | "link": 151
245 | },
246 | {
247 | "name": "landmarks",
248 | "type": "FACELANDMARKS222",
249 | "link": 143
250 | }
251 | ],
252 | "outputs": [
253 | {
254 | "name": "head_pose",
255 | "type": "HEAD_POSE",
256 | "links": [
257 | 183,
258 | 206
259 | ]
260 | }
261 | ],
262 | "properties": {
263 | "cnr_id": "comfyui_hellomeme",
264 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
265 | "Node name for S&R": "GetHeadPose"
266 | },
267 | "widgets_values": [
268 | true
269 | ]
270 | },
271 | {
272 | "id": 81,
273 | "type": "GetFaceLandmarks",
274 | "pos": [
275 | 1466.214111328125,
276 | 510.1761169433594
277 | ],
278 | "size": [
279 | 170.62362670898438,
280 | 46
281 | ],
282 | "flags": {},
283 | "order": 8,
284 | "mode": 0,
285 | "inputs": [
286 | {
287 | "name": "face_toolkits",
288 | "type": "FACE_TOOLKITS",
289 | "link": 193
290 | },
291 | {
292 | "name": "images",
293 | "type": "IMAGE",
294 | "link": 191
295 | }
296 | ],
297 | "outputs": [
298 | {
299 | "name": "landmarks",
300 | "type": "FACELANDMARKS222",
301 | "links": [
302 | 195,
303 | 208
304 | ]
305 | }
306 | ],
307 | "properties": {
308 | "cnr_id": "comfyui_hellomeme",
309 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
310 | "Node name for S&R": "GetFaceLandmarks"
311 | },
312 | "widgets_values": []
313 | },
314 | {
315 | "id": 82,
316 | "type": "GetHeadPose",
317 | "pos": [
318 | 1726.154052734375,
319 | 420.4497375488281
320 | ],
321 | "size": [
322 | 270,
323 | 98
324 | ],
325 | "flags": {},
326 | "order": 9,
327 | "mode": 0,
328 | "inputs": [
329 | {
330 | "name": "face_toolkits",
331 | "type": "FACE_TOOLKITS",
332 | "link": 194
333 | },
334 | {
335 | "name": "images",
336 | "type": "IMAGE",
337 | "link": 192
338 | },
339 | {
340 | "name": "landmarks",
341 | "type": "FACELANDMARKS222",
342 | "link": 195
343 | }
344 | ],
345 | "outputs": [
346 | {
347 | "name": "head_pose",
348 | "type": "HEAD_POSE",
349 | "links": [
350 | 200,
351 | 211
352 | ]
353 | }
354 | ],
355 | "properties": {
356 | "cnr_id": "comfyui_hellomeme",
357 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
358 | "Node name for S&R": "GetHeadPose"
359 | },
360 | "widgets_values": [
361 | false
362 | ]
363 | },
364 | {
365 | "id": 80,
366 | "type": "PreviewImage",
367 | "pos": [
368 | 1597.02294921875,
369 | -149.21035766601562
370 | ],
371 | "size": [
372 | 426.67877197265625,
373 | 454.14093017578125
374 | ],
375 | "flags": {},
376 | "order": 7,
377 | "mode": 0,
378 | "inputs": [
379 | {
380 | "name": "images",
381 | "type": "IMAGE",
382 | "link": 187
383 | }
384 | ],
385 | "outputs": [],
386 | "properties": {
387 | "cnr_id": "comfy-core",
388 | "ver": "0.3.39",
389 | "Node name for S&R": "PreviewImage"
390 | },
391 | "widgets_values": []
392 | },
393 | {
394 | "id": 85,
395 | "type": "HMPipelineImage",
396 | "pos": [
397 | 2126.16357421875,
398 | 485.0890197753906
399 | ],
400 | "size": [
401 | 270,
402 | 306
403 | ],
404 | "flags": {},
405 | "order": 11,
406 | "mode": 0,
407 | "inputs": [
408 | {
409 | "name": "hm_image_pipeline",
410 | "type": "HMIMAGEPIPELINE",
411 | "link": 199
412 | },
413 | {
414 | "name": "ref_head_pose",
415 | "type": "HEAD_POSE",
416 | "link": 200
417 | },
418 | {
419 | "name": "ref_expression",
420 | "type": "EXPRESSION",
421 | "link": 212
422 | },
423 | {
424 | "name": "drive_head_pose",
425 | "type": "HEAD_POSE",
426 | "link": 211
427 | },
428 | {
429 | "name": "drive_expression",
430 | "type": "EXPRESSION",
431 | "link": 213
432 | }
433 | ],
434 | "outputs": [
435 | {
436 | "name": "IMAGE",
437 | "type": "IMAGE",
438 | "links": [
439 | 204
440 | ]
441 | },
442 | {
443 | "name": "LATENT",
444 | "type": "LATENT",
445 | "links": null
446 | }
447 | ],
448 | "properties": {
449 | "cnr_id": "comfyui_hellomeme",
450 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
451 | "Node name for S&R": "HMPipelineImage"
452 | },
453 | "widgets_values": [
454 | 0,
455 | "",
456 | "",
457 | 25,
458 | 431229341494867,
459 | "randomize",
460 | 1.2000000000000002,
461 | 0
462 | ]
463 | },
464 | {
465 | "id": 87,
466 | "type": "GetExpression2",
467 | "pos": [
468 | 1735.521728515625,
469 | 590.853759765625
470 | ],
471 | "size": [
472 | 191.63729858398438,
473 | 66
474 | ],
475 | "flags": {},
476 | "order": 10,
477 | "mode": 0,
478 | "inputs": [
479 | {
480 | "name": "face_toolkits",
481 | "type": "FACE_TOOLKITS",
482 | "link": 210
483 | },
484 | {
485 | "name": "images",
486 | "type": "IMAGE",
487 | "link": 209
488 | },
489 | {
490 | "name": "landmarks",
491 | "type": "FACELANDMARKS222",
492 | "link": 208
493 | }
494 | ],
495 | "outputs": [
496 | {
497 | "name": "expression",
498 | "type": "EXPRESSION",
499 | "links": [
500 | 212,
501 | 213
502 | ]
503 | }
504 | ],
505 | "properties": {
506 | "cnr_id": "comfyui_hellomeme",
507 | "ver": "47fe61445e785922239b772517bd55673d752c7c",
508 | "Node name for S&R": "GetExpression2"
509 | }
510 | },
511 | {
512 | "id": 86,
513 | "type": "PreviewImage",
514 | "pos": [
515 | 2487.072509765625,
516 | 485.347412109375
517 | ],
518 | "size": [
519 | 522.350341796875,
520 | 553.1380004882812
521 | ],
522 | "flags": {},
523 | "order": 12,
524 | "mode": 0,
525 | "inputs": [
526 | {
527 | "name": "images",
528 | "type": "IMAGE",
529 | "link": 204
530 | }
531 | ],
532 | "outputs": [],
533 | "properties": {
534 | "cnr_id": "comfy-core",
535 | "ver": "0.3.39",
536 | "Node name for S&R": "PreviewImage"
537 | },
538 | "widgets_values": []
539 | },
540 | {
541 | "id": 79,
542 | "type": "HMPipelineImage",
543 | "pos": [
544 | 1077.52099609375,
545 | 489.0169372558594
546 | ],
547 | "size": [
548 | 270,
549 | 306
550 | ],
551 | "flags": {},
552 | "order": 6,
553 | "mode": 0,
554 | "inputs": [
555 | {
556 | "name": "hm_image_pipeline",
557 | "type": "HMIMAGEPIPELINE",
558 | "link": 182
559 | },
560 | {
561 | "name": "ref_head_pose",
562 | "type": "HEAD_POSE",
563 | "link": 183
564 | },
565 | {
566 | "name": "ref_expression",
567 | "type": "EXPRESSION",
568 | "link": 184
569 | },
570 | {
571 | "name": "drive_head_pose",
572 | "type": "HEAD_POSE",
573 | "link": 206
574 | },
575 | {
576 | "name": "drive_expression",
577 | "type": "EXPRESSION",
578 | "link": 207
579 | }
580 | ],
581 | "outputs": [
582 | {
583 | "name": "IMAGE",
584 | "type": "IMAGE",
585 | "links": [
586 | 187,
587 | 191,
588 | 192,
589 | 209
590 | ]
591 | },
592 | {
593 | "name": "LATENT",
594 | "type": "LATENT",
595 | "links": null
596 | }
597 | ],
598 | "properties": {
599 | "cnr_id": "comfyui_hellomeme",
600 | "ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
601 | "Node name for S&R": "HMPipelineImage"
602 | },
603 | "widgets_values": [
604 | 0,
605 | "",
606 | "",
607 | 25,
608 | 1116653258512514,
609 | "randomize",
610 | 1.2000000000000002,
611 | 0
612 | ]
613 | }
614 | ],
615 | "links": [
616 | [
617 | 121,
618 | 58,
619 | 0,
620 | 60,
621 | 1,
622 | "IMAGE"
623 | ],
624 | [
625 | 122,
626 | 2,
627 | 0,
628 | 60,
629 | 0,
630 | "FACE_TOOLKITS"
631 | ],
632 | [
633 | 143,
634 | 60,
635 | 0,
636 | 67,
637 | 2,
638 | "FACELANDMARKS222"
639 | ],
640 | [
641 | 147,
642 | 2,
643 | 0,
644 | 67,
645 | 0,
646 | "FACE_TOOLKITS"
647 | ],
648 | [
649 | 151,
650 | 58,
651 | 0,
652 | 67,
653 | 1,
654 | "IMAGE"
655 | ],
656 | [
657 | 173,
658 | 2,
659 | 0,
660 | 75,
661 | 0,
662 | "FACE_TOOLKITS"
663 | ],
664 | [
665 | 175,
666 | 58,
667 | 0,
668 | 75,
669 | 1,
670 | "IMAGE"
671 | ],
672 | [
673 | 178,
674 | 60,
675 | 0,
676 | 75,
677 | 2,
678 | "FACELANDMARKS222"
679 | ],
680 | [
681 | 182,
682 | 78,
683 | 0,
684 | 79,
685 | 0,
686 | "HMIMAGEPIPELINE"
687 | ],
688 | [
689 | 183,
690 | 67,
691 | 0,
692 | 79,
693 | 1,
694 | "HEAD_POSE"
695 | ],
696 | [
697 | 184,
698 | 75,
699 | 0,
700 | 79,
701 | 2,
702 | "EXPRESSION"
703 | ],
704 | [
705 | 187,
706 | 79,
707 | 0,
708 | 80,
709 | 0,
710 | "IMAGE"
711 | ],
712 | [
713 | 191,
714 | 79,
715 | 0,
716 | 81,
717 | 1,
718 | "IMAGE"
719 | ],
720 | [
721 | 192,
722 | 79,
723 | 0,
724 | 82,
725 | 1,
726 | "IMAGE"
727 | ],
728 | [
729 | 193,
730 | 2,
731 | 0,
732 | 81,
733 | 0,
734 | "FACE_TOOLKITS"
735 | ],
736 | [
737 | 194,
738 | 2,
739 | 0,
740 | 82,
741 | 0,
742 | "FACE_TOOLKITS"
743 | ],
744 | [
745 | 195,
746 | 81,
747 | 0,
748 | 82,
749 | 2,
750 | "FACELANDMARKS222"
751 | ],
752 | [
753 | 199,
754 | 78,
755 | 0,
756 | 85,
757 | 0,
758 | "HMIMAGEPIPELINE"
759 | ],
760 | [
761 | 200,
762 | 82,
763 | 0,
764 | 85,
765 | 1,
766 | "HEAD_POSE"
767 | ],
768 | [
769 | 204,
770 | 85,
771 | 0,
772 | 86,
773 | 0,
774 | "IMAGE"
775 | ],
776 | [
777 | 206,
778 | 67,
779 | 0,
780 | 79,
781 | 3,
782 | "HEAD_POSE"
783 | ],
784 | [
785 | 207,
786 | 75,
787 | 0,
788 | 79,
789 | 4,
790 | "EXPRESSION"
791 | ],
792 | [
793 | 208,
794 | 81,
795 | 0,
796 | 87,
797 | 2,
798 | "FACELANDMARKS222"
799 | ],
800 | [
801 | 209,
802 | 79,
803 | 0,
804 | 87,
805 | 1,
806 | "IMAGE"
807 | ],
808 | [
809 | 210,
810 | 2,
811 | 0,
812 | 87,
813 | 0,
814 | "FACE_TOOLKITS"
815 | ],
816 | [
817 | 211,
818 | 82,
819 | 0,
820 | 85,
821 | 3,
822 | "HEAD_POSE"
823 | ],
824 | [
825 | 212,
826 | 87,
827 | 0,
828 | 85,
829 | 2,
830 | "EXPRESSION"
831 | ],
832 | [
833 | 213,
834 | 87,
835 | 0,
836 | 85,
837 | 4,
838 | "EXPRESSION"
839 | ]
840 | ],
841 | "groups": [],
842 | "config": {},
843 | "extra": {
844 | "ds": {
845 | "scale": 0.8264462809917358,
846 | "offset": [
847 | 134.29123083527662,
848 | 160.15280889275093
849 | ]
850 | },
851 | "frontendVersion": "1.19.2"
852 | },
853 | "version": 0.4
854 | }
--------------------------------------------------------------------------------