├── .gitattributes ├── .gitignore ├── DINet ├── DINet.py ├── original_repo_link.txt ├── sync_batchnorm │ ├── __init__.py │ ├── batchnorm.py │ ├── batchnorm_reimpl.py │ ├── comm.py │ ├── replicate.py │ └── unittest.py ├── utils.py ├── wav2vec.py └── wav2vecDS.py ├── LIA ├── LICENSE.md ├── __init__.py ├── encoder.py ├── generator.py ├── styledecoder.py └── utils.py ├── README.md ├── environment.yaml ├── mask.jpg ├── pretrained_models └── put models here ├── requirements.txt ├── run_audio_driven.py ├── run_video_driven.py ├── test ├── driving.mov ├── driving.wav ├── result │ ├── audio_driven_result.avi │ └── video_driven_result.avi └── source.jpg └── utils ├── face_alignment.py ├── retinaface.py └── utils.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.pyc 3 | -------------------------------------------------------------------------------- /DINet/DINet.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import cv2 4 | import numpy as np 5 | import torch 6 | import torch.nn.functional as F 7 | from torch import nn 8 | 9 | from .sync_batchnorm import SynchronizedBatchNorm1d as BatchNorm1d 10 | from .sync_batchnorm import SynchronizedBatchNorm2d as BatchNorm2d 11 | 12 | 13 | def make_coordinate_grid_3d(spatial_size, type): 14 | """ 15 | generate 3D coordinate grid 16 | """ 17 | d, h, w = spatial_size 18 | x = torch.arange(w).type(type) 19 | y = torch.arange(h).type(type) 20 | z = torch.arange(d).type(type) 21 | x = 2 * (x / (w - 1)) - 1 22 | y = 2 * (y / (h - 1)) - 1 23 | z = 2 * (z / (d - 1)) - 1 24 | yy = y.view(1, -1, 1).repeat(d, 1, w) 25 | xx = x.view(1, 1, -1).repeat(d, h, 1) 26 | zz = z.view(-1, 1, 1).repeat(1, h, w) 27 | meshed = torch.cat([xx.unsqueeze_(3), yy.unsqueeze_(3)], 3) 28 | return meshed, zz 29 | 30 | 31 | class ResBlock1d(nn.Module): 32 | """ 33 | basic block 34 | """ 35 | 36 | def __init__(self, in_features, out_features, kernel_size, padding): 37 | super(ResBlock1d, self).__init__() 38 | self.in_features = in_features 39 | self.out_features = out_features 40 | self.conv1 = nn.Conv1d( 41 | in_channels=in_features, 42 | out_channels=in_features, 43 | kernel_size=kernel_size, 44 | padding=padding, 45 | ) 46 | self.conv2 = nn.Conv1d( 47 | in_channels=in_features, 48 | out_channels=out_features, 49 | kernel_size=kernel_size, 50 | padding=padding, 51 | ) 52 | if out_features != in_features: 53 | self.channel_conv = nn.Conv1d(in_features, out_features, 1) 54 | self.norm1 = BatchNorm1d(in_features) 55 | self.norm2 = BatchNorm1d(in_features) 56 | self.relu = nn.ReLU() 57 | 58 | def forward(self, x): 59 | out = self.norm1(x) 60 | out = self.relu(out) 61 | out = self.conv1(out) 62 | out = self.norm2(out) 63 | out = self.relu(out) 64 | out = self.conv2(out) 65 | if self.in_features != self.out_features: 66 | out += self.channel_conv(x) 67 | else: 68 | out += x 69 | return out 70 | 71 | 72 | class ResBlock2d(nn.Module): 73 | """ 74 | basic block 75 | """ 76 | 77 | def __init__(self, in_features, out_features, kernel_size, padding): 78 | super(ResBlock2d, self).__init__() 79 | self.in_features = in_features 80 | self.out_features = out_features 81 | self.conv1 = nn.Conv2d( 82 | in_channels=in_features, 83 | out_channels=in_features, 84 | kernel_size=kernel_size, 85 | padding=padding, 86 | ) 87 | self.conv2 = nn.Conv2d( 88 | in_channels=in_features, 89 | out_channels=out_features, 90 | kernel_size=kernel_size, 91 | padding=padding, 92 | ) 93 | if out_features != in_features: 94 | self.channel_conv = nn.Conv2d(in_features, out_features, 1) 95 | self.norm1 = BatchNorm2d(in_features) 96 | self.norm2 = BatchNorm2d(in_features) 97 | self.relu = nn.ReLU() 98 | 99 | def forward(self, x): 100 | out = self.norm1(x) 101 | out = self.relu(out) 102 | out = self.conv1(out) 103 | out = self.norm2(out) 104 | out = self.relu(out) 105 | out = self.conv2(out) 106 | if self.in_features != self.out_features: 107 | out += self.channel_conv(x) 108 | else: 109 | out += x 110 | return out 111 | 112 | 113 | class UpBlock2d(nn.Module): 114 | """ 115 | basic block 116 | """ 117 | 118 | def __init__(self, in_features, out_features, kernel_size=3, padding=1): 119 | super(UpBlock2d, self).__init__() 120 | self.conv = nn.Conv2d( 121 | in_channels=in_features, 122 | out_channels=out_features, 123 | kernel_size=kernel_size, 124 | padding=padding, 125 | ) 126 | self.norm = BatchNorm2d(out_features) 127 | self.relu = nn.ReLU() 128 | 129 | def forward(self, x): 130 | out = F.interpolate(x, scale_factor=2) 131 | out = self.conv(out) 132 | out = self.norm(out) 133 | out = F.relu(out) 134 | return out 135 | 136 | 137 | class DownBlock1d(nn.Module): 138 | """ 139 | basic block 140 | """ 141 | 142 | def __init__(self, in_features, out_features, kernel_size, padding): 143 | super(DownBlock1d, self).__init__() 144 | self.conv = nn.Conv1d( 145 | in_channels=in_features, 146 | out_channels=out_features, 147 | kernel_size=kernel_size, 148 | padding=padding, 149 | stride=2, 150 | ) 151 | self.norm = BatchNorm1d(out_features) 152 | self.relu = nn.ReLU() 153 | 154 | def forward(self, x): 155 | out = self.conv(x) 156 | out = self.norm(out) 157 | out = self.relu(out) 158 | return out 159 | 160 | 161 | class DownBlock2d(nn.Module): 162 | """ 163 | basic block 164 | """ 165 | 166 | def __init__(self, in_features, out_features, kernel_size=3, padding=1, stride=2): 167 | super(DownBlock2d, self).__init__() 168 | self.conv = nn.Conv2d( 169 | in_channels=in_features, 170 | out_channels=out_features, 171 | kernel_size=kernel_size, 172 | padding=padding, 173 | stride=stride, 174 | ) 175 | self.norm = BatchNorm2d(out_features) 176 | self.relu = nn.ReLU() 177 | 178 | def forward(self, x): 179 | out = self.conv(x) 180 | out = self.norm(out) 181 | out = self.relu(out) 182 | return out 183 | 184 | 185 | class SameBlock1d(nn.Module): 186 | """ 187 | basic block 188 | """ 189 | 190 | def __init__(self, in_features, out_features, kernel_size, padding): 191 | super(SameBlock1d, self).__init__() 192 | self.conv = nn.Conv1d( 193 | in_channels=in_features, 194 | out_channels=out_features, 195 | kernel_size=kernel_size, 196 | padding=padding, 197 | ) 198 | self.norm = BatchNorm1d(out_features) 199 | self.relu = nn.ReLU() 200 | 201 | def forward(self, x): 202 | out = self.conv(x) 203 | out = self.norm(out) 204 | out = self.relu(out) 205 | return out 206 | 207 | 208 | class SameBlock2d(nn.Module): 209 | """ 210 | basic block 211 | """ 212 | 213 | def __init__(self, in_features, out_features, kernel_size=3, padding=1): 214 | super(SameBlock2d, self).__init__() 215 | self.conv = nn.Conv2d( 216 | in_channels=in_features, 217 | out_channels=out_features, 218 | kernel_size=kernel_size, 219 | padding=padding, 220 | ) 221 | self.norm = BatchNorm2d(out_features) 222 | self.relu = nn.ReLU() 223 | 224 | def forward(self, x): 225 | out = self.conv(x) 226 | out = self.norm(out) 227 | out = self.relu(out) 228 | return out 229 | 230 | 231 | class AdaAT(nn.Module): 232 | """ 233 | AdaAT operator 234 | """ 235 | 236 | def __init__(self, para_ch, feature_ch): 237 | super(AdaAT, self).__init__() 238 | self.para_ch = para_ch 239 | self.feature_ch = feature_ch 240 | self.commn_linear = nn.Sequential(nn.Linear(para_ch, para_ch), nn.ReLU()) 241 | self.scale = nn.Sequential(nn.Linear(para_ch, feature_ch), nn.Sigmoid()) 242 | self.rotation = nn.Sequential(nn.Linear(para_ch, feature_ch), nn.Tanh()) 243 | self.translation = nn.Sequential(nn.Linear(para_ch, 2 * feature_ch), nn.Tanh()) 244 | self.tanh = nn.Tanh() 245 | self.sigmoid = nn.Sigmoid() 246 | 247 | def forward(self, feature_map, para_code): 248 | batch, d, h, w = ( 249 | feature_map.size(0), 250 | feature_map.size(1), 251 | feature_map.size(2), 252 | feature_map.size(3), 253 | ) 254 | para_code = self.commn_linear(para_code) 255 | scale = self.scale(para_code).unsqueeze(-1) * 2 256 | angle = self.rotation(para_code).unsqueeze(-1) * 3.14159 # 257 | rotation_matrix = torch.cat( 258 | [torch.cos(angle), -torch.sin(angle), torch.sin(angle), torch.cos(angle)], 259 | -1, 260 | ) 261 | rotation_matrix = rotation_matrix.view(batch, self.feature_ch, 2, 2) 262 | translation = self.translation(para_code).view(batch, self.feature_ch, 2) 263 | grid_xy, grid_z = make_coordinate_grid_3d((d, h, w), feature_map.type()) 264 | grid_xy = grid_xy.unsqueeze(0).repeat(batch, 1, 1, 1, 1) 265 | grid_z = grid_z.unsqueeze(0).repeat(batch, 1, 1, 1) 266 | scale = scale.unsqueeze(2).unsqueeze(3).repeat(1, 1, h, w, 1) 267 | rotation_matrix = ( 268 | rotation_matrix.unsqueeze(2).unsqueeze(3).repeat(1, 1, h, w, 1, 1) 269 | ) 270 | translation = translation.unsqueeze(2).unsqueeze(3).repeat(1, 1, h, w, 1) 271 | trans_grid = ( 272 | torch.matmul(rotation_matrix, grid_xy.unsqueeze(-1)).squeeze(-1) * scale 273 | + translation 274 | ) 275 | full_grid = torch.cat([trans_grid, grid_z.unsqueeze(-1)], -1) 276 | trans_feature = F.grid_sample(feature_map.unsqueeze(1), full_grid).squeeze(1) 277 | return trans_feature 278 | 279 | 280 | class DINet(nn.Module): 281 | def __init__(self, source_channel, ref_channel, audio_channel): 282 | super(DINet, self).__init__() 283 | self.source_in_conv = nn.Sequential( 284 | SameBlock2d(source_channel, 64, kernel_size=7, padding=3), 285 | DownBlock2d(64, 128, kernel_size=3, padding=1), 286 | DownBlock2d(128, 256, kernel_size=3, padding=1), 287 | ) 288 | self.ref_in_conv = nn.Sequential( 289 | SameBlock2d(ref_channel, 64, kernel_size=7, padding=3), 290 | DownBlock2d(64, 128, kernel_size=3, padding=1), 291 | DownBlock2d(128, 256, kernel_size=3, padding=1), 292 | ) 293 | self.trans_conv = nn.Sequential( 294 | # 20 →10 295 | SameBlock2d(512, 128, kernel_size=3, padding=1), 296 | SameBlock2d(128, 128, kernel_size=11, padding=5), 297 | SameBlock2d(128, 128, kernel_size=11, padding=5), 298 | DownBlock2d(128, 128, kernel_size=3, padding=1), 299 | # 10 →5 300 | SameBlock2d(128, 128, kernel_size=7, padding=3), 301 | SameBlock2d(128, 128, kernel_size=7, padding=3), 302 | DownBlock2d(128, 128, kernel_size=3, padding=1), 303 | # 5 →3 304 | SameBlock2d(128, 128, kernel_size=3, padding=1), 305 | DownBlock2d(128, 128, kernel_size=3, padding=1), 306 | # 3 →2 307 | SameBlock2d(128, 128, kernel_size=3, padding=1), 308 | DownBlock2d(128, 128, kernel_size=3, padding=1), 309 | ) 310 | self.audio_encoder = nn.Sequential( 311 | SameBlock1d(audio_channel, 128, kernel_size=5, padding=2), 312 | ResBlock1d(128, 128, 3, 1), 313 | DownBlock1d(128, 128, 3, 1), 314 | ResBlock1d(128, 128, 3, 1), 315 | DownBlock1d(128, 128, 3, 1), 316 | SameBlock1d(128, 128, kernel_size=3, padding=1), 317 | ) 318 | 319 | appearance_conv_list = [] 320 | for i in range(2): 321 | appearance_conv_list.append( 322 | nn.Sequential( 323 | ResBlock2d(256, 256, 3, 1), 324 | ResBlock2d(256, 256, 3, 1), 325 | ResBlock2d(256, 256, 3, 1), 326 | ResBlock2d(256, 256, 3, 1), 327 | ) 328 | ) 329 | self.appearance_conv_list = nn.ModuleList(appearance_conv_list) 330 | self.adaAT = AdaAT(256, 256) 331 | self.out_conv = nn.Sequential( 332 | SameBlock2d(512, 128, kernel_size=3, padding=1), 333 | UpBlock2d(128, 128, kernel_size=3, padding=1), 334 | ResBlock2d(128, 128, 3, 1), 335 | UpBlock2d(128, 128, kernel_size=3, padding=1), 336 | nn.Conv2d(128, 3, kernel_size=(7, 7), padding=(3, 3)), 337 | nn.Sigmoid(), 338 | ) 339 | self.global_avg2d = nn.AdaptiveAvgPool2d(1) 340 | self.global_avg1d = nn.AdaptiveAvgPool1d(1) 341 | 342 | def forward(self, source_img, ref_img, audio_feature): 343 | ## source image encoder 344 | source_in_feature = self.source_in_conv(source_img) 345 | ## reference image encoder 346 | ref_in_feature = self.ref_in_conv(ref_img) 347 | ## alignment encoder 348 | img_para = self.trans_conv(torch.cat([source_in_feature, ref_in_feature], 1)) 349 | img_para = self.global_avg2d(img_para).squeeze(3).squeeze(2) 350 | ## audio encoder 351 | audio_para = self.audio_encoder(audio_feature) 352 | audio_para = self.global_avg1d(audio_para).squeeze(2) 353 | ## concat alignment feature and audio feature 354 | trans_para = torch.cat([img_para, audio_para], 1) 355 | ## use AdaAT do spatial deformation on reference feature maps 356 | ref_trans_feature = self.appearance_conv_list[0](ref_in_feature) 357 | ref_trans_feature = self.adaAT(ref_trans_feature, trans_para) 358 | ref_trans_feature = self.appearance_conv_list[1](ref_trans_feature) 359 | ## feature decoder 360 | merge_feature = torch.cat([source_in_feature, ref_trans_feature], 1) 361 | out = self.out_conv(merge_feature) 362 | return out 363 | -------------------------------------------------------------------------------- /DINet/original_repo_link.txt: -------------------------------------------------------------------------------- 1 | original repo: 2 | https://github.com/MRzzm/DINet 3 | 4 | optimized version: 5 | https://github.com/Elsaam2y/DINet_optimized 6 | -------------------------------------------------------------------------------- /DINet/sync_batchnorm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : __init__.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | from .batchnorm import ( 12 | SynchronizedBatchNorm1d, 13 | SynchronizedBatchNorm2d, 14 | SynchronizedBatchNorm3d, 15 | convert_model, 16 | patch_sync_batchnorm, 17 | set_sbn_eps_mode, 18 | ) 19 | from .replicate import DataParallelWithCallback, patch_replication_callback 20 | -------------------------------------------------------------------------------- /DINet/sync_batchnorm/batchnorm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : batchnorm.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import collections 12 | import contextlib 13 | 14 | import torch 15 | import torch.nn.functional as F 16 | from torch.nn.modules.batchnorm import _BatchNorm 17 | 18 | try: 19 | from torch.nn.parallel._functions import Broadcast, ReduceAddCoalesced 20 | except ImportError: 21 | ReduceAddCoalesced = Broadcast = None 22 | 23 | try: 24 | from jactorch.parallel.comm import SyncMaster 25 | from jactorch.parallel.data_parallel import ( 26 | JacDataParallel as DataParallelWithCallback, 27 | ) 28 | except ImportError: 29 | from .comm import SyncMaster 30 | from .replicate import DataParallelWithCallback 31 | 32 | __all__ = [ 33 | "set_sbn_eps_mode", 34 | "SynchronizedBatchNorm1d", 35 | "SynchronizedBatchNorm2d", 36 | "SynchronizedBatchNorm3d", 37 | "patch_sync_batchnorm", 38 | "convert_model", 39 | ] 40 | 41 | 42 | SBN_EPS_MODE = "clamp" 43 | 44 | 45 | def set_sbn_eps_mode(mode): 46 | global SBN_EPS_MODE 47 | assert mode in ("clamp", "plus") 48 | SBN_EPS_MODE = mode 49 | 50 | 51 | def _sum_ft(tensor): 52 | """sum over the first and last dimention""" 53 | return tensor.sum(dim=0).sum(dim=-1) 54 | 55 | 56 | def _unsqueeze_ft(tensor): 57 | """add new dimensions at the front and the tail""" 58 | return tensor.unsqueeze(0).unsqueeze(-1) 59 | 60 | 61 | _ChildMessage = collections.namedtuple("_ChildMessage", ["sum", "ssum", "sum_size"]) 62 | _MasterMessage = collections.namedtuple("_MasterMessage", ["sum", "inv_std"]) 63 | 64 | 65 | class _SynchronizedBatchNorm(_BatchNorm): 66 | def __init__( 67 | self, 68 | num_features, 69 | eps=1e-5, 70 | momentum=0.1, 71 | affine=True, 72 | track_running_stats=True, 73 | ): 74 | assert ( 75 | ReduceAddCoalesced is not None 76 | ), "Can not use Synchronized Batch Normalization without CUDA support." 77 | 78 | super(_SynchronizedBatchNorm, self).__init__( 79 | num_features, 80 | eps=eps, 81 | momentum=momentum, 82 | affine=affine, 83 | track_running_stats=track_running_stats, 84 | ) 85 | 86 | if not self.track_running_stats: 87 | import warnings 88 | 89 | warnings.warn( 90 | "track_running_stats=False is not supported by the SynchronizedBatchNorm." 91 | ) 92 | 93 | self._sync_master = SyncMaster(self._data_parallel_master) 94 | 95 | self._is_parallel = False 96 | self._parallel_id = None 97 | self._slave_pipe = None 98 | 99 | def forward(self, input): 100 | # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation. 101 | if not (self._is_parallel and self.training): 102 | return F.batch_norm( 103 | input, 104 | self.running_mean, 105 | self.running_var, 106 | self.weight, 107 | self.bias, 108 | self.training, 109 | self.momentum, 110 | self.eps, 111 | ) 112 | 113 | # Resize the input to (B, C, -1). 114 | input_shape = input.size() 115 | assert ( 116 | input.size(1) == self.num_features 117 | ), "Channel size mismatch: got {}, expect {}.".format( 118 | input.size(1), self.num_features 119 | ) 120 | input = input.view(input.size(0), self.num_features, -1) 121 | 122 | # Compute the sum and square-sum. 123 | sum_size = input.size(0) * input.size(2) 124 | input_sum = _sum_ft(input) 125 | input_ssum = _sum_ft(input**2) 126 | 127 | # Reduce-and-broadcast the statistics. 128 | if self._parallel_id == 0: 129 | mean, inv_std = self._sync_master.run_master( 130 | _ChildMessage(input_sum, input_ssum, sum_size) 131 | ) 132 | else: 133 | mean, inv_std = self._slave_pipe.run_slave( 134 | _ChildMessage(input_sum, input_ssum, sum_size) 135 | ) 136 | 137 | # Compute the output. 138 | if self.affine: 139 | # MJY:: Fuse the multiplication for speed. 140 | output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft( 141 | inv_std * self.weight 142 | ) + _unsqueeze_ft(self.bias) 143 | else: 144 | output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std) 145 | 146 | # Reshape it. 147 | return output.view(input_shape) 148 | 149 | def __data_parallel_replicate__(self, ctx, copy_id): 150 | self._is_parallel = True 151 | self._parallel_id = copy_id 152 | 153 | # parallel_id == 0 means master device. 154 | if self._parallel_id == 0: 155 | ctx.sync_master = self._sync_master 156 | else: 157 | self._slave_pipe = ctx.sync_master.register_slave(copy_id) 158 | 159 | def _data_parallel_master(self, intermediates): 160 | """Reduce the sum and square-sum, compute the statistics, and broadcast it.""" 161 | 162 | # Always using same "device order" makes the ReduceAdd operation faster. 163 | # Thanks to:: Tete Xiao (http://tetexiao.com/) 164 | intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device()) 165 | 166 | to_reduce = [i[1][:2] for i in intermediates] 167 | to_reduce = [j for i in to_reduce for j in i] # flatten 168 | target_gpus = [i[1].sum.get_device() for i in intermediates] 169 | 170 | sum_size = sum([i[1].sum_size for i in intermediates]) 171 | sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce) 172 | mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size) 173 | 174 | broadcasted = Broadcast.apply(target_gpus, mean, inv_std) 175 | 176 | outputs = [] 177 | for i, rec in enumerate(intermediates): 178 | outputs.append((rec[0], _MasterMessage(*broadcasted[i * 2 : i * 2 + 2]))) 179 | 180 | return outputs 181 | 182 | def _compute_mean_std(self, sum_, ssum, size): 183 | """Compute the mean and standard-deviation with sum and square-sum. This method 184 | also maintains the moving average on the master device.""" 185 | assert ( 186 | size > 1 187 | ), "BatchNorm computes unbiased standard-deviation, which requires size > 1." 188 | mean = sum_ / size 189 | sumvar = ssum - sum_ * mean 190 | unbias_var = sumvar / (size - 1) 191 | bias_var = sumvar / size 192 | 193 | if hasattr(torch, "no_grad"): 194 | with torch.no_grad(): 195 | self.running_mean = ( 196 | 1 - self.momentum 197 | ) * self.running_mean + self.momentum * mean.data 198 | self.running_var = ( 199 | 1 - self.momentum 200 | ) * self.running_var + self.momentum * unbias_var.data 201 | else: 202 | self.running_mean = ( 203 | 1 - self.momentum 204 | ) * self.running_mean + self.momentum * mean.data 205 | self.running_var = ( 206 | 1 - self.momentum 207 | ) * self.running_var + self.momentum * unbias_var.data 208 | 209 | if SBN_EPS_MODE == "clamp": 210 | return mean, bias_var.clamp(self.eps) ** -0.5 211 | elif SBN_EPS_MODE == "plus": 212 | return mean, (bias_var + self.eps) ** -0.5 213 | else: 214 | raise ValueError("Unknown EPS mode: {}.".format(SBN_EPS_MODE)) 215 | 216 | 217 | class SynchronizedBatchNorm1d(_SynchronizedBatchNorm): 218 | r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a 219 | mini-batch. 220 | 221 | .. math:: 222 | 223 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 224 | 225 | This module differs from the built-in PyTorch BatchNorm1d as the mean and 226 | standard-deviation are reduced across all devices during training. 227 | 228 | For example, when one uses `nn.DataParallel` to wrap the network during 229 | training, PyTorch's implementation normalize the tensor on each device using 230 | the statistics only on that device, which accelerated the computation and 231 | is also easy to implement, but the statistics might be inaccurate. 232 | Instead, in this synchronized version, the statistics will be computed 233 | over all training samples distributed on multiple devices. 234 | 235 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 236 | as the built-in PyTorch implementation. 237 | 238 | The mean and standard-deviation are calculated per-dimension over 239 | the mini-batches and gamma and beta are learnable parameter vectors 240 | of size C (where C is the input size). 241 | 242 | During training, this layer keeps a running estimate of its computed mean 243 | and variance. The running sum is kept with a default momentum of 0.1. 244 | 245 | During evaluation, this running mean/variance is used for normalization. 246 | 247 | Because the BatchNorm is done over the `C` dimension, computing statistics 248 | on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm 249 | 250 | Args: 251 | num_features: num_features from an expected input of size 252 | `batch_size x num_features [x width]` 253 | eps: a value added to the denominator for numerical stability. 254 | Default: 1e-5 255 | momentum: the value used for the running_mean and running_var 256 | computation. Default: 0.1 257 | affine: a boolean value that when set to ``True``, gives the layer learnable 258 | affine parameters. Default: ``True`` 259 | 260 | Shape:: 261 | - Input: :math:`(N, C)` or :math:`(N, C, L)` 262 | - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input) 263 | 264 | Examples: 265 | >>> # With Learnable Parameters 266 | >>> m = SynchronizedBatchNorm1d(100) 267 | >>> # Without Learnable Parameters 268 | >>> m = SynchronizedBatchNorm1d(100, affine=False) 269 | >>> input = torch.autograd.Variable(torch.randn(20, 100)) 270 | >>> output = m(input) 271 | """ 272 | 273 | def _check_input_dim(self, input): 274 | if input.dim() != 2 and input.dim() != 3: 275 | raise ValueError( 276 | "expected 2D or 3D input (got {}D input)".format(input.dim()) 277 | ) 278 | 279 | 280 | class SynchronizedBatchNorm2d(_SynchronizedBatchNorm): 281 | r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch 282 | of 3d inputs 283 | 284 | .. math:: 285 | 286 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 287 | 288 | This module differs from the built-in PyTorch BatchNorm2d as the mean and 289 | standard-deviation are reduced across all devices during training. 290 | 291 | For example, when one uses `nn.DataParallel` to wrap the network during 292 | training, PyTorch's implementation normalize the tensor on each device using 293 | the statistics only on that device, which accelerated the computation and 294 | is also easy to implement, but the statistics might be inaccurate. 295 | Instead, in this synchronized version, the statistics will be computed 296 | over all training samples distributed on multiple devices. 297 | 298 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 299 | as the built-in PyTorch implementation. 300 | 301 | The mean and standard-deviation are calculated per-dimension over 302 | the mini-batches and gamma and beta are learnable parameter vectors 303 | of size C (where C is the input size). 304 | 305 | During training, this layer keeps a running estimate of its computed mean 306 | and variance. The running sum is kept with a default momentum of 0.1. 307 | 308 | During evaluation, this running mean/variance is used for normalization. 309 | 310 | Because the BatchNorm is done over the `C` dimension, computing statistics 311 | on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm 312 | 313 | Args: 314 | num_features: num_features from an expected input of 315 | size batch_size x num_features x height x width 316 | eps: a value added to the denominator for numerical stability. 317 | Default: 1e-5 318 | momentum: the value used for the running_mean and running_var 319 | computation. Default: 0.1 320 | affine: a boolean value that when set to ``True``, gives the layer learnable 321 | affine parameters. Default: ``True`` 322 | 323 | Shape:: 324 | - Input: :math:`(N, C, H, W)` 325 | - Output: :math:`(N, C, H, W)` (same shape as input) 326 | 327 | Examples: 328 | >>> # With Learnable Parameters 329 | >>> m = SynchronizedBatchNorm2d(100) 330 | >>> # Without Learnable Parameters 331 | >>> m = SynchronizedBatchNorm2d(100, affine=False) 332 | >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45)) 333 | >>> output = m(input) 334 | """ 335 | 336 | def _check_input_dim(self, input): 337 | if input.dim() != 4: 338 | raise ValueError("expected 4D input (got {}D input)".format(input.dim())) 339 | 340 | 341 | class SynchronizedBatchNorm3d(_SynchronizedBatchNorm): 342 | r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch 343 | of 4d inputs 344 | 345 | .. math:: 346 | 347 | y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta 348 | 349 | This module differs from the built-in PyTorch BatchNorm3d as the mean and 350 | standard-deviation are reduced across all devices during training. 351 | 352 | For example, when one uses `nn.DataParallel` to wrap the network during 353 | training, PyTorch's implementation normalize the tensor on each device using 354 | the statistics only on that device, which accelerated the computation and 355 | is also easy to implement, but the statistics might be inaccurate. 356 | Instead, in this synchronized version, the statistics will be computed 357 | over all training samples distributed on multiple devices. 358 | 359 | Note that, for one-GPU or CPU-only case, this module behaves exactly same 360 | as the built-in PyTorch implementation. 361 | 362 | The mean and standard-deviation are calculated per-dimension over 363 | the mini-batches and gamma and beta are learnable parameter vectors 364 | of size C (where C is the input size). 365 | 366 | During training, this layer keeps a running estimate of its computed mean 367 | and variance. The running sum is kept with a default momentum of 0.1. 368 | 369 | During evaluation, this running mean/variance is used for normalization. 370 | 371 | Because the BatchNorm is done over the `C` dimension, computing statistics 372 | on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm 373 | or Spatio-temporal BatchNorm 374 | 375 | Args: 376 | num_features: num_features from an expected input of 377 | size batch_size x num_features x depth x height x width 378 | eps: a value added to the denominator for numerical stability. 379 | Default: 1e-5 380 | momentum: the value used for the running_mean and running_var 381 | computation. Default: 0.1 382 | affine: a boolean value that when set to ``True``, gives the layer learnable 383 | affine parameters. Default: ``True`` 384 | 385 | Shape:: 386 | - Input: :math:`(N, C, D, H, W)` 387 | - Output: :math:`(N, C, D, H, W)` (same shape as input) 388 | 389 | Examples: 390 | >>> # With Learnable Parameters 391 | >>> m = SynchronizedBatchNorm3d(100) 392 | >>> # Without Learnable Parameters 393 | >>> m = SynchronizedBatchNorm3d(100, affine=False) 394 | >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10)) 395 | >>> output = m(input) 396 | """ 397 | 398 | def _check_input_dim(self, input): 399 | if input.dim() != 5: 400 | raise ValueError("expected 5D input (got {}D input)".format(input.dim())) 401 | 402 | 403 | @contextlib.contextmanager 404 | def patch_sync_batchnorm(): 405 | import torch.nn as nn 406 | 407 | backup = nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d 408 | 409 | nn.BatchNorm1d = SynchronizedBatchNorm1d 410 | nn.BatchNorm2d = SynchronizedBatchNorm2d 411 | nn.BatchNorm3d = SynchronizedBatchNorm3d 412 | 413 | yield 414 | 415 | nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d = backup 416 | 417 | 418 | def convert_model(module): 419 | """Traverse the input module and its child recursively 420 | and replace all instance of torch.nn.modules.batchnorm.BatchNorm*N*d 421 | to SynchronizedBatchNorm*N*d 422 | 423 | Args: 424 | module: the input module needs to be convert to SyncBN model 425 | 426 | Examples: 427 | >>> import torch.nn as nn 428 | >>> import torchvision 429 | >>> # m is a standard pytorch model 430 | >>> m = torchvision.models.resnet18(True) 431 | >>> m = nn.DataParallel(m) 432 | >>> # after convert, m is using SyncBN 433 | >>> m = convert_model(m) 434 | """ 435 | if isinstance(module, torch.nn.DataParallel): 436 | mod = module.module 437 | mod = convert_model(mod) 438 | mod = DataParallelWithCallback(mod, device_ids=module.device_ids) 439 | return mod 440 | 441 | mod = module 442 | for pth_module, sync_module in zip( 443 | [ 444 | torch.nn.modules.batchnorm.BatchNorm1d, 445 | torch.nn.modules.batchnorm.BatchNorm2d, 446 | torch.nn.modules.batchnorm.BatchNorm3d, 447 | ], 448 | [SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d], 449 | ): 450 | if isinstance(module, pth_module): 451 | mod = sync_module( 452 | module.num_features, module.eps, module.momentum, module.affine 453 | ) 454 | mod.running_mean = module.running_mean 455 | mod.running_var = module.running_var 456 | if module.affine: 457 | mod.weight.data = module.weight.data.clone().detach() 458 | mod.bias.data = module.bias.data.clone().detach() 459 | 460 | for name, child in module.named_children(): 461 | mod.add_module(name, convert_model(child)) 462 | 463 | return mod 464 | -------------------------------------------------------------------------------- /DINet/sync_batchnorm/batchnorm_reimpl.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # File : batchnorm_reimpl.py 4 | # Author : acgtyrant 5 | # Date : 11/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.init as init 14 | 15 | __all__ = ["BatchNorm2dReimpl"] 16 | 17 | 18 | class BatchNorm2dReimpl(nn.Module): 19 | """ 20 | A re-implementation of batch normalization, used for testing the numerical 21 | stability. 22 | 23 | Author: acgtyrant 24 | See also: 25 | https://github.com/vacancy/Synchronized-BatchNorm-PyTorch/issues/14 26 | """ 27 | 28 | def __init__(self, num_features, eps=1e-5, momentum=0.1): 29 | super().__init__() 30 | 31 | self.num_features = num_features 32 | self.eps = eps 33 | self.momentum = momentum 34 | self.weight = nn.Parameter(torch.empty(num_features)) 35 | self.bias = nn.Parameter(torch.empty(num_features)) 36 | self.register_buffer("running_mean", torch.zeros(num_features)) 37 | self.register_buffer("running_var", torch.ones(num_features)) 38 | self.reset_parameters() 39 | 40 | def reset_running_stats(self): 41 | self.running_mean.zero_() 42 | self.running_var.fill_(1) 43 | 44 | def reset_parameters(self): 45 | self.reset_running_stats() 46 | init.uniform_(self.weight) 47 | init.zeros_(self.bias) 48 | 49 | def forward(self, input_): 50 | batchsize, channels, height, width = input_.size() 51 | numel = batchsize * height * width 52 | input_ = input_.permute(1, 0, 2, 3).contiguous().view(channels, numel) 53 | sum_ = input_.sum(1) 54 | sum_of_square = input_.pow(2).sum(1) 55 | mean = sum_ / numel 56 | sumvar = sum_of_square - sum_ * mean 57 | 58 | self.running_mean = ( 59 | 1 - self.momentum 60 | ) * self.running_mean + self.momentum * mean.detach() 61 | unbias_var = sumvar / (numel - 1) 62 | self.running_var = ( 63 | 1 - self.momentum 64 | ) * self.running_var + self.momentum * unbias_var.detach() 65 | 66 | bias_var = sumvar / numel 67 | inv_std = 1 / (bias_var + self.eps).pow(0.5) 68 | output = (input_ - mean.unsqueeze(1)) * inv_std.unsqueeze( 69 | 1 70 | ) * self.weight.unsqueeze(1) + self.bias.unsqueeze(1) 71 | 72 | return ( 73 | output.view(channels, batchsize, height, width) 74 | .permute(1, 0, 2, 3) 75 | .contiguous() 76 | ) 77 | -------------------------------------------------------------------------------- /DINet/sync_batchnorm/comm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : comm.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import collections 12 | import queue 13 | import threading 14 | 15 | __all__ = ["FutureResult", "SlavePipe", "SyncMaster"] 16 | 17 | 18 | class FutureResult(object): 19 | """A thread-safe future implementation. Used only as one-to-one pipe.""" 20 | 21 | def __init__(self): 22 | self._result = None 23 | self._lock = threading.Lock() 24 | self._cond = threading.Condition(self._lock) 25 | 26 | def put(self, result): 27 | with self._lock: 28 | assert self._result is None, "Previous result has't been fetched." 29 | self._result = result 30 | self._cond.notify() 31 | 32 | def get(self): 33 | with self._lock: 34 | if self._result is None: 35 | self._cond.wait() 36 | 37 | res = self._result 38 | self._result = None 39 | return res 40 | 41 | 42 | _MasterRegistry = collections.namedtuple("MasterRegistry", ["result"]) 43 | _SlavePipeBase = collections.namedtuple( 44 | "_SlavePipeBase", ["identifier", "queue", "result"] 45 | ) 46 | 47 | 48 | class SlavePipe(_SlavePipeBase): 49 | """Pipe for master-slave communication.""" 50 | 51 | def run_slave(self, msg): 52 | self.queue.put((self.identifier, msg)) 53 | ret = self.result.get() 54 | self.queue.put(True) 55 | return ret 56 | 57 | 58 | class SyncMaster(object): 59 | """An abstract `SyncMaster` object. 60 | 61 | - During the replication, as the data parallel will trigger an callback of each module, all slave devices should 62 | call `register(id)` and obtain an `SlavePipe` to communicate with the master. 63 | - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected, 64 | and passed to a registered callback. 65 | - After receiving the messages, the master device should gather the information and determine to message passed 66 | back to each slave devices. 67 | """ 68 | 69 | def __init__(self, master_callback): 70 | """ 71 | 72 | Args: 73 | master_callback: a callback to be invoked after having collected messages from slave devices. 74 | """ 75 | self._master_callback = master_callback 76 | self._queue = queue.Queue() 77 | self._registry = collections.OrderedDict() 78 | self._activated = False 79 | 80 | def __getstate__(self): 81 | return {"master_callback": self._master_callback} 82 | 83 | def __setstate__(self, state): 84 | self.__init__(state["master_callback"]) 85 | 86 | def register_slave(self, identifier): 87 | """ 88 | Register an slave device. 89 | 90 | Args: 91 | identifier: an identifier, usually is the device id. 92 | 93 | Returns: a `SlavePipe` object which can be used to communicate with the master device. 94 | 95 | """ 96 | if self._activated: 97 | assert self._queue.empty(), "Queue is not clean before next initialization." 98 | self._activated = False 99 | self._registry.clear() 100 | future = FutureResult() 101 | self._registry[identifier] = _MasterRegistry(future) 102 | return SlavePipe(identifier, self._queue, future) 103 | 104 | def run_master(self, master_msg): 105 | """ 106 | Main entry for the master device in each forward pass. 107 | The messages were first collected from each devices (including the master device), and then 108 | an callback will be invoked to compute the message to be sent back to each devices 109 | (including the master device). 110 | 111 | Args: 112 | master_msg: the message that the master want to send to itself. This will be placed as the first 113 | message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example. 114 | 115 | Returns: the message to be sent back to the master device. 116 | 117 | """ 118 | self._activated = True 119 | 120 | intermediates = [(0, master_msg)] 121 | for i in range(self.nr_slaves): 122 | intermediates.append(self._queue.get()) 123 | 124 | results = self._master_callback(intermediates) 125 | assert results[0][0] == 0, "The first result should belongs to the master." 126 | 127 | for i, res in results: 128 | if i == 0: 129 | continue 130 | self._registry[i].result.put(res) 131 | 132 | for i in range(self.nr_slaves): 133 | assert self._queue.get() is True 134 | 135 | return results[0][1] 136 | 137 | @property 138 | def nr_slaves(self): 139 | return len(self._registry) 140 | -------------------------------------------------------------------------------- /DINet/sync_batchnorm/replicate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : replicate.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import functools 12 | 13 | from torch.nn.parallel.data_parallel import DataParallel 14 | 15 | __all__ = [ 16 | "CallbackContext", 17 | "execute_replication_callbacks", 18 | "DataParallelWithCallback", 19 | "patch_replication_callback", 20 | ] 21 | 22 | 23 | class CallbackContext(object): 24 | pass 25 | 26 | 27 | def execute_replication_callbacks(modules): 28 | """ 29 | Execute an replication callback `__data_parallel_replicate__` on each module created by original replication. 30 | 31 | The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` 32 | 33 | Note that, as all modules are isomorphism, we assign each sub-module with a context 34 | (shared among multiple copies of this module on different devices). 35 | Through this context, different copies can share some information. 36 | 37 | We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback 38 | of any slave copies. 39 | """ 40 | master_copy = modules[0] 41 | nr_modules = len(list(master_copy.modules())) 42 | ctxs = [CallbackContext() for _ in range(nr_modules)] 43 | 44 | for i, module in enumerate(modules): 45 | for j, m in enumerate(module.modules()): 46 | if hasattr(m, "__data_parallel_replicate__"): 47 | m.__data_parallel_replicate__(ctxs[j], i) 48 | 49 | 50 | class DataParallelWithCallback(DataParallel): 51 | """ 52 | Data Parallel with a replication callback. 53 | 54 | An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by 55 | original `replicate` function. 56 | The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` 57 | 58 | Examples: 59 | > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) 60 | > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) 61 | # sync_bn.__data_parallel_replicate__ will be invoked. 62 | """ 63 | 64 | def replicate(self, module, device_ids): 65 | modules = super(DataParallelWithCallback, self).replicate(module, device_ids) 66 | execute_replication_callbacks(modules) 67 | return modules 68 | 69 | 70 | def patch_replication_callback(data_parallel): 71 | """ 72 | Monkey-patch an existing `DataParallel` object. Add the replication callback. 73 | Useful when you have customized `DataParallel` implementation. 74 | 75 | Examples: 76 | > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) 77 | > sync_bn = DataParallel(sync_bn, device_ids=[0, 1]) 78 | > patch_replication_callback(sync_bn) 79 | # this is equivalent to 80 | > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) 81 | > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) 82 | """ 83 | 84 | assert isinstance(data_parallel, DataParallel) 85 | 86 | old_replicate = data_parallel.replicate 87 | 88 | @functools.wraps(old_replicate) 89 | def new_replicate(module, device_ids): 90 | modules = old_replicate(module, device_ids) 91 | execute_replication_callbacks(modules) 92 | return modules 93 | 94 | data_parallel.replicate = new_replicate 95 | -------------------------------------------------------------------------------- /DINet/sync_batchnorm/unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # File : unittest.py 3 | # Author : Jiayuan Mao 4 | # Email : maojiayuan@gmail.com 5 | # Date : 27/01/2018 6 | # 7 | # This file is part of Synchronized-BatchNorm-PyTorch. 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch 9 | # Distributed under MIT License. 10 | 11 | import unittest 12 | 13 | import torch 14 | 15 | 16 | class TorchTestCase(unittest.TestCase): 17 | def assertTensorClose(self, x, y): 18 | adiff = float((x - y).abs().max()) 19 | if (y == 0).all(): 20 | rdiff = "NaN" 21 | else: 22 | rdiff = float((adiff / y).abs().max()) 23 | 24 | message = ("Tensor close check failed\n" "adiff={}\n" "rdiff={}\n").format( 25 | adiff, rdiff 26 | ) 27 | self.assertTrue(torch.allclose(x, y, atol=1e-5, rtol=1e-3), message) 28 | -------------------------------------------------------------------------------- /DINet/utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import random 3 | import numpy as np 4 | 5 | def compute_crop_radius(video_size, landmark_data_clip, random_scale=None): 6 | video_w, video_h = video_size[0], video_size[1] 7 | landmark_max_clip = np.max(landmark_data_clip, axis=1) 8 | if random_scale is None: 9 | random_scale = random.random() / 10 + 1.05 10 | else: 11 | random_scale = random_scale 12 | radius_h = (landmark_max_clip[:, 1] - landmark_data_clip[:, 29, 1]) * random_scale 13 | radius_w = ( 14 | landmark_data_clip[:, 54, 0] - landmark_data_clip[:, 48, 0] 15 | ) * random_scale 16 | radius_clip = np.max(np.stack([radius_h, radius_w], 1), 1) // 2 17 | radius_max = np.max(radius_clip) 18 | radius_max = (np.int32(radius_max / 4) + 1) * 4 19 | radius_max_1_4 = radius_max // 4 20 | clip_min_h = landmark_data_clip[:, 29, 1] - radius_max 21 | clip_max_h = landmark_data_clip[:, 29, 1] + radius_max * 2 + radius_max_1_4 22 | clip_min_w = landmark_data_clip[:, 33, 0] - radius_max - radius_max_1_4 23 | clip_max_w = landmark_data_clip[:, 33, 0] + radius_max + radius_max_1_4 24 | if min(clip_min_h.tolist() + clip_min_w.tolist()) < 0: 25 | return False, None 26 | elif max(clip_max_h.tolist()) > video_h: 27 | return False, None 28 | elif max(clip_max_w.tolist()) > video_w: 29 | return False, None 30 | elif max(radius_clip) > min(radius_clip) * 1.5: 31 | return False, None 32 | else: 33 | return True, radius_max 34 | 35 | def face_crop(image, landmark, crop_radius): 36 | landmark = landmark.astype('int32') 37 | crop_radius_1_4 = crop_radius // 4 38 | cropped = image[ 39 | landmark[29, 1] 40 | - crop_radius : landmark[29, 1] 41 | + crop_radius * 2 42 | + crop_radius_1_4, 43 | landmark[33, 0] 44 | - crop_radius 45 | - crop_radius_1_4 : landmark[33, 0] 46 | + crop_radius 47 | + crop_radius_1_4, 48 | :, 49 | ] 50 | return cropped 51 | 52 | def final_image_fill(full_image, cropped_face, landmark, crop_radius): 53 | landmark = landmark.astype('int32') 54 | crop_radius_1_4 = crop_radius // 4 55 | full_image[ 56 | landmark[29, 1] 57 | - crop_radius : landmark[29, 1] 58 | + crop_radius * 2, 59 | landmark[33, 0] 60 | - crop_radius 61 | - crop_radius_1_4 : landmark[33, 0] 62 | + crop_radius 63 | + crop_radius_1_4, 64 | :, 65 | ] = cropped_face[: crop_radius * 3, :, :] 66 | return full_image 67 | -------------------------------------------------------------------------------- /DINet/wav2vec.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import math 3 | import os 4 | import time 5 | 6 | import numpy as np 7 | import torch 8 | 9 | import torchaudio 10 | 11 | 12 | # This process requires GPU 13 | class Wav2VecFeatureExtractor: 14 | def __init__(self, device): 15 | self.device = device 16 | self.bundle = torchaudio.pipelines.HUBERT_ASR_LARGE 17 | # Build the model and load the pretrained weights 18 | self.model = self.bundle.get_model().to(self.device) 19 | 20 | # fix OOM issues by processing long audio files into smaller chunks 21 | def compute_audio_feature(self, audio_path, chunk_duration=10): 22 | # Load the audio waveform 23 | waveform, sample_rate = torchaudio.load(audio_path) 24 | 25 | # Calculate the number of samples in each chunk 26 | chunk_samples = int(chunk_duration * sample_rate) 27 | 28 | # Initialize a list to store the results for each chunk 29 | all_emissions = [] 30 | 31 | # Iterate over the audio waveform in chunks 32 | for i in range(0, waveform.size(1), chunk_samples): 33 | chunk = waveform[:, i : i + chunk_samples] 34 | 35 | # Move the chunk to the same device as the model 36 | chunk = chunk.to(self.device) 37 | 38 | # Resample the chunk 39 | resampler = torchaudio.transforms.Resample( 40 | sample_rate, self.bundle.sample_rate 41 | ).to(self.device) 42 | chunk = resampler(chunk) 43 | 44 | # Infer the label probability distribution (emissions) for the chunk 45 | emissions, _ = self.model(chunk) 46 | emissions = emissions[0, ::2] 47 | 48 | # Move the emissions back to the CPU 49 | emissions = emissions.to("cpu") 50 | 51 | all_emissions.append(emissions.detach().numpy()) 52 | 53 | return np.concatenate(all_emissions, axis=0) 54 | -------------------------------------------------------------------------------- /DINet/wav2vecDS.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | # Construct the model for mapping the wav2vec features to DS 7 | class _Wav2vecDS(nn.Module): 8 | def __init__(self, input_dim, hidden_dim): 9 | super(_Wav2vecDS, self).__init__() 10 | self.fc1 = nn.Linear(input_dim, hidden_dim) 11 | self.fc2 = nn.Linear(hidden_dim, hidden_dim) 12 | self.fc3 = nn.Linear(hidden_dim, hidden_dim) 13 | self.fc4 = nn.Linear(hidden_dim, input_dim) 14 | 15 | def forward(self, x): 16 | x = torch.relu(self.fc1(x)) 17 | x = torch.relu(self.fc2(x)) 18 | x = torch.relu(self.fc3(x)) 19 | x = self.fc4(x) 20 | return x 21 | 22 | 23 | # mapping the features from wav2vec to DS 24 | class Wav2vecDS(nn.Module): 25 | def __init__(self, model_path=None): 26 | super(Wav2vecDS, self).__init__() 27 | self.input_dim = 29 28 | self.hidden_dim = 512 29 | self.pretrained_model = torch.load(model_path) 30 | 31 | # define the mapping function 32 | def mapping(self, array): 33 | model = _Wav2vecDS(self.input_dim, self.hidden_dim) 34 | model.load_state_dict(self.pretrained_model) 35 | input_data = torch.tensor(array) 36 | # Perform inference 37 | with torch.no_grad(): 38 | output = model(input_data) 39 | return output.numpy() 40 | -------------------------------------------------------------------------------- /LIA/LICENSE.md: -------------------------------------------------------------------------------- 1 | # Attribution-NonCommercial 4.0 International 2 | 3 | Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. 4 | 5 | ### Using Creative Commons Public Licenses 6 | 7 | Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. 8 | 9 | * __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). 10 | 11 | * __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). 12 | 13 | ## Creative Commons Attribution-NonCommercial 4.0 International Public License 14 | 15 | By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. 16 | 17 | ### Section 1 – Definitions. 18 | 19 | a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. 20 | 21 | b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. 22 | 23 | c. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. 24 | 25 | d. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. 26 | 27 | e. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. 28 | 29 | f. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. 30 | 31 | g. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. 32 | 33 | h. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. 34 | 35 | i. __NonCommercial__ means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. 36 | 37 | j. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. 38 | 39 | k. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. 40 | 41 | l. __You__ means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. 42 | 43 | ### Section 2 – Scope. 44 | 45 | a. ___License grant.___ 46 | 47 | 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: 48 | 49 | A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and 50 | 51 | B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only. 52 | 53 | 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. 54 | 55 | 3. __Term.__ The term of this Public License is specified in Section 6(a). 56 | 57 | 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. 58 | 59 | 5. __Downstream recipients.__ 60 | 61 | A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. 62 | 63 | B. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. 64 | 65 | 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). 66 | 67 | b. ___Other rights.___ 68 | 69 | 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. 70 | 71 | 2. Patent and trademark rights are not licensed under this Public License. 72 | 73 | 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes. 74 | 75 | ### Section 3 – License Conditions. 76 | 77 | Your exercise of the Licensed Rights is expressly made subject to the following conditions. 78 | 79 | a. ___Attribution.___ 80 | 81 | 1. If You Share the Licensed Material (including in modified form), You must: 82 | 83 | A. retain the following if it is supplied by the Licensor with the Licensed Material: 84 | 85 | i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); 86 | 87 | ii. a copyright notice; 88 | 89 | iii. a notice that refers to this Public License; 90 | 91 | iv. a notice that refers to the disclaimer of warranties; 92 | 93 | v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; 94 | 95 | B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and 96 | 97 | C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. 98 | 99 | 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. 100 | 101 | 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. 102 | 103 | 4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License. 104 | 105 | ### Section 4 – Sui Generis Database Rights. 106 | 107 | Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: 108 | 109 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; 110 | 111 | b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and 112 | 113 | c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. 114 | 115 | For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. 116 | 117 | ### Section 5 – Disclaimer of Warranties and Limitation of Liability. 118 | 119 | a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ 120 | 121 | b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ 122 | 123 | c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. 124 | 125 | ### Section 6 – Term and Termination. 126 | 127 | a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. 128 | 129 | b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: 130 | 131 | 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or 132 | 133 | 2. upon express reinstatement by the Licensor. 134 | 135 | For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. 136 | 137 | c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. 138 | 139 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. 140 | 141 | ### Section 7 – Other Terms and Conditions. 142 | 143 | a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. 144 | 145 | b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. 146 | 147 | ### Section 8 – Interpretation. 148 | 149 | a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. 150 | 151 | b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. 152 | 153 | c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. 154 | 155 | d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. 156 | 157 | > Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. 158 | > 159 | > Creative Commons may be contacted at creativecommons.org 160 | -------------------------------------------------------------------------------- /LIA/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/LIA/__init__.py -------------------------------------------------------------------------------- /LIA/encoder.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch import nn 4 | from torch.nn import functional as F 5 | 6 | 7 | def fused_leaky_relu(input, bias, negative_slope=0.2, scale=2 ** 0.5): 8 | return F.leaky_relu(input + bias, negative_slope) * scale 9 | 10 | 11 | class FusedLeakyReLU(nn.Module): 12 | def __init__(self, channel, negative_slope=0.2, scale=2 ** 0.5): 13 | super().__init__() 14 | self.bias = nn.Parameter(torch.zeros(1, channel, 1, 1)) 15 | self.negative_slope = negative_slope 16 | self.scale = scale 17 | 18 | def forward(self, input): 19 | out = fused_leaky_relu(input, self.bias, self.negative_slope, self.scale) 20 | return out 21 | 22 | 23 | def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1): 24 | _, minor, in_h, in_w = input.shape 25 | kernel_h, kernel_w = kernel.shape 26 | 27 | out = input.view(-1, minor, in_h, 1, in_w, 1) 28 | out = F.pad(out, [0, up_x - 1, 0, 0, 0, up_y - 1, 0, 0]) 29 | out = out.view(-1, minor, in_h * up_y, in_w * up_x) 30 | 31 | out = F.pad(out, [max(pad_x0, 0), max(pad_x1, 0), max(pad_y0, 0), max(pad_y1, 0)]) 32 | out = out[:, :, max(-pad_y0, 0): out.shape[2] - max(-pad_y1, 0), 33 | max(-pad_x0, 0): out.shape[3] - max(-pad_x1, 0), ] 34 | 35 | out = out.reshape([-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) 36 | w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) 37 | out = F.conv2d(out, w) 38 | out = out.reshape(-1, minor, in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, 39 | in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, ) 40 | 41 | return out[:, :, ::down_y, ::down_x] 42 | 43 | 44 | def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): 45 | return upfirdn2d_native(input, kernel, up, up, down, down, pad[0], pad[1], pad[0], pad[1]) 46 | 47 | 48 | def make_kernel(k): 49 | k = torch.tensor(k, dtype=torch.float32) 50 | 51 | if k.ndim == 1: 52 | k = k[None, :] * k[:, None] 53 | 54 | k /= k.sum() 55 | 56 | return k 57 | 58 | 59 | class Blur(nn.Module): 60 | def __init__(self, kernel, pad, upsample_factor=1): 61 | super().__init__() 62 | 63 | kernel = make_kernel(kernel) 64 | 65 | if upsample_factor > 1: 66 | kernel = kernel * (upsample_factor ** 2) 67 | 68 | self.register_buffer('kernel', kernel) 69 | 70 | self.pad = pad 71 | 72 | def forward(self, input): 73 | return upfirdn2d(input, self.kernel, pad=self.pad) 74 | 75 | 76 | class ScaledLeakyReLU(nn.Module): 77 | def __init__(self, negative_slope=0.2): 78 | super().__init__() 79 | 80 | self.negative_slope = negative_slope 81 | 82 | def forward(self, input): 83 | return F.leaky_relu(input, negative_slope=self.negative_slope) 84 | 85 | 86 | class EqualConv2d(nn.Module): 87 | def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0, bias=True): 88 | super().__init__() 89 | 90 | self.weight = nn.Parameter(torch.randn(out_channel, in_channel, kernel_size, kernel_size)) 91 | self.scale = 1 / math.sqrt(in_channel * kernel_size ** 2) 92 | 93 | self.stride = stride 94 | self.padding = padding 95 | 96 | if bias: 97 | self.bias = nn.Parameter(torch.zeros(out_channel)) 98 | else: 99 | self.bias = None 100 | 101 | def forward(self, input): 102 | 103 | return F.conv2d(input, self.weight * self.scale, bias=self.bias, stride=self.stride, padding=self.padding) 104 | 105 | def __repr__(self): 106 | return ( 107 | f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' 108 | f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' 109 | ) 110 | 111 | 112 | class EqualLinear(nn.Module): 113 | def __init__(self, in_dim, out_dim, bias=True, bias_init=0, lr_mul=1, activation=None): 114 | super().__init__() 115 | 116 | self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) 117 | 118 | if bias: 119 | self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) 120 | else: 121 | self.bias = None 122 | 123 | self.activation = activation 124 | 125 | self.scale = (1 / math.sqrt(in_dim)) * lr_mul 126 | self.lr_mul = lr_mul 127 | 128 | def forward(self, input): 129 | 130 | if self.activation: 131 | out = F.linear(input, self.weight * self.scale) 132 | out = fused_leaky_relu(out, self.bias * self.lr_mul) 133 | else: 134 | out = F.linear(input, self.weight * self.scale, bias=self.bias * self.lr_mul) 135 | 136 | return out 137 | 138 | def __repr__(self): 139 | return (f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})') 140 | 141 | 142 | class ConvLayer(nn.Sequential): 143 | def __init__( 144 | self, 145 | in_channel, 146 | out_channel, 147 | kernel_size, 148 | downsample=False, 149 | blur_kernel=[1, 3, 3, 1], 150 | bias=True, 151 | activate=True, 152 | ): 153 | layers = [] 154 | 155 | if downsample: 156 | factor = 2 157 | p = (len(blur_kernel) - factor) + (kernel_size - 1) 158 | pad0 = (p + 1) // 2 159 | pad1 = p // 2 160 | 161 | layers.append(Blur(blur_kernel, pad=(pad0, pad1))) 162 | 163 | stride = 2 164 | self.padding = 0 165 | 166 | else: 167 | stride = 1 168 | self.padding = kernel_size // 2 169 | 170 | layers.append(EqualConv2d(in_channel, out_channel, kernel_size, padding=self.padding, stride=stride, 171 | bias=bias and not activate)) 172 | 173 | if activate: 174 | if bias: 175 | layers.append(FusedLeakyReLU(out_channel)) 176 | else: 177 | layers.append(ScaledLeakyReLU(0.2)) 178 | 179 | super().__init__(*layers) 180 | 181 | 182 | class ResBlock(nn.Module): 183 | def __init__(self, in_channel, out_channel, blur_kernel=[1, 3, 3, 1]): 184 | super().__init__() 185 | 186 | self.conv1 = ConvLayer(in_channel, in_channel, 3) 187 | self.conv2 = ConvLayer(in_channel, out_channel, 3, downsample=True) 188 | 189 | self.skip = ConvLayer(in_channel, out_channel, 1, downsample=True, activate=False, bias=False) 190 | 191 | def forward(self, input): 192 | out = self.conv1(input) 193 | out = self.conv2(out) 194 | 195 | skip = self.skip(input) 196 | out = (out + skip) / math.sqrt(2) 197 | 198 | return out 199 | 200 | 201 | class EncoderApp(nn.Module): 202 | def __init__(self, size, w_dim=512): 203 | super(EncoderApp, self).__init__() 204 | 205 | channels = { 206 | 4: 512, 207 | 8: 512, 208 | 16: 512, 209 | 32: 512, 210 | 64: 256, 211 | 128: 128, 212 | 256: 64, 213 | 512: 32, 214 | 1024: 16 215 | } 216 | 217 | self.w_dim = w_dim 218 | log_size = int(math.log(size, 2)) 219 | 220 | self.convs = nn.ModuleList() 221 | self.convs.append(ConvLayer(3, channels[size], 1)) 222 | 223 | in_channel = channels[size] 224 | for i in range(log_size, 2, -1): 225 | out_channel = channels[2 ** (i - 1)] 226 | self.convs.append(ResBlock(in_channel, out_channel)) 227 | in_channel = out_channel 228 | 229 | self.convs.append(EqualConv2d(in_channel, self.w_dim, 4, padding=0, bias=False)) 230 | 231 | def forward(self, x): 232 | 233 | res = [] 234 | h = x 235 | for conv in self.convs: 236 | h = conv(h) 237 | res.append(h) 238 | 239 | return res[-1].squeeze(-1).squeeze(-1), res[::-1][2:] 240 | 241 | 242 | class Encoder(nn.Module): 243 | def __init__(self, size, dim=512, dim_motion=20): 244 | super(Encoder, self).__init__() 245 | 246 | # appearance netmork 247 | self.net_app = EncoderApp(size, dim) 248 | 249 | # motion network 250 | fc = [EqualLinear(dim, dim)] 251 | for i in range(3): 252 | fc.append(EqualLinear(dim, dim)) 253 | 254 | fc.append(EqualLinear(dim, dim_motion)) 255 | self.fc = nn.Sequential(*fc) 256 | 257 | def enc_app(self, x): 258 | 259 | h_source = self.net_app(x) 260 | 261 | return h_source 262 | 263 | def enc_motion(self, x): 264 | 265 | h, _ = self.net_app(x) 266 | h_motion = self.fc(h) 267 | 268 | return h_motion 269 | 270 | def forward(self, input_source, input_target, h_start=None): 271 | 272 | if input_target is not None: 273 | 274 | h_source, feats = self.net_app(input_source) 275 | h_target, _ = self.net_app(input_target) 276 | 277 | h_motion_target = self.fc(h_target) 278 | 279 | if h_start is not None: 280 | h_motion_source = self.fc(h_source) 281 | h_motion = [h_motion_target, h_motion_source, h_start] 282 | else: 283 | h_motion = [h_motion_target] 284 | 285 | return h_source, h_motion, feats 286 | else: 287 | h_source, feats = self.net_app(input_source) 288 | 289 | return h_source, None, feats 290 | -------------------------------------------------------------------------------- /LIA/generator.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | from .encoder import Encoder 3 | from .styledecoder import Synthesis 4 | 5 | 6 | class Generator(nn.Module): 7 | def __init__(self, size, style_dim=512, motion_dim=20, channel_multiplier=1., blur_kernel=[1, 3, 3, 1]): 8 | super(Generator, self).__init__() 9 | 10 | # encoder 11 | self.enc = Encoder(size, style_dim, motion_dim) 12 | self.dec = Synthesis(size, style_dim, motion_dim, blur_kernel, channel_multiplier) 13 | 14 | def get_direction(self): 15 | return self.dec.direction(None) 16 | 17 | def synthesis(self, wa, alpha, feat): 18 | img = self.dec(wa, alpha, feat) 19 | 20 | return img 21 | 22 | def forward(self, img_source, img_drive, h_start=None): 23 | wa, alpha, feats = self.enc(img_source, img_drive, h_start) 24 | img_recon = self.dec(wa, alpha, feats) 25 | 26 | return img_recon 27 | -------------------------------------------------------------------------------- /LIA/styledecoder.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch import nn 4 | from torch.nn import functional as F 5 | import numpy as np 6 | 7 | 8 | def fused_leaky_relu(input, bias, negative_slope=0.2, scale=2 ** 0.5): 9 | return F.leaky_relu(input + bias, negative_slope) * scale 10 | 11 | 12 | class FusedLeakyReLU(nn.Module): 13 | def __init__(self, channel, negative_slope=0.2, scale=2 ** 0.5): 14 | super().__init__() 15 | self.bias = nn.Parameter(torch.zeros(1, channel, 1, 1)) 16 | self.negative_slope = negative_slope 17 | self.scale = scale 18 | 19 | def forward(self, input): 20 | # print("FusedLeakyReLU: ", input.abs().mean()) 21 | out = fused_leaky_relu(input, self.bias, self.negative_slope, self.scale) 22 | # print("FusedLeakyReLU: ", out.abs().mean()) 23 | return out 24 | 25 | 26 | def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1): 27 | _, minor, in_h, in_w = input.shape 28 | kernel_h, kernel_w = kernel.shape 29 | 30 | out = input.view(-1, minor, in_h, 1, in_w, 1) 31 | out = F.pad(out, [0, up_x - 1, 0, 0, 0, up_y - 1, 0, 0]) 32 | out = out.view(-1, minor, in_h * up_y, in_w * up_x) 33 | 34 | out = F.pad(out, [max(pad_x0, 0), max(pad_x1, 0), max(pad_y0, 0), max(pad_y1, 0)]) 35 | out = out[:, :, max(-pad_y0, 0): out.shape[2] - max(-pad_y1, 0), 36 | max(-pad_x0, 0): out.shape[3] - max(-pad_x1, 0), ] 37 | 38 | # out = out.permute(0, 3, 1, 2) 39 | out = out.reshape([-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) 40 | w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) 41 | out = F.conv2d(out, w) 42 | out = out.reshape(-1, minor, in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, 43 | in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, ) 44 | # out = out.permute(0, 2, 3, 1) 45 | 46 | return out[:, :, ::down_y, ::down_x] 47 | 48 | 49 | def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): 50 | return upfirdn2d_native(input, kernel, up, up, down, down, pad[0], pad[1], pad[0], pad[1]) 51 | 52 | 53 | class PixelNorm(nn.Module): 54 | def __init__(self): 55 | super().__init__() 56 | 57 | def forward(self, input): 58 | return input * torch.rsqrt(torch.mean(input ** 2, dim=1, keepdim=True) + 1e-8) 59 | 60 | 61 | class MotionPixelNorm(nn.Module): 62 | def __init__(self): 63 | super().__init__() 64 | 65 | def forward(self, input): 66 | return input * torch.rsqrt(torch.mean(input ** 2, dim=2, keepdim=True) + 1e-8) 67 | 68 | 69 | def make_kernel(k): 70 | k = torch.tensor(k, dtype=torch.float32) 71 | 72 | if k.ndim == 1: 73 | k = k[None, :] * k[:, None] 74 | 75 | k /= k.sum() 76 | 77 | return k 78 | 79 | 80 | class Upsample(nn.Module): 81 | def __init__(self, kernel, factor=2): 82 | super().__init__() 83 | 84 | self.factor = factor 85 | kernel = make_kernel(kernel) * (factor ** 2) 86 | self.register_buffer('kernel', kernel) 87 | 88 | p = kernel.shape[0] - factor 89 | 90 | pad0 = (p + 1) // 2 + factor - 1 91 | pad1 = p // 2 92 | 93 | self.pad = (pad0, pad1) 94 | 95 | def forward(self, input): 96 | return upfirdn2d(input, self.kernel, up=self.factor, down=1, pad=self.pad) 97 | 98 | 99 | class Downsample(nn.Module): 100 | def __init__(self, kernel, factor=2): 101 | super().__init__() 102 | 103 | self.factor = factor 104 | kernel = make_kernel(kernel) 105 | self.register_buffer('kernel', kernel) 106 | 107 | p = kernel.shape[0] - factor 108 | 109 | pad0 = (p + 1) // 2 110 | pad1 = p // 2 111 | 112 | self.pad = (pad0, pad1) 113 | 114 | def forward(self, input): 115 | return upfirdn2d(input, self.kernel, up=1, down=self.factor, pad=self.pad) 116 | 117 | 118 | class Blur(nn.Module): 119 | def __init__(self, kernel, pad, upsample_factor=1): 120 | super().__init__() 121 | 122 | kernel = make_kernel(kernel) 123 | 124 | if upsample_factor > 1: 125 | kernel = kernel * (upsample_factor ** 2) 126 | 127 | self.register_buffer('kernel', kernel) 128 | 129 | self.pad = pad 130 | 131 | def forward(self, input): 132 | return upfirdn2d(input, self.kernel, pad=self.pad) 133 | 134 | 135 | class EqualConv2d(nn.Module): 136 | def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0, bias=True): 137 | super().__init__() 138 | 139 | self.weight = nn.Parameter(torch.randn(out_channel, in_channel, kernel_size, kernel_size)) 140 | self.scale = 1 / math.sqrt(in_channel * kernel_size ** 2) 141 | 142 | self.stride = stride 143 | self.padding = padding 144 | 145 | if bias: 146 | self.bias = nn.Parameter(torch.zeros(out_channel)) 147 | else: 148 | self.bias = None 149 | 150 | def forward(self, input): 151 | 152 | return F.conv2d(input, self.weight * self.scale, bias=self.bias, stride=self.stride, padding=self.padding, ) 153 | 154 | def __repr__(self): 155 | return ( 156 | f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' 157 | f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' 158 | ) 159 | 160 | 161 | class EqualLinear(nn.Module): 162 | def __init__(self, in_dim, out_dim, bias=True, bias_init=0, lr_mul=1, activation=None): 163 | super().__init__() 164 | 165 | self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) 166 | 167 | if bias: 168 | self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) 169 | else: 170 | self.bias = None 171 | 172 | self.activation = activation 173 | 174 | self.scale = (1 / math.sqrt(in_dim)) * lr_mul 175 | self.lr_mul = lr_mul 176 | 177 | def forward(self, input): 178 | 179 | if self.activation: 180 | out = F.linear(input, self.weight * self.scale) 181 | out = fused_leaky_relu(out, self.bias * self.lr_mul) 182 | else: 183 | out = F.linear(input, self.weight * self.scale, bias=self.bias * self.lr_mul) 184 | 185 | return out 186 | 187 | def __repr__(self): 188 | return (f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})') 189 | 190 | 191 | class ScaledLeakyReLU(nn.Module): 192 | def __init__(self, negative_slope=0.2): 193 | super().__init__() 194 | 195 | self.negative_slope = negative_slope 196 | 197 | def forward(self, input): 198 | return F.leaky_relu(input, negative_slope=self.negative_slope) 199 | 200 | 201 | class ModulatedConv2d(nn.Module): 202 | def __init__(self, in_channel, out_channel, kernel_size, style_dim, demodulate=True, upsample=False, 203 | downsample=False, blur_kernel=np.array([1, 3, 3, 1]), ): 204 | super().__init__() 205 | 206 | self.eps = 1e-8 207 | self.kernel_size = kernel_size 208 | self.in_channel = in_channel 209 | self.out_channel = out_channel 210 | self.upsample = upsample 211 | self.downsample = downsample 212 | 213 | if upsample: 214 | factor = 2 215 | p = (len(blur_kernel) - factor) - (kernel_size - 1) 216 | pad0 = (p + 1) // 2 + factor - 1 217 | pad1 = p // 2 + 1 218 | 219 | self.blur = Blur(blur_kernel, pad=(pad0, pad1), upsample_factor=factor) 220 | 221 | if downsample: 222 | factor = 2 223 | p = (len(blur_kernel) - factor) + (kernel_size - 1) 224 | pad0 = (p + 1) // 2 225 | pad1 = p // 2 226 | 227 | self.blur = Blur(blur_kernel, pad=(pad0, pad1)) 228 | 229 | fan_in = in_channel * kernel_size ** 2 230 | self.scale = 1 / math.sqrt(fan_in) 231 | self.padding = kernel_size // 2 232 | 233 | self.weight = nn.Parameter(torch.randn(1, out_channel, in_channel, kernel_size, kernel_size)) 234 | 235 | self.modulation = EqualLinear(style_dim, in_channel, bias_init=1) 236 | self.demodulate = demodulate 237 | 238 | def __repr__(self): 239 | return ( 240 | f'{self.__class__.__name__}({self.in_channel}, {self.out_channel}, {self.kernel_size}, ' 241 | f'upsample={self.upsample}, downsample={self.downsample})' 242 | ) 243 | 244 | def forward(self, input, style): 245 | batch, in_channel, height, width = input.shape 246 | 247 | style = self.modulation(style).view(batch, 1, in_channel, 1, 1) 248 | weight = self.scale * self.weight * style 249 | 250 | if self.demodulate: 251 | demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8) 252 | weight = weight * demod.view(batch, self.out_channel, 1, 1, 1) 253 | 254 | weight = weight.view(batch * self.out_channel, in_channel, self.kernel_size, self.kernel_size) 255 | 256 | if self.upsample: 257 | input = input.view(1, batch * in_channel, height, width) 258 | weight = weight.view(batch, self.out_channel, in_channel, self.kernel_size, self.kernel_size) 259 | weight = weight.transpose(1, 2).reshape(batch * in_channel, self.out_channel, self.kernel_size, 260 | self.kernel_size) 261 | out = F.conv_transpose2d(input, weight, padding=0, stride=2, groups=batch) 262 | _, _, height, width = out.shape 263 | out = out.view(batch, self.out_channel, height, width) 264 | out = self.blur(out) 265 | elif self.downsample: 266 | input = self.blur(input) 267 | _, _, height, width = input.shape 268 | input = input.view(1, batch * in_channel, height, width) 269 | out = F.conv2d(input, weight, padding=0, stride=2, groups=batch) 270 | _, _, height, width = out.shape 271 | out = out.view(batch, self.out_channel, height, width) 272 | else: 273 | input = input.view(1, batch * in_channel, height, width) 274 | out = F.conv2d(input, weight, padding=self.padding, groups=batch) 275 | _, _, height, width = out.shape 276 | out = out.view(batch, self.out_channel, height, width) 277 | 278 | return out 279 | 280 | 281 | class NoiseInjection(nn.Module): 282 | def __init__(self): 283 | super().__init__() 284 | 285 | self.weight = nn.Parameter(torch.zeros(1)) 286 | 287 | def forward(self, image, noise=None): 288 | 289 | if noise is None: 290 | return image 291 | else: 292 | return image + self.weight * noise 293 | 294 | 295 | class ConstantInput(nn.Module): 296 | def __init__(self, channel, size=4): 297 | super().__init__() 298 | 299 | self.input = nn.Parameter(torch.randn(1, channel, size, size)) 300 | 301 | def forward(self, input): 302 | batch = input.shape[0] 303 | out = self.input.repeat(batch, 1, 1, 1) 304 | 305 | return out 306 | 307 | 308 | class StyledConv(nn.Module): 309 | def __init__(self, in_channel, out_channel, kernel_size, style_dim, upsample=False, blur_kernel=[1, 3, 3, 1], 310 | demodulate=True): 311 | super().__init__() 312 | 313 | self.conv = ModulatedConv2d( 314 | in_channel, 315 | out_channel, 316 | kernel_size, 317 | style_dim, 318 | upsample=upsample, 319 | blur_kernel=blur_kernel, 320 | demodulate=demodulate, 321 | ) 322 | 323 | self.noise = NoiseInjection() 324 | self.activate = FusedLeakyReLU(out_channel) 325 | 326 | def forward(self, input, style, noise=None): 327 | out = self.conv(input, style) 328 | out = self.noise(out, noise=noise) 329 | out = self.activate(out) 330 | 331 | return out 332 | 333 | 334 | class ConvLayer(nn.Sequential): 335 | def __init__( 336 | self, 337 | in_channel, 338 | out_channel, 339 | kernel_size, 340 | downsample=False, 341 | blur_kernel=[1, 3, 3, 1], 342 | bias=True, 343 | activate=True, 344 | ): 345 | layers = [] 346 | 347 | if downsample: 348 | factor = 2 349 | p = (len(blur_kernel) - factor) + (kernel_size - 1) 350 | pad0 = (p + 1) // 2 351 | pad1 = p // 2 352 | 353 | layers.append(Blur(blur_kernel, pad=(pad0, pad1))) 354 | 355 | stride = 2 356 | self.padding = 0 357 | 358 | else: 359 | stride = 1 360 | self.padding = kernel_size // 2 361 | 362 | layers.append(EqualConv2d(in_channel, out_channel, kernel_size, padding=self.padding, stride=stride, 363 | bias=bias and not activate)) 364 | 365 | if activate: 366 | if bias: 367 | layers.append(FusedLeakyReLU(out_channel)) 368 | else: 369 | layers.append(ScaledLeakyReLU(0.2)) 370 | 371 | super().__init__(*layers) 372 | 373 | 374 | class ToRGB(nn.Module): 375 | def __init__(self, in_channel, style_dim, upsample=True, blur_kernel=[1, 3, 3, 1]): 376 | super().__init__() 377 | 378 | if upsample: 379 | self.upsample = Upsample(blur_kernel) 380 | 381 | self.conv = ConvLayer(in_channel, 3, 1) 382 | self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1)) 383 | 384 | def forward(self, input, skip=None): 385 | out = self.conv(input) 386 | out = out + self.bias 387 | 388 | if skip is not None: 389 | skip = self.upsample(skip) 390 | out = out + skip 391 | 392 | return out 393 | 394 | 395 | class ToFlow(nn.Module): 396 | def __init__(self, in_channel, style_dim, upsample=True, blur_kernel=[1, 3, 3, 1]): 397 | super().__init__() 398 | 399 | if upsample: 400 | self.upsample = Upsample(blur_kernel) 401 | 402 | self.conv = ModulatedConv2d(in_channel, 3, 1, style_dim, demodulate=False) 403 | self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1)) 404 | 405 | def forward(self, input, style, feat, skip=None): 406 | out = self.conv(input, style) 407 | out = out + self.bias 408 | 409 | # warping 410 | xs = np.linspace(-1, 1, input.size(2)) 411 | xs = np.meshgrid(xs, xs) 412 | xs = np.stack(xs, 2) 413 | 414 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 415 | xs = torch.tensor(xs, requires_grad=False).float().unsqueeze(0).repeat(input.size(0), 1, 1, 1).to(device) 416 | 417 | if skip is not None: 418 | skip = self.upsample(skip) 419 | out = out + skip 420 | 421 | sampler = torch.tanh(out[:, 0:2, :, :]) 422 | mask = torch.sigmoid(out[:, 2:3, :, :]) 423 | flow = sampler.permute(0, 2, 3, 1) + xs 424 | 425 | feat_warp = F.grid_sample(feat, flow, align_corners=False) * mask 426 | 427 | return feat_warp, feat_warp + input * (1.0 - mask), out 428 | 429 | #------------------ custom OP ------------------# 430 | def gram_schmidt_qr(A): 431 | m, n = A.shape 432 | Q = torch.zeros(m, n) 433 | R = torch.zeros(n, n) 434 | 435 | for j in range(n): 436 | v = A[:, j] 437 | for i in range(j): 438 | R[i, j] = torch.dot(Q[:, i], A[:, j]) 439 | v -= R[i, j] * Q[:, i] 440 | R[j, j] = torch.norm(v) 441 | Q[:, j] = v / R[j, j] 442 | 443 | return Q, R 444 | 445 | def custom_diag_embed(diagonal): 446 | size = diagonal.size(-1) 447 | eye = torch.eye(size, device=diagonal.device, dtype=diagonal.dtype) 448 | return diagonal.unsqueeze(-1) * eye 449 | #------------------ custom OP ------------------# 450 | 451 | 452 | class Direction(nn.Module): 453 | def __init__(self, motion_dim): 454 | super(Direction, self).__init__() 455 | 456 | self.weight = nn.Parameter(torch.randn(512, motion_dim)) 457 | 458 | def forward(self, input): 459 | # input: (bs*t) x 512 460 | 461 | weight = self.weight + 1e-8 462 | Q, R = torch.linalg.qr(weight, 'reduced') # get eignvector, orthogonal [n1, n2, n3, n4] 463 | #Q, R = gram_schmidt_qr(weight) # custom OP 464 | 465 | if input is None: 466 | return Q 467 | else: 468 | input_diag = torch.diag_embed(input) 469 | out = torch.matmul(input_diag, Q.T) 470 | out = torch.sum(out, dim=1) 471 | 472 | return out 473 | 474 | 475 | class Synthesis(nn.Module): 476 | def __init__(self, size, style_dim, motion_dim, blur_kernel=[1, 3, 3, 1], channel_multiplier=1): 477 | super(Synthesis, self).__init__() 478 | 479 | self.size = size 480 | self.style_dim = style_dim 481 | self.motion_dim = motion_dim 482 | 483 | self.direction = Direction(motion_dim) 484 | 485 | self.channels = { 486 | 4: 512, 487 | 8: 512, 488 | 16: 512, 489 | 32: 512, 490 | 64: 256 * channel_multiplier, 491 | 128: 128 * channel_multiplier, 492 | 256: 64 * channel_multiplier, 493 | 512: 32 * channel_multiplier, 494 | 1024: 16 * channel_multiplier, 495 | } 496 | 497 | self.input = ConstantInput(self.channels[4]) 498 | self.conv1 = StyledConv(self.channels[4], self.channels[4], 3, style_dim, blur_kernel=blur_kernel) 499 | self.to_rgb1 = ToRGB(self.channels[4], style_dim, upsample=False) 500 | 501 | self.log_size = int(math.log(size, 2)) 502 | self.num_layers = (self.log_size - 2) * 2 + 1 503 | 504 | self.convs = nn.ModuleList() 505 | self.upsamples = nn.ModuleList() 506 | self.to_rgbs = nn.ModuleList() 507 | self.to_flows = nn.ModuleList() 508 | 509 | in_channel = self.channels[4] 510 | 511 | for i in range(3, self.log_size + 1): 512 | out_channel = self.channels[2 ** i] 513 | 514 | self.convs.append(StyledConv(in_channel, out_channel, 3, style_dim, upsample=True, 515 | blur_kernel=blur_kernel)) 516 | self.convs.append(StyledConv(out_channel, out_channel, 3, style_dim, blur_kernel=blur_kernel)) 517 | self.to_rgbs.append(ToRGB(out_channel, style_dim)) 518 | 519 | self.to_flows.append(ToFlow(out_channel, style_dim)) 520 | 521 | in_channel = out_channel 522 | 523 | self.n_latent = self.log_size * 2 - 2 524 | 525 | def forward(self, wa, alpha, feats): 526 | 527 | # wa: bs x style_dim 528 | # alpha: bs x style_dim 529 | 530 | bs = wa.size(0) 531 | 532 | if alpha is not None: 533 | # generating moving directions 534 | if len(alpha) > 1: 535 | directions_target = self.direction(alpha[0]) # target 536 | directions_source = self.direction(alpha[1]) # source 537 | directions_start = self.direction(alpha[2]) # start 538 | latent = wa + (directions_target - directions_start) + directions_source 539 | else: 540 | directions = self.direction(alpha[0]) 541 | latent = wa + directions # wa + directions 542 | else: 543 | latent = wa 544 | 545 | inject_index = self.n_latent 546 | latent = latent.unsqueeze(1).repeat(1, inject_index, 1) 547 | 548 | out = self.input(latent) 549 | out = self.conv1(out, latent[:, 0]) 550 | 551 | i = 1 552 | for conv1, conv2, to_rgb, to_flow, feat in zip(self.convs[::2], self.convs[1::2], self.to_rgbs, 553 | self.to_flows, feats): 554 | out = conv1(out, latent[:, i]) 555 | out = conv2(out, latent[:, i + 1]) 556 | if out.size(2) == 8: 557 | out_warp, out, skip_flow = to_flow(out, latent[:, i + 2], feat) 558 | skip = to_rgb(out_warp) 559 | else: 560 | out_warp, out, skip_flow = to_flow(out, latent[:, i + 2], feat, skip_flow) 561 | skip = to_rgb(out_warp, skip) 562 | i += 2 563 | 564 | img = skip 565 | 566 | return img 567 | -------------------------------------------------------------------------------- /LIA/utils.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import torch.nn.functional as F 3 | import torch 4 | 5 | 6 | class AntiAliasInterpolation2d(nn.Module): 7 | """ 8 | Band-limited downsampling, for better preservation of the input signal. 9 | """ 10 | 11 | def __init__(self, channels, scale): 12 | super(AntiAliasInterpolation2d, self).__init__() 13 | sigma = (1 / scale - 1) / 2 14 | kernel_size = 2 * round(sigma * 4) + 1 15 | self.ka = kernel_size // 2 16 | self.kb = self.ka - 1 if kernel_size % 2 == 0 else self.ka 17 | 18 | kernel_size = [kernel_size, kernel_size] 19 | sigma = [sigma, sigma] 20 | # The gaussian kernel is the product of the 21 | # gaussian function of each dimension. 22 | kernel = 1 23 | meshgrids = torch.meshgrid( 24 | [ 25 | torch.arange(size, dtype=torch.float32) 26 | for size in kernel_size 27 | ] 28 | ) 29 | for size, std, mgrid in zip(kernel_size, sigma, meshgrids): 30 | mean = (size - 1) / 2 31 | kernel *= torch.exp(-(mgrid - mean) ** 2 / (2 * std ** 2)) 32 | 33 | # Make sure sum of values in gaussian kernel equals 1. 34 | kernel = kernel / torch.sum(kernel) 35 | # Reshape to depthwise convolutional weight 36 | kernel = kernel.view(1, 1, *kernel.size()) 37 | kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1)) 38 | 39 | self.register_buffer('weight', kernel) 40 | self.groups = channels 41 | self.scale = scale 42 | inv_scale = 1 / scale 43 | self.int_inv_scale = int(inv_scale) 44 | 45 | def forward(self, input): 46 | if self.scale == 1.0: 47 | return input 48 | 49 | out = F.pad(input, (self.ka, self.kb, self.ka, self.kb)) 50 | out = F.conv2d(out, weight=self.weight, groups=self.groups) 51 | out = out[:, :, ::self.int_inv_scale, ::self.int_inv_scale] 52 | 53 | return out 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Portrait-Talker 2 | 3 | Talking head animation 4 | 5 | ## Note: This project is not complete it is still WIP. 6 | 7 | ## Installation 8 | 9 | ### step 1. Clone repo 10 | 11 | ``` 12 | git clone https://github.com/harisreedhar/Portrait-Talker.git 13 | cd Portrait-Talker 14 | ``` 15 | 16 | ### step 2. Download pre-trained models 17 | 18 | `put these models in Portrait-Talker/pretrained_models` 19 | 20 | - [det_10g.onnx](https://huggingface.co/bluefoxcreation/insightface-retinaface-arcface-model/resolve/main/det_10g.onnx) 21 | - [vox512.pt](https://huggingface.co/bluefoxcreation/LIA-512/resolve/main/vox512.pt?download=true) 22 | - [shape_predictor_68_face_landmarks.dat](https://github.com/tzutalin/dlib-android/blob/master/data/shape_predictor_68_face_landmarks.dat) 23 | - [wav2vecDS.pt](https://huggingface.co/bluefoxcreation/DINet_unofficial/resolve/main/wav2vecDS.pt) 24 | - [clip_training_DINet_256mouth.pth](https://huggingface.co/bluefoxcreation/DINet_unofficial/resolve/main/clip_training_DINet_256mouth.pth) 25 | 26 | ### step 3. Create env & Install dependencies 27 | 28 | ``` 29 | conda create -n portrait-talker python=3.10 -y 30 | conda activate portrait-talker 31 | conda install pytorch==2.0.0 torchvision==0.15.0 torchaudio==2.0.0 pytorch-cuda=11.8 -c pytorch -c nvidia 32 | pip install -r requirements.txt 33 | ``` 34 | 35 | ### step 4. Test Run 36 | 37 | ``` 38 | python run_video_driven.py --source_path ./test/source.jpg --driving_path ./test/driving.mov 39 | ``` 40 | 41 | ### or 42 | 43 | ``` 44 | python run_audio_driven.py --source_path ./test/source.jpg --driving_path ./test/driving.wav 45 | ``` 46 | 47 | ## Acknowledgment 48 | 49 | - [LIA](https://github.com/wyhsirius/LIA/) 50 | - [DINet](https://github.com/MRzzm/DINet) 51 | - [Insightface](https://github.com/deepinsight/insightface/tree/master/python-package/insightface) 52 | - [Dlib](https://github.com/davisking/dlib) 53 | -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- 1 | name: portrait-talker 2 | channels: 3 | - pytorch 4 | - nvidia 5 | - defaults 6 | dependencies: 7 | - _libgcc_mutex=0.1=main 8 | - _openmp_mutex=5.1=1_gnu 9 | - blas=1.0=mkl 10 | - brotlipy=0.7.0=py310h7f8727e_1002 11 | - bzip2=1.0.8=h7b6447c_0 12 | - ca-certificates=2023.05.30=h06a4308_0 13 | - certifi=2023.7.22=py310h06a4308_0 14 | - cffi=1.15.1=py310h5eee18b_3 15 | - charset-normalizer=2.0.4=pyhd3eb1b0_0 16 | - cryptography=41.0.2=py310h22a60cf_0 17 | - cuda-cudart=11.8.89=0 18 | - cuda-cupti=11.8.87=0 19 | - cuda-libraries=11.8.0=0 20 | - cuda-nvrtc=11.8.89=0 21 | - cuda-nvtx=11.8.86=0 22 | - cuda-runtime=11.8.0=0 23 | - ffmpeg=4.3=hf484d3e_0 24 | - filelock=3.9.0=py310h06a4308_0 25 | - freetype=2.12.1=h4a9f257_0 26 | - giflib=5.2.1=h5eee18b_3 27 | - gmp=6.2.1=h295c915_3 28 | - gmpy2=2.1.2=py310heeb90bb_0 29 | - gnutls=3.6.15=he1e5248_0 30 | - idna=3.4=py310h06a4308_0 31 | - intel-openmp=2023.1.0=hdb19cb5_46305 32 | - jinja2=3.1.2=py310h06a4308_0 33 | - jpeg=9e=h5eee18b_1 34 | - lame=3.100=h7b6447c_0 35 | - lcms2=2.12=h3be6417_0 36 | - ld_impl_linux-64=2.38=h1181459_1 37 | - lerc=3.0=h295c915_0 38 | - libcublas=11.11.3.6=0 39 | - libcufft=10.9.0.58=0 40 | - libcufile=1.7.2.10=0 41 | - libcurand=10.3.3.141=0 42 | - libcusolver=11.4.1.48=0 43 | - libcusparse=11.7.5.86=0 44 | - libdeflate=1.17=h5eee18b_0 45 | - libffi=3.4.4=h6a678d5_0 46 | - libgcc-ng=11.2.0=h1234567_1 47 | - libgomp=11.2.0=h1234567_1 48 | - libiconv=1.16=h7f8727e_2 49 | - libidn2=2.3.4=h5eee18b_0 50 | - libnpp=11.8.0.86=0 51 | - libnvjpeg=11.9.0.86=0 52 | - libpng=1.6.39=h5eee18b_0 53 | - libstdcxx-ng=11.2.0=h1234567_1 54 | - libtasn1=4.19.0=h5eee18b_0 55 | - libtiff=4.5.0=h6a678d5_2 56 | - libunistring=0.9.10=h27cfd23_0 57 | - libuuid=1.41.5=h5eee18b_0 58 | - libwebp=1.2.4=h11a3e52_1 59 | - libwebp-base=1.2.4=h5eee18b_1 60 | - lz4-c=1.9.4=h6a678d5_0 61 | - markupsafe=2.1.1=py310h7f8727e_0 62 | - mkl=2023.1.0=h213fc3f_46343 63 | - mkl-service=2.4.0=py310h5eee18b_1 64 | - mkl_fft=1.3.6=py310h1128e8f_1 65 | - mkl_random=1.2.2=py310h1128e8f_1 66 | - mpc=1.1.0=h10f8cd9_1 67 | - mpfr=4.0.2=hb69a4c5_1 68 | - mpmath=1.3.0=py310h06a4308_0 69 | - ncurses=6.4=h6a678d5_0 70 | - nettle=3.7.3=hbbd107a_1 71 | - networkx=3.1=py310h06a4308_0 72 | - numpy=1.25.2=py310h5f9d8c6_0 73 | - numpy-base=1.25.2=py310hb5e798b_0 74 | - openh264=2.1.1=h4ff587b_0 75 | - openssl=3.0.10=h7f8727e_2 76 | - pillow=9.4.0=py310h6a678d5_0 77 | - pip=23.2.1=py310h06a4308_0 78 | - pycparser=2.21=pyhd3eb1b0_0 79 | - pyopenssl=23.2.0=py310h06a4308_0 80 | - pysocks=1.7.1=py310h06a4308_0 81 | - python=3.10.12=h955ad1f_0 82 | - pytorch=2.0.0=py3.10_cuda11.8_cudnn8.7.0_0 83 | - pytorch-cuda=11.8=h7e8668a_5 84 | - pytorch-mutex=1.0=cuda 85 | - readline=8.2=h5eee18b_0 86 | - requests=2.31.0=py310h06a4308_0 87 | - setuptools=68.0.0=py310h06a4308_0 88 | - sqlite=3.41.2=h5eee18b_0 89 | - sympy=1.11.1=py310h06a4308_0 90 | - tbb=2021.8.0=hdb19cb5_0 91 | - tk=8.6.12=h1ccaba5_0 92 | - torchaudio=2.0.0=py310_cu118 93 | - torchtriton=2.0.0=py310 94 | - torchvision=0.15.0=py310_cu118 95 | - typing_extensions=4.7.1=py310h06a4308_0 96 | - tzdata=2023c=h04d1e81_0 97 | - urllib3=1.26.16=py310h06a4308_0 98 | - wheel=0.38.4=py310h06a4308_0 99 | - xz=5.4.2=h5eee18b_0 100 | - zlib=1.2.13=h5eee18b_0 101 | - zstd=1.5.5=hc292b87_0 102 | - pip: 103 | - coloredlogs==15.0.1 104 | - dlib==19.24.2 105 | - flatbuffers==23.5.26 106 | - humanfriendly==10.0 107 | - onnxruntime-gpu==1.15.0 108 | - opencv-python==4.8.0.76 109 | - opencv-python-headless==4.8.0.76 110 | - protobuf==4.24.2 111 | - tqdm==4.66.1 112 | prefix: /home/harisree/anaconda3/envs/portrait-talker 113 | -------------------------------------------------------------------------------- /mask.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/mask.jpg -------------------------------------------------------------------------------- /pretrained_models/put models here: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | onnxruntime-gpu==1.15.0 2 | opencv-python=4.7.0.72 3 | opencv-python-headless==4.7.0.72 4 | tqdm==4.65.0 5 | dlib==19.24 6 | scipy==1.10.1 7 | -------------------------------------------------------------------------------- /run_audio_driven.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import glob 4 | import dlib 5 | import torch 6 | import random 7 | import warnings 8 | import argparse 9 | import subprocess 10 | import numpy as np 11 | from tqdm import tqdm 12 | from collections import OrderedDict 13 | from utils.utils import is_image, is_video, create_directory 14 | 15 | from DINet.DINet import DINet 16 | from DINet.wav2vec import Wav2VecFeatureExtractor 17 | from DINet.wav2vecDS import Wav2vecDS 18 | from DINet.utils import compute_crop_radius, face_crop, final_image_fill 19 | 20 | 21 | warnings.filterwarnings("ignore", category=UserWarning, module="torch") 22 | 23 | 24 | DLIB_68 = "./pretrained_models/shape_predictor_68_face_landmarks.dat" 25 | DINET_PATH = "./pretrained_models/clip_training_DINet_256mouth.pth" 26 | WAV2VECDS_PATH = "./pretrained_models/wav2vecDS.pt" 27 | 28 | 29 | def extract_frames_from_video(video_path, save_dir): 30 | videoCapture = cv2.VideoCapture(video_path) 31 | fps = videoCapture.get(cv2.CAP_PROP_FPS) 32 | if int(fps) != 25: 33 | print("Warning: the input video is not 25 fps, it would be better to trans it to 25 fps!") 34 | frames = int(videoCapture.get(cv2.CAP_PROP_FRAME_COUNT)) 35 | frame_height = int(videoCapture.get(cv2.CAP_PROP_FRAME_HEIGHT)) 36 | frame_width = int(videoCapture.get(cv2.CAP_PROP_FRAME_WIDTH)) 37 | os.makedirs(save_dir, exist_ok=True) 38 | ffmpeg_command = ["ffmpeg", "-i", video_path, os.path.join(save_dir, "%06d.png")] 39 | subprocess.run(ffmpeg_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) 40 | return frame_width, frame_height 41 | 42 | 43 | def get_landmarks(video_frame_paths): 44 | face_detector = dlib.get_frontal_face_detector() 45 | shape_predictor = dlib.shape_predictor(DLIB_68) 46 | progress_bar = tqdm(total=len(video_frame_paths), desc='[ Getting face landmarks ]', unit='frame', dynamic_ncols=True) 47 | landmarks = [] 48 | for frame_index, image_path in enumerate(video_frame_paths): 49 | frame = cv2.imread(image_path) 50 | gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 51 | faces = face_detector(gray_frame) 52 | if len(faces) == 0: 53 | print(f"Error: No Face detected on frame {frame_index}!") 54 | break 55 | if len(faces) > 1: 56 | print(f"Warning: More than one face detected on frame {frame_index}. Using first detected one!") 57 | face = faces[0] 58 | landmark68 = shape_predictor(gray_frame, face) 59 | x1, y1, x2, y2 = face.left(), face.top(), face.right(), face.bottom() 60 | landmarks.append([np.array([lmk.x, lmk.y]) for lmk in landmark68.parts()]) 61 | progress_bar.update(1) 62 | progress_bar.close() 63 | landmarks = np.array(landmarks) 64 | return landmarks 65 | 66 | 67 | class Demo(): 68 | def __init__(self, args): 69 | self.args = args 70 | self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") 71 | 72 | print('[ Loading models ]') 73 | 74 | # audio to vec model 75 | self.feature_extractor = Wav2VecFeatureExtractor(self.device) 76 | self.audio_mapping = Wav2vecDS(model_path=WAV2VECDS_PATH) 77 | 78 | # dinet model 79 | self.dinet = DINet(3, 15, 29).to(self.device) 80 | state_dict = torch.load(DINET_PATH)["state_dict"]["net_g"] 81 | new_state_dict = OrderedDict() 82 | for k, v in state_dict.items(): 83 | name = k[7:] 84 | new_state_dict[name] = v 85 | self.dinet.load_state_dict(new_state_dict) 86 | self.dinet.eval() 87 | 88 | def run(self): 89 | 90 | print('[ Extracting driving audio features ]') 91 | 92 | ds_feature = self.feature_extractor.compute_audio_feature(self.args.driving_path, chunk_duration=10) 93 | ds_feature = self.audio_mapping.mapping(ds_feature) 94 | res_frame_length = ds_feature.shape[0] 95 | ds_feature_padding = np.pad(ds_feature, ((2, 2), (0, 0)), mode="edge") 96 | 97 | if is_video(self.args.source_path): 98 | temp_dir = os.path.join(self.args.save_dir, "temp_sequence") 99 | create_directory(temp_dir, remove_existing=True) 100 | video_size = extract_frames_from_video(self.args.source_path, temp_dir) 101 | video_frame_path_list = glob.glob(os.path.join(temp_dir, "*.png")) 102 | video_frame_path_list.sort() 103 | video_landmark_data = get_landmarks(video_frame_path_list).astype(np.int32) 104 | if self.args.landmark_smoothness > 0: 105 | from scipy.ndimage import gaussian_filter1d 106 | video_landmark_data = gaussian_filter1d(video_landmark_data, sigma=self.args.landmark_smoothness, axis=0, mode='nearest').astype(np.int32) 107 | 108 | elif is_image(self.args.source_path): 109 | image = cv2.imread(self.args.source_path) 110 | video_size = image.shape[1], image.shape[0] 111 | video_frame_path_list = [self.args.source_path] * res_frame_length 112 | video_landmark_data = np.array([get_landmarks([self.args.source_path])[0]] * res_frame_length).astype(np.int32) 113 | else: 114 | print(f"Unknown source {self.args.source_path}") 115 | 116 | video_frame_path_list_cycle = video_frame_path_list + video_frame_path_list[::-1] 117 | video_landmark_data_cycle = np.concatenate([video_landmark_data, np.flip(video_landmark_data, 0)], 0) 118 | video_frame_path_list_cycle_length = len(video_frame_path_list_cycle) 119 | 120 | if video_frame_path_list_cycle_length >= res_frame_length: 121 | res_video_frame_path_list = video_frame_path_list_cycle[:res_frame_length] 122 | res_video_landmark_data = video_landmark_data_cycle[:res_frame_length, :, :] 123 | else: 124 | divisor = res_frame_length // video_frame_path_list_cycle_length 125 | remainder = res_frame_length % video_frame_path_list_cycle_length 126 | res_video_frame_path_list = (video_frame_path_list_cycle * divisor + video_frame_path_list_cycle[:remainder]) 127 | res_video_landmark_data = np.concatenate([video_landmark_data_cycle] * divisor + [video_landmark_data_cycle[:remainder, :, :]], 0,) 128 | 129 | res_video_frame_path_list_pad = ([video_frame_path_list_cycle[0]] * 2 + res_video_frame_path_list + [video_frame_path_list_cycle[-1]] * 2) 130 | res_video_landmark_data_pad = np.pad(res_video_landmark_data, ((2, 2), (0, 0), (0, 0)), mode="edge") 131 | 132 | assert (ds_feature_padding.shape[0] == len(res_video_frame_path_list_pad) == res_video_landmark_data_pad.shape[0]) 133 | pad_length = ds_feature_padding.shape[0] 134 | 135 | m_2 = self.args.mouth_region_size // 2 136 | m_4 = self.args.mouth_region_size // 4 137 | m_8 = self.args.mouth_region_size // 8 138 | 139 | ref_img_list = [] 140 | resize_shape = (int(self.args.mouth_region_size + m_4), int((m_2) * 3 + m_8)) 141 | ref_index_list = random.sample(range(5, len(res_video_frame_path_list_pad) - 2), 5) 142 | 143 | for ref_index in ref_index_list: 144 | crop_flag, crop_radius = compute_crop_radius(video_size, res_video_landmark_data_pad[ref_index - 5 : ref_index, :, :]) 145 | if not crop_flag: 146 | raise ValueError("DINET cannot handle videos with large changes in facial size!!") 147 | 148 | ref_img = cv2.imread(res_video_frame_path_list_pad[ref_index - 3])[:, :, ::-1] 149 | ref_landmark = res_video_landmark_data_pad[ref_index - 3, :, :] 150 | ref_img_crop = face_crop(ref_img, ref_landmark, crop_radius) 151 | ref_img_crop = cv2.resize(ref_img_crop, resize_shape) 152 | ref_img_crop = ref_img_crop / 255.0 153 | ref_img_list.append(ref_img_crop) 154 | 155 | ref_video_frame = np.concatenate(ref_img_list, 2) 156 | ref_img_tensor = (torch.from_numpy(ref_video_frame).permute(2, 0, 1).unsqueeze(0).float().to(self.device)) 157 | 158 | fourcc = cv2.VideoWriter_fourcc(*'XVID') 159 | os.makedirs(self.args.save_dir, exist_ok=True) 160 | save_path = os.path.join(self.args.save_dir, 'audio_driven_result_without_audio.avi') 161 | video_writer = cv2.VideoWriter(save_path, fourcc, 25, video_size) 162 | progress_bar = tqdm(total=res_frame_length, desc='[ Processing frames ]', unit='frame', dynamic_ncols=True) 163 | 164 | for clip_end_index in range(5, pad_length, 1): 165 | crop_flag, crop_radius = compute_crop_radius(video_size, res_video_landmark_data_pad[clip_end_index - 5 : clip_end_index, :, :], random_scale=1.05) 166 | if not crop_flag: 167 | raise ("DINET can not handle videos with large change of facial size!!") 168 | 169 | frame_data = cv2.imread(res_video_frame_path_list_pad[clip_end_index - 3])[:, :, ::-1] 170 | frame_landmark = res_video_landmark_data_pad[clip_end_index - 3, :, :] 171 | 172 | crop_frame_data = face_crop(frame_data, frame_landmark, crop_radius) 173 | crop_frame_h, crop_frame_w = crop_frame_data.shape[0], crop_frame_data.shape[1] 174 | crop_frame_data = cv2.resize(crop_frame_data, resize_shape) 175 | crop_frame_data = crop_frame_data / 255.0 176 | crop_frame_data[m_2 : m_2 + self.args.mouth_region_size, m_8 : m_8 + self.args.mouth_region_size, :,] = 0 177 | 178 | crop_frame_tensor = ( 179 | torch.from_numpy(crop_frame_data) 180 | .float() 181 | .to(self.device) 182 | .permute(2, 0, 1) 183 | .unsqueeze(0) 184 | ) 185 | deepspeech_tensor = ( 186 | torch.from_numpy(ds_feature_padding[clip_end_index - 5 : clip_end_index, :]) 187 | .permute(1, 0) 188 | .unsqueeze(0) 189 | .float() 190 | .to(self.device) 191 | ) 192 | 193 | with torch.no_grad(): 194 | pre_frame = self.dinet(crop_frame_tensor, ref_img_tensor, deepspeech_tensor) 195 | pre_frame = (pre_frame.squeeze(0).permute(1, 2, 0).detach().cpu().numpy() * 255) 196 | 197 | pre_frame_resize = cv2.resize(pre_frame, (crop_frame_w, crop_frame_h)) 198 | final_frame = final_image_fill(frame_data.copy(), pre_frame_resize, frame_landmark, crop_radius) 199 | 200 | video_writer.write(final_frame[:, :, ::-1]) 201 | progress_bar.update(1) 202 | 203 | video_writer.release() 204 | progress_bar.close() 205 | 206 | print('[ Merging audio ]') 207 | 208 | video_with_audio_path = save_path.replace("_without_audio", "") 209 | if os.path.exists(video_with_audio_path): 210 | os.remove(video_with_audio_path) 211 | 212 | ffmpeg_command = f"ffmpeg -i {save_path} -i {self.args.driving_path} -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {video_with_audio_path}" 213 | subprocess.call(ffmpeg_command, shell=True) 214 | 215 | if os.path.exists(save_path) and os.path.exists(video_with_audio_path): 216 | os.remove(save_path) 217 | 218 | 219 | if __name__ == '__main__': 220 | parser = argparse.ArgumentParser() 221 | parser.add_argument("--source_path", type=str, default='') 222 | parser.add_argument("--driving_path", type=str, default='') 223 | parser.add_argument("--save_dir", type=str, default='./test/result') 224 | parser.add_argument("--mouth_region_size", type=int, default=256) 225 | parser.add_argument("--landmark_smoothness", type=float, default=0) 226 | args = parser.parse_args() 227 | 228 | demo = Demo(args) 229 | demo.run() 230 | -------------------------------------------------------------------------------- /run_video_driven.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import torch 4 | import torch.nn as nn 5 | import torchvision 6 | import argparse 7 | import numpy as np 8 | from tqdm import tqdm 9 | import onnxruntime as ort 10 | from LIA.generator import Generator 11 | from utils.retinaface import RetinaFace 12 | from utils.face_alignment import get_cropped_head 13 | 14 | 15 | LIA_PATH = "./pretrained_models/vox512.pt" 16 | RETINAFACE_PATH = "./pretrained_models/det_10g.onnx" 17 | MASK = cv2.imread("./mask.jpg") 18 | 19 | 20 | def paste_back(img, face, matrix): 21 | inverse_affine = cv2.invertAffineTransform(matrix) 22 | h, w = img.shape[0:2] 23 | face_h, face_w = face.shape[0:2] 24 | inv_restored = cv2.warpAffine(face, inverse_affine, (w, h)) 25 | inv_restored = inv_restored.astype('float32') 26 | mask = MASK.copy().astype('float32') / 255 27 | mask = cv2.resize(mask, (face_w, face_h)) 28 | inv_mask = cv2.warpAffine(mask, inverse_affine, (w, h)) 29 | img = inv_mask * inv_restored + (1 - inv_mask) * img 30 | return img.clip(0, 255).astype('uint8') 31 | 32 | 33 | def process_source(model, img_path, size, crop_scale=1.6): 34 | ori_img = cv2.imread(img_path) 35 | bboxes, kpss = model.detect(ori_img, det_thresh=0.6) 36 | assert len(kpss) != 0, "No face detected" 37 | aimg, mat = get_cropped_head(ori_img, kpss[0], size=size, scale=crop_scale) 38 | aimg = np.transpose(aimg[:,:,::-1], (2, 0, 1)) / 255 39 | aimg_tensor = torch.from_numpy(aimg).unsqueeze(0).float() 40 | aimg_tensor_norm = (aimg_tensor - 0.5) * 2.0 41 | return aimg_tensor_norm, ori_img, mat 42 | 43 | 44 | class Demo(nn.Module): 45 | def __init__(self, args): 46 | super(Demo, self).__init__() 47 | self.size = 512 48 | self.device = 'cuda' if torch.cuda.is_available() else 'cpu' 49 | 50 | print('[ loading model ]') 51 | 52 | detector = RetinaFace(model_file=RETINAFACE_PATH, provider=["CUDAExecutionProvider", "CPUExecutionProvider"], session_options=None) 53 | self.gen = Generator(self.size, 512, 20, 1).to(self.device) 54 | weight = torch.load(LIA_PATH, map_location=lambda storage, loc: storage)['gen'] 55 | self.gen.load_state_dict(weight) 56 | self.gen.eval() 57 | 58 | print('[ loading data ]') 59 | 60 | self.source, self.original_image, self.matrix = process_source(detector, args.source_path, self.size, crop_scale=args.crop_scale) 61 | self.source = self.source.to(self.device) 62 | 63 | def run(self): 64 | print('[ running ]') 65 | 66 | cap = cv2.VideoCapture(args.driving_path) 67 | fps = cap.get(cv2.CAP_PROP_FPS) 68 | total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) 69 | 70 | fourcc = cv2.VideoWriter_fourcc(*'XVID') 71 | h, w = self.original_image.shape[:2] 72 | os.makedirs(args.save_dir, exist_ok=True) 73 | save_path = os.path.join(args.save_dir, 'video_driven_result.avi') 74 | out = cv2.VideoWriter(save_path, fourcc, int(fps), (w,h)) 75 | 76 | with torch.no_grad(): 77 | h_start = None 78 | for index in tqdm(range(total_frames), desc='[ Processing frames ]', unit='frame'): 79 | ret, frame = cap.read() 80 | if ret: 81 | frame = cv2.resize(frame, (self.size, self.size)) 82 | frame_np = np.expand_dims(frame, axis = 0).astype('float32') 83 | frame_np = (frame_np / 255 - 0.5) * 2.0 84 | frame_np = frame_np.transpose(0, 3, 1, 2) 85 | frame_torch = torch.from_numpy(frame_np).to(self.device) 86 | if index == 0: 87 | h_start = self.gen.enc.enc_motion(frame_torch) 88 | frame_recon = self.gen(self.source, frame_torch, h_start) 89 | frame_recon_np = frame_recon.permute(0, 2, 3, 1).cpu().numpy()[0] 90 | frame_recon_np = ((frame_recon_np[:,:,::-1] + 1) / 2) * 255 91 | pasted = paste_back(self.original_image, frame_recon_np, self.matrix) 92 | out.write(pasted.astype('uint8')) 93 | else: 94 | break 95 | 96 | cap.release() 97 | out.release() 98 | cv2.destroyAllWindows() 99 | 100 | 101 | if __name__ == '__main__': 102 | parser = argparse.ArgumentParser() 103 | parser.add_argument("--source_path", type=str, default='') 104 | parser.add_argument("--driving_path", type=str, default='') 105 | parser.add_argument("--save_dir", type=str, default='./test/result') 106 | parser.add_argument("--crop_scale", type=float, default=1.25) 107 | args = parser.parse_args() 108 | 109 | demo = Demo(args) 110 | demo.run() 111 | -------------------------------------------------------------------------------- /test/driving.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/test/driving.mov -------------------------------------------------------------------------------- /test/driving.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/test/driving.wav -------------------------------------------------------------------------------- /test/result/audio_driven_result.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/test/result/audio_driven_result.avi -------------------------------------------------------------------------------- /test/result/video_driven_result.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/test/result/video_driven_result.avi -------------------------------------------------------------------------------- /test/source.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harisreedhar/Portrait-Talker/1a8e3c0be017d40e6b6e1aa74f88db993d3e1042/test/source.jpg -------------------------------------------------------------------------------- /utils/face_alignment.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | 5 | def align_crop(img, landmark, size): 6 | template_ffhq = np.array( 7 | [ 8 | [192.98138, 239.94708], 9 | [318.90277, 240.19366], 10 | [256.63416, 314.01935], 11 | [201.26117, 371.41043], 12 | [313.08905, 371.15118] 13 | ]) 14 | template_ffhq *= (512 / size) 15 | matrix = cv2.estimateAffinePartial2D(landmark, template_ffhq, method=cv2.RANSAC, ransacReprojThreshold=100)[0] 16 | warped = cv2.warpAffine(img, matrix, (size, size), borderMode=cv2.BORDER_REPLICATE) 17 | return warped, matrix 18 | 19 | 20 | def get_cropped_head(img, landmark, scale=1.4, size=512): 21 | center = np.mean(landmark, axis=0) 22 | landmark = center + (landmark - center) * scale 23 | return align_crop(img, landmark, size) 24 | -------------------------------------------------------------------------------- /utils/retinaface.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Organization : insightface.ai 3 | # @Author : Jia Guo 4 | # @Time : 2021-09-18 5 | # @Function : 6 | 7 | from __future__ import division 8 | import datetime 9 | import numpy as np 10 | import onnxruntime 11 | import os 12 | import cv2 13 | import sys 14 | 15 | def softmax(z): 16 | assert len(z.shape) == 2 17 | s = np.max(z, axis=1) 18 | s = s[:, np.newaxis] # necessary step to do broadcasting 19 | e_x = np.exp(z - s) 20 | div = np.sum(e_x, axis=1) 21 | div = div[:, np.newaxis] # dito 22 | return e_x / div 23 | 24 | def distance2bbox(points, distance, max_shape=None): 25 | """Decode distance prediction to bounding box. 26 | 27 | Args: 28 | points (Tensor): Shape (n, 2), [x, y]. 29 | distance (Tensor): Distance from the given point to 4 30 | boundaries (left, top, right, bottom). 31 | max_shape (tuple): Shape of the image. 32 | 33 | Returns: 34 | Tensor: Decoded bboxes. 35 | """ 36 | x1 = points[:, 0] - distance[:, 0] 37 | y1 = points[:, 1] - distance[:, 1] 38 | x2 = points[:, 0] + distance[:, 2] 39 | y2 = points[:, 1] + distance[:, 3] 40 | if max_shape is not None: 41 | x1 = x1.clamp(min=0, max=max_shape[1]) 42 | y1 = y1.clamp(min=0, max=max_shape[0]) 43 | x2 = x2.clamp(min=0, max=max_shape[1]) 44 | y2 = y2.clamp(min=0, max=max_shape[0]) 45 | return np.stack([x1, y1, x2, y2], axis=-1) 46 | 47 | def distance2kps(points, distance, max_shape=None): 48 | """Decode distance prediction to bounding box. 49 | 50 | Args: 51 | points (Tensor): Shape (n, 2), [x, y]. 52 | distance (Tensor): Distance from the given point to 4 53 | boundaries (left, top, right, bottom). 54 | max_shape (tuple): Shape of the image. 55 | 56 | Returns: 57 | Tensor: Decoded bboxes. 58 | """ 59 | preds = [] 60 | for i in range(0, distance.shape[1], 2): 61 | px = points[:, i%2] + distance[:, i] 62 | py = points[:, i%2+1] + distance[:, i+1] 63 | if max_shape is not None: 64 | px = px.clamp(min=0, max=max_shape[1]) 65 | py = py.clamp(min=0, max=max_shape[0]) 66 | preds.append(px) 67 | preds.append(py) 68 | return np.stack(preds, axis=-1) 69 | 70 | class RetinaFace: 71 | def __init__(self, model_file=None, provider=["CPUExecutionProvider"], session_options=None): 72 | self.model_file = model_file 73 | self.session_options = session_options 74 | if self.session_options is None: 75 | self.session_options = onnxruntime.SessionOptions() 76 | self.session = onnxruntime.InferenceSession(self.model_file, providers=provider, sess_options=self.session_options) 77 | self.center_cache = {} 78 | self.nms_thresh = 0.4 79 | self.det_thresh = 0.5 80 | self._init_vars() 81 | 82 | def _init_vars(self): 83 | input_cfg = self.session.get_inputs()[0] 84 | input_shape = input_cfg.shape 85 | #print(input_shape) 86 | if isinstance(input_shape[2], str): 87 | self.input_size = None 88 | else: 89 | self.input_size = tuple(input_shape[2:4][::-1]) 90 | #print('image_size:', self.image_size) 91 | input_name = input_cfg.name 92 | self.input_shape = input_shape 93 | outputs = self.session.get_outputs() 94 | output_names = [] 95 | for o in outputs: 96 | output_names.append(o.name) 97 | self.input_name = input_name 98 | self.output_names = output_names 99 | self.input_mean = 127.5 100 | self.input_std = 128.0 101 | #print(self.output_names) 102 | #assert len(outputs)==10 or len(outputs)==15 103 | self.use_kps = False 104 | self._anchor_ratio = 1.0 105 | self._num_anchors = 1 106 | if len(outputs)==6: 107 | self.fmc = 3 108 | self._feat_stride_fpn = [8, 16, 32] 109 | self._num_anchors = 2 110 | elif len(outputs)==9: 111 | self.fmc = 3 112 | self._feat_stride_fpn = [8, 16, 32] 113 | self._num_anchors = 2 114 | self.use_kps = True 115 | elif len(outputs)==10: 116 | self.fmc = 5 117 | self._feat_stride_fpn = [8, 16, 32, 64, 128] 118 | self._num_anchors = 1 119 | elif len(outputs)==15: 120 | self.fmc = 5 121 | self._feat_stride_fpn = [8, 16, 32, 64, 128] 122 | self._num_anchors = 1 123 | self.use_kps = True 124 | 125 | def prepare(self, **kwargs): 126 | nms_thresh = kwargs.get('nms_thresh', None) 127 | if nms_thresh is not None: 128 | self.nms_thresh = nms_thresh 129 | det_thresh = kwargs.get('det_thresh', None) 130 | if det_thresh is not None: 131 | self.det_thresh = det_thresh 132 | input_size = kwargs.get('input_size', None) 133 | if input_size is not None: 134 | if self.input_size is not None: 135 | print('warning: det_size is already set in detection model, ignore') 136 | else: 137 | self.input_size = input_size 138 | 139 | def forward(self, img, threshold): 140 | scores_list = [] 141 | bboxes_list = [] 142 | kpss_list = [] 143 | input_size = tuple(img.shape[0:2][::-1]) 144 | blob = cv2.dnn.blobFromImage(img, 1.0/self.input_std, input_size, (self.input_mean, self.input_mean, self.input_mean), swapRB=True) 145 | net_outs = self.session.run(self.output_names, {self.input_name : blob}) 146 | 147 | input_height = blob.shape[2] 148 | input_width = blob.shape[3] 149 | fmc = self.fmc 150 | for idx, stride in enumerate(self._feat_stride_fpn): 151 | scores = net_outs[idx] 152 | bbox_preds = net_outs[idx+fmc] 153 | bbox_preds = bbox_preds * stride 154 | if self.use_kps: 155 | kps_preds = net_outs[idx+fmc*2] * stride 156 | height = input_height // stride 157 | width = input_width // stride 158 | K = height * width 159 | key = (height, width, stride) 160 | if key in self.center_cache: 161 | anchor_centers = self.center_cache[key] 162 | else: 163 | anchor_centers = np.stack(np.mgrid[:height, :width][::-1], axis=-1).astype(np.float32) 164 | anchor_centers = (anchor_centers * stride).reshape( (-1, 2) ) 165 | if self._num_anchors>1: 166 | anchor_centers = np.stack([anchor_centers]*self._num_anchors, axis=1).reshape( (-1,2) ) 167 | if len(self.center_cache)<100: 168 | self.center_cache[key] = anchor_centers 169 | 170 | pos_inds = np.where(scores>=threshold)[0] 171 | bboxes = distance2bbox(anchor_centers, bbox_preds) 172 | pos_scores = scores[pos_inds] 173 | pos_bboxes = bboxes[pos_inds] 174 | scores_list.append(pos_scores) 175 | bboxes_list.append(pos_bboxes) 176 | if self.use_kps: 177 | kpss = distance2kps(anchor_centers, kps_preds) 178 | kpss = kpss.reshape( (kpss.shape[0], -1, 2) ) 179 | pos_kpss = kpss[pos_inds] 180 | kpss_list.append(pos_kpss) 181 | return scores_list, bboxes_list, kpss_list 182 | 183 | def detect(self, img, input_size = (640,640), max_num=0, metric='default', det_thresh=0.5): 184 | assert input_size is not None or self.input_size is not None 185 | input_size = self.input_size if input_size is None else input_size 186 | 187 | im_ratio = float(img.shape[0]) / img.shape[1] 188 | model_ratio = float(input_size[1]) / input_size[0] 189 | if im_ratio>model_ratio: 190 | new_height = input_size[1] 191 | new_width = int(new_height / im_ratio) 192 | else: 193 | new_width = input_size[0] 194 | new_height = int(new_width * im_ratio) 195 | det_scale = float(new_height) / img.shape[0] 196 | resized_img = cv2.resize(img, (new_width, new_height)) 197 | det_img = np.zeros( (input_size[1], input_size[0], 3), dtype=np.uint8 ) 198 | det_img[:new_height, :new_width, :] = resized_img 199 | 200 | scores_list, bboxes_list, kpss_list = self.forward(det_img, det_thresh) 201 | 202 | scores = np.vstack(scores_list) 203 | scores_ravel = scores.ravel() 204 | order = scores_ravel.argsort()[::-1] 205 | bboxes = np.vstack(bboxes_list) / det_scale 206 | if self.use_kps: 207 | kpss = np.vstack(kpss_list) / det_scale 208 | pre_det = np.hstack((bboxes, scores)).astype(np.float32, copy=False) 209 | pre_det = pre_det[order, :] 210 | keep = self.nms(pre_det) 211 | det = pre_det[keep, :] 212 | if self.use_kps: 213 | kpss = kpss[order,:,:] 214 | kpss = kpss[keep,:,:] 215 | else: 216 | kpss = None 217 | if max_num > 0 and det.shape[0] > max_num: 218 | area = (det[:, 2] - det[:, 0]) * (det[:, 3] - 219 | det[:, 1]) 220 | img_center = img.shape[0] // 2, img.shape[1] // 2 221 | offsets = np.vstack([ 222 | (det[:, 0] + det[:, 2]) / 2 - img_center[1], 223 | (det[:, 1] + det[:, 3]) / 2 - img_center[0] 224 | ]) 225 | offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) 226 | if metric=='max': 227 | values = area 228 | else: 229 | values = area - offset_dist_squared * 2.0 # some extra weight on the centering 230 | bindex = np.argsort( 231 | values)[::-1] # some extra weight on the centering 232 | bindex = bindex[0:max_num] 233 | det = det[bindex, :] 234 | if kpss is not None: 235 | kpss = kpss[bindex, :] 236 | return det, kpss 237 | 238 | def nms(self, dets): 239 | thresh = self.nms_thresh 240 | x1 = dets[:, 0] 241 | y1 = dets[:, 1] 242 | x2 = dets[:, 2] 243 | y2 = dets[:, 3] 244 | scores = dets[:, 4] 245 | 246 | areas = (x2 - x1 + 1) * (y2 - y1 + 1) 247 | order = scores.argsort()[::-1] 248 | 249 | keep = [] 250 | while order.size > 0: 251 | i = order[0] 252 | keep.append(i) 253 | xx1 = np.maximum(x1[i], x1[order[1:]]) 254 | yy1 = np.maximum(y1[i], y1[order[1:]]) 255 | xx2 = np.minimum(x2[i], x2[order[1:]]) 256 | yy2 = np.minimum(y2[i], y2[order[1:]]) 257 | 258 | w = np.maximum(0.0, xx2 - xx1 + 1) 259 | h = np.maximum(0.0, yy2 - yy1 + 1) 260 | inter = w * h 261 | ovr = inter / (areas[i] + areas[order[1:]] - inter) 262 | 263 | inds = np.where(ovr <= thresh)[0] 264 | order = order[inds + 1] 265 | 266 | return keep 267 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | image_extensions = [ 5 | ".bmp", 6 | ".jpeg", 7 | ".jpg", 8 | ".jpe", 9 | ".jp2", 10 | ".png", 11 | ".tiff", 12 | ".tif", 13 | ".webp", 14 | ".pic", 15 | ".ico", 16 | ] 17 | 18 | video_extensions = [ 19 | ".avi", 20 | ".mkv", 21 | ".mp4", 22 | ".mov", 23 | ".wmv", 24 | ".flv", 25 | ".3gp", 26 | ".mpg", 27 | ".mpeg", 28 | ".m4v", 29 | ".asf", 30 | ".ts", 31 | ".vob", 32 | ".webm", 33 | ".gif", 34 | ] 35 | 36 | def is_file(file_path): 37 | if not file_path: 38 | return False 39 | return os.path.isfile(file_path) 40 | 41 | def is_image(file_path): 42 | if is_file(file_path): 43 | file_ext = os.path.splitext(file_path)[1].lower() 44 | return file_ext in image_extensions 45 | return False 46 | 47 | def is_video(file_path): 48 | if is_file(file_path): 49 | file_ext = os.path.splitext(file_path)[1].lower() 50 | return file_ext in video_extensions 51 | return False 52 | 53 | def create_directory(directory_path, remove_existing=True): 54 | if os.path.exists(directory_path) and remove_existing: 55 | shutil.rmtree(directory_path) 56 | 57 | if not os.path.exists(directory_path): 58 | os.mkdir(directory_path) 59 | return directory_path 60 | else: 61 | counter = 1 62 | while True: 63 | new_directory_path = f"{directory_path}_{counter}" 64 | if not os.path.exists(new_directory_path): 65 | os.mkdir(new_directory_path) 66 | return new_directory_path 67 | counter += 1 --------------------------------------------------------------------------------