├── Design Choices.txt ├── README.md ├── src_baseline ├── __pycache__ │ └── opts.cpython-36.pyc ├── datasets │ ├── PoseTransfer_Dataset.py │ └── __pycache__ │ │ └── PoseTransfer_Dataset.cpython-36.pyc ├── logs │ ├── baseline_fasion │ ├── baseline_fasion_check │ ├── baseline_fasion_reinit │ ├── baseline_fasion_unitTest │ ├── baseline_h36m │ ├── baseline_h36m_check │ ├── baseline_h36m_reinit │ ├── deformable_fasion │ ├── deformable_h36m │ ├── stacked_fasion │ └── stacked_h36m ├── main.py ├── models │ ├── __pycache__ │ │ ├── networks.cpython-36.pyc │ │ └── pose_gan.cpython-36.pyc │ ├── networks.py │ └── pose_gan.py ├── opts.py ├── unitTests.py └── utils │ ├── __pycache__ │ ├── pose_transform.cpython-36.pyc │ └── pose_utils.cpython-36.pyc │ ├── create_pairs.py │ ├── pose_transform.py │ ├── pose_utils.py │ └── proc_bg_h36m.py └── src_deformable ├── __init__.py ├── __pycache__ └── opts.cpython-36.pyc ├── commands ├── datasets ├── PoseTransfer_Dataset.py └── __pycache__ │ └── PoseTransfer_Dataset.cpython-36.pyc ├── logs ├── baseline_fasion ├── baseline_h36m ├── deformable_fasion ├── deformable_fasion_clean ├── deformable_fasion_unif_init ├── deformable_h36m ├── deformable_h36m_extreme ├── deformable_h36m_reinit ├── full_fasion ├── full_fasion_stable ├── gen_full_fasion ├── gen_full_h36m ├── stacked_deformable_fasion ├── stacked_deformable_fasion_pretrained ├── stacked_deformable_h36m ├── stacked_deformable_h36m_pretrained ├── stacked_deformable_market ├── stacked_fasion └── stacked_h36m ├── main.py ├── models ├── __pycache__ │ ├── networks.cpython-36.pyc │ └── pose_gan.cpython-36.pyc ├── networks.py ├── networks.pyc └── pose_gan.py ├── opts.py ├── opts.pyc ├── test.py ├── tests ├── test_warp.png ├── test_warp_cv.png ├── test_warp_pytorch.png └── test_warp_sk.png ├── unitTests.py └── utils ├── __pycache__ ├── pose_transform.cpython-36.pyc └── pose_utils.cpython-36.pyc ├── create_pairs.py ├── pose_transform.py ├── pose_utils.py └── proc_bg_h36m.py /Design Choices.txt: -------------------------------------------------------------------------------- 1 | -> Single kind of discriminator . . simplest one 2 | -> No validation epoch 3 | -> need to periodically change code base for efficiency and elegance, currently speed s of the essensce 4 | -> admiration for code recedes in the light of execution of ideas and evaluation of resutls 5 | essential to get feedback from running code 6 | -> training loop needs to cleanly load inputs for gen and disc . . currently exploiting iterators 7 | kinda ugly for training ratio, and otherwise 8 | -> transposeconv not working due to inappropraite padding . . silly dimension mismatch . . look at convolution arithmetic and other implentation for resolution 9 | -> taking stacked architecture where each stack has different weights 10 | -> for human 3.6 interpolations may be taken directly from video frames or vae interpolations 11 | -> for fasion, interpolations are linear where the missing annotations are synthesized/vanished midway through the interpolation 12 | 13 | Changes for stacked/staged arch : 14 | -> Changed dataset to pass interpol poses as additional input 15 | -> added stacked gen module in networks 16 | -> changed pose_gan (gen_update, disc_update, init) for new gan type 17 | -> changed main training loop to pass interpol pose to gen_update nad disc_update 18 | 19 | ToDO 20 | -> add logging through tensorboard to analyze the losses 21 | -> model not fitting in memory . . use same stack everywhere ? still not fitting, don't keep interpolated inputs in memory ? 22 | -> taking long to train . . overfit on small train dataset ? 23 | -> Add intermediate supervision 24 | -> add code for nearest neighbour/feature loss, inception score 25 | -> add code for dsc . . important, else variations in hands arms won't be captured 26 | -> that should be it for iterative approach idea .. consider different kind of arch ? 27 | 28 | Issues 29 | 30 | -> deformable skip connections not working in pytorch due to incomplete backprop through affine layer 31 | set requires_grad = True for all tensors, inspect computational graph for unit testing 32 | Note - Variable is deprecated in Pytorch 0.4.0 33 | 34 | ToDO 35 | 36 | -> Stacked variant of Deformable GAN 37 | -> Verify nn loss and ad losses for defo GAN before proceeding. 38 | -> Use shg for intermediate supervision on Stacked experiment. 39 | -> Fit model in memory by lowering resolution, precomputation etc. 40 | -> Use pretrained models to initialzie rnn weights .. should increase performance drastically 41 | 42 | Issues 43 | 44 | -> Market dataset goes to nan very soon when using stacked deformable architecture. This is model divergence :/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pose-Transfer 2 | 3 | PyTorch Implementation of Deformable GAN https://arxiv.org/abs/1801.00055 4 | 5 | Also check out the original repo in Keras, by AliaksandrSiarohin - https://github.com/AliaksandrSiarohin/pose-gan 6 | 7 | ![alt text](https://github.com/AliaksandrSiarohin/pose-gan/blob/master/sup-mat/teaser.jpg) 8 | 9 | ## Baseline Model ( run in src_baseline ) 10 | 11 | ### Fasion : 12 | 13 | CUDA_VISIBLE_DEVICES=1 python main.py --l1_penalty_weight 10 --batch_size 4 --number_of_epochs 90 --gen_type baseline --expID baseline_fasion --pose_dim 18 --dataset fasion 14 | 15 | ### Human 3.6 : 16 | 17 | CUDA_VISIBLE_DEVICES=0 python main.py --l1_penalty_weight 10 --batch_size 4 --number_of_epochs 90 --gen_type baseline --expID baseline_h36m --dataset h36m 18 | 19 | ## Deformable Model ( run in src_deformable ) 20 | 21 | ### Fasion : 22 | 23 | CUDA_VISIBLE_DEVICES=1 python main.py --warp_skip mask --l1_penalty_weight 0.01 --batch_size 2 --number_of_epochs 90 --gen_type baseline --expID full_fasion --pose_dim 18 --dataset fasion --nn_loss_area_size 5 --batch_size 2 --content_loss_layer block1_conv2 24 | 25 | ### Human 3.6 : 26 | 27 | CUDA_VISIBLE_DEVICES=1 python main.py --warp_skip mask --l1_penalty_weight 0.01 --batch_size 2 --number_of_epochs 90 --gen_type baseline --expID full_h36m --dataset h36m --nn_loss_area_size 5 --batch_size 2 --content_loss_layer block1_conv2 28 | 29 | Specify data directory by passing data_Dir option. 30 | 31 | ## ToDo : 32 | 33 | -> Add code for loading annotations and train/test pairs for Fasion and Human 3.6 dataset. 34 | -------------------------------------------------------------------------------- /src_baseline/__pycache__/opts.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_baseline/__pycache__/opts.cpython-36.pyc -------------------------------------------------------------------------------- /src_baseline/datasets/PoseTransfer_Dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import matplotlib 4 | matplotlib.use('Agg') 5 | from utils import pose_utils 6 | from utils import pose_transform 7 | from skimage.io import imread 8 | import torch.utils.data as data 9 | import pandas as pd 10 | import os 11 | 12 | 13 | # class PoseTransfer_Dataset(data.Dataset): 14 | # def __init__(self, opt, split): 15 | # self.split = split 16 | # 17 | # self.pose_dim = opt['pose_dim'] 18 | # self._batch_size = 1 if (self.split=='test' or self.split=='val') else opt['batch_size'] 19 | # self._image_size = opt['image_size'] 20 | # 21 | # self._images_dir_train = opt['images_dir_train'] 22 | # self._images_dir_test = opt['images_dir_test'] 23 | # 24 | # self._pairs_file_train = pd.read_csv(opt['pairs_file_train']) 25 | # self._pairs_file_test = pd.read_csv(opt['pairs_file_test']) 26 | # 27 | # # self._pairs_file_test_iterative = pd.read_csv(opt['pairs_file_test_iterative']) 28 | # # self._pairs_file_train_iterative = pd.read_csv(opt['pairs_file_train_iterative']) 29 | # 30 | # self._annotations_file_test = pd.read_csv(opt['annotations_file_train'], sep=':') 31 | # self._annotations_file_train = pd.read_csv(opt['annotations_file_test'], sep=':') 32 | # 33 | # self._annotations_file = pd.concat([self._annotations_file_test, self._annotations_file_train], 34 | # axis=0, ignore_index=True) 35 | # 36 | # self._annotations_file = self._annotations_file.set_index('name') 37 | # 38 | # if (split == 'train'): 39 | # self.length = len(self._pairs_file_train) 40 | # else: 41 | # self.length = len(self._pairs_file_test) 42 | # 43 | # self._use_input_pose = opt['use_input_pose'] 44 | # 45 | # self._warp_skip = opt['warp_skip'] 46 | # 47 | # # self._disc_type = opt['disc_type'] 48 | # 49 | # self._tmp_pose = opt['tmp_pose_dir'] 50 | # # self.frame_diff = opt['frame_diff'] 51 | # 52 | # if not os.path.exists(self._tmp_pose): 53 | # os.makedirs(self._tmp_pose) 54 | # 55 | # print ("Statistics for loaded dataset : Human 3.6") 56 | # print ("Number of images: %s" % len(self._annotations_file)) 57 | # print ("Number of pairs train: %s" % len(self._pairs_file_train)) 58 | # print ("Number of pairs test: %s" % len(self._pairs_file_test)) 59 | # 60 | # # print ("Number of pairs test iterative: %s" % len(self._pairs_file_train_iterative)) 61 | # # print ("Number of pairs test iterative: %s" % len(self._pairs_file_test_iterative)) 62 | # 63 | # # pair is now a pandas series 64 | # def compute_pose_map(self, pair, direction): 65 | # assert direction in ['to', 'from'] 66 | # pose_map = np.empty(list(self._image_size) + [self.pose_dim]) 67 | # row = self._annotations_file.loc[pair[direction]] 68 | # # file_name = self._tmp_pose + pair[direction] + '.npy' 69 | # # if os.path.exists(file_name): 70 | # # pose = np.load(file_name) 71 | # # else: 72 | # kp_array = pose_utils.load_pose_cords_from_strings(row['keypoints_y'], row['keypoints_x']) 73 | # pose_map = pose_utils.cords_to_map(kp_array, self._image_size) 74 | # # np.save(file_name, pose) 75 | # # channels must be first 76 | # return np.transpose(pose_map, [2,0,1] ) 77 | # 78 | # def compute_cord_warp(self, pair): 79 | # if self._warp_skip == 'full': 80 | # warp = [np.empty([1, 8])] 81 | # else: 82 | # warp = [np.empty([10, 8]), 83 | # np.empty([10] + list(self._image_size))] 84 | # fr = self._annotations_file.loc[p['from']] 85 | # to = self._annotations_file.loc[p['to']] 86 | # kp_array1 = pose_utils.load_pose_cords_from_strings(fr['keypoints_y'], 87 | # fr['keypoints_x']) 88 | # kp_array2 = pose_utils.load_pose_cords_from_strings(to['keypoints_y'], 89 | # to['keypoints_x']) 90 | # if self._warp_skip == 'mask': 91 | # warp[0] = pose_transform.affine_transforms(kp_array1, kp_array2) 92 | # warp[1] = pose_transform.pose_masks(kp_array2, self._image_size) 93 | # else: 94 | # warp[0] = pose_transform.estimate_uniform_transform(kp_array1, kp_array2) 95 | # return warp 96 | # 97 | # def load_image(self, pair, direction='from'): 98 | # assert direction in ['to', 'from'] 99 | # if os.path.exists(os.path.join(self._images_dir_train, pair[direction])): 100 | # image = imread(os.path.join(self._images_dir_train, pair[direction])) 101 | # elif os.path.exists(os.path.join(self._images_dir_test, pair[direction])): 102 | # image = imread(os.path.join(self._images_dir_test, pair[direction])) 103 | # else: 104 | # # blank image if no file present on disk . . hacky way 105 | # image = np.zeros([3, self._image_size,self._image_size]) 106 | # return np.transpose(pose_utils._preprocess_image(image), [2,0,1]) 107 | # 108 | # # returns a dictionary with the generator sample and the discriminator sample 109 | # # training loop will update generator and discriminator accordingly 110 | # def __getitem__(self, index): 111 | # if self.split == 'val' or self.split == 'test': 112 | # pair = self._pairs_file_test.iloc[index] 113 | # else: 114 | # pair = self._pairs_file_train.iloc[index] 115 | # result = [self.load_image(pair, 'from')] 116 | # if self._use_input_pose: 117 | # result.append(self.compute_pose_map(pair, 'from')) 118 | # result.append(self.compute_pose_map(pair, 'to')) 119 | # target = self.load_image(pair, 'to') 120 | # 121 | # input = torch.from_numpy(np.concatenate(result, axis=0)).float() 122 | # target = torch.from_numpy(target).float() 123 | # # if(torch.cuda.is_available()): 124 | # # input = input.cuda() 125 | # 126 | # # TODO 127 | # # if self._warp_skip != 'none' and (not for_discriminator or self._disc_type == 'warp'): 128 | # # result += self.compute_cord_warp(pair) 129 | # return input, target 130 | # 131 | # def __len__(self): 132 | # return self.length 133 | 134 | 135 | class PoseTransfer_Dataset(data.Dataset): 136 | def __init__(self, opt, split): 137 | self.split = split 138 | 139 | self.pose_dim = opt['pose_dim'] 140 | self.num_stacks = opt['num_stacks'] 141 | self._batch_size = 1 if (self.split=='test' or self.split=='val') else opt['batch_size'] 142 | self._image_size = opt['image_size'] 143 | 144 | self._images_dir_train = opt['images_dir_train'] 145 | self._images_dir_test = opt['images_dir_test'] 146 | 147 | if(opt['checkMode'] == 0): 148 | self._pairs_file_train = pd.read_csv(opt['pairs_file_train']) 149 | self._pairs_file_test = pd.read_csv(opt['pairs_file_test']) 150 | else: 151 | self._pairs_file_train = pd.read_csv(opt['pairs_file_train_check']) 152 | self._pairs_file_test = pd.read_csv(opt['pairs_file_test_check']) 153 | 154 | if (opt['checkMode'] == 0): 155 | self.pairs_file_test_interpol = pd.read_csv(opt['pairs_file_test_interpol']) 156 | self.pairs_file_train_interpol = pd.read_csv(opt['pairs_file_train_interpol']) 157 | else: 158 | self.pairs_file_test_interpol = pd.read_csv(opt['pairs_file_test_check']) 159 | self.pairs_file_train_interpol = pd.read_csv(opt['pairs_file_train_check']) 160 | 161 | self._annotations_file_test = pd.read_csv(opt['annotations_file_train'], sep=':') 162 | self._annotations_file_train = pd.read_csv(opt['annotations_file_test'], sep=':') 163 | 164 | self._annotations_file = pd.concat([self._annotations_file_test, self._annotations_file_train], 165 | axis=0, ignore_index=True) 166 | 167 | self._annotations_file = self._annotations_file.set_index('name') 168 | 169 | if (split == 'train'): 170 | self.length = len(self.pairs_file_train_interpol) 171 | else: 172 | self.length = len(self.pairs_file_test_interpol) 173 | 174 | self._use_input_pose = opt['use_input_pose'] 175 | 176 | self._warp_skip = opt['warp_skip'] 177 | 178 | # self._disc_type = opt['disc_type'] 179 | 180 | self._tmp_pose = opt['tmp_pose_dir'] 181 | # self.frame_diff = opt['frame_diff'] 182 | 183 | # use this if enough space available 184 | # if not os.path.exists(self._tmp_pose): 185 | # os.makedirs(self._tmp_pose) 186 | 187 | print ("Statistics for loaded dataset : {}".format(opt['dataset'])) 188 | print ("Number of images: %s" % len(self._annotations_file)) 189 | # print ("Number of pairs train: %s" % len(self._pairs_file_train_iterative)) 190 | # print ("Number of pairs test: %s" % len(self._pairs_file_test_iterative)) 191 | 192 | print ("Number of pairs train : %s" % len(self.pairs_file_train_interpol)) 193 | print ("Number of pairs test : %s" % len(self.pairs_file_test_interpol)) 194 | 195 | # pair is now a pandas series 196 | def compute_pose_map(self, pair, direction): 197 | assert direction in ['to', 'from'] 198 | pose_map = np.empty(list(self._image_size) + [self.pose_dim]) 199 | row = self._annotations_file.loc[pair[direction]] 200 | # file_name = self._tmp_pose + pair[direction] + '.npy' 201 | # if os.path.exists(file_name): 202 | # pose = np.load(file_name) 203 | # else: 204 | kp_array = pose_utils.load_pose_cords_from_strings(row['keypoints_y'], row['keypoints_x']) 205 | pose_map = pose_utils.cords_to_map(kp_array, self._image_size) 206 | # np.save(file_name, pose) 207 | 208 | # channels must be first for pytorch ( cord to map yields channles in last dim - old keras code ) 209 | return np.transpose(pose_map, [2,0,1] ) 210 | 211 | def compute_interpol_map(self, inp_map, tg_map): 212 | # map to cord expects channels in last dim ( old keras code ) 213 | inp_pos = pose_utils.map_to_cord(np.transpose(inp_map, [1,2,0]), self.pose_dim) 214 | tg_pos = pose_utils.map_to_cord(np.transpose(tg_map, [1,2,0]), self.pose_dim) 215 | pose_maps = [] 216 | # compute interpol poses equal to num_stacks, with final pose being equal to the target psoe 217 | for i in range(1,self.num_stacks+1): 218 | interpol_pose = pose_utils.compute_interpol_pose(inp_pos,tg_pos,i, self.num_stacks, self.pose_dim) 219 | interpol_pose_map = pose_utils.cords_to_map(interpol_pose, self._image_size) 220 | pose_maps.append(np.transpose(interpol_pose_map, [2,0,1])) 221 | return pose_maps 222 | 223 | def compute_cord_warp(self, pair): 224 | if self._warp_skip == 'full': 225 | warp = [np.empty([1, 8])] 226 | else: 227 | warp = [np.empty([10, 8]), 228 | np.empty([10] + list(self._image_size))] 229 | fr = self._annotations_file.loc[p['from']] 230 | to = self._annotations_file.loc[p['to']] 231 | kp_array1 = pose_utils.load_pose_cords_from_strings(fr['keypoints_y'], 232 | fr['keypoints_x']) 233 | kp_array2 = pose_utils.load_pose_cords_from_strings(to['keypoints_y'], 234 | to['keypoints_x']) 235 | if self._warp_skip == 'mask': 236 | warp[0] = pose_transform.affine_transforms(kp_array1, kp_array2) 237 | warp[1] = pose_transform.pose_masks(kp_array2, self._image_size) 238 | else: 239 | warp[0] = pose_transform.estimate_uniform_transform(kp_array1, kp_array2) 240 | return warp 241 | 242 | def load_image(self, pair, direction='from'): 243 | assert direction in ['to', 'from'] 244 | if os.path.exists(os.path.join(self._images_dir_train, pair[direction])): 245 | image = imread(os.path.join(self._images_dir_train, pair[direction])) 246 | elif os.path.exists(os.path.join(self._images_dir_test, pair[direction])): 247 | image = imread(os.path.join(self._images_dir_test, pair[direction])) 248 | else: 249 | # blank image if no file present on disk . . hacky way 250 | image = np.zeros([3, self._image_size,self._image_size]) 251 | return np.transpose(pose_utils._preprocess_image(image), [2,0,1]) 252 | 253 | # returns a dictionary with the generator sample and the discriminator sample 254 | # training loop will update generator and discriminator accordingly 255 | def __getitem__(self, index): 256 | if self.split == 'val' or self.split == 'test': 257 | pair = self.pairs_file_test_interpol.iloc[index] 258 | else: 259 | pair = self.pairs_file_train_interpol.iloc[index] 260 | result = [self.load_image(pair, 'from')] 261 | input_pose_map = self.compute_pose_map(pair, 'from') 262 | if self._use_input_pose: 263 | result.append(input_pose_map) 264 | target_pose_map = self.compute_pose_map(pair, 'to') 265 | result.append(target_pose_map) 266 | interpol_pose_map = self.compute_interpol_map(input_pose_map,target_pose_map) 267 | target = self.load_image(pair, 'to') 268 | 269 | input = torch.from_numpy(np.concatenate(result, axis=0)).float() 270 | target = torch.from_numpy(target).float() 271 | interpol_pose_map = torch.from_numpy(np.concatenate(interpol_pose_map, axis=0)).float() 272 | # if(torch.cuda.is_available()): 273 | # input = input.cuda() 274 | 275 | # TODO 276 | # if self._warp_skip != 'none' and (not for_discriminator or self._disc_type == 'warp'): 277 | # result += self.compute_cord_warp(pair) 278 | return input, target, interpol_pose_map 279 | 280 | def __len__(self): 281 | return self.length 282 | 283 | -------------------------------------------------------------------------------- /src_baseline/datasets/__pycache__/PoseTransfer_Dataset.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_baseline/datasets/__pycache__/PoseTransfer_Dataset.cpython-36.pyc -------------------------------------------------------------------------------- /src_baseline/logs/baseline_h36m_reinit: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | Model options . . 3 | annotations_file_test: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-test.csv 4 | annotations_file_test_paf: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-paf-test0.csv 5 | annotations_file_train: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-train.csv 6 | annotations_file_train_paf: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-paf-train0.csv 7 | batch_size: 4 8 | checkpoint_ratio: 5 9 | checkpoints_dir: ../exp/baseline_h36m_reinit/models 10 | compute_h36m_paf_split: 0 11 | content_loss_layer: none 12 | data_Dir: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/ 13 | dataset: h36m 14 | disc_type: call 15 | discriminator_checkpoint: None 16 | display_ratio: 50 17 | expID: baseline_h36m_reinit 18 | frame_diff: 10 19 | gan_penalty_weight: 1 20 | gen_type: baseline 21 | generated_images_dir: output/generated_images/h36m-restricted 22 | generator_checkpoint: None 23 | image_size: (224, 224) 24 | images_dir_test: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-dataset/test 25 | images_dir_train: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-dataset/train 26 | images_for_test: 12000 27 | iters_per_epoch: 1000 28 | l1_penalty_weight: 10.0 29 | learning_rate: 0.0002 30 | load_generated_images: 0 31 | log_file: output/full/fasion/log 32 | lstruct_penalty_weight: 0 33 | nn_loss_area_size: 1 34 | num_stacks: 4 35 | number_of_epochs: 90 36 | output_dir: ../exp/baseline_h36m_reinit/results 37 | pairs_file_test: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-test.csv 38 | pairs_file_test_interpol: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-test-interpol.csv 39 | pairs_file_test_iterative: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-test-iterative.csv 40 | pairs_file_train: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-train.csv 41 | pairs_file_train_interpol: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-train-interpol.csv 42 | pairs_file_train_iterative: /home/project/saurabh/pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-train-iterative.csv 43 | pose_dim: 16 44 | pose_estimator: pose_estimator.h5 45 | resume: 0 46 | saveDir: ../exp/baseline_h36m_reinit 47 | start_epoch: 0 48 | tmp_pose_dir: tmp/h36m/ 49 | training_ratio: 1 50 | tv_penalty_weight: 0 51 | use_dropout_test: 0 52 | use_input_pose: True 53 | warp_agg: max 54 | warp_skip: none 55 | (224, 224) 56 | Statistics for loaded dataset : Human 3.6 57 | Number of images: 92586 58 | Number of pairs train interpol: 73812 59 | Number of pairs test interpol: 12000 60 | Statistics for loaded dataset : Human 3.6 61 | Number of images: 92586 62 | Number of pairs train interpol: 73812 63 | Number of pairs test interpol: 12000 64 | ---------- Networks initialized ------------- 65 | Generator( 66 | (encoder): encoder( 67 | (net): ModuleList( 68 | (0): Conv2d(35, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 69 | (1): Block( 70 | (net): Sequential( 71 | (0): LeakyReLU(negative_slope=0.2) 72 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 73 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 74 | ) 75 | ) 76 | (2): Block( 77 | (net): Sequential( 78 | (0): LeakyReLU(negative_slope=0.2) 79 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 80 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 81 | ) 82 | ) 83 | (3): Block( 84 | (net): Sequential( 85 | (0): LeakyReLU(negative_slope=0.2) 86 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 87 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 88 | ) 89 | ) 90 | (4): Block( 91 | (net): Sequential( 92 | (0): LeakyReLU(negative_slope=0.2) 93 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 94 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 95 | ) 96 | ) 97 | (5): Block( 98 | (net): Sequential( 99 | (0): LeakyReLU(negative_slope=0.2) 100 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 101 | ) 102 | ) 103 | ) 104 | ) 105 | (decoder): decoder( 106 | (net): ModuleList( 107 | (0): Block( 108 | (net): Sequential( 109 | (0): ReLU() 110 | (1): ConvTranspose2d(512, 512, kernel_size=(4, 4), stride=(2, 2)) 111 | (2): Cropping2D() 112 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 113 | (4): Dropout2d(p=0.5) 114 | ) 115 | ) 116 | (1): Block( 117 | (net): Sequential( 118 | (0): ReLU() 119 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2)) 120 | (2): Cropping2D() 121 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 122 | (4): Dropout2d(p=0.5) 123 | ) 124 | ) 125 | (2): Block( 126 | (net): Sequential( 127 | (0): ReLU() 128 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2)) 129 | (2): Cropping2D() 130 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 131 | (4): Dropout2d(p=0.5) 132 | ) 133 | ) 134 | (3): Block( 135 | (net): Sequential( 136 | (0): ReLU() 137 | (1): ConvTranspose2d(768, 256, kernel_size=(4, 4), stride=(2, 2)) 138 | (2): Cropping2D() 139 | (3): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 140 | ) 141 | ) 142 | (4): Block( 143 | (net): Sequential( 144 | (0): ReLU() 145 | (1): ConvTranspose2d(384, 128, kernel_size=(4, 4), stride=(2, 2)) 146 | (2): Cropping2D() 147 | (3): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 148 | ) 149 | ) 150 | (5): ReLU() 151 | (6): Conv2d(192, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 152 | (7): Tanh() 153 | ) 154 | ) 155 | ) 156 | Total number of parameters: 36074051 157 | Discriminator( 158 | (net): Sequential( 159 | (0): Conv2d(38, 64, kernel_size=(4, 4), stride=(2, 2)) 160 | (1): Block( 161 | (net): Sequential( 162 | (0): LeakyReLU(negative_slope=0.2) 163 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 164 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 165 | ) 166 | ) 167 | (2): Block( 168 | (net): Sequential( 169 | (0): LeakyReLU(negative_slope=0.2) 170 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 171 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 172 | ) 173 | ) 174 | (3): Block( 175 | (net): Sequential( 176 | (0): LeakyReLU(negative_slope=0.2) 177 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 178 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 179 | ) 180 | ) 181 | (4): Block( 182 | (net): Sequential( 183 | (0): LeakyReLU(negative_slope=0.2) 184 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 185 | ) 186 | ) 187 | (5): Sigmoid() 188 | (6): Flatten() 189 | ) 190 | ) 191 | Total number of parameters: 2800577 192 | ----------------------------------------------- 193 | Num iterations : 1000 194 | Epoch : 1 | Progress : 0.00 | Total Loss : 12.3395 | Gen Total Loss : 10.7509, Gen Ad Loss : 1.2211, Gen LL Loss : 9.5298 | Disc Total Loss : 1.5886, Disc True Loss : 0.5828, Disc Fake Loss : 1.0058 195 | -------------------------------------------------------------------------------- /src_baseline/logs/deformable_fasion: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | /home/project/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. 3 | from ._conv import register_converters as _register_converters 4 | Using TensorFlow backend. 5 | /home/project/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6 6 | return f(*args, **kwds) 7 | /home/project/saurabh/pose-transfer/src_baseline/models/networks.py:28: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_. 8 | init.normal(m.weight.data, 0.0, 0.02) 9 | Model options . . 10 | annotations_file_test: ../../pose-gan-clean/pose-gan/data/fasion-annotation-test.csv 11 | annotations_file_test_paf: ../../pose-gan-clean/pose-gan/data/fasion-annotation-paf-test0.csv 12 | annotations_file_train: ../../pose-gan-clean/pose-gan/data/fasion-annotation-train.csv 13 | annotations_file_train_paf: ../../pose-gan-clean/pose-gan/data/fasion-annotation-paf-train0.csv 14 | batch_size: 2 15 | checkpoint_ratio: 5 16 | checkpoints_dir: ../exp/deformable_fasion/models 17 | compute_h36m_paf_split: 0 18 | content_loss_layer: none 19 | data_Dir: ../../pose-gan-clean/pose-gan/data/ 20 | dataset: fasion 21 | disc_type: call 22 | discriminator_checkpoint: None 23 | display_ratio: 50 24 | expID: deformable_fasion 25 | frame_diff: 10 26 | gan_penalty_weight: 1 27 | gen_type: baseline 28 | generated_images_dir: output/generated_images/fasion-restricted 29 | generator_checkpoint: None 30 | image_size: (256, 256) 31 | images_dir_test: ../../pose-gan-clean/pose-gan/data/fasion-dataset/test 32 | images_dir_train: ../../pose-gan-clean/pose-gan/data/fasion-dataset/train 33 | images_for_test: 12000 34 | iters_per_epoch: 1000 35 | l1_penalty_weight: 10.0 36 | learning_rate: 0.0002 37 | load_generated_images: 0 38 | log_file: output/full/fasion/log 39 | lstruct_penalty_weight: 0 40 | nn_loss_area_size: 1 41 | num_stacks: 4 42 | number_of_epochs: 90 43 | output_dir: ../exp/deformable_fasion/results 44 | pairs_file_test: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test.csv 45 | pairs_file_test_interpol: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test-interpol.csv 46 | pairs_file_test_iterative: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test-iterative.csv 47 | pairs_file_train: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train.csv 48 | pairs_file_train_interpol: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train-interpol.csv 49 | pairs_file_train_iterative: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train-iterative.csv 50 | pose_dim: 18 51 | pose_estimator: pose_estimator.h5 52 | resume: 0 53 | saveDir: ../exp/deformable_fasion 54 | start_epoch: 0 55 | tmp_pose_dir: tmp/fasion/ 56 | training_ratio: 1 57 | tv_penalty_weight: 0 58 | use_dropout_test: 0 59 | use_input_pose: True 60 | warp_agg: max 61 | warp_skip: mask 62 | (256, 256) 63 | Statistics for loaded dataset : Human 3.6 64 | Number of images: 52712 65 | Number of pairs train interpol: 101268 66 | Number of pairs test interpol: 8670 67 | Statistics for loaded dataset : Human 3.6 68 | Number of images: 52712 69 | Number of pairs train interpol: 101268 70 | Number of pairs test interpol: 8670 71 | ---------- Networks initialized ------------- 72 | Generator( 73 | (encoder): encoder( 74 | (net): ModuleList( 75 | (0): Conv2d(39, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 76 | (1): Block( 77 | (net): Sequential( 78 | (0): LeakyReLU(negative_slope=0.2) 79 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 80 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 81 | ) 82 | ) 83 | (2): Block( 84 | (net): Sequential( 85 | (0): LeakyReLU(negative_slope=0.2) 86 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 87 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 88 | ) 89 | ) 90 | (3): Block( 91 | (net): Sequential( 92 | (0): LeakyReLU(negative_slope=0.2) 93 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 94 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 95 | ) 96 | ) 97 | (4): Block( 98 | (net): Sequential( 99 | (0): LeakyReLU(negative_slope=0.2) 100 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 101 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 102 | ) 103 | ) 104 | (5): Block( 105 | (net): Sequential( 106 | (0): LeakyReLU(negative_slope=0.2) 107 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 108 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 109 | ) 110 | ) 111 | (6): Block( 112 | (net): Sequential( 113 | (0): LeakyReLU(negative_slope=0.2) 114 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 115 | ) 116 | ) 117 | ) 118 | ) 119 | (decoder): decoder( 120 | (net): ModuleList( 121 | (0): Block( 122 | (net): Sequential( 123 | (0): ReLU() 124 | (1): ConvTranspose2d(512, 512, kernel_size=(4, 4), stride=(2, 2)) 125 | (2): Cropping2D() 126 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 127 | (4): Dropout2d(p=0.5) 128 | ) 129 | ) 130 | (1): Block( 131 | (net): Sequential( 132 | (0): ReLU() 133 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2)) 134 | (2): Cropping2D() 135 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 136 | (4): Dropout2d(p=0.5) 137 | ) 138 | ) 139 | (2): Block( 140 | (net): Sequential( 141 | (0): ReLU() 142 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2)) 143 | (2): Cropping2D() 144 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 145 | (4): Dropout2d(p=0.5) 146 | ) 147 | ) 148 | (3): Block( 149 | (net): Sequential( 150 | (0): ReLU() 151 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2)) 152 | (2): Cropping2D() 153 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 154 | ) 155 | ) 156 | (4): Block( 157 | (net): Sequential( 158 | (0): ReLU() 159 | (1): ConvTranspose2d(768, 256, kernel_size=(4, 4), stride=(2, 2)) 160 | (2): Cropping2D() 161 | (3): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 162 | ) 163 | ) 164 | (5): Block( 165 | (net): Sequential( 166 | (0): ReLU() 167 | (1): ConvTranspose2d(384, 128, kernel_size=(4, 4), stride=(2, 2)) 168 | (2): Cropping2D() 169 | (3): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 170 | ) 171 | ) 172 | (6): ReLU() 173 | (7): Conv2d(192, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 174 | (8): Tanh() 175 | ) 176 | ) 177 | ) 178 | Total number of parameters: 48660291 179 | Discriminator( 180 | (net): Sequential( 181 | (0): Conv2d(42, 64, kernel_size=(4, 4), stride=(2, 2)) 182 | (1): Block( 183 | (net): Sequential( 184 | (0): LeakyReLU(negative_slope=0.2) 185 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 186 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 187 | ) 188 | ) 189 | (2): Block( 190 | (net): Sequential( 191 | (0): LeakyReLU(negative_slope=0.2) 192 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 193 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 194 | ) 195 | ) 196 | (3): Block( 197 | (net): Sequential( 198 | (0): LeakyReLU(negative_slope=0.2) 199 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 200 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 201 | ) 202 | ) 203 | (4): Block( 204 | (net): Sequential( 205 | (0): LeakyReLU(negative_slope=0.2) 206 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) 207 | ) 208 | ) 209 | (5): Sigmoid() 210 | (6): Flatten() 211 | ) 212 | ) 213 | Total number of parameters: 2804673 214 | ----------------------------------------------- 215 | Num iterations : 1000 216 | Traceback (most recent call last): 217 | File "main.py", line 122, in 218 | main() 219 | File "main.py", line 85, in main 220 | model.dis_update(Variable(input.cuda()),Variable(target.cuda()), Variable(interpol_pose.cuda()), Variable(real_inp.cuda()), Variable(real_target.cuda()), vars(opt)) 221 | File "/home/project/saurabh/pose-transfer/src_baseline/models/pose_gan.py", line 124, in dis_update 222 | self.dis_loss = loss.data.cpu().numpy()[0] 223 | IndexError: too many indices for array 224 | -------------------------------------------------------------------------------- /src_baseline/logs/deformable_h36m: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | -------------------------------------------------------------------------------- /src_baseline/main.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | matplotlib.use('Agg') 3 | import pylab as plt 4 | import os 5 | import sys 6 | import torch 7 | import numpy as np 8 | import cv2 9 | import time 10 | import datetime 11 | from opts import opts 12 | from torch.autograd import Variable 13 | from datasets.PoseTransfer_Dataset import PoseTransfer_Dataset 14 | from models.networks import Generator, Discriminator 15 | from models.pose_gan import Pose_GAN 16 | from utils import pose_utils 17 | 18 | def load_sample(iter,dataloader,name): 19 | try: 20 | input, target, interpol_pose = iter.next() 21 | except: 22 | # resetting iterator 23 | print("Resetting iterator for loader :", name) 24 | iter = dataloader.__iter__() 25 | input, target, interpol_pose = iter.next() 26 | return input, target, interpol_pose, iter 27 | 28 | def main(): 29 | opt = opts().parse() 30 | print("Model options . .") 31 | for k, v in sorted(vars(opt).items()): 32 | print(' %s: %s' % (str(k), str(v))) 33 | 34 | print(opt.image_size) 35 | 36 | # gen_loader_train = torch.utils.data.DataLoader( 37 | # H36M_Dataset(vars(opt), split ='train'), 38 | # batch_size = opt.batch_size, 39 | # shuffle = True, 40 | # ) 41 | # 42 | # gen_loader_test = torch.utils.data.DataLoader( 43 | # H36M_Dataset(vars(opt), split='test'), 44 | # batch_size=opt.batch_size, 45 | # shuffle=True, 46 | # ) 47 | 48 | # disc_loader_train = torch.utils.data.DataLoader( 49 | # H36M_Dataset(vars(opt), split='train'), 50 | # batch_size=opt.batch_size, 51 | # shuffle=True, 52 | # ) 53 | 54 | loader_train = torch.utils.data.DataLoader( 55 | PoseTransfer_Dataset(vars(opt), split ='train'), 56 | batch_size = opt.batch_size, 57 | shuffle = True, 58 | ) 59 | 60 | loader_test = torch.utils.data.DataLoader( 61 | PoseTransfer_Dataset(vars(opt), split='test'), 62 | batch_size=opt.batch_size, 63 | shuffle=True, 64 | ) 65 | 66 | model = Pose_GAN(opt).cuda() 67 | start_epoch = 1 68 | if(opt.resume==1): 69 | start_epoch = model.resume(opt.checkpoints_dir) 70 | 71 | # discriminator and generator updates 72 | # note - iterators initialized out of epoch loop 73 | loader_train_iter = loader_train.__iter__() 74 | loader_test_iter = loader_test.__iter__() 75 | for epoch in range(start_epoch, opt.number_of_epochs + 1): 76 | # num_iter = len(gen_loader_train) 77 | gen_losses, disc_losses = [],[] 78 | num_iterations = opt.iters_per_epoch 79 | print("Num iterations : ", num_iterations) 80 | # for it, input in enumerate(gen_loader_train): 81 | for it in range(num_iterations): 82 | for _ in range(opt.training_ratio): 83 | input, target, interpol_pose, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train') 84 | real_inp, real_target, interpol_pose, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train') 85 | disc_loss = model.dis_update(Variable(input.cuda()),Variable(target.cuda()), Variable(interpol_pose.cuda()), Variable(real_inp.cuda()), Variable(real_target.cuda()), vars(opt)) 86 | disc_losses.append(disc_loss) 87 | input, target, interpol_pose, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train') 88 | out, outputs, gen_loss = model.gen_update(Variable(input.cuda()), Variable(target.cuda()), Variable(interpol_pose.cuda()), vars(opt)) 89 | gen_losses.append(gen_loss) 90 | 91 | if(it%opt.display_ratio==0): 92 | 93 | # print losses 94 | # gen_total_loss, gen_ll_loss, gen_ad_loss, disc_total_loss, disc_true_loss, disc_fake_loss = model.gen_total_loss, model.gen_ll_loss, model.gen_ad_loss, model.dis_total_loss, model.dis_true_loss, model.dis_fake_loss 95 | # averaging loss over all iterations in this epoch 96 | gen_total_loss, gen_ll_loss, gen_ad_loss = np.mean(np.array(gen_losses), axis=0) 97 | disc_total_loss, disc_true_loss, disc_fake_loss = np.mean(np.array(disc_losses), axis=0) 98 | total_loss = gen_total_loss + disc_total_loss 99 | print("Epoch : {8:d} | Progress : {0:.2f} | Total Loss : {1:.4f} | Gen Total Loss : {2:.4f}, Gen Ad Loss : {3:.4f}, Gen LL Loss : {4:.4f} | Disc Total Loss : {5:.4f}, Disc True Loss : {6:.4f}, Disc Fake Loss : {7:.4f} ".format(it / num_iterations, total_loss, gen_total_loss, gen_ad_loss, gen_ll_loss, disc_total_loss, disc_true_loss, disc_fake_loss, epoch)) 100 | sys.stdout.flush() 101 | # saving seen images 102 | if(opt.gen_type=='baseline'): 103 | images = pose_utils.display(input,target,out.data.cpu(),opt.use_input_pose,opt.pose_dim) 104 | else: 105 | images = pose_utils.display_stacked(input, interpol_pose, target, outputs, opt.num_stacks, opt.use_input_pose, opt.pose_dim) 106 | title = "epoch_{0}_{1}.png".format(str(epoch).zfill(3), str(it).zfill(5)) 107 | plt.imsave(os.path.join(opt.output_dir, 'train', title), images, cmap=plt.cm.gray) 108 | 109 | # saving results for unseen images 110 | input, target, interpol_pose, loader_test_iter = load_sample(loader_test_iter, loader_test, 'test') 111 | 112 | if(opt.gen_type=='baseline'): 113 | out = model.gen(Variable(input.cuda())) 114 | images = pose_utils.display(input, target, out.data.cpu(), opt.use_input_pose, opt.pose_dim) 115 | elif(opt.gen_type=='stacked'): 116 | out = model.gen(Variable(input.cuda()), Variable(interpol_pose.cuda())) 117 | images = pose_utils.display_stacked(input, interpol_pose, target, out, opt.num_stacks, opt.use_input_pose, opt.pose_dim) 118 | else: 119 | raise Exception('Invalid gen type !!') 120 | title = "epoch_{0}_{1}.png".format(str(epoch).zfill(3), str(it).zfill(5)) 121 | plt.imsave(os.path.join(opt.output_dir, 'test', title), images, cmap=plt.cm.gray) 122 | 123 | if(epoch%opt.checkpoint_ratio==0): 124 | model.save(opt.checkpoints_dir, epoch) 125 | 126 | main() -------------------------------------------------------------------------------- /src_baseline/models/__pycache__/networks.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_baseline/models/__pycache__/networks.cpython-36.pyc -------------------------------------------------------------------------------- /src_baseline/models/__pycache__/pose_gan.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_baseline/models/__pycache__/pose_gan.cpython-36.pyc -------------------------------------------------------------------------------- /src_baseline/models/pose_gan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import os 4 | import itertools 5 | import torch.nn as nn 6 | from torch.autograd import Variable 7 | from utils import pose_utils 8 | from models.networks import Generator, Stacked_Generator, Discriminator, gaussian_weights_init, print_network, xavier_weights_init 9 | 10 | class Pose_GAN(nn.Module): 11 | def __init__(self, opt): 12 | super(Pose_GAN, self).__init__() 13 | 14 | # load generator and discriminator models 15 | # adding extra layers for larger image size 16 | if(opt.checkMode == 0): 17 | nfilters_decoder = (512, 512, 512, 256, 128, 3) if max(opt.image_size) < 256 else (512, 512, 512, 512, 256, 128, 3) 18 | nfilters_encoder = (64, 128, 256, 512, 512, 512) if max(opt.image_size) < 256 else (64, 128, 256, 512, 512, 512, 512) 19 | else: 20 | nfilters_decoder = (128, 3) if max(opt.image_size) < 256 else (256, 128, 3) 21 | nfilters_encoder = (64, 128) if max(opt.image_size) < 256 else (64, 128, 256) 22 | 23 | if (opt.use_input_pose): 24 | input_nc = 3 + 2*opt.pose_dim 25 | else: 26 | input_nc = 3 + opt.pose_dim 27 | 28 | self.num_stacks = opt.num_stacks 29 | self.batch_size = opt.batch_size 30 | self.pose_dim = opt.pose_dim 31 | if(opt.gen_type=='stacked'): 32 | self.gen = Stacked_Generator(input_nc, opt.num_stacks, opt.pose_dim, nfilters_encoder, nfilters_decoder, use_input_pose=opt.use_input_pose) 33 | elif(opt.gen_type=='baseline'): 34 | self.gen = Generator(input_nc, nfilters_encoder, nfilters_decoder, use_input_pose=opt.use_input_pose) 35 | else: 36 | raise Exception('Invalid gen_type') 37 | # discriminator also sees the output image for the target pose 38 | self.disc = Discriminator(input_nc + 3, use_input_pose=opt.use_input_pose, checkMode=opt.checkMode) 39 | print('---------- Networks initialized -------------') 40 | print_network(self.gen) 41 | print_network(self.disc) 42 | print('-----------------------------------------------') 43 | # Setup the optimizers 44 | lr = opt.learning_rate 45 | self.disc_opt = torch.optim.Adam(self.disc.parameters(), lr=lr, betas=(0.5, 0.999)) 46 | self.gen_opt = torch.optim.Adam(self.gen.parameters(), lr=lr, betas=(0.5, 0.999)) 47 | 48 | # Network weight initialization 49 | self.gen.cuda() 50 | self.disc.cuda() 51 | self.disc.apply(xavier_weights_init) 52 | self.gen.apply(xavier_weights_init) 53 | 54 | # Setup the loss function for training 55 | self.ll_loss_criterion = torch.nn.L1Loss() 56 | 57 | # add code for intermediate supervision for the interpolated poses using pretrained pose-estimator 58 | def gen_update(self, input, target, interpol_pose, opt): 59 | self.gen.zero_grad() 60 | 61 | if(opt['gen_type']=='stacked'): 62 | outputs_gen = self.gen(input, interpol_pose) 63 | out_gen = outputs_gen[-1] 64 | else: 65 | out_gen = self.gen(input) 66 | outputs_gen = [] 67 | 68 | inp_img, inp_pose, out_pose = pose_utils.get_imgpose(input, opt['use_input_pose'], opt['pose_dim']) 69 | 70 | inp_dis = torch.cat([inp_img, inp_pose, out_gen, out_pose], dim=1) 71 | out_dis = self.disc(inp_dis) 72 | 73 | # computing adversarial loss 74 | for it in range(out_dis.shape[0]): 75 | out = out_dis[it, :] 76 | all_ones = Variable(torch.ones((out.size(0))).cuda()) 77 | if it==0: 78 | # ad_loss = nn.functional.binary_cross_entropy(out, all_ones) 79 | ad_loss = -torch.mean(torch.log(out + 1e-7)) 80 | else: 81 | # ad_loss += nn.functional.binary_cross_entropy(out, all_ones) 82 | ad_loss += -torch.mean(torch.log(out + 1e-7) 83 | ) 84 | ll_loss = self.ll_loss_criterion(out_gen, target) 85 | ad_loss = ad_loss * opt['gan_penalty_weight'] / self.batch_size 86 | ll_loss = ll_loss * opt['l1_penalty_weight'] 87 | total_loss = ad_loss + ll_loss 88 | total_loss.backward() 89 | self.gen_opt.step() 90 | self.gen_ll_loss = ll_loss.item() 91 | self.gen_ad_loss = ad_loss.item() 92 | self.gen_total_loss = total_loss.item() 93 | return out_gen, outputs_gen, [self.gen_total_loss, self.gen_ll_loss, self.gen_ad_loss ] 94 | 95 | def dis_update(self, input, target, interpol_pose, real_inp, real_target, opt): 96 | self.disc.zero_grad() 97 | 98 | if (opt['gen_type'] == 'stacked'): 99 | out_gen = self.gen(input, interpol_pose) 100 | out_gen = out_gen[-1] 101 | else: 102 | out_gen = self.gen(input) 103 | 104 | inp_img, inp_pose, out_pose = pose_utils.get_imgpose(input, opt['use_input_pose'], opt['pose_dim']) 105 | 106 | fake_disc_inp = torch.cat([inp_img, inp_pose, out_gen, out_pose], dim=1) 107 | r_inp_img, r_inp_pose, r_out_pose = pose_utils.get_imgpose(real_inp, opt['use_input_pose'], opt['pose_dim']) 108 | real_disc_inp = torch.cat([r_inp_img, r_inp_pose, real_target, r_out_pose], dim=1) 109 | data_dis = torch.cat((real_disc_inp, fake_disc_inp), 0) 110 | res_dis = self.disc(data_dis) 111 | 112 | for it in range(res_dis.shape[0]): 113 | out = res_dis[it,:] 114 | if(it Args:\n') 143 | for k, v in sorted(vars(self.opt).items()): 144 | opt_file.write(' %s: %s\n' % (str(k), str(v))) 145 | opt_file.write('==> Args:\n') 146 | return self.opt 147 | -------------------------------------------------------------------------------- /src_baseline/unitTests.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | matplotlib.use('Agg') 3 | import pylab as plt 4 | import torch 5 | import numpy as np 6 | import cv2 7 | import time 8 | import datetime 9 | from opts import opts 10 | import tqdm 11 | from datasets.PoseTransfer_Dataset import PoseTransfer_Dataset 12 | from utils import pose_utils 13 | from models.networks import Generator, Discriminator, torch_summarize, keras_to_pytorch 14 | from models.pose_gan import Pose_GAN 15 | import sys 16 | sys.path.append('../../pose-gan-clean/pose-gan-h36m-fg/') 17 | from conditional_gan import CGAN 18 | from pose_dataset import PoseHMDataset 19 | from opt import cmd 20 | 21 | opt = opts().parse() 22 | 23 | # <---------------------------------------------------> 24 | # intialzie dataloader 25 | 26 | # disc_loader = torch.utils.data.DataLoader( 27 | # PoseTransfer_Dataset(vars(opt), 'train'), 28 | # batch_size=opt.batch_size, 29 | # shuffle=False, 30 | # ) 31 | # 32 | # disc_iter = disc_loader.__iter__() 33 | 34 | # <------------------------------------------------------> 35 | # visualize interpolated pose 36 | 37 | # 38 | # input, target, interpol_pose = disc_iter.next() 39 | # print(batch.shape) 40 | # dis_images = pose_utils.display(input, target, opt.use_input_pose, opt.pose_dim) 41 | 42 | # interpol_pose = pose_utils.postProcess(interpol_pose) 43 | # result = [] 44 | # for i in range(opt.num_stacks): 45 | # pose_batch = interpol_pose[:, :, :, i * opt.pose_dim:(i + 1) * opt.pose_dim] 46 | # pose_images = np.array([pose_utils.draw_pose_from_map(pose.numpy(), opt.pose_dim)[0] for pose in pose_batch]) 47 | # result.append(pose_images) 48 | # interpol_pose = np.concatenate(result, axis=0) 49 | # interpol_pose = pose_utils.make_grid(torch.from_numpy(interpol_pose), None, row=opt.batch_size, col=opt.num_stacks) 50 | # 51 | # 52 | # plt.imsave('test_interpol.png', interpol_pose , cmap=plt.cm.gray) 53 | # plt.imsave('test_image_pose.png', dis_images, cmap=plt.cm.gray) 54 | 55 | 56 | # <------------------------------------------------------> 57 | # load images at particular index and initialzie model 58 | 59 | # index = 20 60 | # for _ in range(index): 61 | # input, target, _ = disc_iter.next() 62 | # # 63 | # model = Pose_GAN(opt).cuda() 64 | # print(torch_summarize(model)) 65 | 66 | # out = model.gen(input.cuda()) 67 | # images = pose_utils.display(input, target, out.data.cpu(), opt.use_input_pose, opt.pose_dim) 68 | # plt.imsave('test_model.png', images, cmap=plt.cm.gray) 69 | 70 | from keras.models import load_model 71 | 72 | # <------------------------------------------------------> 73 | # covnert mdoel from keras and save the model 74 | 75 | # config = --dataset fasion --pose_dim 18 --data_Dir ../../pose-gan-clean/pose-gan/data/ 76 | 77 | # keras_gen = load_model('../../pose-gan-clean/pose-gan/output/baseline-fasion/epoch_089_generator.h5') 78 | # pytorch_gen = model.gen 79 | # pytorch_gen, _ = keras_to_pytorch(pytorch_gen, keras_gen.layers, 0) 80 | # 81 | # del keras_gen 82 | # del model 83 | 84 | 85 | # keras_disc = load_model('../../pose-gan-clean/pose-gan/output/baseline-fasion/epoch_089_discriminator.h5') 86 | # pytorch_disc = model.disc 87 | # pytorch_disc, _ = keras_to_pytorch(pytorch_disc, keras_disc.layers, 0) 88 | 89 | # <------------------------------------------------------> 90 | # load the model 91 | 92 | # pytorch_disc = pytorch_disc.cuda() 93 | # torch.save(pytorch_disc.state_dict(), "disc.pkl") 94 | 95 | # pytorch_gen = model.gen 96 | # pytorch_gen.load_state_dict(torch.load('gen.pkl')) 97 | # pytorch_gen.eval() 98 | 99 | # keras_weights = [] 100 | # for layer in keras_gen.layers: 101 | # keras_weights.append(layer.get_weights()) 102 | 103 | # out = pytorch_gen(input.cuda()) 104 | # images = pose_utils.display(input, target, out.data.cpu(), opt.use_input_pose, opt.pose_dim) 105 | # plt.imsave('test_pytorch_model.png', images, cmap=plt.cm.gray) 106 | 107 | # <------------------------------------------------> 108 | # ----> keras unit test 109 | 110 | # args = cmd.args() 111 | # args.dataset = 'fasion' 112 | # generator = load_model('../../pose-gan-clean/pose-gan/output/baseline-fasion/epoch_089_generator.h5') 113 | # dataset = PoseHMDataset(test_phase=False, **vars(args)) 114 | # 115 | # index = 11 116 | # for _ in range(index): 117 | # generator_batch = dataset.next_generator_sample() 118 | # 119 | # import tensorflow as tf 120 | # sess = tf.Session() 121 | # with sess.as_default(): 122 | # out_batch = generator.predict_on_batch(generator_batch) 123 | # 124 | # image_test = dataset.display(out_batch,generator_batch) 125 | # plt.imsave('test_keras_model.png', image_test, cmap=plt.cm.gray) 126 | 127 | 128 | # <------------------------------------------------------> 129 | # test forward pass for both models 130 | 131 | # import cv2 132 | # pytorch_out = plt.imread('test_pytorch_model.png') 133 | # keras_out = plt.imread('test_keras_model.png') 134 | # 135 | # pytorch_out = pytorch_out[:,:-256] 136 | # keras_out = keras_out[:,:-256] 137 | # 138 | # 139 | # f, axarr = plt.subplots(2,1) 140 | # axarr[0] = plt.imshow(pytorch_out) 141 | # plt.show() 142 | # # cv2.waitKey(0) 143 | # axarr[1] = plt.imshow(keras_out) 144 | # plt.show() 145 | # print(np.max(pytorch_out), np.max(keras_out)) 146 | # print(np.min(pytorch_out), np.min(keras_out)) 147 | # print(np.mean(pytorch_out), np.mean(keras_out)) 148 | # print(np.mean(np.absolute(pytorch_out-keras_out))) 149 | # print(np.max(pytorch_out - keras_out)) 150 | 151 | # loss = torch.nn.L1Loss(torch.from_numpy(pytorch_out), torch.from_numpy(keras_out)) 152 | # print(loss) 153 | # print(np.allclose(pytorch_out,keras_out)) 154 | 155 | # <------------------------------------------------------> 156 | # test losses for both models 157 | 158 | # batch_size = 4 159 | # out_size = 49 160 | # 161 | # fake_out_dis = torch.from_numpy(np.random.rand(4,49)) 162 | # fake_out_dis.requires_grad = True 163 | # out_dis = fake_out_dis 164 | # for it in range(out_dis.shape[0]): 165 | # out = out_dis[it, :] 166 | # # all_ones = Variable(torch.ones((out.size(0))).cuda()) 167 | # if it == 0: 168 | # # ad_loss = nn.functional.binary_cross_entropy(out, all_ones) 169 | # ad_loss = -torch.mean(torch.log(out + 1e-7)) 170 | # else: 171 | # # ad_loss += nn.functional.binary_cross_entropy(out, all_ones) 172 | # ad_loss += -torch.mean(torch.log(out + 1e-7) 173 | # ) 174 | # ad_loss = ad_loss * opt.gan_penalty_weight / batch_size 175 | # print(ad_loss) 176 | # 177 | # from keras import backend as K 178 | # import tensorflow as tf 179 | # sess = tf.Session() 180 | # with sess.as_default(): 181 | # keras_loss = -K.mean(K.log(fake_out_dis.detach().numpy() + 1e-7)).eval() 182 | # print(keras_loss) 183 | -------------------------------------------------------------------------------- /src_baseline/utils/__pycache__/pose_transform.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_baseline/utils/__pycache__/pose_transform.cpython-36.pyc -------------------------------------------------------------------------------- /src_baseline/utils/__pycache__/pose_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_baseline/utils/__pycache__/pose_utils.cpython-36.pyc -------------------------------------------------------------------------------- /src_baseline/utils/create_pairs.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | import pandas as pd 4 | import pose_transform 5 | import pose_utils 6 | from itertools import permutations 7 | from opts import opts 8 | import re 9 | 10 | opt = opts().parse() 11 | 12 | def make_pair_nonvid(df): 13 | persons = df.apply(lambda x: '_'.join(x['name'].split('_')[0:1]), axis=1) 14 | df['person'] = persons 15 | fr, to = [], [] 16 | for person in pd.unique(persons): 17 | pairs = list(zip(*list(permutations(df[df['person'] == person]['name'], 2)))) 18 | if len(pairs) != 0: 19 | fr += list(pairs[0]) 20 | to += list(pairs[1]) 21 | pair_df = pd.DataFrame(index=range(len(fr))) 22 | pair_df['from'] = fr 23 | pair_df['to'] = to 24 | return pair_df 25 | 26 | def make_pairs(df): 27 | persons = df.apply(lambda x: '_'.join(x['name'].split('_')[:-1]), axis=1) 28 | df['person'] = persons 29 | fr, to = [], [] 30 | for person in pd.unique(persons): 31 | pair_fr = df[df['person'] == person] 32 | # print(person,pair_fr.shape[0]) 33 | num_rows = pair_fr.shape[0] 34 | i = 0 35 | for index, row in pair_fr.iterrows(): 36 | if(i+2 threshold)) 61 | for x_i, y_i, z_i in zip(x, y, z): 62 | all_peaks[z_i].append([x_i, y_i]) 63 | 64 | x_values = [] 65 | y_values = [] 66 | 67 | for i in range(pose_dim): 68 | if len(all_peaks[i]) != 0: 69 | x_values.append(all_peaks[i][0][0]) 70 | y_values.append(all_peaks[i][0][1]) 71 | else: 72 | x_values.append(MISSING_VALUE) 73 | y_values.append(MISSING_VALUE) 74 | 75 | return np.concatenate([np.expand_dims(y_values, -1), np.expand_dims(x_values, -1)], axis=1) 76 | 77 | 78 | def cords_to_map(cords, img_size, sigma=6): 79 | result = np.zeros(img_size + cords.shape[0:1], dtype='float32') 80 | for i, point in enumerate(cords): 81 | if point[0] == MISSING_VALUE or point[1] == MISSING_VALUE: 82 | continue 83 | xx, yy = np.meshgrid(np.arange(img_size[1]), np.arange(img_size[0])) 84 | result[..., i] = np.exp(-((yy - point[0]) ** 2 + (xx - point[1]) ** 2) / (2 * sigma ** 2)) 85 | return result 86 | 87 | 88 | def compute_interpol_pose(inp_pos,tg_pos,index,num_stacks,pose_dim): 89 | assert index <= num_stacks 90 | if(pose_dim==16): 91 | interpol_pose = inp_pos + (tg_pos-inp_pos)*index/num_stacks 92 | # bad logic to circumvent missing annot . . synthesize and vanish missing annot after half sequence is completed 93 | elif(pose_dim==18): 94 | interpol_pose = np.zeros([pose_dim,2], dtype='float32') 95 | for i in range(pose_dim): 96 | # inp pose has missing annot and tg pose has it 97 | if ((inp_pos[i,0] == MISSING_VALUE or inp_pos[i,1]== MISSING_VALUE) and 98 | (tg_pos[i,0] != MISSING_VALUE and tg_pos[i,1]!= MISSING_VALUE)): 99 | if(index<=num_stacks//2): 100 | interpol_pose[i] = MISSING_VALUE 101 | else: 102 | interpol_pose[i] = tg_pos[i] 103 | # tg pose has missing annot and inp pose has it 104 | elif ((tg_pos[i, 0] == MISSING_VALUE or tg_pos[i, 1] == MISSING_VALUE) and ( 105 | inp_pos[i, 0] != MISSING_VALUE and inp_pos[i, 1] != MISSING_VALUE)): 106 | if (index <= num_stacks // 2): 107 | interpol_pose[i] = inp_pos[i] 108 | else: 109 | interpol_pose[i] = MISSING_VALUE 110 | # annot missing in both poses 111 | elif ((tg_pos[i, 0] == MISSING_VALUE or tg_pos[i, 1] == MISSING_VALUE) and ( 112 | inp_pos[i, 0] == MISSING_VALUE or inp_pos[i, 1] == MISSING_VALUE)): 113 | interpol_pose[i] = MISSING_VALUE 114 | # normal interpol when annot are present in both cases 115 | else: 116 | interpol_pose[i] = inp_pos[i] + (tg_pos[i]-inp_pos[i])*index/num_stacks 117 | return interpol_pose 118 | 119 | def draw_pose_from_cords(pose_joints, pose_dim, img_size, radius=2, draw_joints=True, img = np.zeros([224,224,3])): 120 | colors = np.zeros(shape=img_size + (3, ), dtype=np.uint8) 121 | # interesting case of numpy array assignment semantics here . . the below ass is a view, hence the overliad poses error that destroyed the precious time of an incomporable specimen on this overpopulated information craving filth obsessed world 122 | # colors = img 123 | mask = np.zeros(shape=img_size, dtype=bool) 124 | 125 | if draw_joints: 126 | if(pose_dim==16): 127 | for f, t in LIMB_SEQ: 128 | from_missing = pose_joints[f][0] == MISSING_VALUE or pose_joints[f][1] == MISSING_VALUE 129 | to_missing = pose_joints[t][0] == MISSING_VALUE or pose_joints[t][1] == MISSING_VALUE 130 | if from_missing or to_missing: 131 | continue 132 | yy, xx, val = line_aa(pose_joints[f][0], pose_joints[f][1], pose_joints[t][0], pose_joints[t][1]) 133 | colors[yy, xx] = np.expand_dims(val, 1) * 255 134 | mask[yy, xx] = True 135 | else: 136 | for f, t in LIMB_SEQ_PAF: 137 | from_missing = pose_joints[f][0] == MISSING_VALUE or pose_joints[f][1] == MISSING_VALUE 138 | to_missing = pose_joints[t][0] == MISSING_VALUE or pose_joints[t][1] == MISSING_VALUE 139 | if from_missing or to_missing: 140 | continue 141 | yy, xx, val = line_aa(pose_joints[f][0], pose_joints[f][1], pose_joints[t][0], pose_joints[t][1]) 142 | colors[yy, xx] = np.expand_dims(val, 1) * 255 143 | mask[yy, xx] = True 144 | 145 | for i, joint in enumerate(pose_joints): 146 | if pose_joints[i][0] == MISSING_VALUE or pose_joints[i][1] == MISSING_VALUE: 147 | continue 148 | yy, xx = circle(joint[0], joint[1], radius=radius, shape=img_size) 149 | colors[yy, xx] = COLORS[i] 150 | mask[yy, xx] = True 151 | 152 | return colors, mask 153 | 154 | 155 | def draw_pose_from_map(pose_map, pose_dim, threshold=0.1, **kwargs): 156 | cords = map_to_cord(pose_map, pose_dim, threshold=threshold) 157 | return draw_pose_from_cords(cords, pose_dim, pose_map.shape[:2], **kwargs) 158 | 159 | 160 | def load_pose_cords_from_strings(y_str, x_str): 161 | y_cords = json.loads(y_str) 162 | x_cords = json.loads(x_str) 163 | return np.concatenate([np.expand_dims(y_cords, -1), np.expand_dims(x_cords, -1)], axis=1) 164 | 165 | def mean_inputation(X): 166 | X = X.copy() 167 | for i in range(X.shape[1]): 168 | for j in range(X.shape[2]): 169 | val = np.mean(X[:, i, j][X[:, i, j] != -1]) 170 | X[:, i, j][X[:, i, j] == -1] = val 171 | return X 172 | 173 | def draw_legend(): 174 | handles = [mpatches.Patch(color=np.array(color) / 255.0, label=name) for color, name in zip(COLORS, LABELS)] 175 | plt.legend(handles=handles, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) 176 | 177 | def produce_ma_mask(kp_array, img_size, point_radius=4): 178 | from skimage.morphology import dilation, erosion, square 179 | mask = np.zeros(shape=img_size, dtype=bool) 180 | limbs = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], 181 | [10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], 182 | [1,16], [16,18], [2,17], [2,18], [9,12], [12,6], [9,3], [17,18]] 183 | limbs = np.array(limbs) - 1 184 | for f, t in limbs: 185 | from_missing = kp_array[f][0] == MISSING_VALUE or kp_array[f][1] == MISSING_VALUE 186 | to_missing = kp_array[t][0] == MISSING_VALUE or kp_array[t][1] == MISSING_VALUE 187 | if from_missing or to_missing: 188 | continue 189 | 190 | norm_vec = kp_array[f] - kp_array[t] 191 | norm_vec = np.array([-norm_vec[1], norm_vec[0]]) 192 | norm_vec = point_radius * norm_vec / np.linalg.norm(norm_vec) 193 | 194 | 195 | vetexes = np.array([ 196 | kp_array[f] + norm_vec, 197 | kp_array[f] - norm_vec, 198 | kp_array[t] - norm_vec, 199 | kp_array[t] + norm_vec 200 | ]) 201 | yy, xx = polygon(vetexes[:, 0], vetexes[:, 1], shape=img_size) 202 | mask[yy, xx] = True 203 | 204 | for i, joint in enumerate(kp_array): 205 | if kp_array[i][0] == MISSING_VALUE or kp_array[i][1] == MISSING_VALUE: 206 | continue 207 | yy, xx = circle(joint[0], joint[1], radius=point_radius, shape=img_size) 208 | mask[yy, xx] = True 209 | 210 | mask = dilation(mask, square(5)) 211 | mask = erosion(mask, square(5)) 212 | return mask 213 | 214 | 215 | def _preprocess_image(image): 216 | return (image / 255 - 0.5) * 2 217 | 218 | def _deprocess_image(image): 219 | return (255 * (image + 1) / 2).byte() 220 | 221 | # channels last for display by pyplot/cv2 222 | def postProcess(image): 223 | return image.permute(0,2,3,1) 224 | 225 | 226 | def get_imgpose(input, use_input_pose, pose_dim): 227 | inp_img = input[:, :3] 228 | inp_pose = input[:, 3:3 + pose_dim] if use_input_pose else None 229 | tg_pose_index = 3 + pose_dim if use_input_pose else 6 230 | tg_pose = input[:, tg_pose_index:] 231 | 232 | return inp_img, inp_pose, tg_pose 233 | 234 | def display(input_batch, target_batch, output_batch, _use_input_pose, pose_dim): # output_batch for result image 235 | row = input_batch.shape[0] 236 | col = 1 237 | 238 | inp_img, inp_pose, tg_pose = get_imgpose(input_batch, _use_input_pose, pose_dim) 239 | inp_img = postProcess(_deprocess_image(inp_img)) 240 | tg_pose = postProcess(tg_pose) 241 | tg_img = postProcess(_deprocess_image(target_batch)) 242 | 243 | res_img = postProcess(_deprocess_image(output_batch)) 244 | 245 | # changing original code due to extra named argument 246 | inp_img = make_grid(inp_img, None, row=row, col=col) 247 | 248 | pose_images = np.array([draw_pose_from_map(pose.numpy(), pose_dim)[0] for pose in tg_pose]) 249 | tg_pose = make_grid(torch.from_numpy(pose_images), None, row=row, col=col) 250 | 251 | tg_img = make_grid(tg_img, None, row=row, col=col) 252 | res_img = make_grid(res_img, None, row=row, col=col) 253 | 254 | return np.concatenate(np.array([inp_img, tg_pose, tg_img, res_img]), axis=1) #res_img]) , axis=1) 255 | 256 | 257 | def display_stacked(input_batch, interpol_batch, target_batch, output_batch, num_stacks, _use_input_pose, pose_dim): 258 | row = input_batch.shape[0] 259 | col = 1 260 | 261 | inp_img, inp_pose, tg_pose = get_imgpose(input_batch, _use_input_pose, pose_dim) 262 | inp_img = postProcess(_deprocess_image(inp_img)) 263 | interpol_pose = postProcess(interpol_batch) 264 | tg_img = postProcess(_deprocess_image(target_batch)) 265 | 266 | res_img = [postProcess(_deprocess_image(output.data.cpu())) for output in output_batch] 267 | 268 | # changing original code due to extra named argument 269 | inp_img = make_grid(inp_img, None, row=row, col=col) 270 | 271 | result = [] 272 | for i in range(num_stacks): 273 | pose_batch = interpol_pose[:,:,:,i*pose_dim:(i+1)*pose_dim] 274 | pose_images = np.array([draw_pose_from_map(pose.numpy(), pose_dim)[0] for pose in pose_batch]) 275 | result.append(pose_images) 276 | interpol_pose = np.concatenate(result, axis=0) 277 | # ith interpol pose for jth sample = interpol_pose[(i-1)*batch_size+j] 278 | interpol_pose = make_grid(torch.from_numpy(interpol_pose), None, row=row, col=num_stacks) 279 | 280 | tg_img = make_grid(tg_img, None, row=row, col=col) 281 | 282 | res_img = np.concatenate(res_img,axis=0) 283 | # ith res_img for jth sample = res_img[(i-1)*batch_size+j] 284 | res_img = make_grid(torch.from_numpy(res_img), None, row=row, col=num_stacks) 285 | 286 | # print(inp_img.shape, interpol_pose.shape, tg_img.shape, res_img.shape) 287 | return np.concatenate([inp_img, interpol_pose, tg_img, res_img] , axis=1) #res_img]) , axis=1) 288 | 289 | 290 | 291 | def make_grid(output_batch, input_batch = None, row=8, col=8): 292 | batch = output_batch.numpy() 293 | height, width = batch.shape[1], batch.shape[2] 294 | total_width, total_height = width * col, height * row 295 | result_image = np.empty((total_height, total_width, batch.shape[3]), dtype=batch.dtype) 296 | batch_index = 0 297 | # fill rows first then columns 298 | for i in range(col): 299 | for j in range(row): 300 | result_image[(j * height):((j+1)*height), (i * width):((i+1)*width)] = batch[batch_index] 301 | batch_index += 1 302 | return result_image 303 | -------------------------------------------------------------------------------- /src_baseline/utils/proc_bg_h36m.py: -------------------------------------------------------------------------------- 1 | import scipy.io as sio 2 | import numpy as np 3 | import h5py 4 | import os 5 | import cv2 6 | import xml.etree.ElementTree as ElementTree 7 | 8 | subject_list = [1, 5, 6, 7, 8, 9, 11] 9 | action_list = np.arange(2, 17) 10 | subaction_list = np.arange(1, 3) 11 | camera_list = np.arange(1, 5) 12 | 13 | root_dir = '/home/project/datasets/Human3.6/' 14 | xml_path = "/home/project/datasets/Human3.6/metadata.xml" 15 | SAVE_PATH = root_dir + '/images_fg/' 16 | 17 | annot_path = '/home/project/Pose-Estimation/images/' 18 | annot_name = 'matlab_meta.mat' 19 | 20 | xml_file = ElementTree.parse(xml_path) 21 | xml_mapping = xml_file.find('mapping') 22 | xml_cameras = xml_file.find('dbcameras') 23 | 24 | for subject in subject_list: 25 | for action in action_list: 26 | for subaction in subaction_list: 27 | for camera in camera_list: 28 | 29 | if (subject < 11): 30 | print("Pass suubject :{0:d}".format(subject)) 31 | continue 32 | 33 | vid_folder = root_dir + "S" + str(subject) + "/Videos/" 34 | bg_vid_folder = root_dir + "/S" + str(subject) + "/ground_truth_bs/" 35 | save_folder_name = 's_{:02d}_act_{:02d}_subact_{:02d}_ca_{:02d}'.format(subject, action, subaction, 36 | camera) 37 | 38 | if (os.path.exists(os.path.join(SAVE_PATH, save_folder_name)) != True): 39 | os.makedirs(os.path.join(SAVE_PATH, save_folder_name)) 40 | 41 | # fetch relevant video . . code converted from matlab code in h36m release code 42 | vid_name = xml_mapping[int(action * 2 + subaction - 2)][int(subject + 1)].text + '.' + str( 43 | xml_cameras[0][int(camera - 1)].text) + '.mp4' 44 | 45 | print("Processing file S{0:d}: {1} ".format(subject, vid_name)) 46 | 47 | try: 48 | vid_file = os.path.join(vid_folder, vid_name) 49 | bg_vid_file = os.path.join(bg_vid_folder, vid_name) 50 | 51 | vidcap = cv2.VideoCapture(vid_file) 52 | bg_vidcap = cv2.VideoCapture(bg_vid_file) 53 | except: 54 | print("Pass vid : {}".format(vid_name)) 55 | continue 56 | 57 | # process annot file for fetching bounding box annotaions 58 | annot_file = annot_path + save_folder_name + '/' + annot_name 59 | try: 60 | data = sio.loadmat(annot_file) 61 | except: 62 | continue 63 | bboxx = data['bbox'].transpose(1, 0) 64 | 65 | index, num_proc = 0, 0 66 | while (True): 67 | success_img, image = vidcap.read() 68 | success_bg_img, bg_image = bg_vidcap.read() 69 | 70 | index += 1 71 | 72 | if (success_img != success_bg_img): 73 | print('Invalid background frame for {0} : {1:06d}'.format(vid_file, index)) 74 | break 75 | 76 | success = success_bg_img and success_img 77 | 78 | if (success): 79 | 80 | if ((index - 1) % 5 != 0): 81 | continue 82 | 83 | num_proc += 1 84 | 85 | bg_image[bg_image > 100] = 1 86 | fg_image = np.multiply(image, bg_image) 87 | 88 | # bounding box computation . . converted from matlab code directly 89 | bb = bboxx[index - 1, :].astype(np.int32) 90 | bb[bb < 0] = 0 91 | bb[2] = min(bb[2], image.shape[1]); 92 | bb[3] = min(bb[3], image.shape[0]); 93 | bb = np.round(bb) 94 | 95 | if (bb[3] - bb[1] > bb[2] - bb[0]): 96 | PAD = ((bb[3] - bb[1]) - (bb[2] - bb[0])) / 2; 97 | bb[2] = bb[2] + PAD; 98 | bb[0] = bb[0] - PAD; 99 | else: 100 | PAD = ((bb[2] - bb[0]) - (bb[3] - bb[1])) / 2; 101 | bb[3] = bb[3] + PAD; 102 | bb[1] = bb[1] - PAD; 103 | 104 | bb[bb < 0] = 0 105 | bb[2] = min(bb[2], image.shape[1]); 106 | bb[3] = min(bb[3], image.shape[0]); 107 | bb = np.round(bb) 108 | 109 | fg_image = cv2.resize(fg_image[bb[1]:bb[3], bb[0]:bb[2], :], (224, 224)) 110 | file_name = '{}/{}/{}_{:06d}.jpg'.format(SAVE_PATH, save_folder_name, save_folder_name, index) 111 | cv2.imwrite(file_name, fg_image) 112 | # cv2.imshow(file_name, fg_image) 113 | # cv2.waitKey(0) 114 | # cv2.destroyAllWindows() 115 | else: 116 | break 117 | 118 | print("Finished . Processed {0:d} frames".format(num_proc)) 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /src_deformable/__init__.py: -------------------------------------------------------------------------------- 1 | from datasets.PoseTransfer_Dataset import PoseTransfer_Dataset 2 | from models.networks import Deformable_Generator, Discriminator 3 | from models.pose_gan import DeformablePose_GAN -------------------------------------------------------------------------------- /src_deformable/__pycache__/opts.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/__pycache__/opts.cpython-36.pyc -------------------------------------------------------------------------------- /src_deformable/commands: -------------------------------------------------------------------------------- 1 | fasion : baseline 2 | CUDA_VISIBLE_DEVICES=1 nohup python main.py --l1_penalty_weight 10 --batch_size 4 --data_Dir ../../pose-gan-clean/pose-gan/data/ --number_of_epochs 90 --gen_type baseline --expID baseline_fasion --pose_dim 18 --dataset fasion > logs/baseline_fasion 2>&1 & 3 | 4 | fasion : deformable 5 | CUDA_VISIBLE_DEVICES=1 nohup python main.py --warp_skip mask --l1_penalty_weight 10 --batch_size 2 --number_of_epochs 90 --gen_type baseline --expID deformable_fasion --pose_dim 18 --dataset fasion > logs/deformable_fasion 2>&1 & 6 | 7 | fasion : full 8 | CUDA_VISIBLE_DEVICES=1 nohup python main.py --warp_skip mask --l1_penalty_weight 100 --batch_size 2 --number_of_epochs 90 --gen_type baseline --expID deformable_fasion --pose_dim 18 --dataset fasion --nn_loss_area_size 5 --batch_size 2 --content_loss_layer block1_conv2> logs/deformable_fasion 2>&1 & 9 | 10 | fasion : stacked_deformable 11 | CUDA_VISIBLE_DEVICES=1 nohup python main.py --warp_skip mask --l1_penalty_weight 10 --batch_size 2 --number_of_epochs 90 --gen_type stacked --num_stacks 1 --expID stacked_deformable_fasion --pose_dim 18 --dataset fasion > logs/stacked_deformable_fasion 2>&1 & 12 | 13 | h36m : deformable 14 | CUDA_VISIBLE_DEVICES=0 nohup python main.py --warp_skip mask --l1_penalty_weight 10 --batch_size 4 --number_of_epochs 90 --gen_type baseline --expID deformable_h36m > logs/deformable_h36m 2>&1 & 15 | 16 | h36m : stacked_deformable 17 | CUDA_VISIBLE_DEVICES=1 nohup python main.py --warp_skip mask --l1_penalty_weight 10 --batch_size 2 --number_of_epochs 90 --gen_type stacked --expID stacked_deformable_h36m --num_stacks 1 > logs/stacked_deformable_h36m 2>&1 & -------------------------------------------------------------------------------- /src_deformable/datasets/__pycache__/PoseTransfer_Dataset.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/datasets/__pycache__/PoseTransfer_Dataset.cpython-36.pyc -------------------------------------------------------------------------------- /src_deformable/logs/deformable_fasion: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | Model options . . 3 | annotations_file_test: ../../pose-gan-clean/pose-gan/data/fasion-annotation-test.csv 4 | annotations_file_test_paf: ../../pose-gan-clean/pose-gan/data/fasion-annotation-paf-test0.csv 5 | annotations_file_train: ../../pose-gan-clean/pose-gan/data/fasion-annotation-train.csv 6 | annotations_file_train_paf: ../../pose-gan-clean/pose-gan/data/fasion-annotation-paf-train0.csv 7 | batch_size: 2 8 | checkpoint_ratio: 5 9 | checkpoints_dir: ../exp/deformable_fasion/models 10 | compute_h36m_paf_split: 0 11 | content_loss_layer: none 12 | data_Dir: ../../pose-gan-clean/pose-gan/data/ 13 | dataset: fasion 14 | disc_type: call 15 | discriminator_checkpoint: None 16 | display_ratio: 50 17 | expID: deformable_fasion 18 | frame_diff: 10 19 | gan_penalty_weight: 1 20 | gen_type: baseline 21 | generated_images_dir: output/generated_images/fasion-restricted 22 | generator_checkpoint: None 23 | image_size: (256, 256) 24 | images_dir_test: ../../pose-gan-clean/pose-gan/data/fasion-dataset/test 25 | images_dir_train: ../../pose-gan-clean/pose-gan/data/fasion-dataset/train 26 | images_for_test: 12000 27 | iters_per_epoch: 1000 28 | l1_penalty_weight: 10.0 29 | learning_rate: 0.0002 30 | load_generated_images: 0 31 | log_file: output/full/fasion/log 32 | lstruct_penalty_weight: 0 33 | nn_loss_area_size: 1 34 | num_stacks: 4 35 | number_of_epochs: 90 36 | output_dir: ../exp/deformable_fasion/results 37 | pairs_file_test: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test.csv 38 | pairs_file_test_interpol: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test-interpol.csv 39 | pairs_file_test_iterative: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test-iterative.csv 40 | pairs_file_train: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train.csv 41 | pairs_file_train_interpol: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train-interpol.csv 42 | pairs_file_train_iterative: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train-iterative.csv 43 | pose_dim: 18 44 | pose_estimator: pose_estimator.h5 45 | resume: 0 46 | saveDir: ../exp/deformable_fasion 47 | start_epoch: 0 48 | tmp_pose_dir: tmp/fasion/ 49 | training_ratio: 1 50 | tv_penalty_weight: 0 51 | use_dropout_test: 0 52 | use_input_pose: True 53 | warp_agg: max 54 | warp_skip: mask 55 | (256, 256) 56 | Statistics for loaded dataset : Human 3.6 57 | Number of images: 52712 58 | Number of pairs train: 101268 59 | Number of pairs test: 8670 60 | Statistics for loaded dataset : Human 3.6 61 | Number of images: 52712 62 | Number of pairs train: 101268 63 | Number of pairs test: 8670 64 | ---------- Networks initialized ------------- 65 | Deformable_Generator( 66 | (encoder_app): encoder( 67 | (net): ModuleList( 68 | (0): Conv2d(21, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 69 | (1): Block( 70 | (net): Sequential( 71 | (0): LeakyReLU(negative_slope=0.2) 72 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 73 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 74 | ) 75 | ) 76 | (2): Block( 77 | (net): Sequential( 78 | (0): LeakyReLU(negative_slope=0.2) 79 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 80 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 81 | ) 82 | ) 83 | (3): Block( 84 | (net): Sequential( 85 | (0): LeakyReLU(negative_slope=0.2) 86 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 87 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 88 | ) 89 | ) 90 | (4): Block( 91 | (net): Sequential( 92 | (0): LeakyReLU(negative_slope=0.2) 93 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 94 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 95 | ) 96 | ) 97 | (5): Block( 98 | (net): Sequential( 99 | (0): LeakyReLU(negative_slope=0.2) 100 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 101 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 102 | ) 103 | ) 104 | (6): Block( 105 | (net): Sequential( 106 | (0): LeakyReLU(negative_slope=0.2) 107 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 108 | ) 109 | ) 110 | ) 111 | ) 112 | (encoder_pose): encoder( 113 | (net): ModuleList( 114 | (0): Conv2d(18, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 115 | (1): Block( 116 | (net): Sequential( 117 | (0): LeakyReLU(negative_slope=0.2) 118 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 119 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 120 | ) 121 | ) 122 | (2): Block( 123 | (net): Sequential( 124 | (0): LeakyReLU(negative_slope=0.2) 125 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 126 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 127 | ) 128 | ) 129 | (3): Block( 130 | (net): Sequential( 131 | (0): LeakyReLU(negative_slope=0.2) 132 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 133 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 134 | ) 135 | ) 136 | (4): Block( 137 | (net): Sequential( 138 | (0): LeakyReLU(negative_slope=0.2) 139 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 140 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 141 | ) 142 | ) 143 | (5): Block( 144 | (net): Sequential( 145 | (0): LeakyReLU(negative_slope=0.2) 146 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 147 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 148 | ) 149 | ) 150 | (6): Block( 151 | (net): Sequential( 152 | (0): LeakyReLU(negative_slope=0.2) 153 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 154 | ) 155 | ) 156 | ) 157 | ) 158 | (decoder): decoder( 159 | (net): ModuleList( 160 | (0): Block( 161 | (net): Sequential( 162 | (0): ReLU() 163 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 164 | (2): Cropping2D() 165 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 166 | (4): Dropout2d(p=0.5) 167 | ) 168 | ) 169 | (1): Block( 170 | (net): Sequential( 171 | (0): ReLU() 172 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 173 | (2): Cropping2D() 174 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 175 | (4): Dropout2d(p=0.5) 176 | ) 177 | ) 178 | (2): Block( 179 | (net): Sequential( 180 | (0): ReLU() 181 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 182 | (2): Cropping2D() 183 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 184 | (4): Dropout2d(p=0.5) 185 | ) 186 | ) 187 | (3): Block( 188 | (net): Sequential( 189 | (0): ReLU() 190 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 191 | (2): Cropping2D() 192 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 193 | ) 194 | ) 195 | (4): Block( 196 | (net): Sequential( 197 | (0): ReLU() 198 | (1): ConvTranspose2d(1024, 256, kernel_size=(4, 4), stride=(2, 2), bias=False) 199 | (2): Cropping2D() 200 | (3): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 201 | ) 202 | ) 203 | (5): Block( 204 | (net): Sequential( 205 | (0): ReLU() 206 | (1): ConvTranspose2d(512, 128, kernel_size=(4, 4), stride=(2, 2), bias=False) 207 | (2): Cropping2D() 208 | (3): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 209 | ) 210 | ) 211 | (6): ReLU() 212 | (7): Conv2d(256, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 213 | (8): Tanh() 214 | ) 215 | ) 216 | ) 217 | Total number of parameters: 82080579 218 | Discriminator( 219 | (net): Sequential( 220 | (0): Conv2d(42, 64, kernel_size=(4, 4), stride=(2, 2)) 221 | (1): Block( 222 | (net): Sequential( 223 | (0): LeakyReLU(negative_slope=0.2) 224 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 225 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 226 | ) 227 | ) 228 | (2): Block( 229 | (net): Sequential( 230 | (0): LeakyReLU(negative_slope=0.2) 231 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 232 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 233 | ) 234 | ) 235 | (3): Block( 236 | (net): Sequential( 237 | (0): LeakyReLU(negative_slope=0.2) 238 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 239 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 240 | ) 241 | ) 242 | (4): Block( 243 | (net): Sequential( 244 | (0): LeakyReLU(negative_slope=0.2) 245 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 246 | ) 247 | ) 248 | (5): Sigmoid() 249 | (6): Flatten() 250 | ) 251 | ) 252 | Total number of parameters: 2803776 253 | ----------------------------------------------- 254 | Num iterations : 1000 255 | Epoch : 1 | Progress : 0.00 | Total Loss : 12.5045 | Gen Total Loss : 9.3643, Gen Ad Loss : 1.9070, Gen LL Loss : 0.7457 | Disc Loss : 3.1401 256 | -------------------------------------------------------------------------------- /src_deformable/logs/deformable_fasion_unif_init: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | Model options . . 3 | annotations_file_test: ../../pose-gan-clean/pose-gan/data/fasion-annotation-test.csv 4 | annotations_file_test_paf: ../../pose-gan-clean/pose-gan/data/fasion-annotation-paf-test0.csv 5 | annotations_file_train: ../../pose-gan-clean/pose-gan/data/fasion-annotation-train.csv 6 | annotations_file_train_paf: ../../pose-gan-clean/pose-gan/data/fasion-annotation-paf-train0.csv 7 | batch_size: 2 8 | checkpoint_ratio: 5 9 | checkpoints_dir: ../exp/deformable_fasion_unif_init/models 10 | compute_h36m_paf_split: 0 11 | content_loss_layer: none 12 | data_Dir: ../../pose-gan-clean/pose-gan/data/ 13 | dataset: fasion 14 | disc_type: call 15 | discriminator_checkpoint: None 16 | display_ratio: 50 17 | expID: deformable_fasion_unif_init 18 | frame_diff: 10 19 | gan_penalty_weight: 1 20 | gen_type: baseline 21 | generated_images_dir: output/generated_images/fasion-restricted 22 | generator_checkpoint: None 23 | image_size: (256, 256) 24 | images_dir_test: ../../pose-gan-clean/pose-gan/data/fasion-dataset/test 25 | images_dir_train: ../../pose-gan-clean/pose-gan/data/fasion-dataset/train 26 | images_for_test: 12000 27 | iters_per_epoch: 1000 28 | l1_penalty_weight: 10.0 29 | learning_rate: 0.0002 30 | load_generated_images: 0 31 | log_file: output/full/fasion/log 32 | lstruct_penalty_weight: 0 33 | nn_loss_area_size: 1 34 | num_stacks: 4 35 | number_of_epochs: 90 36 | output_dir: ../exp/deformable_fasion_unif_init/results 37 | pairs_file_test: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test.csv 38 | pairs_file_test_interpol: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test-interpol.csv 39 | pairs_file_test_iterative: ../../pose-gan-clean/pose-gan/data/fasion-pairs-test-iterative.csv 40 | pairs_file_train: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train.csv 41 | pairs_file_train_interpol: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train-interpol.csv 42 | pairs_file_train_iterative: ../../pose-gan-clean/pose-gan/data/fasion-pairs-train-iterative.csv 43 | pose_dim: 18 44 | pose_estimator: pose_estimator.h5 45 | resume: 0 46 | saveDir: ../exp/deformable_fasion_unif_init 47 | start_epoch: 0 48 | tmp_pose_dir: tmp/fasion/ 49 | training_ratio: 1 50 | tv_penalty_weight: 0 51 | use_dropout_test: 0 52 | use_input_pose: True 53 | warp_agg: max 54 | warp_skip: mask 55 | (256, 256) 56 | Statistics for loaded dataset : Human 3.6 57 | Number of images: 52712 58 | Number of pairs train: 101268 59 | Number of pairs test: 8670 60 | Statistics for loaded dataset : Human 3.6 61 | Number of images: 52712 62 | Number of pairs train: 101268 63 | Number of pairs test: 8670 64 | ---------- Networks initialized ------------- 65 | Deformable_Generator( 66 | (encoder_app): encoder( 67 | (net): ModuleList( 68 | (0): Conv2d(21, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 69 | (1): Block( 70 | (net): Sequential( 71 | (0): LeakyReLU(negative_slope=0.2) 72 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 73 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 74 | ) 75 | ) 76 | (2): Block( 77 | (net): Sequential( 78 | (0): LeakyReLU(negative_slope=0.2) 79 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 80 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 81 | ) 82 | ) 83 | (3): Block( 84 | (net): Sequential( 85 | (0): LeakyReLU(negative_slope=0.2) 86 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 87 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 88 | ) 89 | ) 90 | (4): Block( 91 | (net): Sequential( 92 | (0): LeakyReLU(negative_slope=0.2) 93 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 94 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 95 | ) 96 | ) 97 | (5): Block( 98 | (net): Sequential( 99 | (0): LeakyReLU(negative_slope=0.2) 100 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 101 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 102 | ) 103 | ) 104 | (6): Block( 105 | (net): Sequential( 106 | (0): LeakyReLU(negative_slope=0.2) 107 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 108 | ) 109 | ) 110 | ) 111 | ) 112 | (encoder_pose): encoder( 113 | (net): ModuleList( 114 | (0): Conv2d(18, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 115 | (1): Block( 116 | (net): Sequential( 117 | (0): LeakyReLU(negative_slope=0.2) 118 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 119 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 120 | ) 121 | ) 122 | (2): Block( 123 | (net): Sequential( 124 | (0): LeakyReLU(negative_slope=0.2) 125 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 126 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 127 | ) 128 | ) 129 | (3): Block( 130 | (net): Sequential( 131 | (0): LeakyReLU(negative_slope=0.2) 132 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 133 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 134 | ) 135 | ) 136 | (4): Block( 137 | (net): Sequential( 138 | (0): LeakyReLU(negative_slope=0.2) 139 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 140 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 141 | ) 142 | ) 143 | (5): Block( 144 | (net): Sequential( 145 | (0): LeakyReLU(negative_slope=0.2) 146 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 147 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 148 | ) 149 | ) 150 | (6): Block( 151 | (net): Sequential( 152 | (0): LeakyReLU(negative_slope=0.2) 153 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 154 | ) 155 | ) 156 | ) 157 | ) 158 | (decoder): decoder( 159 | (net): ModuleList( 160 | (0): Block( 161 | (net): Sequential( 162 | (0): ReLU() 163 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 164 | (2): Cropping2D() 165 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 166 | (4): Dropout2d(p=0.5) 167 | ) 168 | ) 169 | (1): Block( 170 | (net): Sequential( 171 | (0): ReLU() 172 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 173 | (2): Cropping2D() 174 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 175 | (4): Dropout2d(p=0.5) 176 | ) 177 | ) 178 | (2): Block( 179 | (net): Sequential( 180 | (0): ReLU() 181 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 182 | (2): Cropping2D() 183 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 184 | (4): Dropout2d(p=0.5) 185 | ) 186 | ) 187 | (3): Block( 188 | (net): Sequential( 189 | (0): ReLU() 190 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 191 | (2): Cropping2D() 192 | (3): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 193 | ) 194 | ) 195 | (4): Block( 196 | (net): Sequential( 197 | (0): ReLU() 198 | (1): ConvTranspose2d(1024, 256, kernel_size=(4, 4), stride=(2, 2), bias=False) 199 | (2): Cropping2D() 200 | (3): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 201 | ) 202 | ) 203 | (5): Block( 204 | (net): Sequential( 205 | (0): ReLU() 206 | (1): ConvTranspose2d(512, 128, kernel_size=(4, 4), stride=(2, 2), bias=False) 207 | (2): Cropping2D() 208 | (3): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 209 | ) 210 | ) 211 | (6): ReLU() 212 | (7): Conv2d(256, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 213 | (8): Tanh() 214 | ) 215 | ) 216 | )/home/project/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. 217 | from ._conv import register_converters as _register_converters 218 | Using TensorFlow backend. 219 | /home/project/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6 220 | return f(*args, **kwds) 221 | 222 | Total number of parameters: 82080579 223 | Discriminator( 224 | (net): Sequential( 225 | (0): Conv2d(42, 64, kernel_size=(4, 4), stride=(2, 2)) 226 | (1): Block( 227 | (net): Sequential( 228 | (0): LeakyReLU(negative_slope=0.2) 229 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 230 | (2): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 231 | ) 232 | ) 233 | (2): Block( 234 | (net): Sequential( 235 | (0): LeakyReLU(negative_slope=0.2) 236 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 237 | (2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 238 | ) 239 | ) 240 | (3): Block( 241 | (net): Sequential( 242 | (0): LeakyReLU(negative_slope=0.2) 243 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 244 | (2): InstanceNorm2d(512, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False) 245 | ) 246 | ) 247 | (4): Block( 248 | (net): Sequential( 249 | (0): LeakyReLU(negative_slope=0.2) 250 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 251 | ) 252 | ) 253 | (5): Sigmoid() 254 | (6): Flatten() 255 | ) 256 | ) 257 | Total number of parameters: 2803776 258 | ----------------------------------------------- 259 | Traceback (most recent call last): 260 | File "main.py", line 132, in 261 | main() 262 | File "main.py", line 72, in main 263 | model = DeformablePose_GAN(opt) 264 | File "/home/project/saurabh/pose-transfer/src_deformable/models/pose_gan.py", line 45, in __init__ 265 | self.gen.cuda() 266 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 249, in cuda 267 | return self._apply(lambda t: t.cuda(device)) 268 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 176, in _apply 269 | module._apply(fn) 270 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 176, in _apply 271 | module._apply(fn) 272 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 176, in _apply 273 | module._apply(fn) 274 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 182, in _apply 275 | param.data = fn(param.data) 276 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 249, in 277 | return self._apply(lambda t: t.cuda(device)) 278 | File "/home/project/anaconda3/lib/python3.6/site-packages/torch/cuda/__init__.py", line 146, in _lazy_init 279 | def _lazy_init(): 280 | KeyboardInterrupt 281 | -------------------------------------------------------------------------------- /src_deformable/logs/gen_full_fasion: -------------------------------------------------------------------------------- 1 | Statistics for loaded dataset : fasion 2 | Number of images: 52712 3 | Number of pairs train: 101268 4 | Number of pairs test: 8670 5 | ---------- Networks initialized ------------- 6 | Deformable_Generator( 7 | (encoder_app): encoder( 8 | (net): ModuleList( 9 | (0): Conv2d(21, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 10 | (1): Block( 11 | (net): ModuleList( 12 | (0): LeakyReLU(negative_slope=0.2) 13 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 14 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 15 | ) 16 | ) 17 | (2): Block( 18 | (net): ModuleList( 19 | (0): LeakyReLU(negative_slope=0.2) 20 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 21 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 22 | ) 23 | ) 24 | (3): Block( 25 | (net): ModuleList( 26 | (0): LeakyReLU(negative_slope=0.2) 27 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 28 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 29 | ) 30 | ) 31 | (4): Block( 32 | (net): ModuleList( 33 | (0): LeakyReLU(negative_slope=0.2) 34 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 35 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 36 | ) 37 | ) 38 | (5): Block( 39 | (net): ModuleList( 40 | (0): LeakyReLU(negative_slope=0.2) 41 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 42 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 43 | ) 44 | ) 45 | (6): Block( 46 | (net): ModuleList( 47 | (0): LeakyReLU(negative_slope=0.2) 48 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 49 | ) 50 | ) 51 | ) 52 | ) 53 | (encoder_pose): encoder( 54 | (net): ModuleList( 55 | (0): Conv2d(18, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 56 | (1): Block( 57 | (net): ModuleList( 58 | (0): LeakyReLU(negative_slope=0.2) 59 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 60 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 61 | ) 62 | ) 63 | (2): Block( 64 | (net): ModuleList( 65 | (0): LeakyReLU(negative_slope=0.2) 66 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 67 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 68 | ) 69 | ) 70 | (3): Block( 71 | (net): ModuleList( 72 | (0): LeakyReLU(negative_slope=0.2) 73 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 74 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 75 | ) 76 | ) 77 | (4): Block( 78 | (net): ModuleList( 79 | (0): LeakyReLU(negative_slope=0.2) 80 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 81 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 82 | ) 83 | ) 84 | (5): Block( 85 | (net): ModuleList( 86 | (0): LeakyReLU(negative_slope=0.2) 87 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 88 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 89 | ) 90 | ) 91 | (6): Block( 92 | (net): ModuleList( 93 | (0): LeakyReLU(negative_slope=0.2) 94 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 95 | ) 96 | ) 97 | ) 98 | ) 99 | (decoder): decoder( 100 | (net): ModuleList( 101 | (0): Block( 102 | (net): ModuleList( 103 | (0): ReLU() 104 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 105 | (2): Cropping2D() 106 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 107 | (4): Dropout2d(p=0.5) 108 | ) 109 | ) 110 | (1): Block( 111 | (net): ModuleList( 112 | (0): ReLU() 113 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 114 | (2): Cropping2D() 115 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 116 | (4): Dropout2d(p=0.5) 117 | ) 118 | ) 119 | (2): Block( 120 | (net): ModuleList( 121 | (0): ReLU() 122 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 123 | (2): Cropping2D() 124 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 125 | (4): Dropout2d(p=0.5) 126 | ) 127 | ) 128 | (3): Block( 129 | (net): ModuleList( 130 | (0): ReLU() 131 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 132 | (2): Cropping2D() 133 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 134 | ) 135 | ) 136 | (4): Block( 137 | (net): ModuleList( 138 | (0): ReLU() 139 | (1): ConvTranspose2d(1024, 256, kernel_size=(4, 4), stride=(2, 2), bias=False) 140 | (2): Cropping2D() 141 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 142 | ) 143 | ) 144 | (5): Block( 145 | (net): ModuleList( 146 | (0): ReLU() 147 | (1): ConvTranspose2d(512, 128, kernel_size=(4, 4), stride=(2, 2), bias=False) 148 | (2): Cropping2D() 149 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 150 | ) 151 | ) 152 | (6): ReLU() 153 | (7): Conv2d(256, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 154 | (8): Tanh() 155 | ) 156 | ) 157 | ) 158 | Total number of parameters: 82080611 159 | Discriminator( 160 | (net): Sequential( 161 | (0): Conv2d(42, 64, kernel_size=(4, 4), stride=(2, 2)) 162 | (1): Block( 163 | (net): ModuleList( 164 | (0): LeakyReLU(negative_slope=0.2) 165 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 166 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 167 | ) 168 | ) 169 | (2): Block( 170 | (net): ModuleList( 171 | (0): LeakyReLU(negative_slope=0.2) 172 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 173 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 174 | ) 175 | ) 176 | (3): Block( 177 | (net): ModuleList( 178 | (0): LeakyReLU(negative_slope=0.2) 179 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 180 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 181 | ) 182 | ) 183 | (4): Block( 184 | (net): ModuleList( 185 | (0): LeakyReLU(negative_slope=0.2) 186 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 187 | ) 188 | ) 189 | (5): Sigmoid() 190 | (6): Flatten() 191 | ) 192 | ) 193 | Total number of parameters: 2803782 194 | ----------------------------------------------- 195 | Resume gen from epoch 90 196 | Resume disc from epoch 90 197 | 0.0 198 | 0.011534025374855825 199 | 0.02306805074971165 200 | 0.03460207612456748 201 | 0.0461361014994233 202 | 0.05767012687427912 203 | 0.06920415224913495 204 | 0.08073817762399077 205 | 0.0922722029988466 206 | 0.10380622837370242 207 | 0.11534025374855825 208 | 0.12687427912341406 209 | 0.1384083044982699 210 | 0.14994232987312572 211 | 0.16147635524798154 212 | 0.17301038062283736 213 | 0.1845444059976932 214 | 0.19607843137254902 215 | 0.20761245674740483 216 | 0.21914648212226068 217 | 0.2306805074971165 218 | 0.2422145328719723 219 | 0.2537485582468281 220 | 0.26528258362168394 221 | 0.2768166089965398 222 | 0.28835063437139563 223 | 0.29988465974625145 224 | 0.31141868512110726 225 | 0.3229527104959631 226 | 0.3344867358708189 227 | 0.3460207612456747 228 | 0.3575547866205306 229 | 0.3690888119953864 230 | 0.3806228373702422 231 | 0.39215686274509803 232 | 0.40369088811995385 233 | 0.41522491349480967 234 | 0.42675893886966554 235 | 0.43829296424452135 236 | 0.44982698961937717 237 | 0.461361014994233 238 | 0.4728950403690888 239 | 0.4844290657439446 240 | 0.49596309111880044 241 | 0.5074971164936563 242 | 0.5190311418685121 243 | 0.5305651672433679 244 | 0.5420991926182238 245 | 0.5536332179930796 246 | 0.5651672433679354 247 | 0.5767012687427913 248 | 0.5882352941176471 249 | 0.5997693194925029 250 | 0.6113033448673587 251 | 0.6228373702422145 252 | 0.6343713956170703 253 | 0.6459054209919262 254 | 0.657439446366782 255 | 0.6689734717416378 256 | 0.6805074971164936 257 | 0.6920415224913494 258 | 0.7035755478662054 259 | 0.7151095732410612 260 | 0.726643598615917 261 | 0.7381776239907728 262 | 0.7497116493656286 263 | 0.7612456747404844 264 | 0.7727797001153403 265 | 0.7843137254901961 266 | 0.7958477508650519 267 | 0.8073817762399077 268 | 0.8189158016147635 269 | 0.8304498269896193 270 | 0.8419838523644751 271 | 0.8535178777393311 272 | 0.8650519031141869 273 | 0.8765859284890427 274 | 0.8881199538638985 275 | 0.8996539792387543 276 | 0.9111880046136102 277 | 0.922722029988466 278 | 0.9342560553633218 279 | 0.9457900807381776 280 | 0.9573241061130334 281 | 0.9688581314878892 282 | 0.9803921568627451 283 | Using TensorFlow backend. 284 | /home/safeer/anaconda3/envs/saurabh/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6 285 | return f(*args, **kwds) 286 | /home/safeer/link_project/saurabh/pose-transfer/src_deformable/models/networks.py:29: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_. 287 | init.xavier_uniform(m.weight.data) 288 | 0.9919261822376009 289 | -------------------------------------------------------------------------------- /src_deformable/logs/gen_full_h36m: -------------------------------------------------------------------------------- 1 | Statistics for loaded dataset : h36m 2 | Number of images: 92586 3 | Number of pairs train: 77412 4 | Number of pairs test: 12000 5 | ---------- Networks initialized ------------- 6 | Deformable_Generator( 7 | (encoder_app): encoder( 8 | (net): ModuleList( 9 | (0): Conv2d(19, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 10 | (1): Block( 11 | (net): ModuleList( 12 | (0): LeakyReLU(negative_slope=0.2) 13 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 14 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 15 | ) 16 | ) 17 | (2): Block( 18 | (net): ModuleList( 19 | (0): LeakyReLU(negative_slope=0.2) 20 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 21 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 22 | ) 23 | ) 24 | (3): Block( 25 | (net): ModuleList( 26 | (0): LeakyReLU(negative_slope=0.2) 27 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 28 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 29 | ) 30 | ) 31 | (4): Block( 32 | (net): ModuleList( 33 | (0): LeakyReLU(negative_slope=0.2) 34 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 35 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 36 | ) 37 | ) 38 | (5): Block( 39 | (net): ModuleList( 40 | (0): LeakyReLU(negative_slope=0.2) 41 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 42 | ) 43 | ) 44 | ) 45 | ) 46 | (encoder_pose): encoder( 47 | (net): ModuleList( 48 | (0): Conv2d(16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 49 | (1): Block( 50 | (net): ModuleList( 51 | (0): LeakyReLU(negative_slope=0.2) 52 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 53 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 54 | ) 55 | ) 56 | (2): Block( 57 | (net): ModuleList( 58 | (0): LeakyReLU(negative_slope=0.2) 59 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 60 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 61 | ) 62 | ) 63 | (3): Block( 64 | (net): ModuleList( 65 | (0): LeakyReLU(negative_slope=0.2) 66 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 67 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 68 | ) 69 | ) 70 | (4): Block( 71 | (net): ModuleList( 72 | (0): LeakyReLU(negative_slope=0.2) 73 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 74 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 75 | ) 76 | ) 77 | (5): Block( 78 | (net): ModuleList( 79 | (0): LeakyReLU(negative_slope=0.2) 80 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 81 | ) 82 | ) 83 | ) 84 | ) 85 | (decoder): decoder( 86 | (net): ModuleList( 87 | (0): Block( 88 | (net): ModuleList( 89 | (0): ReLU() 90 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 91 | (2): Cropping2D() 92 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 93 | (4): Dropout2d(p=0.5) 94 | ) 95 | ) 96 | (1): Block( 97 | (net): ModuleList( 98 | (0): ReLU() 99 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 100 | (2): Cropping2D() 101 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 102 | (4): Dropout2d(p=0.5) 103 | ) 104 | ) 105 | (2): Block( 106 | (net): ModuleList( 107 | (0): ReLU() 108 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 109 | (2): Cropping2D() 110 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 111 | (4): Dropout2d(p=0.5) 112 | ) 113 | ) 114 | (3): Block( 115 | (net): ModuleList( 116 | (0): ReLU() 117 | (1): ConvTranspose2d(1024, 256, kernel_size=(4, 4), stride=(2, 2), bias=False) 118 | (2): Cropping2D() 119 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 120 | ) 121 | ) 122 | (4): Block( 123 | (net): ModuleList( 124 | (0): ReLU() 125 | (1): ConvTranspose2d(512, 128, kernel_size=(4, 4), stride=(2, 2), bias=False) 126 | (2): Cropping2D() 127 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 128 | ) 129 | ) 130 | (5): ReLU() 131 | (6): Conv2d(256, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 132 | (7): Tanh() 133 | ) 134 | ) 135 | ) 136 | Total number of parameters: 61106781 137 | Discriminator( 138 | (net): Sequential( 139 | (0): Conv2d(38, 64, kernel_size=(4, 4), stride=(2, 2)) 140 | (1): Block( 141 | (net): ModuleList( 142 | (0): LeakyReLU(negative_slope=0.2) 143 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 144 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 145 | ) 146 | ) 147 | (2): Block( 148 | (net): ModuleList( 149 | (0): LeakyReLU(negative_slope=0.2) 150 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 151 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 152 | ) 153 | ) 154 | (3): Block( 155 | (net): ModuleList( 156 | (0): LeakyReLU(negative_slope=0.2) 157 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 158 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 159 | ) 160 | ) 161 | (4): Block( 162 | (net): ModuleList( 163 | (0): LeakyReLU(negative_slope=0.2) 164 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 165 | ) 166 | ) 167 | (5): Sigmoid() 168 | (6): Flatten() 169 | ) 170 | ) 171 | Total number of parameters: 2799686 172 | ----------------------------------------------- 173 | Resume gen from epoch 90 174 | Resume disc from epoch 90 175 | 0.0 176 | 0.016666666666666666 177 | 0.03333333333333333 178 | 0.05 179 | 0.06666666666666667 180 | 0.08333333333333333 181 | 0.1 182 | 0.11666666666666667 183 | 0.13333333333333333 184 | 0.15 185 | 0.16666666666666666 186 | 0.18333333333333332 187 | 0.2 188 | 0.21666666666666667 189 | 0.23333333333333334 190 | 0.25 191 | 0.26666666666666666 192 | 0.2833333333333333 193 | 0.3 194 | 0.31666666666666665 195 | 0.3333333333333333 196 | 0.35 197 | 0.36666666666666664 198 | 0.38333333333333336 199 | 0.4 200 | 0.4166666666666667 201 | 0.43333333333333335 202 | 0.45 203 | 0.4666666666666667 204 | 0.48333333333333334 205 | 0.5 206 | 0.5166666666666667 207 | 0.5333333333333333 208 | 0.55 209 | 0.5666666666666667 210 | 0.5833333333333334 211 | 0.6 212 | 0.6166666666666667 213 | 0.6333333333333333 214 | 0.65 215 | 0.6666666666666666 216 | 0.6833333333333333 217 | 0.7 218 | 0.7166666666666667 219 | 0.7333333333333333 220 | 0.75 221 | 0.7666666666666667 222 | 0.7833333333333333 223 | 0.8 224 | 0.8166666666666667 225 | 0.8333333333333334 226 | 0.85 227 | 0.8666666666666667 228 | 0.8833333333333333 229 | 0.9 230 | 0.9166666666666666 231 | 0.9333333333333333 232 | 0.95 233 | 0.9666666666666667 234 | /home/project/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. 235 | from ._conv import register_converters as _register_converters 236 | Using TensorFlow backend. 237 | /home/project/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6 238 | return f(*args, **kwds) 239 | /home/project/saurabh/pose-transfer/src_deformable/models/networks.py:29: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_. 240 | init.xavier_uniform(m.weight.data) 241 | 0.9833333333333333 242 | -------------------------------------------------------------------------------- /src_deformable/logs/stacked_deformable_h36m: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | Model options . . 3 | annotations_file_test: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-test.csv 4 | annotations_file_test_paf: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-paf-test0.csv 5 | annotations_file_train: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-train.csv 6 | annotations_file_train_paf: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-annotation-paf-train0.csv 7 | batch_size: 2 8 | checkpoint_ratio: 5 9 | checkpoints_dir: ../exp/stacked_deformable_h36m/models 10 | compute_h36m_paf_split: 0 11 | content_loss_layer: none 12 | data_Dir: ../../pose-gan-clean/pose-gan-h36m-fg/data/ 13 | dataset: h36m 14 | disc_type: call 15 | discriminator_checkpoint: None 16 | display_ratio: 50 17 | expID: stacked_deformable_h36m 18 | frame_diff: 10 19 | gan_penalty_weight: 1 20 | gen_type: stacked 21 | generated_images_dir: ../exp/stacked_deformable_h36m/results/generated 22 | generator_checkpoint: None 23 | image_size: (224, 224) 24 | images_dir_test: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-dataset/test 25 | images_dir_train: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-dataset/train 26 | images_for_test: 12000 27 | iters_per_epoch: 1000 28 | l1_penalty_weight: 10.0 29 | learning_rate: 0.0002 30 | load_generated_images: 0 31 | log_file: output/full/fasion/log 32 | lstruct_penalty_weight: 0 33 | nn_loss_area_size: 1 34 | num_stacks: 2 35 | number_of_epochs: 90 36 | output_dir: ../exp/stacked_deformable_h36m/results 37 | pairs_file_test: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-test.csv 38 | pairs_file_test_interpol: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-test-interpol.csv 39 | pairs_file_test_iterative: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-test-iterative.csv 40 | pairs_file_train: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-train.csv 41 | pairs_file_train_interpol: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-train-interpol.csv 42 | pairs_file_train_iterative: ../../pose-gan-clean/pose-gan-h36m-fg/data/h36m-pairs-train-iterative.csv 43 | pose_dim: 16 44 | pose_estimator: pose_estimator.h5 45 | resume: 0 46 | saveDir: ../exp/stacked_deformable_h36m 47 | start_epoch: 0 48 | tmp_pose_dir: tmp/h36m/ 49 | training_ratio: 1 50 | tv_penalty_weight: 0 51 | use_dropout_test: 0 52 | use_input_pose: True 53 | warp_agg: max 54 | warp_skip: mask 55 | Statistics for loaded dataset : h36m 56 | Number of images: 92586 57 | Number of pairs train: 73812 58 | Number of pairs test: 12000 59 | Statistics for loaded dataset : h36m 60 | Number of images: 92586 61 | Number of pairs train: 73812 62 | Number of pairs test: 12000 63 | ---------- Networks initialized ------------- 64 | Stacked_Generator( 65 | (generator): Deformable_Generator( 66 | (encoder_app): encoder( 67 | (net): ModuleList( 68 | (0): Conv2d(19, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 69 | (1): Block( 70 | (net): ModuleList( 71 | (0): LeakyReLU(negative_slope=0.2) 72 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 73 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 74 | ) 75 | ) 76 | (2): Block( 77 | (net): ModuleList( 78 | (0): LeakyReLU(negative_slope=0.2) 79 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 80 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 81 | ) 82 | ) 83 | (3): Block( 84 | (net): ModuleList( 85 | (0): LeakyReLU(negative_slope=0.2) 86 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 87 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 88 | ) 89 | ) 90 | (4): Block( 91 | (net): ModuleList( 92 | (0): LeakyReLU(negative_slope=0.2) 93 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 94 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 95 | ) 96 | ) 97 | (5): Block( 98 | (net): ModuleList( 99 | (0): LeakyReLU(negative_slope=0.2) 100 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 101 | ) 102 | ) 103 | ) 104 | ) 105 | (encoder_pose): encoder( 106 | (net): ModuleList( 107 | (0): Conv2d(16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 108 | (1): Block( 109 | (net): ModuleList( 110 | (0): LeakyReLU(negative_slope=0.2) 111 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 112 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 113 | ) 114 | ) 115 | (2): Block( 116 | (net): ModuleList( 117 | (0): LeakyReLU(negative_slope=0.2) 118 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 119 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 120 | ) 121 | ) 122 | (3): Block( 123 | (net): ModuleList( 124 | (0): LeakyReLU(negative_slope=0.2) 125 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 126 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 127 | ) 128 | ) 129 | (4): Block( 130 | (net): ModuleList( 131 | (0): LeakyReLU(negative_slope=0.2) 132 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 133 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 134 | ) 135 | ) 136 | (5): Block( 137 | (net): ModuleList( 138 | (0): LeakyReLU(negative_slope=0.2) 139 | (1): Conv2d(512, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 140 | ) 141 | ) 142 | ) 143 | ) 144 | (decoder): decoder( 145 | (net): ModuleList( 146 | (0): Block( 147 | (net): ModuleList( 148 | (0): ReLU() 149 | (1): ConvTranspose2d(1024, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 150 | (2): Cropping2D() 151 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 152 | (4): Dropout2d(p=0.5) 153 | ) 154 | ) 155 | (1): Block( 156 | (net): ModuleList( 157 | (0): ReLU() 158 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 159 | (2): Cropping2D() 160 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 161 | (4): Dropout2d(p=0.5) 162 | ) 163 | ) 164 | (2): Block( 165 | (net): ModuleList( 166 | (0): ReLU() 167 | (1): ConvTranspose2d(1536, 512, kernel_size=(4, 4), stride=(2, 2), bias=False) 168 | (2): Cropping2D() 169 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 170 | (4): Dropout2d(p=0.5) 171 | ) 172 | ) 173 | (3): Block( 174 | (net): ModuleList( 175 | (0): ReLU() 176 | (1): ConvTranspose2d(1024, 256, kernel_size=(4, 4), stride=(2, 2), bias=False) 177 | (2): Cropping2D() 178 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 179 | ) 180 | ) 181 | (4): Block( 182 | (net): ModuleList( 183 | (0): ReLU() 184 | (1): ConvTranspose2d(512, 128, kernel_size=(4, 4), stride=(2, 2), bias=False) 185 | (2): Cropping2D() 186 | (3): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 187 | ) 188 | ) 189 | (5): ReLU() 190 | (6): Conv2d(256, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) 191 | (7): Tanh() 192 | ) 193 | ) 194 | ) 195 | ) 196 | Total number of parameters: 61106781 197 | Discriminator( 198 | (net): Sequential( 199 | (0): Conv2d(38, 64, kernel_size=(4, 4), stride=(2, 2)) 200 | (1): Block( 201 | (net): ModuleList( 202 | (0): LeakyReLU(negative_slope=0.2) 203 | (1): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 204 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 205 | ) 206 | ) 207 | (2): Block( 208 | (net): ModuleList( 209 | (0): LeakyReLU(negative_slope=0.2) 210 | (1): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 211 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 212 | ) 213 | ) 214 | (3): Block( 215 | (net): ModuleList( 216 | (0): LeakyReLU(negative_slope=0.2) 217 | (1): Conv2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 218 | (2): InstanceNorm3d(1, eps=0.001, momentum=0.1, affine=True, track_running_stats=False) 219 | ) 220 | ) 221 | (4): Block( 222 | (net): ModuleList( 223 | (0): LeakyReLU(negative_slope=0.2) 224 | (1): Conv2d(512, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False) 225 | ) 226 | ) 227 | (5): Sigmoid() 228 | (6): Flatten() 229 | ) 230 | ) 231 | Total number of parameters: 2799686 232 | ----------------------------------------------- 233 | Num iterations : 1000 234 | Epoch : 1 | Progress : 0.00 | Total Loss : 12.1988 | Gen Total Loss : 10.6425, Gen Ad Loss : 1.5654, Gen LL Loss : 9.0771 | Disc Total Loss : 1.5563, Disc True Loss : 0.8132, Disc Fake Loss : 0.7432 235 | -------------------------------------------------------------------------------- /src_deformable/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | # shitty thing overriding src code in current dir 4 | if('/home/anurag/multiperson/detectron/lib' in sys.path): 5 | sys.path.remove('/home/anurag/multiperson/detectron/lib') 6 | 7 | import matplotlib 8 | matplotlib.use('Agg') 9 | import pylab as plt 10 | import os 11 | import sys 12 | import torch 13 | import numpy as np 14 | import cv2 15 | import time 16 | import datetime 17 | from opts import opts 18 | # from tqdm import tqdm 19 | from datasets.PoseTransfer_Dataset import PoseTransfer_Dataset 20 | from models.networks import Deformable_Generator, Discriminator 21 | from models.pose_gan import DeformablePose_GAN 22 | from utils import pose_utils 23 | 24 | def load_sample(iter,dataloader,name, gen_type): 25 | if (gen_type == 'baseline'): 26 | try: 27 | input, target, warps, masks = iter.next() 28 | except: 29 | # resetting iterator 30 | print("Resetting iterator for loader :", name) 31 | iter = dataloader.__iter__() 32 | input, target, warps, masks = iter.next() 33 | return input, target, warps, masks, iter 34 | elif (gen_type == 'stacked'): 35 | try: 36 | input, target, interpol_pose, interpol_warps, interpol_masks = iter.next() 37 | except: 38 | # resetting iterator 39 | print("Resetting iterator for loader :", name) 40 | iter = dataloader.__iter__() 41 | input, target, interpol_pose, interpol_warps, interpol_masks = iter.next() 42 | return input, target, interpol_pose, interpol_warps, interpol_masks, iter 43 | 44 | def main(): 45 | opt = opts().parse() 46 | print("Model options . .") 47 | for k, v in sorted(vars(opt).items()): 48 | print(' %s: %s' % (str(k), str(v))) 49 | 50 | loader_train = torch.utils.data.DataLoader( 51 | PoseTransfer_Dataset(vars(opt), split ='train'), 52 | batch_size = opt.batch_size, 53 | shuffle = True, 54 | ) 55 | 56 | loader_test = torch.utils.data.DataLoader( 57 | PoseTransfer_Dataset(vars(opt), split='test'), 58 | batch_size=opt.batch_size, 59 | shuffle=True, 60 | ) 61 | 62 | model = DeformablePose_GAN(opt).cuda() 63 | start_epoch = 1 64 | if(opt.resume==1): 65 | start_epoch = model.resume(opt.checkpoints_dir) 66 | 67 | # discriminator and generator updates; iterators initialized out of epoch loop 68 | loader_train_iter = loader_train.__iter__() 69 | loader_test_iter = loader_test.__iter__() 70 | for epoch in range(start_epoch, opt.number_of_epochs + 1): 71 | # num_iter = len(gen_loader_train) 72 | gen_losses, disc_losses = [], [] 73 | num_iterations = opt.iters_per_epoch 74 | print("Num iterations : ", num_iterations) 75 | 76 | # note warps must be converted into float 77 | for it in range(num_iterations): 78 | for _ in range(opt.training_ratio): 79 | if(opt.gen_type=='baseline'): 80 | 81 | input, target, warps, masks, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train', opt.gen_type) 82 | real_inp, real_target, _, _, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train', opt.gen_type) 83 | other_inputs = {'warps': warps.float().cuda(), 'masks': masks.cuda()} 84 | 85 | disc_loss = model.dis_update(input.cuda(),target.cuda(), other_inputs, 86 | real_inp.cuda(), real_target.cuda(), vars(opt)) 87 | disc_losses.append(disc_loss) 88 | 89 | elif (opt.gen_type == 'stacked'): 90 | 91 | input, target, interpol_pose, interpol_warps, interpol_masks, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train', opt.gen_type) 92 | other_inputs = {'interpol_pose' : interpol_pose.cuda(), 'interpol_warps': interpol_warps.float().cuda(), 'interpol_masks': interpol_masks.cuda()} 93 | real_inp, real_target, _, _,_, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train', opt.gen_type) 94 | 95 | # debugging statements 96 | # print(interpol_warps.shape) 97 | # print(interpol_masks.shape) 98 | 99 | disc_loss = model.dis_update(input.cuda(), target.cuda(), other_inputs, 100 | real_inp.cuda(), real_target.cuda(), vars(opt)) 101 | disc_losses.append(disc_loss) 102 | 103 | if (opt.gen_type == 'baseline'): 104 | 105 | input, target, warps, masks, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train', opt.gen_type) 106 | other_inputs = { 'warps' : warps.float().cuda(), 'masks' : masks.cuda() } 107 | out, outputs, gen_loss = model.gen_update(input.cuda(), target.cuda(), other_inputs, vars(opt)) 108 | gen_losses.append(gen_loss) 109 | 110 | elif (opt.gen_type == 'stacked'): 111 | 112 | input, target, interpol_pose, interpol_warps, interpol_masks, loader_train_iter = load_sample(loader_train_iter, loader_train, 'train', opt.gen_type) 113 | other_inputs = {'interpol_pose': interpol_pose.cuda(), 'interpol_warps': interpol_warps.float().cuda(), 'interpol_masks': interpol_masks.cuda()} 114 | out, outputs, gen_loss = model.gen_update(input.cuda(), target.cuda(), other_inputs, vars(opt)) 115 | gen_losses.append(gen_loss) 116 | 117 | if(it%opt.display_ratio==0): 118 | 119 | # print losses 120 | # edit later to visualize easily 121 | gen_total_loss, gen_ll_loss, gen_ad_loss = np.mean(np.array(gen_losses), axis=0) 122 | disc_total_loss, disc_true_loss, disc_fake_loss = np.mean(np.array(disc_losses), axis=0) 123 | total_loss = gen_total_loss + disc_total_loss 124 | print("Epoch : {8:d} | Progress : {0:.2f} | Total Loss : {1:.4f} | Gen Total Loss : {2:.4f}, Gen Ad Loss : {3:.4f}, Gen LL Loss : {4:.4f} | Disc Total Loss : {5:.4f}, Disc True Loss : {6:.4f}, Disc Fake Loss : {7:.4f} ".format(it/num_iterations, total_loss, gen_total_loss, gen_ad_loss, gen_ll_loss, disc_total_loss, disc_true_loss, disc_fake_loss, epoch)) 125 | 126 | # required to intermittently print output when using nohup 127 | sys.stdout.flush() 128 | 129 | # saving seen images 130 | 131 | if(opt.gen_type=='baseline'): 132 | images = pose_utils.display(input,target,out.data.cpu(),opt.use_input_pose,opt.pose_dim) 133 | else: 134 | images = pose_utils.display_stacked(input, interpol_pose, target, outputs, opt.num_stacks, opt.use_input_pose, opt.pose_dim) 135 | 136 | title = "epoch_{0}_{1}.png".format(str(epoch).zfill(3), str(it).zfill(5)) 137 | plt.imsave(os.path.join(opt.output_dir, 'train', title), images, cmap=plt.cm.gray) 138 | 139 | # saving results for unseen images 140 | 141 | if(opt.gen_type=='baseline'): 142 | input, target, warps, masks, loader_test_iter = load_sample(loader_test_iter, loader_test, 'test', opt.gen_type) 143 | out = model.gen(input.cuda(), warps.float().cuda(), masks.cuda()) 144 | images = pose_utils.display(input, target, out.data.cpu(), opt.use_input_pose, opt.pose_dim) 145 | 146 | elif(opt.gen_type=='stacked'): 147 | input, target, interpol_pose, interpol_warps, interpol_masks, loader_test_iter = load_sample(loader_test_iter, loader_test, 'test', opt.gen_type) 148 | out = model.gen(input.cuda(),interpol_pose.cuda(), interpol_warps.float().cuda(), interpol_masks.cuda()) 149 | images = pose_utils.display_stacked(input, interpol_pose, target, out, opt.num_stacks, opt.use_input_pose, opt.pose_dim) 150 | 151 | else: 152 | raise Exception('Invalid gen type !!') 153 | title = "epoch_{0}_{1}.png".format(str(epoch).zfill(3), str(it).zfill(5)) 154 | plt.imsave(os.path.join(opt.output_dir, 'test', title), images, cmap=plt.cm.gray) 155 | 156 | if(epoch%opt.checkpoint_ratio==0): 157 | model.save(opt.checkpoints_dir, epoch) 158 | 159 | main() -------------------------------------------------------------------------------- /src_deformable/models/__pycache__/networks.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/models/__pycache__/networks.cpython-36.pyc -------------------------------------------------------------------------------- /src_deformable/models/__pycache__/pose_gan.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/models/__pycache__/pose_gan.cpython-36.pyc -------------------------------------------------------------------------------- /src_deformable/models/networks.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/models/networks.pyc -------------------------------------------------------------------------------- /src_deformable/models/pose_gan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import os 4 | import itertools 5 | import torch.nn as nn 6 | from torch.autograd import Variable 7 | from utils import pose_utils 8 | from models.networks import Deformable_Generator, Stacked_Generator, Discriminator, gaussian_weights_init, print_network, xavier_weights_init 9 | from torchvision.models import vgg19 10 | 11 | class DeformablePose_GAN(nn.Module): 12 | def __init__(self, opt): 13 | super(DeformablePose_GAN, self).__init__() 14 | 15 | # load generator and discriminator models 16 | # adding extra layers for larger image size 17 | nfilters_decoder = (512, 512, 512, 256, 128, 3) if max(opt.image_size) < 256 else (512, 512, 512, 512, 256, 128, 3) 18 | nfilters_encoder = (64, 128, 256, 512, 512, 512) if max(opt.image_size) < 256 else (64, 128, 256, 512, 512, 512, 512) 19 | 20 | if (opt.use_input_pose): 21 | input_nc = 3 + 2*opt.pose_dim 22 | else: 23 | input_nc = 3 + opt.pose_dim 24 | 25 | self.batch_size = opt.batch_size 26 | self.num_stacks = opt.num_stacks 27 | self.pose_dim = opt.pose_dim 28 | if(opt.gen_type=='stacked'): 29 | self.gen = Stacked_Generator(input_nc, opt.num_stacks, opt.image_size, opt.pose_dim, nfilters_encoder, nfilters_decoder, opt.warp_skip, use_input_pose=opt.use_input_pose) 30 | # hack to get better results 31 | pretrained_gen_path = '../exp/' + 'full_' + opt.dataset + '/models/gen_090.pkl' 32 | self.gen.generator.load_state_dict(torch.load(pretrained_gen_path)) 33 | print("Loaded generator from pretrained model ") 34 | elif(opt.gen_type=='baseline'): 35 | self.gen = Deformable_Generator(input_nc, self.pose_dim, opt.image_size, nfilters_encoder, nfilters_decoder, opt.warp_skip, use_input_pose=opt.use_input_pose) 36 | else: 37 | raise Exception('Invalid gen_type') 38 | # discriminator also sees the output image for the target pose 39 | self.disc = Discriminator(input_nc + 3, use_input_pose=opt.use_input_pose) 40 | pretrained_disc_path = '../exp/' + 'full_' + opt.dataset + '/models/disc_090.pkl' 41 | print("Loaded discriminator from pretrained model ") 42 | self.disc.load_state_dict(torch.load(pretrained_disc_path)) 43 | 44 | print('---------- Networks initialized -------------') 45 | # print_network(self.gen) 46 | # print_network(self.disc) 47 | print('-----------------------------------------------') 48 | # Setup the optimizers 49 | lr = opt.learning_rate 50 | self.disc_opt = torch.optim.Adam(self.disc.parameters(), lr=lr, betas=(0.5, 0.999)) 51 | self.gen_opt = torch.optim.Adam(self.gen.parameters(), lr=lr, betas=(0.5, 0.999)) 52 | 53 | self.content_loss_layer = opt.content_loss_layer 54 | self.nn_loss_area_size = opt.nn_loss_area_size 55 | if self.content_loss_layer != 'none': 56 | self.content_model = vgg19(pretrained=True) 57 | # Setup the loss function for training 58 | # Network weight initialization 59 | self.gen.cuda() 60 | self.disc.cuda() 61 | self._nn_loss_area_size = opt.nn_loss_area_size 62 | # applying xavier_uniform, equivalent to glorot unigorm, as in Keras Defo GAN 63 | # skipping as models are pretrained 64 | # self.disc.apply(xavier_weights_init) 65 | # self.gen.apply(xavier_weights_init) 66 | self.ll_loss_criterion = torch.nn.L1Loss() 67 | 68 | # add code for intermediate supervision for the interpolated poses using pretrained pose-estimator 69 | def gen_update(self, input, target, other_inputs, opt): 70 | self.gen.zero_grad() 71 | 72 | if(opt['gen_type']=='stacked'): 73 | interpol_pose = other_inputs['interpol_pose'] 74 | interpol_warps = other_inputs['interpol_warps'] 75 | interpol_masks = other_inputs['interpol_masks'] 76 | outputs_gen = self.gen(input, interpol_pose, interpol_warps, interpol_masks) 77 | out_gen = outputs_gen[-1] 78 | else: 79 | warps = other_inputs['warps'] 80 | masks = other_inputs['masks'] 81 | out_gen = self.gen(input, warps, masks) 82 | outputs_gen = [] 83 | 84 | inp_img, inp_pose, out_pose = pose_utils.get_imgpose(input, opt['use_input_pose'], opt['pose_dim']) 85 | 86 | inp_dis = torch.cat([inp_img, inp_pose, out_gen, out_pose], dim=1) 87 | out_dis = self.disc(inp_dis) 88 | 89 | # computing adversarial loss 90 | for it in range(out_dis.shape[0]): 91 | out = out_dis[it, :] 92 | all_ones = Variable(torch.ones((out.size(0))).cuda()) 93 | if it==0: 94 | # ad_loss = nn.functional.binary_cross_entropy(out, all_ones) 95 | ad_loss = -torch.mean(torch.log(out + 1e-7)) 96 | else: 97 | # ad_loss += nn.functional.binary_cross_entropy(out, all_ones) 98 | ad_loss += -torch.mean(torch.log(out + 1e-7)) 99 | 100 | if self.content_loss_layer != 'none': 101 | content_out_gen = pose_utils.Feature_Extractor(self.content_model, input=out_gen, layer_name=self.content_loss_layer ) 102 | content_target = pose_utils.Feature_Extractor(self.content_model, input=target, layer_name=self.content_loss_layer ) 103 | ll_loss = self.nn_loss(content_out_gen, content_target, self.nn_loss_area_size, self.nn_loss_area_size) 104 | else: 105 | ll_loss = self.ll_loss_criterion(out_gen, target) 106 | 107 | ad_loss = ad_loss*opt['gan_penalty_weight']/self.batch_size 108 | ll_loss = ll_loss*opt['l1_penalty_weight'] 109 | total_loss = ad_loss + ll_loss 110 | total_loss.backward() 111 | self.gen_opt.step() 112 | self.gen_ll_loss = ll_loss.item() 113 | self.gen_ad_loss = ad_loss.item() 114 | self.gen_total_loss = total_loss.item() 115 | return out_gen, outputs_gen, [self.gen_total_loss, self.gen_ll_loss, self.gen_ad_loss] 116 | 117 | def dis_update(self, input, target, other_inputs, real_inp, real_target, opt): 118 | self.disc.zero_grad() 119 | 120 | if (opt['gen_type'] == 'stacked'): 121 | interpol_pose = other_inputs['interpol_pose'] 122 | interpol_warps = other_inputs['interpol_warps'] 123 | interpol_masks = other_inputs['interpol_masks'] 124 | out_gen = self.gen(input, interpol_pose, interpol_warps, interpol_masks) 125 | out_gen = out_gen[-1] 126 | else: 127 | warps = other_inputs['warps'] 128 | masks = other_inputs['masks'] 129 | out_gen = self.gen(input, warps, masks) 130 | 131 | inp_img, inp_pose, out_pose = pose_utils.get_imgpose(input, opt['use_input_pose'], opt['pose_dim']) 132 | 133 | fake_disc_inp = torch.cat([inp_img, inp_pose, out_gen, out_pose], dim=1) 134 | r_inp_img, r_inp_pose, r_out_pose = pose_utils.get_imgpose(real_inp, opt['use_input_pose'], opt['pose_dim']) 135 | real_disc_inp = torch.cat([r_inp_img, r_inp_pose, real_target, r_out_pose], dim=1) 136 | data_dis = torch.cat((real_disc_inp, fake_disc_inp), 0) 137 | res_dis = self.disc(data_dis) 138 | 139 | # print(res_dis.shape) 140 | for it in range(res_dis.shape[0]): 141 | out = res_dis[it, :] 142 | if (it < opt['batch_size']): 143 | out_true_n = out.size(0) 144 | # real inputs should be 1 145 | # all1 = Variable(torch.ones((out_true_n)).cuda()) 146 | if it == 0: 147 | # ad_true_loss = nn.functional.binary_cross_entropy(out, all1) 148 | ad_true_loss = -torch.mean(torch.log(out + 1e-7)) 149 | else: 150 | # ad_true_loss += nn.functional.binary_cross_entropy(out, all1) 151 | ad_true_loss += -torch.mean(torch.log(out + 1e-7)) 152 | else: 153 | out_fake_n = out.size(0) 154 | # fake inputs should be 0, appear after batch_size iters 155 | # all0 = Variable(torch.zeros((out_fake_n)).cuda()) 156 | if it == opt['batch_size']: 157 | # ad_true_loss = -torch.mean(torch.log(out + 1e-7))= nn.functional.binary_cross_entropy(out, all0) 158 | ad_fake_loss = -torch.mean(torch.log(1 - out + 1e-7)) 159 | else: 160 | ad_fake_loss += -torch.mean(torch.log(1 - out + 1e-7)) 161 | 162 | ad_true_loss = ad_true_loss*opt['gan_penalty_weight']/self.batch_size 163 | ad_fake_loss = ad_fake_loss*opt['gan_penalty_weight']/self.batch_size 164 | ad_loss = ad_true_loss + ad_fake_loss 165 | loss = ad_loss 166 | loss.backward() 167 | self.disc_opt.step() 168 | self.dis_total_loss = loss.item() 169 | self.dis_true_loss = ad_true_loss.item() 170 | self.dis_fake_loss = ad_fake_loss.item() 171 | return [self.dis_total_loss, self.dis_true_loss, self.dis_fake_loss] 172 | 173 | def nn_loss(self, predicted, ground_truth, nh=3, nw=3): 174 | v_pad = nh // 2 175 | h_pad = nw // 2 176 | val_pad = nn.ConstantPad2d((v_pad, v_pad, h_pad, h_pad), -10000)(ground_truth) 177 | 178 | reference_tensors = [] 179 | for i_begin in range(0, nh): 180 | i_end = i_begin - nh + 1 181 | i_end = None if i_end == 0 else i_end 182 | for j_begin in range(0, nw): 183 | j_end = j_begin - nw + 1 184 | j_end = None if j_end == 0 else j_end 185 | sub_tensor = val_pad[:, :, i_begin:i_end, j_begin:j_end] 186 | reference_tensors.append(sub_tensor.unsqueeze(-1)) 187 | reference = torch.cat(reference_tensors, dim=-1) 188 | ground_truth = ground_truth.unsqueeze(dim=-1) 189 | 190 | predicted = predicted.unsqueeze(-1) 191 | abs = torch.abs(reference - predicted) 192 | # sum along channels 193 | norms = torch.sum(abs, dim=1) 194 | # min over neighbourhood 195 | loss,_ = torch.min(norms, dim=-1) 196 | # loss = torch.sum(loss)/self.batch_size 197 | loss = torch.mean(loss) 198 | 199 | return loss 200 | 201 | def resume(self, save_dir): 202 | last_model_name = pose_utils.get_model_list(save_dir,"gen") 203 | if last_model_name is None: 204 | return 1 205 | self.gen.load_state_dict(torch.load(last_model_name)) 206 | epoch = int(last_model_name[-7:-4]) 207 | print('Resume gen from epoch %d' % epoch) 208 | last_model_name = pose_utils.get_model_list(save_dir, "dis") 209 | if last_model_name is None: 210 | return 1 211 | epoch = int(last_model_name[-7:-4]) 212 | self.disc.load_state_dict(torch.load(last_model_name)) 213 | print('Resume disc from epoch %d' % epoch) 214 | return epoch 215 | 216 | def save(self, save_dir, epoch): 217 | gen_filename = os.path.join(save_dir, 'gen_{0:03d}.pkl'.format(epoch)) 218 | disc_filename = os.path.join(save_dir, 'disc_{0:03d}.pkl'.format(epoch)) 219 | torch.save(self.gen.state_dict(), gen_filename) 220 | torch.save(self.disc.state_dict(), disc_filename) 221 | 222 | def normalize_image(self, x): 223 | return x[:,0:3,:,:] 224 | 225 | -------------------------------------------------------------------------------- /src_deformable/opts.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | class opts(): 5 | def __init__(self): 6 | self.parser = argparse.ArgumentParser() 7 | 8 | def init(self): 9 | """ 10 | Define args that is used in project 11 | """ 12 | parser = argparse.ArgumentParser(description="Pose guided image generation usign deformable skip layers") 13 | # adding arguments for expID and data directories lacking in originial codebase 14 | self.parser.add_argument('--expID', default='default', help='Experiment ID') 15 | self.parser.add_argument("--data_Dir", default='../../pose-gan-clean/pose-gan-h36m-fg/data/', help="Directory with annotations and data") 16 | 17 | self.parser.add_argument("--output_dir", default='output/displayed_samples', help="Directory with generated sample images") 18 | self.parser.add_argument("--batch_size", default=4, type=int, help='Size of the batch') 19 | self.parser.add_argument("--log_file", default="output/full/fasion/log", help="log file") 20 | self.parser.add_argument("--training_ratio", default=1, type=int, 21 | help="The training ratio is the number of discriminator updates per generator update.") 22 | 23 | self.parser.add_argument("--resume", default=0, type=int, help="resume from checkpoint") 24 | 25 | self.parser.add_argument("--learning_rate", default=2e-4, type=float, help='learning rate of optimizer') 26 | self.parser.add_argument("--l1_penalty_weight", default=100, type=float, help='Weight of l1 loss') 27 | self.parser.add_argument('--gan_penalty_weight', default=1, type=float, help='Weight of GAN loss') 28 | self.parser.add_argument('--tv_penalty_weight', default=0, type=float, help='Weight of total variation loss') 29 | self.parser.add_argument('--lstruct_penalty_weight', default=0, type=float, help="Weight of lstruct") 30 | 31 | self.parser.add_argument("--number_of_epochs", default=500, type=int, help="Number of training epochs") 32 | 33 | self.parser.add_argument("--content_loss_layer", default='none', help='Name of content layer (vgg19)' 34 | ' e.g. block4_conv1 or none') 35 | # change pose_utils and pose_transfroms to honor this 36 | self.parser.add_argument("--pose_dim", default=16, type=int, help="Dimensionality of pose vector") 37 | self.parser.add_argument("--iters_per_epoch", default=1000, type=int, help="Dimensionality of pose vector") 38 | self.parser.add_argument("--checkpoints_dir", default="output/checkpoints", help="Folder with checkpoints") 39 | self.parser.add_argument("--checkpoint_ratio", default=5, type=int, help="Number of epochs between consecutive checkpoints") 40 | self.parser.add_argument("--generator_checkpoint", default=None, help="Previosly saved model of generator") 41 | self.parser.add_argument("--discriminator_checkpoint", default=None, help="Previosly saved model of discriminator") 42 | self.parser.add_argument("--nn_loss_area_size", default=1, type=int, help="Use nearest neighbour loss") 43 | self.parser.add_argument('--dataset', default='h36m', choices=['market', 'fasion', 'fasion128', 'fasion128128', 'h36m'], 44 | help='Market or fasion or h36m') 45 | 46 | self.parser.add_argument("--frame_diff", default=10, type=int, help='Number of frames for iterative testing . . increment by 1 to get 10 frames displacement from initial frame and adding another stack') 47 | self.parser.add_argument("--num_stacks", default=4, type=int, help='Number of stacks for interpol arch') 48 | 49 | self.parser.add_argument('--compute_h36m_paf_split', default=0, type=int, 50 | help='which split to compute annot for') 51 | 52 | self.parser.add_argument("--display_ratio", default=50, type=int, help='Number of epochs between ploting') 53 | self.parser.add_argument("--start_epoch", default=0, type=int, help='Start epoch for starting from checkpoint') 54 | self.parser.add_argument("--pose_estimator", default='pose_estimator.h5', 55 | help='Pretrained model for cao pose estimator') 56 | 57 | self.parser.add_argument("--images_for_test", default=12000, type=int, help="Number of images for testing") 58 | 59 | self.parser.add_argument("--use_input_pose", default=True, type=int, help='Feed to generator input pose') 60 | self.parser.add_argument("--warp_skip", default='mask', choices=['none', 'full', 'mask'], 61 | help="Type of warping skip layers to use.") 62 | self.parser.add_argument("--warp_agg", default='max', choices=['max', 'avg'], 63 | help="Type of aggregation.") 64 | 65 | self.parser.add_argument("--disc_type", default='call', choices=['call', 'sim', 'warp'], 66 | help="Type of discriminator call - concat all, sim - siamease, sharewarp - warp.") 67 | self.parser.add_argument("--gen_type", default='baseline', choices=['baseline', 'stacked'], 68 | help="Type of generator") 69 | 70 | self.parser.add_argument("--generated_images_dir", default='output/generated_images', 71 | help='Folder with generated images from training dataset') 72 | 73 | self.parser.add_argument('--load_generated_images', default=0, type=int, 74 | help='Load images from generated_images_dir or generate') 75 | 76 | self.parser.add_argument('--use_dropout_test', default=0, type=int, 77 | help='To use dropout when generate images') 78 | 79 | def parse(self): 80 | self.init() 81 | self.opt = self.parser.parse_args() 82 | self.opt.saveDir = os.path.join("../exp/", self.opt.expID) 83 | self.opt.output_dir = os.path.join("../exp/", self.opt.expID, "results") 84 | self.opt.checkpoints_dir = os.path.join("../exp/", self.opt.expID, 'models') 85 | self.opt.generated_images_dir = os.path.join("../exp/", self.opt.expID, 'results', 'generated') 86 | 87 | args = dict((name, getattr(self.opt, name)) for name in dir(self.opt) 88 | if not name.startswith('_')) 89 | 90 | if self.opt.dataset == 'fasion': 91 | self.opt.image_size = (256, 256) 92 | elif self.opt.dataset == 'h36m': 93 | self.opt.image_size = (224, 224) 94 | elif self.opt.dataset == 'fasion128128': 95 | self.opt.image_size = (128, 128) 96 | else: 97 | self.opt.image_size = (128, 64) 98 | 99 | 100 | self.opt.images_dir_train = self.opt.data_Dir + self.opt.dataset + '-dataset/train' 101 | self.opt.images_dir_test = self.opt.data_Dir + self.opt.dataset + '-dataset/test' 102 | 103 | # for shg ( 16 joints ) human 3.6 annotations 104 | self.opt.annotations_file_train = self.opt.data_Dir + self.opt.dataset + '-annotation-train.csv' 105 | self.opt.annotations_file_test = self.opt.data_Dir + self.opt.dataset + '-annotation-test.csv' 106 | 107 | # for paf ( 18 joints ) human 3.6 annotations 108 | self.opt.annotations_file_train_paf = self.opt.data_Dir + self.opt.dataset + '-annotation-paf-train' + str( 109 | self.opt.compute_h36m_paf_split) + '.csv' 110 | self.opt.annotations_file_test_paf = self.opt.data_Dir + self.opt.dataset + '-annotation-paf-test' + str( 111 | self.opt.compute_h36m_paf_split) + '.csv' 112 | 113 | self.opt.pairs_file_train = self.opt.data_Dir + self.opt.dataset + '-pairs-train.csv' 114 | self.opt.pairs_file_test = self.opt.data_Dir + self.opt.dataset + '-pairs-test.csv' 115 | 116 | self.opt.pairs_file_train_iterative = self.opt.data_Dir + self.opt.dataset + '-pairs-train-iterative.csv' 117 | self.opt.pairs_file_test_iterative = self.opt.data_Dir + self.opt.dataset + '-pairs-test-iterative.csv' 118 | 119 | self.opt.pairs_file_train_interpol = self.opt.data_Dir + self.opt.dataset + '-pairs-train-interpol.csv' 120 | self.opt.pairs_file_test_interpol = self.opt.data_Dir + self.opt.dataset + '-pairs-test-interpol.csv' 121 | 122 | self.opt.tmp_pose_dir = 'tmp/' + self.opt.dataset + '/' 123 | 124 | if not os.path.exists(self.opt.saveDir): 125 | os.makedirs(self.opt.saveDir) 126 | 127 | if not os.path.exists(self.opt.output_dir): 128 | os.makedirs(self.opt.output_dir) 129 | os.makedirs(os.path.join(self.opt.output_dir, 'train')) 130 | os.makedirs(os.path.join(self.opt.output_dir, 'test')) 131 | os.makedirs(self.opt.generated_images_dir) 132 | 133 | if not os.path.exists(self.opt.checkpoints_dir): 134 | os.makedirs(self.opt.checkpoints_dir) 135 | 136 | file_name = os.path.join(self.opt.saveDir, 'opt.txt') 137 | with open(file_name, 'wt') as opt_file: 138 | opt_file.write('==> Args:\n') 139 | for k, v in sorted(vars(self.opt).items()): 140 | opt_file.write(' %s: %s\n' % (str(k), str(v))) 141 | opt_file.write('==> Args:\n') 142 | return self.opt 143 | -------------------------------------------------------------------------------- /src_deformable/opts.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/opts.pyc -------------------------------------------------------------------------------- /src_deformable/test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | # shitty thing overriding src code in current dir 4 | if('/home/anurag/multiperson/detectron/lib' in sys.path): 5 | sys.path.remove('/home/anurag/multiperson/detectron/lib') 6 | 7 | import matplotlib 8 | matplotlib.use('Agg') 9 | import pylab as plt 10 | import os 11 | import sys 12 | import torch 13 | import numpy as np 14 | import cv2 15 | import time 16 | import datetime 17 | from opts import opts 18 | # from tqdm import tqdm 19 | from datasets.PoseTransfer_Dataset import PoseTransfer_Dataset 20 | from models.networks import Deformable_Generator, Discriminator 21 | from models.pose_gan import DeformablePose_GAN 22 | from utils import pose_utils 23 | # from tqdm import tqdm 24 | 25 | opt = opts().parse() 26 | 27 | dataset = PoseTransfer_Dataset(vars(opt), split='test') 28 | loader_test = torch.utils.data.DataLoader( 29 | dataset, 30 | batch_size=opt.batch_size, 31 | shuffle=True, 32 | ) 33 | 34 | test_iter = loader_test.__iter__() 35 | 36 | model = DeformablePose_GAN(opt).cuda() 37 | epoch = model.resume(opt.checkpoints_dir) 38 | 39 | num_iterations = dataset._pairs_file_test.shape[0]//opt.batch_size 40 | for it in range(num_iterations): 41 | if(it%50==0): 42 | sys.stdout.flush() 43 | print(it / num_iterations) 44 | input, target, warps, masks = test_iter.__next__() 45 | if (opt.gen_type == 'baseline'): 46 | out = model.gen(input.cuda(), warps.float().cuda(), masks.cuda()) 47 | images = pose_utils.display(input, target, out.data.cpu(), opt.use_input_pose, opt.pose_dim) 48 | elif (opt.gen_type == 'stacked'): 49 | out = model.gen(input.cuda(), ) 50 | images = pose_utils.display_stacked(input, target, out, opt.num_stacks, opt.use_input_pose, opt.pose_dim) 51 | else: 52 | raise Exception('Invalid gen type !!') 53 | title = "{0}.png".format(str(it).zfill(5)) 54 | plt.imsave(os.path.join(opt.generated_images_dir, title), images, cmap=plt.cm.gray) -------------------------------------------------------------------------------- /src_deformable/tests/test_warp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/tests/test_warp.png -------------------------------------------------------------------------------- /src_deformable/tests/test_warp_cv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/tests/test_warp_cv.png -------------------------------------------------------------------------------- /src_deformable/tests/test_warp_pytorch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/tests/test_warp_pytorch.png -------------------------------------------------------------------------------- /src_deformable/tests/test_warp_sk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/tests/test_warp_sk.png -------------------------------------------------------------------------------- /src_deformable/unitTests.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | # shitty thing overriding src code in current dir 4 | if('/home/anurag/multiperson/detectron/lib' in sys.path): 5 | sys.path.remove('/home/anurag/multiperson/detectron/lib') 6 | 7 | import matplotlib 8 | matplotlib.use('Agg') 9 | import pylab as plt 10 | import torch 11 | from torch.autograd import Variable 12 | import skimage.transform as transform 13 | import numpy as np 14 | import cv2 15 | import time 16 | import datetime 17 | from opts import opts 18 | # import tqdm 19 | from datasets.PoseTransfer_Dataset import PoseTransfer_Dataset 20 | from utils import pose_utils 21 | from models.networks import Deformable_Generator, Discriminator, torch_summarize, keras_to_pytorch 22 | from models.pose_gan import DeformablePose_GAN 23 | # from utils.pose_transform import AffineTransformLayer 24 | import sys 25 | 26 | # <---------------------------------------------------> 27 | # <---------------------------------------------------> 28 | # daata loader, used everywhere in number of unit tests 29 | 30 | # opt = opts().parse() 31 | # 32 | # disc_loader = torch.utils.data.DataLoader( 33 | # PoseTransfer_Dataset(vars(opt), 'train'), 34 | # batch_size=4, 35 | # shuffle=False, 36 | # ) 37 | # 38 | # disc_iter = disc_loader.__iter__() 39 | 40 | # <---------------------------------------------------> 41 | # <---------------------------------------------------> 42 | # unit tests for visualization and Affine layer 43 | 44 | # input, target, interpol_pose = disc_iter.next() 45 | # # print(batch.shape) 46 | # dis_images = pose_utils.display(input, target, opt.use_input_pose, opt.pose_dim) 47 | # 48 | # interpol_pose = pose_utils.postProcess(interpol_pose) 49 | # result = [] 50 | # for i in range(opt.num_stacks): 51 | # pose_batch = interpol_pose[:, :, :, i * opt.pose_dim:(i + 1) * opt.pose_dim] 52 | # pose_images = np.array([pose_utils.draw_pose_from_map(pose.numpy(), opt.pose_dim)[0] for pose in pose_batch]) 53 | # result.append(pose_images) 54 | # interpol_pose = np.concatenate(result, axis=0) 55 | # interpol_pose = pose_utils.make_grid(torch.from_numpy(interpol_pose), None, row=opt.batch_size, col=opt.num_stacks) 56 | # 57 | # 58 | # plt.imsave('test_interpol.png', interpol_pose , cmap=plt.cm.gray) 59 | # plt.imsave('test_image_pose.png', dis_images, cmap=plt.cm.gray) 60 | 61 | # input, target, warps, masks = disc_iter.next() 62 | # warps = warps[:,:,:-1].float() 63 | # masks = masks.float() 64 | 65 | # <------------------------------------------------------> 66 | # <---------------------------------------------------> 67 | # test 1 for affinetransform layer 68 | 69 | # 70 | # index = 10 71 | # for _ in range(index): 72 | # input, target, warps, masks = disc_iter.next() 73 | # inp_img, inp_pose, tg_pose = pose_utils.get_imgpose(input, True, 16) 74 | # print(inp_img[0].shape) 75 | # warp_skip = 'mask' 76 | # warps = warps.float() 77 | # masks = masks.float() 78 | # input.requires_grad = True 79 | # image_size = input.shape[2:] 80 | 81 | # gradcheck for affinelayer 82 | # affine_model = AffineTransformLayer(10 if warp_skip == 'mask' else 1, image_size, warp_skip) 83 | from torch.autograd import gradcheck 84 | # test = gradcheck(affine_model, (input.cuda(), warps.float().cuda(), masks.cuda()), eps=1e-6, atol=1e-4) 85 | # print(test) 86 | 87 | # out = AffineTransformLayer(10 if warp_skip == 'mask' else 1, image_size, warp_skip)(inp_img, warps, masks) 88 | # inp_img = pose_utils.postProcess(pose_utils._deprocess_image(inp_img)) 89 | # target = pose_utils.postProcess(pose_utils._deprocess_image(target)) 90 | # out = pose_utils.postProcess(pose_utils._deprocess_image(out.data.cpu())) 91 | # print(torch.min(out), torch.max(out)) 92 | # print(torch.min(inp_img), torch.max(inp_img)) 93 | # img = pose_utils.make_grid(torch.cat([inp_img, out, target], dim=0), None, row=opt.batch_size, col=3, order=0) 94 | # plt.imsave('test_warp.png', img , cmap=plt.cm.gray) 95 | # # # 96 | # def _deprocess_image(image): 97 | # return (255 * (image + 1) / 2).astype(np.uint8) 98 | 99 | # <------------------------------------------------------> 100 | # <---------------------------------------------------> 101 | # test 2 for cv2 warp affine and skimage affine 102 | 103 | # import cv2 104 | # input, target, warps, masks = disc_iter.next() 105 | # inp_img, inp_pose, tg_pose = pose_utils.get_imgpose(input, True, 16) 106 | # image = np.transpose(inp_img[0].numpy(), [1,2,0]) 107 | # # print(image.shape) 108 | # target_image = np.transpose(target[0], [1,2,0]) 109 | # # M = np.reshape(warps[0,8,:6].numpy(), [2,3]) 110 | # # M_sk = np.reshape(warps[0,8].numpy(), [3,3]) 111 | # 112 | # M = np.reshape(warps[0,0,:6].numpy(), [2,3]) 113 | # M_sk = np.reshape(warps[0,0].numpy(), [3,3]) 114 | # 115 | # 116 | # warp_img = cv2.warpAffine(image,M,(224,224)) 117 | # warp_img_sk = transform.warp(image,inverse_map=M_sk) 118 | # if(np.min(warp_img)==0 and np.max(warp_img)==0): 119 | # warp_img[...] = -1 120 | # # print(warp_img.shape) 121 | # # print(target_image.shape) 122 | # 123 | # # cv2 warpAffine works on multiple channel images . . method verified by observing warped images 124 | # res = np.concatenate([_deprocess_image(image),_deprocess_image(warp_img), _deprocess_image(target_image.numpy())], axis=0) 125 | # plt.imsave('test_warp_cv.png', res , cmap=plt.cm.gray) 126 | # res = np.concatenate([_deprocess_image(image),_deprocess_image(warp_img_sk), _deprocess_image(target_image.numpy())], axis=0) 127 | # plt.imsave('test_warp_sk.png', res , cmap=plt.cm.gray) 128 | 129 | # <------------------------------------------------------> 130 | # <---------------------------------------------------> 131 | # check for autograd backprop through affine layer 132 | 133 | # opt = opts().parse() 134 | # model = DeformablePose_GAN(opt) 135 | # with torch.autograd.profiler.profile() as prof: 136 | # out = model.gen(Variable(input.cuda()), Variable(warps.cuda()), Variable(masks.cuda())) 137 | # grad_fn = out.grad_fn 138 | # while(grad_fn!=None): 139 | # print(grad_fn) 140 | # grad_fn = grad_fn.next_functions[0][0] 141 | # # print(prof) 142 | 143 | # import torch.nn.functional as F 144 | # 145 | # input, target, warps, masks = disc_iter.next() 146 | # inp_img, inp_pose, tg_pose = pose_utils.get_imgpose(input, True, 16) 147 | # # transforms = torch.ones([1,2,3]).float() 148 | # # transforms = torch.Tensor([[[1,0,0], [0,1,0]]]).float() 149 | # transforms = warps[0,6,:6].view(-1,2,3).float()#/224 150 | # transforms[:,:,2] /= 224 151 | # 152 | # inp_img = inp_img[0].unsqueeze(0) 153 | # target = target[0].unsqueeze(0) 154 | # grid = F.affine_grid(transforms,inp_img.shape) 155 | # print(torch.max(grid), torch.min(grid), torch.mean(grid)) 156 | # warped_map = F.grid_sample(inp_img, grid) 157 | # warped_map = pose_utils.postProcess(pose_utils._deprocess_image(warped_map)) 158 | # 159 | # inp_img = pose_utils.postProcess(pose_utils._deprocess_image(inp_img)) 160 | # target = pose_utils.postProcess(pose_utils._deprocess_image(target)) 161 | # 162 | # img = pose_utils.make_grid(torch.cat([inp_img, warped_map, target], dim=0), None, row=1, col=3, order=1) 163 | # 164 | # plt.imsave('test_warp_pytorch.png', img, cmap=plt.cm.gray) 165 | 166 | # pose_utils.Feature_Extractor() 167 | 168 | # <------------------------------------------------------> 169 | # <---------------------------------------------------> 170 | # test for pose estimator 171 | 172 | # index = 10 173 | # for _ in range(index): 174 | # input, target, warps, masks = disc_iter.next() 175 | # inp_img, inp_pose, tg_pose = pose_utils.get_imgpose(input, True, 16) 176 | # print(inp_img[0].shape) 177 | # warp_skip = 'mask' 178 | # warps = warps.float() 179 | # masks = masks.float() 180 | # input.requires_grad = True 181 | # image_size = input.shape[2:] 182 | # 183 | # # pose_model = torch.load('pose_model.pth') 184 | # # out_pose = pose_model(inp_img) 185 | # 186 | # inp_img = pose_utils.postProcess(pose_utils._deprocess_image(inp_img)) 187 | # target = pose_utils.postProcess(pose_utils._deprocess_image(target)) 188 | # out_pose = pose_utils.postProcess(pose_utils._deprocess_image(out.data.cpu())) 189 | # img = pose_utils.make_grid(torch.cat([inp_img, out_pose, target], dim=0), None, row=opt.batch_size, col=3, order=0) 190 | # plt.imsave('test_warp.png', img , cmap=plt.cm.gray) 191 | 192 | # <------------------------------------------------------> 193 | # <---------------------------------------------------> 194 | # load images at particular index and initialzie model 195 | 196 | # index = 2 197 | # for _ in range(index): 198 | # input, target, warps, masks = disc_iter.next() 199 | # 200 | # model = DeformablePose_GAN(opt).cuda() 201 | # print(torch_summarize(model)) 202 | 203 | # <------------------------------------------------------> 204 | # <---------------------------------------------------> 205 | # covnert mdoel from keras and save the model 206 | 207 | # from keras.models import load_model 208 | # import sys 209 | # sys.path.append('../../pose-gan-clean/pose-gan-h36m-fg/') 210 | # from conditional_gan import CGAN 211 | # from pose_transform_dummy import AffineTransformLayer 212 | # appending sys path to load keras model luibratries 213 | 214 | # config = --dataset fasion --pose_dim 18 --data_Dir ../../pose-gan-clean/pose-gan/data/ 215 | 216 | # keras_gen = load_model('../../pose-gan-clean/pose-gan-h36m-fg/output/full/h36m-fg-dsc/epoch_089_generator.h5', custom_objects={'AffineTransformLayer': AffineTransformLayer}) 217 | # 218 | # 219 | # keras_weights = [] 220 | # for layer in keras_gen.layers: 221 | # keras_weights.append(layer.get_weights()) 222 | # 223 | # model_layers = keras_gen.layers[4:] 224 | # del keras_gen 225 | # 226 | # # 48 for fasion, 40 for h36m 227 | # model_layers_app = [model_layers[2*index] for index in range(20)] 228 | # model_layers_pose = [model_layers[2*index+1] for index in range(20)] 229 | # model_layers[:20] = model_layers_app 230 | # model_layers[20:40] = model_layers_pose 231 | # 232 | # pytorch_gen = model.gen 233 | # pytorch_gen, _ = keras_to_pytorch(pytorch_gen, model_layers, 0) 234 | # 235 | # print(torch_summarize(pytorch_gen)) 236 | # torch.save(pytorch_gen.state_dict(), "gen.pkl") 237 | 238 | # <------------------------------------------------------> 239 | # <---------------------------------------------------> 240 | # load gen model and evalautte qualitatively 241 | 242 | 243 | # pytorch_gen = model.gen 244 | # pytorch_gen.load_state_dict(torch.load('gen.pkl')) 245 | # pytorch_gen.eval() 246 | # 247 | # out = pytorch_gen(input.cuda(), warps.float().cuda(), masks.cuda()) 248 | # images = pose_utils.display(input, target, out.data.cpu(), opt.use_input_pose, opt.pose_dim) 249 | # plt.imsave('test_defo_pytorch_model.png', images, cmap=plt.cm.gray) 250 | 251 | 252 | # <------------------------------------------------------> 253 | # <---------------------------------------------------> 254 | # convert disc model 255 | 256 | 257 | # keras_disc = load_model('../../pose-gan-clean/pose-gan-h36m-fg/output/full/h36m-fg-dsc/epoch_089_discriminator.h5', custom_objects={'AffineTransformLayer': AffineTransformLayer}) 258 | # pytorch_disc = model.disc 259 | # pytorch_disc, _ = keras_to_pytorch(pytorch_disc, keras_disc.layers, 0) 260 | # torch.save(pytorch_disc.state_dict(), "disc.pkl") -------------------------------------------------------------------------------- /src_deformable/utils/__pycache__/pose_transform.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/utils/__pycache__/pose_transform.cpython-36.pyc -------------------------------------------------------------------------------- /src_deformable/utils/__pycache__/pose_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saurabhsharma1993/pose-transfer/e7df4fa34b578949a271bde250047b917eab3efb/src_deformable/utils/__pycache__/pose_utils.cpython-36.pyc -------------------------------------------------------------------------------- /src_deformable/utils/create_pairs.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | import pandas as pd 4 | import pose_transform 5 | import pose_utils 6 | from itertools import permutations 7 | from opts import opts 8 | import re 9 | 10 | opt = opts().parse() 11 | 12 | def make_pair_nonvid(df): 13 | persons = df.apply(lambda x: '_'.join(x['name'].split('_')[0:1]), axis=1) 14 | df['person'] = persons 15 | fr, to = [], [] 16 | for person in pd.unique(persons): 17 | pairs = list(zip(*list(permutations(df[df['person'] == person]['name'], 2)))) 18 | if len(pairs) != 0: 19 | fr += list(pairs[0]) 20 | to += list(pairs[1]) 21 | pair_df = pd.DataFrame(index=range(len(fr))) 22 | pair_df['from'] = fr 23 | pair_df['to'] = to 24 | return pair_df 25 | 26 | def make_pairs(df): 27 | persons = df.apply(lambda x: '_'.join(x['name'].split('_')[:-1]), axis=1) 28 | df['person'] = persons 29 | fr, to = [], [] 30 | for person in pd.unique(persons): 31 | pair_fr = df[df['person'] == person] 32 | # print(person,pair_fr.shape[0]) 33 | num_rows = pair_fr.shape[0] 34 | i = 0 35 | for index, row in pair_fr.iterrows(): 36 | if(i+2 100] = 1 86 | fg_image = np.multiply(image, bg_image) 87 | 88 | # bounding box computation . . converted from matlab code directly 89 | bb = bboxx[index - 1, :].astype(np.int32) 90 | bb[bb < 0] = 0 91 | bb[2] = min(bb[2], image.shape[1]); 92 | bb[3] = min(bb[3], image.shape[0]); 93 | bb = np.round(bb) 94 | 95 | if (bb[3] - bb[1] > bb[2] - bb[0]): 96 | PAD = ((bb[3] - bb[1]) - (bb[2] - bb[0])) / 2; 97 | bb[2] = bb[2] + PAD; 98 | bb[0] = bb[0] - PAD; 99 | else: 100 | PAD = ((bb[2] - bb[0]) - (bb[3] - bb[1])) / 2; 101 | bb[3] = bb[3] + PAD; 102 | bb[1] = bb[1] - PAD; 103 | 104 | bb[bb < 0] = 0 105 | bb[2] = min(bb[2], image.shape[1]); 106 | bb[3] = min(bb[3], image.shape[0]); 107 | bb = np.round(bb) 108 | 109 | fg_image = cv2.resize(fg_image[bb[1]:bb[3], bb[0]:bb[2], :], (224, 224)) 110 | file_name = '{}/{}/{}_{:06d}.jpg'.format(SAVE_PATH, save_folder_name, save_folder_name, index) 111 | cv2.imwrite(file_name, fg_image) 112 | # cv2.imshow(file_name, fg_image) 113 | # cv2.waitKey(0) 114 | # cv2.destroyAllWindows() 115 | else: 116 | break 117 | 118 | print("Finished . Processed {0:d} frames".format(num_proc)) 119 | 120 | 121 | 122 | --------------------------------------------------------------------------------