├── Data ├── Data1OutputPRNet.avi ├── Data1OutputTPS.avi ├── Data1OutputTri.avi ├── Data2OutputPRNet.avi ├── Data2OutputTPS.avi ├── Data2OutputTri.avi ├── Data3OutputPRNet.avi ├── Data3OutputTPS.avi ├── Data3OutputTri.avi ├── Test1.mp4 ├── Test2.mp4 └── Test3.mp4 ├── README.md ├── Report.pdf ├── Wrapper.py ├── codes ├── Data │ ├── net-data │ │ ├── 256_256_resfcn256_weight.index │ │ └── mmod_human_face_detector.dat │ └── uv-data │ │ ├── canonical_vertices.npy │ │ ├── face_ind.txt │ │ ├── triangles.txt │ │ ├── uv_face.png │ │ ├── uv_face_eyes.png │ │ ├── uv_face_mask.png │ │ ├── uv_kpt_ind.txt │ │ ├── uv_kpt_mask.png │ │ └── uv_weight_mask.png ├── Face_Swap.py ├── PRNet.py ├── __init__.py ├── api.py ├── predictor.py └── utils │ ├── cv_plot.py │ ├── estimate_pose.py │ ├── render.py │ ├── render_app.py │ ├── rotate_vertices.py │ └── write.py └── combined.png /Data/Data1OutputPRNet.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data1OutputPRNet.avi -------------------------------------------------------------------------------- /Data/Data1OutputTPS.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data1OutputTPS.avi -------------------------------------------------------------------------------- /Data/Data1OutputTri.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data1OutputTri.avi -------------------------------------------------------------------------------- /Data/Data2OutputPRNet.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data2OutputPRNet.avi -------------------------------------------------------------------------------- /Data/Data2OutputTPS.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data2OutputTPS.avi -------------------------------------------------------------------------------- /Data/Data2OutputTri.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data2OutputTri.avi -------------------------------------------------------------------------------- /Data/Data3OutputPRNet.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data3OutputPRNet.avi -------------------------------------------------------------------------------- /Data/Data3OutputTPS.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data3OutputTPS.avi -------------------------------------------------------------------------------- /Data/Data3OutputTri.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Data3OutputTri.avi -------------------------------------------------------------------------------- /Data/Test1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Test1.mp4 -------------------------------------------------------------------------------- /Data/Test2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Test2.mp4 -------------------------------------------------------------------------------- /Data/Test3.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Data/Test3.mp4 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face-Swap 2 | Python end-to-end pipeline to swap faces in videos and images. 3 | 4 | 5 | The aim of this code is to implement an end-to-end pipeline to swap faces in a video just like Snapchat’s face swap filter or this face swap website. The following figure shows an illustration of the face swap where we swap Ron Weasley's face with Hermione Granger's face. 6 | 7 | ![Repo List](combined.png) 8 | 9 | 10 | # Landmark detection and Triangulation 11 | 12 | 1- If you want to see triangulation and landmark detection on an arbitrary image run this: 13 | 14 | 15 | ```shell 16 | python Wrapper.py --triangulation [path to the image] 17 | ``` 18 | 19 | # Face swap between two images 20 | Download the PRN trained model at BaiduDrive or GoogleDrive, and put it into codes/Data/net-data 21 | 22 | Link: https://drive.google.com/file/d/1UoE-XuW1SDLUjZmJPkIZ1MLxvQFgmTFH/view 23 | 24 | 2- To swap faces of two arbitrary images run this: 25 | 26 | ```shell 27 | python Wrapper.py --image_swap true --src_image [path to source image] --dst_image [path to destination image] --method [metod key] 28 | ``` 29 | 30 | Note: Method key should be choosen between tri, tps, or prnet. 31 | 32 | "tri" stands for Delaunay Triangulation. 33 | 34 | "tps" stands for Thin Plate Splines. 35 | 36 | "prnet" stands for PR Net. 37 | 38 | # Face swap in a video and a target iamge 39 | 3- Two swap a face in a video with an target image run this: 40 | 41 | ```shell 42 | python Wrapper.py --video [path to the .mp4 video] --image [path to destination image] --method [metod key] --name [output video name] 43 | ``` 44 | 45 | Note: You can set frame per second by setting --fps [your desire frame per second] 46 | 47 | 48 | # Face swap in a single video 49 | 4- Two swap two faces in a video run this: 50 | 51 | ```shell 52 | python Wrapper.py --video [path to the .mp4 video] --tf true --method [metod key] --name [output video name] 53 | ``` 54 | 55 | Note: You can set frame per second by setting --fps [your desire frame per second] 56 | 57 | 58 | # Appendix 59 | 60 | Note1: For the full project description click [here](https://cmsc733.github.io/2019/proj/p2/). 61 | 62 | Note2: For the full documentation click [here](https://github.com/hsouri/Face-Swap/blob/main/Report.pdf). 63 | -------------------------------------------------------------------------------- /Report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/Report.pdf -------------------------------------------------------------------------------- /Wrapper.py: -------------------------------------------------------------------------------- 1 | """ 2 | CMSC733 Spring 2020: Classical and Deep Learning Approaches for 3 | Geometric Computer Vision 4 | Project2: FaceSwap 5 | 6 | Author: 7 | 8 | Hossein Souri (hsouri@umiacs.umd.edu) 9 | PhD Student in Computer Vision and Machine Learning 10 | University of Maryland, College Park 11 | 12 | cite: 13 | https://github.com/italojs/facial-landmarks-recognition 14 | https://github.com/wuhuikai/FaceSwap/blob/master/face_swap.py 15 | https://cmsc733.github.io/2019/proj/p2-results/ 16 | https://github.com/YadiraF/PRNet 17 | 18 | """ 19 | 20 | 21 | import cv2 22 | import imutils 23 | import numpy as np 24 | from scipy.spatial import Delaunay 25 | from scipy import interpolate 26 | import dlib 27 | from imutils import face_utils 28 | import matplotlib.pyplot as plt 29 | import sys 30 | import math 31 | import argparse 32 | import copy 33 | from codes.api import PRN 34 | from codes.Face_Swap import face_swap 35 | from codes.PRNet import PRNet_process 36 | import os 37 | 38 | 39 | 40 | 41 | 42 | def main(): 43 | 44 | 45 | Parser = argparse.ArgumentParser() 46 | Parser.add_argument('--video', default='', help='The path to a video to do the face swap') 47 | Parser.add_argument('--image', default='Rambo.jpg', help='the path to an image you want to replace in a video') 48 | Parser.add_argument('--method', default='tri', help='swap method. Choose between tps, tri, and prnet') 49 | Parser.add_argument('--image_swap', default='', help='set to True if you like to swap two arbitrary images ' 50 | '(you should then set --src_image and --dst_image paths)') 51 | Parser.add_argument('--triangulation', default='', help='path to the image you want to plot the ' 52 | 'landmarks and triangulation') 53 | Parser.add_argument('--src_image', default='hermione.jpg', help='the path to an image you want to replace to another ' 54 | 'image') 55 | Parser.add_argument('--dst_image', default='ron.jpg', help='the path to an image you want to swap with source ' 56 | 'image') 57 | Parser.add_argument('--fps', default=30, type=int, help='frame per second') 58 | Parser.add_argument('--tf', default='', help='set to True if you want to swap two faces within video') 59 | Parser.add_argument('--name', default='Output', type=str, help='Output video name') 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | args = Parser.parse_args() 69 | fs = face_swap() 70 | prn = None 71 | if args.method == 'prnet': 72 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 73 | prn = PRN(is_dlib=True) 74 | 75 | if args.triangulation: 76 | src_image = cv2.imread(args.triangulation) 77 | fs.Delaunay_triangulation(src_image) 78 | 79 | if args.image_swap: 80 | dst_image = cv2.imread(args.dst_image) 81 | src_image = cv2.imread(args.src_image) 82 | 83 | if args.method =='tri': 84 | fs.DLN_swap(src_image, dst_image, plot=True) 85 | elif args.method =='tps': 86 | fs.TPS_swap(dst_image, src_image, plot=True) 87 | elif args.method == 'prnet': 88 | _, _ = PRNet_process(prn, dst_image, src_image, two_faces=False, plot=True) 89 | 90 | 91 | 92 | 93 | if args.video: 94 | image = cv2.imread(args.image) 95 | cap = cv2.VideoCapture(args.video) 96 | frame_index = 0 97 | 98 | _, video_image = cap.read() 99 | height = video_image.shape[0] 100 | width = video_image.shape[1] 101 | fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G') 102 | out = cv2.VideoWriter(args.name + '.avi', fourcc, args.fps, (width, height)) 103 | 104 | 105 | while True: 106 | _, video_image = cap.read() 107 | 108 | operation, new_image = fs.swap_operation(args, copy.deepcopy(video_image), image, prn=prn) 109 | cv2.imwrite('./video_results/frame_' + str(frame_index) + '.jpg', new_image) 110 | print('frame ' + str(frame_index) + ' has been saved successfully') 111 | out.write(new_image) 112 | frame_index += 1 113 | if not operation: 114 | continue 115 | key = cv2.waitKey(5) & 0xFF 116 | if key == ord('c'): 117 | cv2.destroyAllWindows() 118 | break 119 | cap.release() 120 | 121 | else: 122 | print('Nothing to do!') 123 | 124 | 125 | if __name__ == '__main__': 126 | main() 127 | -------------------------------------------------------------------------------- /codes/Data/net-data/256_256_resfcn256_weight.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/net-data/256_256_resfcn256_weight.index -------------------------------------------------------------------------------- /codes/Data/net-data/mmod_human_face_detector.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/net-data/mmod_human_face_detector.dat -------------------------------------------------------------------------------- /codes/Data/uv-data/canonical_vertices.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/uv-data/canonical_vertices.npy -------------------------------------------------------------------------------- /codes/Data/uv-data/uv_face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/uv-data/uv_face.png -------------------------------------------------------------------------------- /codes/Data/uv-data/uv_face_eyes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/uv-data/uv_face_eyes.png -------------------------------------------------------------------------------- /codes/Data/uv-data/uv_face_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/uv-data/uv_face_mask.png -------------------------------------------------------------------------------- /codes/Data/uv-data/uv_kpt_ind.txt: -------------------------------------------------------------------------------- 1 | 1.500000000000000000e+01 2.200000000000000000e+01 2.600000000000000000e+01 3.200000000000000000e+01 4.500000000000000000e+01 6.700000000000000000e+01 9.100000000000000000e+01 1.120000000000000000e+02 1.280000000000000000e+02 1.430000000000000000e+02 1.640000000000000000e+02 1.880000000000000000e+02 2.100000000000000000e+02 2.230000000000000000e+02 2.290000000000000000e+02 2.330000000000000000e+02 2.400000000000000000e+02 5.800000000000000000e+01 7.100000000000000000e+01 8.500000000000000000e+01 9.700000000000000000e+01 1.060000000000000000e+02 1.490000000000000000e+02 1.580000000000000000e+02 1.700000000000000000e+02 1.840000000000000000e+02 1.970000000000000000e+02 1.280000000000000000e+02 1.280000000000000000e+02 1.280000000000000000e+02 1.280000000000000000e+02 1.170000000000000000e+02 1.220000000000000000e+02 1.280000000000000000e+02 1.330000000000000000e+02 1.380000000000000000e+02 7.800000000000000000e+01 8.600000000000000000e+01 9.500000000000000000e+01 1.020000000000000000e+02 9.600000000000000000e+01 8.700000000000000000e+01 1.530000000000000000e+02 1.600000000000000000e+02 1.690000000000000000e+02 1.770000000000000000e+02 1.680000000000000000e+02 1.590000000000000000e+02 1.080000000000000000e+02 1.160000000000000000e+02 1.240000000000000000e+02 1.280000000000000000e+02 1.310000000000000000e+02 1.390000000000000000e+02 1.460000000000000000e+02 1.370000000000000000e+02 1.320000000000000000e+02 1.280000000000000000e+02 1.230000000000000000e+02 1.180000000000000000e+02 1.100000000000000000e+02 1.220000000000000000e+02 1.280000000000000000e+02 1.330000000000000000e+02 1.450000000000000000e+02 1.320000000000000000e+02 1.280000000000000000e+02 1.230000000000000000e+02 2 | 9.600000000000000000e+01 1.180000000000000000e+02 1.410000000000000000e+02 1.650000000000000000e+02 1.830000000000000000e+02 1.900000000000000000e+02 1.880000000000000000e+02 1.870000000000000000e+02 1.930000000000000000e+02 1.870000000000000000e+02 1.880000000000000000e+02 1.900000000000000000e+02 1.830000000000000000e+02 1.650000000000000000e+02 1.410000000000000000e+02 1.180000000000000000e+02 9.600000000000000000e+01 4.900000000000000000e+01 4.200000000000000000e+01 3.900000000000000000e+01 4.000000000000000000e+01 4.200000000000000000e+01 4.200000000000000000e+01 4.000000000000000000e+01 3.900000000000000000e+01 4.200000000000000000e+01 4.900000000000000000e+01 5.900000000000000000e+01 7.300000000000000000e+01 8.600000000000000000e+01 9.600000000000000000e+01 1.110000000000000000e+02 1.130000000000000000e+02 1.150000000000000000e+02 1.130000000000000000e+02 1.110000000000000000e+02 6.700000000000000000e+01 6.000000000000000000e+01 6.100000000000000000e+01 6.500000000000000000e+01 6.800000000000000000e+01 6.900000000000000000e+01 6.500000000000000000e+01 6.100000000000000000e+01 6.000000000000000000e+01 6.700000000000000000e+01 6.900000000000000000e+01 6.800000000000000000e+01 1.420000000000000000e+02 1.310000000000000000e+02 1.270000000000000000e+02 1.280000000000000000e+02 1.270000000000000000e+02 1.310000000000000000e+02 1.420000000000000000e+02 1.480000000000000000e+02 1.500000000000000000e+02 1.500000000000000000e+02 1.500000000000000000e+02 1.480000000000000000e+02 1.410000000000000000e+02 1.350000000000000000e+02 1.340000000000000000e+02 1.350000000000000000e+02 1.420000000000000000e+02 1.430000000000000000e+02 1.420000000000000000e+02 1.430000000000000000e+02 3 | -------------------------------------------------------------------------------- /codes/Data/uv-data/uv_kpt_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/uv-data/uv_kpt_mask.png -------------------------------------------------------------------------------- /codes/Data/uv-data/uv_weight_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/codes/Data/uv-data/uv_weight_mask.png -------------------------------------------------------------------------------- /codes/Face_Swap.py: -------------------------------------------------------------------------------- 1 | """ 2 | CMSC733 Spring 2020: Classical and Deep Learning Approaches for 3 | Geometric Computer Vision 4 | Project2: FaceSwap 5 | 6 | Author: 7 | 8 | Hossein Souri (hsouri@umiacs.umd.edu) 9 | PhD Student in Computer Vision and Machine Learning 10 | University of Maryland, College Park 11 | 12 | cite: 13 | https://github.com/italojs/facial-landmarks-recognition 14 | https://github.com/wuhuikai/FaceSwap/blob/master/face_swap.py 15 | https://cmsc733.github.io/2019/proj/p2-results/ 16 | https://github.com/YadiraF/PRNet 17 | 18 | """ 19 | 20 | import cv2 21 | import imutils 22 | import numpy as np 23 | from scipy.spatial import Delaunay 24 | from scipy import interpolate 25 | import dlib 26 | from imutils import face_utils 27 | import matplotlib.pyplot as plt 28 | import sys 29 | import math 30 | import argparse 31 | import copy 32 | from codes.PRNet import PRNet_process 33 | 34 | 35 | class face_swap: 36 | 37 | def __init__(self): 38 | self.path = "codes/shape_predictor_68_face_landmarks.dat" 39 | self.detector = dlib.get_frontal_face_detector() 40 | self.predictor = dlib.shape_predictor(self.path) 41 | self.edge = 40 42 | 43 | 44 | def get_gray(self, image): 45 | return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 46 | 47 | def rand_color(self): 48 | return (np.random.randint(0, 255), np.random.randint(0, 255), np.random.randint(0, 255)) 49 | 50 | def landmarks_detection(self, image, plot=True): 51 | gray = self.get_gray(image) 52 | 53 | rects = self.detector(gray, 1) 54 | 55 | if not rects.__len__() : 56 | return None, None, None 57 | shapes = [] 58 | points = [] 59 | 60 | for (i, rect) in enumerate(rects): 61 | point = [] 62 | shape = self.predictor(gray, rect) 63 | shape = face_utils.shape_to_np(shape) 64 | shapes.append(shape) 65 | 66 | for (x, y) in shape: 67 | point.append((x, y)) 68 | 69 | if plot: 70 | for (x, y) in shape: 71 | cv2.circle(image, (x, y), 3, self.rand_color(), -1) 72 | 73 | points.append(point) 74 | if plot: 75 | cv2.imwrite("Landmarks.png", image) 76 | 77 | return True, points, shapes 78 | 79 | def draw_delaunay(self, img, subdiv, plot=True): 80 | 81 | triangleList = subdiv.getTriangleList() 82 | 83 | for t in triangleList: 84 | pt1 = (t[0], t[1]) 85 | pt2 = (t[2], t[3]) 86 | pt3 = (t[4], t[5]) 87 | 88 | cv2.line(img, pt1, pt2, self.rand_color(), 1, cv2.LINE_AA, 0) 89 | cv2.line(img, pt2, pt3, self.rand_color(), 1, cv2.LINE_AA, 0) 90 | cv2.line(img, pt3, pt1, self.rand_color(), 1, cv2.LINE_AA, 0) 91 | 92 | if plot: 93 | cv2.imwrite("Triangulation.png", img) 94 | 95 | 96 | def Delaunay_triangulation(self, image, plot=True): 97 | size = image.shape 98 | rect = (0, 0, size[1], size[0]) 99 | subdiv = cv2.Subdiv2D(rect) 100 | check, points , shapes = self.landmarks_detection(image, plot=True) 101 | for point in points[0]: 102 | subdiv.insert(point) 103 | 104 | if plot: 105 | self.draw_delaunay(image.copy(), subdiv) 106 | 107 | def bounding_box(self, image, shapes): 108 | upper_left_x, upper_left_y = np.min(shapes, 0) 109 | lower_right_x, lower_right_y = np.max(shapes, 0) 110 | shift = np.array([[upper_left_x, upper_left_y]]) 111 | box = (upper_left_x - 1, upper_left_y -1, lower_right_x + 1, lower_right_y + 1) 112 | 113 | return box, shapes - shift, image[box[1]:box[3], box[0]:box[2]] 114 | 115 | 116 | def bilinear_interpolate(self, img, coords): 117 | """ Interpolates over every image channel 118 | http://en.wikipedia.org/wiki/Bilinear_interpolation 119 | :param img: max 3 channel image 120 | :param coords: 2 x _m_ array. 1st row = xcoords, 2nd row = ycoords 121 | :returns: array of interpolated pixels with same shape as coords 122 | """ 123 | int_coords = np.int32(coords) 124 | x0, y0 = int_coords 125 | dx, dy = coords - int_coords 126 | 127 | # 4 Neighour pixels 128 | q11 = img[y0, x0] 129 | q21 = img[y0, x0 + 1] 130 | q12 = img[y0 + 1, x0] 131 | q22 = img[y0 + 1, x0 + 1] 132 | 133 | btm = q21.T * dx + q11.T * (1 - dx) 134 | top = q22.T * dx + q12.T * (1 - dx) 135 | inter_pixel = top * dy + btm * (1 - dy) 136 | 137 | return inter_pixel.T 138 | 139 | 140 | def Barycentric(self, tri, tri_area, src_features, dst_features): 141 | A_delta = np.vstack((src_features[tri].T, [1, 1, 1])) 142 | B_delta = np.vstack((dst_features[tri].T, [1, 1, 1])) 143 | x, y, z = A_delta @ np.linalg.inv(B_delta) @ np.vstack((tri_area.T, np.ones(tri_area.__len__()))) 144 | return np.array([x, y]) 145 | 146 | 147 | 148 | def get_box(self, features): 149 | box = [] 150 | 151 | for col in range(np.min(features[:, 1]), np.max(features[:, 1]) + 1): 152 | for row in range(np.min(features[:, 0]), np.max(features[:, 0]) + 1): 153 | box.append((row, col)) 154 | 155 | return np.asarray(box, dtype=np.uint32) 156 | 157 | def mask_from_points(self,size, points, erode_flag=1): 158 | radius = 10 # kernel size 159 | kernel = np.ones((radius, radius), np.uint8) 160 | 161 | mask = np.zeros(size, np.uint8) 162 | cv2.fillConvexPoly(mask, cv2.convexHull(points), 255) 163 | if erode_flag: 164 | mask = cv2.erode(mask, kernel, iterations=1) 165 | 166 | return mask 167 | 168 | def mask_from_points(self,size, points, erode_flag=1): 169 | radius = 10 # kernel size 170 | kernel = np.ones((radius, radius), np.uint8) 171 | 172 | mask = np.zeros(size, np.uint8) 173 | cv2.fillConvexPoly(mask, cv2.convexHull(points), 255) 174 | if erode_flag: 175 | mask = cv2.erode(mask, kernel, iterations=1) 176 | 177 | return mask 178 | 179 | def apply_mask(self,img, mask): 180 | 181 | """ Apply mask to supplied image 182 | :param img: max 3 channel image 183 | :param mask: [0-255] values in mask 184 | :returns: new image with mask applied 185 | """ 186 | masked_img = cv2.bitwise_and(img, img, mask=mask) 187 | 188 | return masked_img 189 | 190 | def correct_colours(self,im1, im2, landmarks1): 191 | COLOUR_CORRECT_BLUR_FRAC = 0.75 192 | LEFT_EYE_POINTS = list(range(42, 48)) 193 | RIGHT_EYE_POINTS = list(range(36, 42)) 194 | 195 | blur_amount = COLOUR_CORRECT_BLUR_FRAC * np.linalg.norm( 196 | np.mean(landmarks1[LEFT_EYE_POINTS], axis=0) - 197 | np.mean(landmarks1[RIGHT_EYE_POINTS], axis=0)) 198 | blur_amount = int(blur_amount) 199 | if blur_amount % 2 == 0: 200 | blur_amount += 1 201 | im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0) 202 | im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0) 203 | 204 | # Avoid divide-by-zero errors. 205 | im2_blur = im2_blur.astype(int) 206 | im2_blur += 128 * (im2_blur <= 1) 207 | 208 | result = im2.astype(np.float64) * im1_blur.astype(np.float64) / im2_blur.astype(np.float64) 209 | result = np.clip(result, 0, 255).astype(np.uint8) 210 | 211 | return result 212 | 213 | 214 | 215 | def blending(self, dst_points, dst_face, warped_src_face): 216 | h, w = dst_face.shape[:2] 217 | ## Mask for blending 218 | mask = self.mask_from_points((h, w), dst_points) 219 | mask_src = np.mean(warped_src_face, axis=2) > 0 220 | mask = np.asarray(mask * mask_src, dtype=np.uint8) 221 | ## Correct color 222 | # if args.correct_color: 223 | warped_src_face = self.apply_mask(warped_src_face, mask) 224 | dst_face_masked = self.apply_mask(dst_face, mask) 225 | warped_src_face = self.correct_colours(dst_face_masked, warped_src_face, dst_points) 226 | 227 | ## Shrink the mask 228 | kernel = np.ones((10, 10), np.uint8) 229 | mask = cv2.erode(mask, kernel, iterations=1) 230 | ##Poisson Blending 231 | r = cv2.boundingRect(mask) 232 | center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2))) 233 | output = cv2.seamlessClone(warped_src_face, dst_face, mask, center, cv2.NORMAL_CLONE) 234 | 235 | 236 | return output 237 | 238 | 239 | def replace(self, dst_image, box, blended_source): 240 | dst_image_copy = dst_image.copy() 241 | dst_image_copy[box[1]:box[3], box[0]:box[2]] = blended_source 242 | return dst_image_copy 243 | 244 | def DLN_swap(self, src_image, dst_image, plot=False): 245 | 246 | check, points, shapes = self.landmarks_detection(src_image, plot=False) 247 | if not check: 248 | return None, None 249 | src_box_coordinates, src_features, src_cropped = self.bounding_box(src_image, shapes[0]) 250 | 251 | check, points, shapes = self.landmarks_detection(dst_image, plot=False) 252 | if not check: 253 | return None, None 254 | dst_box_coordinates, dst_features, dst_cropped = self.bounding_box(dst_image, shapes[0]) 255 | 256 | 257 | 258 | 259 | triangles = Delaunay(dst_features) 260 | 261 | dst_box = self.get_box(dst_features) 262 | 263 | box_inices = triangles.find_simplex(dst_box) 264 | 265 | warped_image = np.zeros((dst_cropped.shape[0], dst_cropped.shape[1], 3), dtype=np.uint8) 266 | 267 | for tri_index, tri in enumerate(triangles.simplices): 268 | tri_area = dst_box[box_inices == tri_index] 269 | x, y = tri_area.T 270 | 271 | barycentric = self.Barycentric(tri, tri_area, src_features, dst_features) 272 | warped_image[y, x] = self.bilinear_interpolate(src_cropped, barycentric) 273 | 274 | if plot: 275 | self.save_image('warped', warped_image) 276 | 277 | blended_src_image = self.blending(dst_features, dst_cropped, warped_image) 278 | 279 | des_swaped_image = self.replace(dst_image, dst_box_coordinates, blended_src_image) 280 | 281 | if plot: 282 | # self.save_image('warped', warped_image) 283 | # self.save_image('blended', blended_src_image) 284 | self.save_image('tri_swaped_image', des_swaped_image) 285 | 286 | return True, des_swaped_image 287 | 288 | 289 | def U(self, r): 290 | return (r ** 2) * (math.log(r ** 2)) 291 | 292 | def lp_norm(self, p1, p2, order=2): 293 | 294 | if np.linalg.norm((-p1 + p2), ord=order) == 0: 295 | return sys.float_info.epsilon 296 | return np.linalg.norm((-p1 + p2), ord=order) 297 | 298 | 299 | def get_K(self, features): 300 | len = features.__len__() 301 | K = [[self.U(self.lp_norm(features[i], features[j], order=2)) for i in range(len)] for j in range(len)] 302 | return np.array(K) 303 | 304 | def get_fat_mat(self, K, P): 305 | fat = np.concatenate((np.concatenate((K, P), axis=1), np.concatenate((P.T, np.zeros([3, 3])), axis=1)), axis=0) 306 | return fat 307 | 308 | 309 | def TPS_weights(self, src_features, dst_features, _lambda=sys.float_info.epsilon): 310 | _lambda = 200 311 | num_points = src_features.__len__() 312 | P = np.concatenate((src_features, np.ones((num_points, 1))), axis=1) 313 | K = self.get_K(src_features) 314 | Inverse = np.linalg.inv(self.get_fat_mat(K, P) + _lambda * np.identity(num_points + 3)) 315 | v = np.concatenate((dst_features, np.array([0, 0, 0])), axis=0) 316 | w = Inverse @ v 317 | return w 318 | 319 | def f(self, point, features, w): 320 | K = np.zeros([features.shape[0], 1]) 321 | for i in range(features.shape[0]): 322 | K[i] = self.U(self.lp_norm(features[i], point, order=2)) 323 | f = w[-1] + w[-3] * point[0] + w[-2] * point[1] + K.T @ w[0:-3] 324 | return f 325 | 326 | def get_coordinates(self, features): 327 | grid_x = np.arange(min(features[:, 0]), max(features[:, 0])) 328 | grid_y = np.arange(min(features[:, 1]), max(features[:, 1])) 329 | row, col = np.mgrid[grid_x[0]: grid_x[-1] + 1, grid_y[0]: grid_y[-1] + 1] 330 | return np.vstack((row.ravel(), col.ravel())).T 331 | 332 | 333 | def get_color(self, src_features, min , max, coords, w): 334 | colors = np.zeros_like(coords[:, 0]) 335 | for i in range(coords.shape[0]): 336 | colors[i] = self.f(coords[i, :], src_features, w) 337 | colors[colors < min] = min 338 | colors[colors > max] = max 339 | return colors 340 | 341 | 342 | def TPS_blend(self,src_image, src_warped_img, masked_warped_img, plot=False): 343 | r = cv2.boundingRect(masked_warped_img) 344 | center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2))) 345 | blended_image = cv2.seamlessClone(src_warped_img.copy(), src_image, masked_warped_img, center, cv2.NORMAL_CLONE) 346 | if plot: 347 | self.save_image("tps_swaped_image", blended_image) 348 | 349 | return blended_image 350 | 351 | def TPS_swap(self, src_image, dst_image, plot=False): 352 | check1, _, src_shapes = self.landmarks_detection(src_image, plot=False) 353 | # src_box_coordinates, src_features, src_cropped = self.bounding_box(src_image, shapes[0]) 354 | 355 | check2, _, dst_shapes = self.landmarks_detection(dst_image, plot=False) 356 | # dst_box_coordinates, dst_features, dst_cropped = self.bounding_box(dst_image, shapes[0]) 357 | 358 | if (not check1) or (not check2): 359 | return None, None 360 | 361 | w_x = self.TPS_weights(src_shapes[0], dst_shapes[0][:, 0]) 362 | w_y = self.TPS_weights(src_shapes[0], dst_shapes[0][:, 1]) 363 | coords = self.get_coordinates(src_shapes[0]) 364 | color_x = self.get_color(src_shapes[0], min(dst_shapes[0][:, 0]), max(dst_shapes[0][:, 0]), coords, w_x) 365 | color_y = self.get_color(src_shapes[0], min(dst_shapes[0][:, 1]), max(dst_shapes[0][:, 1]), coords, w_y) 366 | 367 | w, h = dst_image.shape[:2] 368 | # ## Mask for blending 369 | mask = self.mask_from_points((w, h), dst_shapes[0]) 370 | src_warped_img = src_image.copy() 371 | masked_warped_img = np.zeros_like(src_warped_img[:, :, 0]) 372 | 373 | for index in range(color_x.shape[0]): 374 | if mask[color_y[index], color_x[index]] > 0: 375 | src_warped_img[coords[index, 1], coords[index, 0], :] = dst_image[color_y[index], color_x[index], :] 376 | masked_warped_img[coords[index, 1], coords[index, 0]] = 255 377 | 378 | if plot: 379 | self.save_image('tps_warped', src_warped_img) 380 | 381 | 382 | 383 | return True, self.TPS_blend(src_image, src_warped_img, masked_warped_img, plot=plot) 384 | 385 | 386 | def save_image(self, name, image): 387 | cv2.imwrite(name + ".png", image) 388 | 389 | def get_face(self, image, rect, edge=25): 390 | return image[rect.top() - edge: rect.bottom() + edge, rect.left() - edge: rect.right() + edge, :] 391 | 392 | 393 | 394 | 395 | def get_images(self, rects, video_image, image, edge=25): 396 | 397 | if len(rects) == 1: 398 | src_image = self.get_face(video_image, rects[0], edge=edge) 399 | dst_image = image 400 | return src_image, dst_image 401 | 402 | elif len(rects)>1: 403 | src_image = self.get_face(video_image, rects[0], edge=edge) 404 | dst_image = self.get_face(video_image, rects[1], edge=edge) 405 | return copy.deepcopy(src_image), copy.deepcopy(dst_image) 406 | 407 | 408 | 409 | def resize(self, image, rect, edge=25): 410 | return cv2.resize(image, 411 | ((rect.right() + edge) - (rect.left() - edge), (rect.bottom() + edge) - (rect.top() - edge))) 412 | 413 | def replace_output(self, image, output, rect, edge=25): 414 | image[rect.top() - edge:rect.bottom() + edge, rect.left() - edge:rect.right() + edge, :] = output 415 | 416 | 417 | def swap_operation(self, args, video_image, image, prn=None): 418 | 419 | 420 | if args.method == 'prnet': 421 | 422 | ckeck, output = PRNet_process(prn, video_image, image, args.tf) 423 | if ckeck is None: 424 | return None, video_image 425 | return True, output 426 | 427 | 428 | edge = self.edge 429 | rects = self.detector(video_image, 1) 430 | if not rects.__len__(): 431 | return None, video_image 432 | 433 | copy_video_image = copy.deepcopy(video_image) 434 | src_image, dst_image = self.get_images(rects, copy_video_image, image, edge=edge) 435 | 436 | 437 | if args.method == 'tps': 438 | 439 | check, output1 = self.TPS_swap(src_image, dst_image) 440 | if not check: 441 | return None, video_image 442 | output1 = self.resize(output1, rects[0], edge=edge) 443 | self.replace_output(video_image, output1, rects[0], edge=edge) 444 | 445 | if len(rects) > 1: 446 | check, output2 = self.TPS_swap(dst_image, src_image) 447 | if not check: 448 | return None, video_image 449 | output2 = self.resize(output2, rects[1], edge=edge) 450 | self.replace_output(video_image, output2, rects[1], edge=edge) 451 | 452 | 453 | elif args.method =='tri': 454 | 455 | ckeck, output1 = self.DLN_swap(dst_image, src_image) 456 | if not ckeck: 457 | return None, video_image 458 | output1 = self.resize(output1, rects[0], edge=edge) 459 | self.replace_output(video_image, output1, rects[0], edge=edge) 460 | if len(rects) > 1: 461 | check, output2 = self.DLN_swap(src_image, dst_image) 462 | if not ckeck: 463 | return None, video_image 464 | output2 = self.resize(output2, rects[1], edge=edge) 465 | self.replace_output(video_image, output2, rects[1], edge=edge) 466 | 467 | 468 | 469 | 470 | return True, video_image -------------------------------------------------------------------------------- /codes/PRNet.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | CMSC733 Spring 2020: Classical and Deep Learning Approaches for 4 | Geometric Computer Vision 5 | Project2: FaceSwap 6 | 7 | Author: 8 | 9 | Hossein Souri (hsouri@umiacs.umd.edu) 10 | PhD Student in Computer Vision and Machine Learning 11 | University of Maryland, College Park 12 | 13 | cite: 14 | 15 | https://github.com/YadiraF/PRNet 16 | 17 | """ 18 | 19 | 20 | 21 | 22 | import numpy as np 23 | import os 24 | from glob import glob 25 | import scipy.io as sio 26 | from skimage.io import imread, imsave 27 | from skimage.transform import rescale, resize 28 | from time import time 29 | import argparse 30 | import ast 31 | import matplotlib.pyplot as plt 32 | import argparse 33 | 34 | from codes.utils.render import render_texture 35 | import cv2 36 | 37 | 38 | def PRNet_process(prn, image, ref_image, two_faces, plot=False): 39 | 40 | 41 | [h, w, _] = image.shape 42 | 43 | # -- 1. 3d reconstruction -> get texture. 44 | 45 | if two_faces: 46 | 47 | all_poses = prn.process2(image) 48 | if all_poses == None: 49 | return None, image 50 | 51 | if len(all_poses) == 2: 52 | _, output = PRNet_swap(prn, image, image, all_poses[0], all_poses[1], h, w) 53 | _, output = PRNet_swap(prn, output, image, all_poses[1], all_poses[0], h, w, plot=plot) 54 | else: 55 | return None, image 56 | 57 | return all_poses, output 58 | 59 | else: 60 | 61 | pos = prn.process(image) 62 | if pos is None: 63 | return pos, image 64 | ref_pos = prn.process(ref_image) 65 | _, output = PRNet_swap(prn, image, ref_image, pos, ref_pos, h, w, plot=plot) 66 | return pos, output 67 | 68 | 69 | 70 | def PRNet_swap(prn, image, ref_image, pos, ref_pos, h, w, plot=False): 71 | 72 | 73 | # -- 1. 3d reconstruction -> get texture. 74 | 75 | vertices = prn.get_vertices(pos) 76 | image = image / 255. 77 | texture = cv2.remap(image, pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, 78 | borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) 79 | 80 | 81 | ref_image = ref_image / 255. 82 | ref_texture = cv2.remap(ref_image, ref_pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, 83 | borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) 84 | ref_vertices = prn.get_vertices(ref_pos) 85 | new_texture = ref_texture # (texture + ref_texture)/2. 86 | 87 | # -- 3. remap to input image.(render) 88 | vis_colors = np.ones((vertices.shape[0], 1)) 89 | face_mask = render_texture(vertices.T, vis_colors.T, prn.triangles.T, h, w, c=1) 90 | face_mask = np.squeeze(face_mask > 0).astype(np.float32) 91 | 92 | new_colors = prn.get_colors_from_texture(new_texture) 93 | new_image = render_texture(vertices.T, new_colors.T, prn.triangles.T, h, w, c=3) 94 | new_image = image * (1 - face_mask[:, :, np.newaxis]) + new_image * face_mask[:, :, np.newaxis] 95 | 96 | # Possion Editing for blending image 97 | vis_ind = np.argwhere(face_mask > 0) 98 | vis_min = np.min(vis_ind, 0) 99 | vis_max = np.max(vis_ind, 0) 100 | center = (int((vis_min[1] + vis_max[1]) / 2 + 0.5), int((vis_min[0] + vis_max[0]) / 2 + 0.5)) 101 | output = cv2.seamlessClone((new_image * 255).astype(np.uint8), (image * 255).astype(np.uint8), 102 | (face_mask * 255).astype(np.uint8), center, cv2.NORMAL_CLONE) 103 | 104 | if plot: 105 | cv2.imwrite('PRNet_swaped_image.png', output) 106 | 107 | return pos, output -------------------------------------------------------------------------------- /codes/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /codes/api.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | from skimage.io import imread, imsave 4 | from skimage.transform import estimate_transform, warp 5 | from time import time 6 | 7 | from codes.predictor import PosPrediction 8 | 9 | class PRN: 10 | ''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network 11 | Args: 12 | is_dlib(bool, optional): If true, dlib is used for detecting faces. 13 | prefix(str, optional): If run at another folder, the absolute path is needed to load the data. 14 | ''' 15 | def __init__(self, is_dlib = False, prefix = '.'): 16 | 17 | # resolution of input and output image size. 18 | self.resolution_inp = 256 19 | self.resolution_op = 256 20 | 21 | #---- load detectors 22 | if is_dlib: 23 | import dlib 24 | detector_path = os.path.join(prefix, 'codes/Data/net-data/mmod_human_face_detector.dat') 25 | self.face_detector = dlib.cnn_face_detection_model_v1( 26 | detector_path) 27 | 28 | #---- load PRN 29 | self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op) 30 | prn_path = os.path.join(prefix, 'codes/Data/net-data/256_256_resfcn256_weight') 31 | if not os.path.isfile(prn_path + '.data-00000-of-00001'): 32 | print("please download PRN trained model first.") 33 | exit() 34 | self.pos_predictor.restore(prn_path) 35 | 36 | # uv file 37 | self.uv_kpt_ind = np.loadtxt(prefix + '/codes/Data/uv-data/uv_kpt_ind.txt').astype(np.int32) # 2 x 68 get kpt 38 | self.face_ind = np.loadtxt(prefix + '/codes/Data/uv-data/face_ind.txt').astype(np.int32) # get valid vertices in the pos map 39 | self.triangles = np.loadtxt(prefix + '/codes/Data/uv-data/triangles.txt').astype(np.int32) # ntri x 3 40 | 41 | self.uv_coords = self.generate_uv_coords() 42 | 43 | def generate_uv_coords(self): 44 | resolution = self.resolution_op 45 | uv_coords = np.meshgrid(range(resolution),range(resolution)) 46 | uv_coords = np.transpose(np.array(uv_coords), [1,2,0]) 47 | uv_coords = np.reshape(uv_coords, [resolution**2, -1]); 48 | uv_coords = uv_coords[self.face_ind, :] 49 | uv_coords = np.hstack((uv_coords[:,:2], np.zeros([uv_coords.shape[0], 1]))) 50 | return uv_coords 51 | 52 | def dlib_detect(self, image): 53 | return self.face_detector(image, 1) 54 | 55 | def net_forward(self, image): 56 | ''' The core of out method: regress the position map of a given image. 57 | Args: 58 | image: (256,256,3) array. value range: 0~1 59 | Returns: 60 | pos: the 3D position map. (256, 256, 3) array. 61 | ''' 62 | return self.pos_predictor.predict(image) 63 | 64 | def process(self, input, image_info = None): 65 | ''' process image with crop operation. 66 | Args: 67 | input: (h,w,3) array or str(image path). image value range:1~255. 68 | image_info(optional): the bounding box information of faces. if None, will use dlib to detect face. 69 | 70 | Returns: 71 | pos: the 3D position map. (256, 256, 3). 72 | ''' 73 | if isinstance(input, str): 74 | try: 75 | image = imread(input) 76 | except IOError: 77 | print("error opening file: ", input) 78 | return None 79 | else: 80 | image = input 81 | 82 | if image.ndim < 3: 83 | image = np.tile(image[:,:,np.newaxis], [1,1,3]) 84 | 85 | if image_info is not None: 86 | if np.max(image_info.shape) > 4: # key points to get bounding box 87 | kpt = image_info 88 | if kpt.shape[0] > 3: 89 | kpt = kpt.T 90 | left = np.min(kpt[0, :]); right = np.max(kpt[0, :]); 91 | top = np.min(kpt[1,:]); bottom = np.max(kpt[1,:]) 92 | else: # bounding box 93 | bbox = image_info 94 | left = bbox[0]; right = bbox[1]; top = bbox[2]; bottom = bbox[3] 95 | old_size = (right - left + bottom - top)/2 96 | center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0]) 97 | size = int(old_size*1.6) 98 | else: 99 | detected_faces = self.dlib_detect(image) 100 | if len(detected_faces) == 0: 101 | print('warning: no detected face') 102 | return None 103 | 104 | d = detected_faces[0].rect ## only use the first detected face (assume that each input image only contains one face) 105 | left = d.left(); right = d.right(); top = d.top(); bottom = d.bottom() 106 | old_size = (right - left + bottom - top)/2 107 | center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size*0.14]) 108 | size = int(old_size*1.58) 109 | 110 | # crop image 111 | src_pts = np.array([[center[0]-size/2, center[1]-size/2], [center[0] - size/2, center[1]+size/2], [center[0]+size/2, center[1]-size/2]]) 112 | DST_PTS = np.array([[0,0], [0,self.resolution_inp - 1], [self.resolution_inp - 1, 0]]) 113 | tform = estimate_transform('similarity', src_pts, DST_PTS) 114 | 115 | image = image/255. 116 | cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp)) 117 | 118 | # run our net 119 | #st = time() 120 | cropped_pos = self.net_forward(cropped_image) 121 | #print 'net time:', time() - st 122 | 123 | # restore 124 | cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T 125 | z = cropped_vertices[2,:].copy()/tform.params[0,0] 126 | cropped_vertices[2,:] = 1 127 | vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices) 128 | vertices = np.vstack((vertices[:2,:], z)) 129 | pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3]) 130 | 131 | return pos 132 | 133 | def process2(self, input, image_info=None): 134 | ''' process image with crop operation. 135 | Args: 136 | input: (h,w,3) array or str(image path). image value range:1~255. 137 | image_info(optional): the bounding box information of faces. if None, will use dlib to detect face. 138 | Returns: 139 | pos: the 3D position map. (256, 256, 3). 140 | ''' 141 | if isinstance(input, str): 142 | try: 143 | image = imread(input) 144 | except IOError: 145 | print("error opening file: ", input) 146 | return None 147 | else: 148 | image = input 149 | 150 | if image.ndim < 3: 151 | image = np.tile(image[:, :, np.newaxis], [1, 1, 3]) 152 | 153 | if image_info is not None: 154 | if np.max(image_info.shape) > 4: # key points to get bounding box 155 | kpt = image_info 156 | if kpt.shape[0] > 3: 157 | kpt = kpt.T 158 | left = np.min(kpt[0, :]); 159 | right = np.max(kpt[0, :]); 160 | top = np.min(kpt[1, :]); 161 | bottom = np.max(kpt[1, :]) 162 | else: # bounding box 163 | bbox = image_info 164 | left = bbox[0]; 165 | right = bbox[1]; 166 | top = bbox[2]; 167 | bottom = bbox[3] 168 | old_size = (right - left + bottom - top) / 2 169 | center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0]) 170 | size = int(old_size * 1.6) 171 | 172 | src_pts = np.array( 173 | [[center[0] - size / 2, center[1] - size / 2], [center[0] - size / 2, center[1] + size / 2], 174 | [center[0] + size / 2, center[1] - size / 2]]) 175 | DST_PTS = np.array([[0, 0], [0, self.resolution_inp - 1], [self.resolution_inp - 1, 0]]) 176 | tform = estimate_transform('similarity', src_pts, DST_PTS) 177 | 178 | image = image / 255. 179 | cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp)) 180 | 181 | # run our net 182 | # st = time() 183 | cropped_pos = self.net_forward(cropped_image) 184 | # print 'net time:', time() - st 185 | 186 | # restore 187 | cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T 188 | z = cropped_vertices[2, :].copy() / tform.params[0, 0] 189 | cropped_vertices[2, :] = 1 190 | vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices) 191 | vertices = np.vstack((vertices[:2, :], z)) 192 | pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3]) 193 | 194 | return pos 195 | 196 | else: 197 | detected_faces = self.dlib_detect(image) 198 | if len(detected_faces) != 2: 199 | print('warning: 2 faces not detected') 200 | return None 201 | 202 | print("{} faces detected".format(len(detected_faces))) 203 | all_poses = [] 204 | 205 | for i in range(len(detected_faces)): 206 | d = detected_faces[i].rect 207 | left = d.left(); 208 | right = d.right(); 209 | top = d.top(); 210 | bottom = d.bottom() 211 | old_size = (right - left + bottom - top) / 2 212 | center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size * 0.14]) 213 | size = int(old_size * 1.58) 214 | 215 | src_pts = np.array( 216 | [[center[0] - size / 2, center[1] - size / 2], [center[0] - size / 2, center[1] + size / 2], 217 | [center[0] + size / 2, center[1] - size / 2]]) 218 | DST_PTS = np.array([[0, 0], [0, self.resolution_inp - 1], [self.resolution_inp - 1, 0]]) 219 | tform = estimate_transform('similarity', src_pts, DST_PTS) 220 | 221 | t = image / 255. 222 | cropped_image = warp(t, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp)) 223 | 224 | # run our net 225 | # st = time() 226 | cropped_pos = self.net_forward(cropped_image) 227 | # print 'net time:', time() - st 228 | 229 | # restore 230 | cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T 231 | z = cropped_vertices[2, :].copy() / tform.params[0, 0] 232 | cropped_vertices[2, :] = 1 233 | vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices) 234 | vertices = np.vstack((vertices[:2, :], z)) 235 | pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3]) 236 | all_poses.append(pos) 237 | 238 | return all_poses 239 | 240 | def get_landmarks(self, pos): 241 | ''' 242 | Args: 243 | pos: the 3D position map. shape = (256, 256, 3). 244 | Returns: 245 | kpt: 68 3D landmarks. shape = (68, 3). 246 | ''' 247 | kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :] 248 | return kpt 249 | 250 | 251 | def get_vertices(self, pos): 252 | ''' 253 | Args: 254 | pos: the 3D position map. shape = (256, 256, 3). 255 | Returns: 256 | vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here. 257 | ''' 258 | all_vertices = np.reshape(pos, [self.resolution_op**2, -1]) 259 | vertices = all_vertices[self.face_ind, :] 260 | 261 | return vertices 262 | 263 | def get_colors_from_texture(self, texture): 264 | ''' 265 | Args: 266 | texture: the texture map. shape = (256, 256, 3). 267 | Returns: 268 | colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here. 269 | ''' 270 | all_colors = np.reshape(texture, [self.resolution_op**2, -1]) 271 | colors = all_colors[self.face_ind, :] 272 | 273 | return colors 274 | 275 | 276 | def get_colors(self, image, vertices): 277 | ''' 278 | Args: 279 | pos: the 3D position map. shape = (256, 256, 3). 280 | Returns: 281 | colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here. 282 | ''' 283 | [h, w, _] = image.shape 284 | vertices[:,0] = np.minimum(np.maximum(vertices[:,0], 0), w - 1) # x 285 | vertices[:,1] = np.minimum(np.maximum(vertices[:,1], 0), h - 1) # y 286 | ind = np.round(vertices).astype(np.int32) 287 | colors = image[ind[:,1], ind[:,0], :] # n x 3 288 | 289 | return colors 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | -------------------------------------------------------------------------------- /codes/predictor.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.layers as tcl 3 | from tensorflow.contrib.framework import arg_scope 4 | import numpy as np 5 | 6 | def resBlock(x, num_outputs, kernel_size = 4, stride=1, activation_fn=tf.nn.relu, normalizer_fn=tcl.batch_norm, scope=None): 7 | assert num_outputs%2==0 #num_outputs must be divided by channel_factor(2 here) 8 | with tf.variable_scope(scope, 'resBlock'): 9 | shortcut = x 10 | if stride != 1 or x.get_shape()[3] != num_outputs: 11 | shortcut = tcl.conv2d(shortcut, num_outputs, kernel_size=1, stride=stride, 12 | activation_fn=None, normalizer_fn=None, scope='shortcut') 13 | x = tcl.conv2d(x, num_outputs/2, kernel_size=1, stride=1, padding='SAME') 14 | x = tcl.conv2d(x, num_outputs/2, kernel_size=kernel_size, stride=stride, padding='SAME') 15 | x = tcl.conv2d(x, num_outputs, kernel_size=1, stride=1, activation_fn=None, padding='SAME', normalizer_fn=None) 16 | 17 | x += shortcut 18 | x = normalizer_fn(x) 19 | x = activation_fn(x) 20 | return x 21 | 22 | 23 | class resfcn256(object): 24 | def __init__(self, resolution_inp = 256, resolution_op = 256, channel = 3, name = 'resfcn256'): 25 | self.name = name 26 | self.channel = channel 27 | self.resolution_inp = resolution_inp 28 | self.resolution_op = resolution_op 29 | 30 | def __call__(self, x, is_training = True): 31 | with tf.variable_scope(self.name) as scope: 32 | with arg_scope([tcl.batch_norm], is_training=is_training, scale=True): 33 | with arg_scope([tcl.conv2d, tcl.conv2d_transpose], activation_fn=tf.nn.relu, 34 | normalizer_fn=tcl.batch_norm, 35 | biases_initializer=None, 36 | padding='SAME', 37 | weights_regularizer=tcl.l2_regularizer(0.0002)): 38 | size = 16 39 | # x: s x s x 3 40 | se = tcl.conv2d(x, num_outputs=size, kernel_size=4, stride=1) # 256 x 256 x 16 41 | se = resBlock(se, num_outputs=size * 2, kernel_size=4, stride=2) # 128 x 128 x 32 42 | se = resBlock(se, num_outputs=size * 2, kernel_size=4, stride=1) # 128 x 128 x 32 43 | se = resBlock(se, num_outputs=size * 4, kernel_size=4, stride=2) # 64 x 64 x 64 44 | se = resBlock(se, num_outputs=size * 4, kernel_size=4, stride=1) # 64 x 64 x 64 45 | se = resBlock(se, num_outputs=size * 8, kernel_size=4, stride=2) # 32 x 32 x 128 46 | se = resBlock(se, num_outputs=size * 8, kernel_size=4, stride=1) # 32 x 32 x 128 47 | se = resBlock(se, num_outputs=size * 16, kernel_size=4, stride=2) # 16 x 16 x 256 48 | se = resBlock(se, num_outputs=size * 16, kernel_size=4, stride=1) # 16 x 16 x 256 49 | se = resBlock(se, num_outputs=size * 32, kernel_size=4, stride=2) # 8 x 8 x 512 50 | se = resBlock(se, num_outputs=size * 32, kernel_size=4, stride=1) # 8 x 8 x 512 51 | 52 | pd = tcl.conv2d_transpose(se, size * 32, 4, stride=1) # 8 x 8 x 512 53 | pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=2) # 16 x 16 x 256 54 | pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=1) # 16 x 16 x 256 55 | pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=1) # 16 x 16 x 256 56 | pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=2) # 32 x 32 x 128 57 | pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=1) # 32 x 32 x 128 58 | pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=1) # 32 x 32 x 128 59 | pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=2) # 64 x 64 x 64 60 | pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=1) # 64 x 64 x 64 61 | pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=1) # 64 x 64 x 64 62 | 63 | pd = tcl.conv2d_transpose(pd, size * 2, 4, stride=2) # 128 x 128 x 32 64 | pd = tcl.conv2d_transpose(pd, size * 2, 4, stride=1) # 128 x 128 x 32 65 | pd = tcl.conv2d_transpose(pd, size, 4, stride=2) # 256 x 256 x 16 66 | pd = tcl.conv2d_transpose(pd, size, 4, stride=1) # 256 x 256 x 16 67 | 68 | pd = tcl.conv2d_transpose(pd, 3, 4, stride=1) # 256 x 256 x 3 69 | pd = tcl.conv2d_transpose(pd, 3, 4, stride=1) # 256 x 256 x 3 70 | pos = tcl.conv2d_transpose(pd, 3, 4, stride=1, activation_fn = tf.nn.sigmoid)#, padding='SAME', weights_initializer=tf.random_normal_initializer(0, 0.02)) 71 | 72 | return pos 73 | @property 74 | def vars(self): 75 | return [var for var in tf.global_variables() if self.name in var.name] 76 | 77 | 78 | class PosPrediction(): 79 | def __init__(self, resolution_inp = 256, resolution_op = 256): 80 | # -- hyper settings 81 | self.resolution_inp = resolution_inp 82 | self.resolution_op = resolution_op 83 | self.MaxPos = resolution_inp*1.1 84 | 85 | # network type 86 | self.network = resfcn256(self.resolution_inp, self.resolution_op) 87 | 88 | # net forward 89 | self.x = tf.placeholder(tf.float32, shape=[None, self.resolution_inp, self.resolution_inp, 3]) 90 | self.x_op = self.network(self.x, is_training = False) 91 | self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) 92 | 93 | def restore(self, model_path): 94 | tf.train.Saver(self.network.vars).restore(self.sess, model_path) 95 | 96 | def predict(self, image): 97 | pos = self.sess.run(self.x_op, 98 | feed_dict = {self.x: image[np.newaxis, :,:,:]}) 99 | pos = np.squeeze(pos) 100 | return pos*self.MaxPos 101 | 102 | def predict_batch(self, images): 103 | pos = self.sess.run(self.x_op, 104 | feed_dict = {self.x: images}) 105 | return pos*self.MaxPos 106 | 107 | -------------------------------------------------------------------------------- /codes/utils/cv_plot.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | end_list = np.array([17, 22, 27, 42, 48, 31, 36, 68], dtype = np.int32) - 1 5 | def plot_kpt(image, kpt): 6 | ''' Draw 68 key points 7 | Args: 8 | image: the input image 9 | kpt: (68, 3). 10 | ''' 11 | image = image.copy() 12 | kpt = np.round(kpt).astype(np.int32) 13 | for i in range(kpt.shape[0]): 14 | st = kpt[i, :2] 15 | image = cv2.circle(image,(st[0], st[1]), 1, (0,0,255), 2) 16 | if i in end_list: 17 | continue 18 | ed = kpt[i + 1, :2] 19 | image = cv2.line(image, (st[0], st[1]), (ed[0], ed[1]), (255, 255, 255), 1) 20 | return image 21 | 22 | 23 | def plot_vertices(image, vertices): 24 | image = image.copy() 25 | vertices = np.round(vertices).astype(np.int32) 26 | for i in range(0, vertices.shape[0], 2): 27 | st = vertices[i, :2] 28 | image = cv2.circle(image,(st[0], st[1]), 1, (255,0,0), -1) 29 | return image 30 | 31 | 32 | def plot_pose_box(image, P, kpt, color=(0, 255, 0), line_width=2): 33 | ''' Draw a 3D box as annotation of pose. Ref:https://github.com/yinguobing/head-pose-estimation/blob/master/pose_estimator.py 34 | Args: 35 | image: the input image 36 | P: (3, 4). Affine Camera Matrix. 37 | kpt: (68, 3). 38 | ''' 39 | image = image.copy() 40 | 41 | point_3d = [] 42 | rear_size = 90 43 | rear_depth = 0 44 | point_3d.append((-rear_size, -rear_size, rear_depth)) 45 | point_3d.append((-rear_size, rear_size, rear_depth)) 46 | point_3d.append((rear_size, rear_size, rear_depth)) 47 | point_3d.append((rear_size, -rear_size, rear_depth)) 48 | point_3d.append((-rear_size, -rear_size, rear_depth)) 49 | 50 | front_size = 105 51 | front_depth = 110 52 | point_3d.append((-front_size, -front_size, front_depth)) 53 | point_3d.append((-front_size, front_size, front_depth)) 54 | point_3d.append((front_size, front_size, front_depth)) 55 | point_3d.append((front_size, -front_size, front_depth)) 56 | point_3d.append((-front_size, -front_size, front_depth)) 57 | point_3d = np.array(point_3d, dtype=np.float).reshape(-1, 3) 58 | 59 | # Map to 2d image points 60 | point_3d_homo = np.hstack((point_3d, np.ones([point_3d.shape[0],1]))) #n x 4 61 | point_2d = point_3d_homo.dot(P.T)[:,:2] 62 | point_2d[:,:2] = point_2d[:,:2] - np.mean(point_2d[:4,:2], 0) + np.mean(kpt[:27,:2], 0) 63 | point_2d = np.int32(point_2d.reshape(-1, 2)) 64 | 65 | # Draw all the lines 66 | cv2.polylines(image, [point_2d], True, color, line_width, cv2.LINE_AA) 67 | cv2.line(image, tuple(point_2d[1]), tuple( 68 | point_2d[6]), color, line_width, cv2.LINE_AA) 69 | cv2.line(image, tuple(point_2d[2]), tuple( 70 | point_2d[7]), color, line_width, cv2.LINE_AA) 71 | cv2.line(image, tuple(point_2d[3]), tuple( 72 | point_2d[8]), color, line_width, cv2.LINE_AA) 73 | 74 | return image -------------------------------------------------------------------------------- /codes/utils/estimate_pose.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from math import cos, sin, atan2, asin 3 | 4 | 5 | def isRotationMatrix(R): 6 | ''' checks if a matrix is a valid rotation matrix(whether orthogonal or not) 7 | ''' 8 | Rt = np.transpose(R) 9 | shouldBeIdentity = np.dot(Rt, R) 10 | I = np.identity(3, dtype = R.dtype) 11 | n = np.linalg.norm(I - shouldBeIdentity) 12 | return n < 1e-6 13 | 14 | 15 | def matrix2angle(R): 16 | ''' compute three Euler angles from a Rotation Matrix. Ref: http://www.gregslabaugh.net/publications/euler.pdf 17 | Args: 18 | R: (3,3). rotation matrix 19 | Returns: 20 | x: yaw 21 | y: pitch 22 | z: roll 23 | ''' 24 | # assert(isRotationMatrix(R)) 25 | 26 | if R[2,0] !=1 or R[2,0] != -1: 27 | x = asin(R[2,0]) 28 | y = atan2(R[2,1]/cos(x), R[2,2]/cos(x)) 29 | z = atan2(R[1,0]/cos(x), R[0,0]/cos(x)) 30 | 31 | else:# Gimbal lock 32 | z = 0 #can be anything 33 | if R[2,0] == -1: 34 | x = np.pi/2 35 | y = z + atan2(R[0,1], R[0,2]) 36 | else: 37 | x = -np.pi/2 38 | y = -z + atan2(-R[0,1], -R[0,2]) 39 | 40 | return x, y, z 41 | 42 | 43 | def P2sRt(P): 44 | ''' decompositing camera matrix P. 45 | Args: 46 | P: (3, 4). Affine Camera Matrix. 47 | Returns: 48 | s: scale factor. 49 | R: (3, 3). rotation matrix. 50 | t2d: (2,). 2d translation. 51 | ''' 52 | t2d = P[:2, 3] 53 | R1 = P[0:1, :3] 54 | R2 = P[1:2, :3] 55 | s = (np.linalg.norm(R1) + np.linalg.norm(R2))/2.0 56 | r1 = R1/np.linalg.norm(R1) 57 | r2 = R2/np.linalg.norm(R2) 58 | r3 = np.cross(r1, r2) 59 | 60 | R = np.concatenate((r1, r2, r3), 0) 61 | return s, R, t2d 62 | 63 | 64 | def compute_similarity_transform(points_static, points_to_transform): 65 | #http://nghiaho.com/?page_id=671 66 | p0 = np.copy(points_static).T 67 | p1 = np.copy(points_to_transform).T 68 | 69 | t0 = -np.mean(p0, axis=1).reshape(3,1) 70 | t1 = -np.mean(p1, axis=1).reshape(3,1) 71 | t_final = t1 -t0 72 | 73 | p0c = p0+t0 74 | p1c = p1+t1 75 | 76 | covariance_matrix = p0c.dot(p1c.T) 77 | U,S,V = np.linalg.svd(covariance_matrix) 78 | R = U.dot(V) 79 | if np.linalg.det(R) < 0: 80 | R[:,2] *= -1 81 | 82 | rms_d0 = np.sqrt(np.mean(np.linalg.norm(p0c, axis=0)**2)) 83 | rms_d1 = np.sqrt(np.mean(np.linalg.norm(p1c, axis=0)**2)) 84 | 85 | s = (rms_d0/rms_d1) 86 | P = np.c_[s*np.eye(3).dot(R), t_final] 87 | return P 88 | 89 | def estimate_pose(vertices): 90 | canonical_vertices = np.load('Data/uv-data/canonical_vertices.npy') 91 | P = compute_similarity_transform(vertices, canonical_vertices) 92 | _,R,_ = P2sRt(P) # decompose affine matrix to s, R, t 93 | pose = matrix2angle(R) 94 | 95 | return P, pose 96 | -------------------------------------------------------------------------------- /codes/utils/render.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Author: YadiraF 3 | Mail: fengyao@sjtu.edu.cn 4 | ''' 5 | import numpy as np 6 | 7 | def isPointInTri(point, tri_points): 8 | ''' Judge whether the point is in the triangle 9 | Method: 10 | http://blackpawn.com/texts/pointinpoly/ 11 | Args: 12 | point: [u, v] or [x, y] 13 | tri_points: three vertices(2d points) of a triangle. 2 coords x 3 vertices 14 | Returns: 15 | bool: true for in triangle 16 | ''' 17 | tp = tri_points 18 | 19 | # vectors 20 | v0 = tp[:,2] - tp[:,0] 21 | v1 = tp[:,1] - tp[:,0] 22 | v2 = point - tp[:,0] 23 | 24 | # dot products 25 | dot00 = np.dot(v0.T, v0) 26 | dot01 = np.dot(v0.T, v1) 27 | dot02 = np.dot(v0.T, v2) 28 | dot11 = np.dot(v1.T, v1) 29 | dot12 = np.dot(v1.T, v2) 30 | 31 | # barycentric coordinates 32 | if dot00*dot11 - dot01*dot01 == 0: 33 | inverDeno = 0 34 | else: 35 | inverDeno = 1/(dot00*dot11 - dot01*dot01) 36 | 37 | u = (dot11*dot02 - dot01*dot12)*inverDeno 38 | v = (dot00*dot12 - dot01*dot02)*inverDeno 39 | 40 | # check if point in triangle 41 | return (u >= 0) & (v >= 0) & (u + v < 1) 42 | 43 | def get_point_weight(point, tri_points): 44 | ''' Get the weights of the position 45 | Methods: https://gamedev.stackexchange.com/questions/23743/whats-the-most-efficient-way-to-find-barycentric-coordinates 46 | -m1.compute the area of the triangles formed by embedding the point P inside the triangle 47 | -m2.Christer Ericson's book "Real-Time Collision Detection". faster, so I used this. 48 | Args: 49 | point: [u, v] or [x, y] 50 | tri_points: three vertices(2d points) of a triangle. 2 coords x 3 vertices 51 | Returns: 52 | w0: weight of v0 53 | w1: weight of v1 54 | w2: weight of v3 55 | ''' 56 | tp = tri_points 57 | # vectors 58 | v0 = tp[:,2] - tp[:,0] 59 | v1 = tp[:,1] - tp[:,0] 60 | v2 = point - tp[:,0] 61 | 62 | # dot products 63 | dot00 = np.dot(v0.T, v0) 64 | dot01 = np.dot(v0.T, v1) 65 | dot02 = np.dot(v0.T, v2) 66 | dot11 = np.dot(v1.T, v1) 67 | dot12 = np.dot(v1.T, v2) 68 | 69 | # barycentric coordinates 70 | if dot00*dot11 - dot01*dot01 == 0: 71 | inverDeno = 0 72 | else: 73 | inverDeno = 1/(dot00*dot11 - dot01*dot01) 74 | 75 | u = (dot11*dot02 - dot01*dot12)*inverDeno 76 | v = (dot00*dot12 - dot01*dot02)*inverDeno 77 | 78 | w0 = 1 - u - v 79 | w1 = v 80 | w2 = u 81 | 82 | return w0, w1, w2 83 | 84 | 85 | def render_texture(vertices, colors, triangles, h, w, c = 3): 86 | ''' render mesh by z buffer 87 | Args: 88 | vertices: 3 x nver 89 | colors: 3 x nver 90 | triangles: 3 x ntri 91 | h: height 92 | w: width 93 | ''' 94 | # initial 95 | image = np.zeros((h, w, c)) 96 | 97 | depth_buffer = np.zeros([h, w]) - 999999. 98 | # triangle depth: approximate the depth to the average value of z in each vertex(v0, v1, v2), since the vertices are closed to each other 99 | tri_depth = (vertices[2, triangles[0,:]] + vertices[2,triangles[1,:]] + vertices[2, triangles[2,:]])/3. 100 | tri_tex = (colors[:, triangles[0,:]] + colors[:,triangles[1,:]] + colors[:, triangles[2,:]])/3. 101 | 102 | for i in range(triangles.shape[1]): 103 | tri = triangles[:, i] # 3 vertex indices 104 | 105 | # the inner bounding box 106 | umin = max(int(np.ceil(np.min(vertices[0,tri]))), 0) 107 | umax = min(int(np.floor(np.max(vertices[0,tri]))), w-1) 108 | 109 | vmin = max(int(np.ceil(np.min(vertices[1,tri]))), 0) 110 | vmax = min(int(np.floor(np.max(vertices[1,tri]))), h-1) 111 | 112 | if umax depth_buffer[v, u] and isPointInTri([u,v], vertices[:2, tri]): 118 | depth_buffer[v, u] = tri_depth[i] 119 | image[v, u, :] = tri_tex[:, i] 120 | return image 121 | 122 | 123 | def map_texture(src_image, src_vertices, dst_vertices, dst_triangle_buffer, triangles, h, w, c = 3, mapping_type = 'bilinear'): 124 | ''' 125 | Args: 126 | triangles: 3 x ntri 127 | 128 | # src 129 | src_image: height x width x nchannels 130 | src_vertices: 3 x nver 131 | 132 | # dst 133 | dst_vertices: 3 x nver 134 | dst_triangle_buffer: height x width. the triangle index of each pixel in dst image 135 | 136 | Returns: 137 | dst_image: height x width x nchannels 138 | 139 | ''' 140 | [sh, sw, sc] = src_image.shape 141 | dst_image = np.zeros((h, w, c)) 142 | for y in range(h): 143 | for x in range(w): 144 | tri_ind = dst_triangle_buffer[y,x] 145 | if tri_ind < 0: # no tri in dst image 146 | continue 147 | #if src_triangles_vis[tri_ind]: # the corresponding triangle in src image is invisible 148 | # continue 149 | 150 | # then. For this triangle index, map corresponding pixels(in triangles) in src image to dst image 151 | # Two Methods: 152 | # M1. Calculate the corresponding affine matrix from src triangle to dst triangle. Then find the corresponding src position of this dst pixel. 153 | # -- ToDo 154 | # M2. Calculate the relative position of three vertices in dst triangle, then find the corresponding src position relative to three src vertices. 155 | tri = triangles[:, tri_ind] 156 | # dst weight, here directly use the center to approximate because the tri is small 157 | # if tri_ind < 366: 158 | # print tri_ind 159 | w0, w1, w2 = get_point_weight([x, y], dst_vertices[:2, tri]) 160 | # else: 161 | # w0 = w1 = w2 = 1./3 162 | # src 163 | src_texel = w0*src_vertices[:2, tri[0]] + w1*src_vertices[:2, tri[1]] + w2*src_vertices[:2, tri[2]] # 164 | # 165 | if src_texel[0] < 0 or src_texel[0]> sw-1 or src_texel[1]<0 or src_texel[1] > sh-1: 166 | dst_image[y, x, :] = 0 167 | continue 168 | # As the coordinates of the transformed pixel in the image will most likely not lie on a texel, we have to choose how to 169 | # calculate the pixel colors depending on the next texels 170 | # there are three different texture interpolation methods: area, bilinear and nearest neighbour 171 | # print y, x, src_texel 172 | # nearest neighbour 173 | if mapping_type == 'nearest': 174 | dst_image[y, x, :] = src_image[int(round(src_texel[1])), int(round(src_texel[0])), :] 175 | # bilinear 176 | elif mapping_type == 'bilinear': 177 | # next 4 pixels 178 | ul = src_image[int(np.floor(src_texel[1])), int(np.floor(src_texel[0])), :] 179 | ur = src_image[int(np.floor(src_texel[1])), int(np.ceil(src_texel[0])), :] 180 | dl = src_image[int(np.ceil(src_texel[1])), int(np.floor(src_texel[0])), :] 181 | dr = src_image[int(np.ceil(src_texel[1])), int(np.ceil(src_texel[0])), :] 182 | 183 | yd = src_texel[1] - np.floor(src_texel[1]) 184 | xd = src_texel[0] - np.floor(src_texel[0]) 185 | dst_image[y, x, :] = ul*(1-xd)*(1-yd) + ur*xd*(1-yd) + dl*(1-xd)*yd + dr*xd*yd 186 | 187 | return dst_image 188 | 189 | 190 | def get_depth_buffer(vertices, triangles, h, w): 191 | ''' 192 | Args: 193 | vertices: 3 x nver 194 | triangles: 3 x ntri 195 | h: height 196 | w: width 197 | Returns: 198 | depth_buffer: height x width 199 | ToDo: 200 | whether to add x, y by 0.5? the center of the pixel? 201 | m3. like somewhere is wrong 202 | # Each triangle has 3 vertices & Each vertex has 3 coordinates x, y, z. 203 | # Here, the bigger the z, the fronter the point. 204 | ''' 205 | # initial 206 | depth_buffer = np.zeros([h, w]) - 999999. #+ np.min(vertices[2,:]) - 999999. # set the initial z to the farest position 207 | 208 | ## calculate the depth(z) of each triangle 209 | #-m1. z = the center of shpere(through 3 vertices) 210 | #center3d = (vertices[:, triangles[0,:]] + vertices[:,triangles[1,:]] + vertices[:, triangles[2,:]])/3. 211 | #tri_depth = np.sum(center3d**2, axis = 0) 212 | #-m2. z = the center of z(v0, v1, v2) 213 | tri_depth = (vertices[2, triangles[0,:]] + vertices[2,triangles[1,:]] + vertices[2, triangles[2,:]])/3. 214 | 215 | for i in range(triangles.shape[1]): 216 | tri = triangles[:, i] # 3 vertex indices 217 | 218 | # the inner bounding box 219 | umin = max(int(np.ceil(np.min(vertices[0,tri]))), 0) 220 | umax = min(int(np.floor(np.max(vertices[0,tri]))), w-1) 221 | 222 | vmin = max(int(np.ceil(np.min(vertices[1,tri]))), 0) 223 | vmax = min(int(np.floor(np.max(vertices[1,tri]))), h-1) 224 | 225 | if umax depth_buffer[v, u]: # and is_pointIntri([u,v], vertices[:2, tri]): 234 | depth_buffer[v, u] = tri_depth[i] 235 | 236 | return depth_buffer 237 | 238 | 239 | def get_triangle_buffer(vertices, triangles, h, w): 240 | ''' 241 | Args: 242 | vertices: 3 x nver 243 | triangles: 3 x ntri 244 | h: height 245 | w: width 246 | Returns: 247 | depth_buffer: height x width 248 | ToDo: 249 | whether to add x, y by 0.5? the center of the pixel? 250 | m3. like somewhere is wrong 251 | # Each triangle has 3 vertices & Each vertex has 3 coordinates x, y, z. 252 | # Here, the bigger the z, the fronter the point. 253 | ''' 254 | # initial 255 | depth_buffer = np.zeros([h, w]) - 999999. #+ np.min(vertices[2,:]) - 999999. # set the initial z to the farest position 256 | triangle_buffer = np.zeros_like(depth_buffer, dtype = np.int32) - 1 # if -1, the pixel has no triangle correspondance 257 | 258 | ## calculate the depth(z) of each triangle 259 | #-m1. z = the center of shpere(through 3 vertices) 260 | #center3d = (vertices[:, triangles[0,:]] + vertices[:,triangles[1,:]] + vertices[:, triangles[2,:]])/3. 261 | #tri_depth = np.sum(center3d**2, axis = 0) 262 | #-m2. z = the center of z(v0, v1, v2) 263 | tri_depth = (vertices[2, triangles[0,:]] + vertices[2,triangles[1,:]] + vertices[2, triangles[2,:]])/3. 264 | 265 | for i in range(triangles.shape[1]): 266 | tri = triangles[:, i] # 3 vertex indices 267 | 268 | # the inner bounding box 269 | umin = max(int(np.ceil(np.min(vertices[0,tri]))), 0) 270 | umax = min(int(np.floor(np.max(vertices[0,tri]))), w-1) 271 | 272 | vmin = max(int(np.ceil(np.min(vertices[1,tri]))), 0) 273 | vmax = min(int(np.floor(np.max(vertices[1,tri]))), h-1) 274 | 275 | if umax depth_buffer[v, u] and isPointInTri([u,v], vertices[:2, tri]): 284 | depth_buffer[v, u] = tri_depth[i] 285 | triangle_buffer[v, u] = i 286 | 287 | return triangle_buffer 288 | 289 | 290 | def vis_of_vertices(vertices, triangles, h, w, depth_buffer = None): 291 | ''' 292 | Args: 293 | vertices: 3 x nver 294 | triangles: 3 x ntri 295 | depth_buffer: height x width 296 | Returns: 297 | vertices_vis: nver. the visibility of each vertex 298 | ''' 299 | if depth_buffer == None: 300 | depth_buffer = get_depth_buffer(vertices, triangles, h, w) 301 | 302 | vertices_vis = np.zeros(vertices.shape[1], dtype = bool) 303 | 304 | depth_tmp = np.zeros_like(depth_buffer) - 99999 305 | for i in range(vertices.shape[1]): 306 | vertex = vertices[:, i] 307 | 308 | if np.floor(vertex[0]) < 0 or np.ceil(vertex[0]) > w-1 or np.floor(vertex[1]) < 0 or np.ceil(vertex[1]) > h-1: 309 | continue 310 | 311 | # bilinear interp 312 | # ul = depth_buffer[int(np.floor(vertex[1])), int(np.floor(vertex[0]))] 313 | # ur = depth_buffer[int(np.floor(vertex[1])), int(np.ceil(vertex[0]))] 314 | # dl = depth_buffer[int(np.ceil(vertex[1])), int(np.floor(vertex[0]))] 315 | # dr = depth_buffer[int(np.ceil(vertex[1])), int(np.ceil(vertex[0]))] 316 | 317 | # yd = vertex[1] - np.floor(vertex[1]) 318 | # xd = vertex[0] - np.floor(vertex[0]) 319 | 320 | # vertex_depth = ul*(1-xd)*(1-yd) + ur*xd*(1-yd) + dl*(1-xd)*yd + dr*xd*yd 321 | 322 | # nearest 323 | px = int(np.round(vertex[0])) 324 | py = int(np.round(vertex[1])) 325 | 326 | # if (vertex[2] > depth_buffer[ul[0], ul[1]]) & (vertex[2] > depth_buffer[ur[0], ur[1]]) & (vertex[2] > depth_buffer[dl[0], dl[1]]) & (vertex[2] > depth_buffer[dr[0], dr[1]]): 327 | if vertex[2] < depth_tmp[py, px]: 328 | continue 329 | 330 | # if vertex[2] > depth_buffer[py, px]: 331 | # vertices_vis[i] = True 332 | # depth_tmp[py, px] = vertex[2] 333 | # elif np.abs(vertex[2] - depth_buffer[py, px]) < 1: 334 | # vertices_vis[i] = True 335 | 336 | threshold = 2 # need to be optimized. 337 | if np.abs(vertex[2] - depth_buffer[py, px]) < threshold: 338 | # if np.abs(vertex[2] - vertex_depth) < threshold: 339 | vertices_vis[i] = True 340 | depth_tmp[py, px] = vertex[2] 341 | 342 | return vertices_vis 343 | -------------------------------------------------------------------------------- /codes/utils/render_app.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from utils.render import vis_of_vertices, render_texture 3 | from scipy import ndimage 4 | 5 | def get_visibility(vertices, triangles, h, w): 6 | triangles = triangles.T 7 | vertices_vis = vis_of_vertices(vertices.T, triangles, h, w) 8 | vertices_vis = vertices_vis.astype(bool) 9 | for k in range(2): 10 | tri_vis = vertices_vis[triangles[0,:]] | vertices_vis[triangles[1,:]] | vertices_vis[triangles[2,:]] 11 | ind = triangles[:, tri_vis] 12 | vertices_vis[ind] = True 13 | # for k in range(2): 14 | # tri_vis = vertices_vis[triangles[0,:]] & vertices_vis[triangles[1,:]] & vertices_vis[triangles[2,:]] 15 | # ind = triangles[:, tri_vis] 16 | # vertices_vis[ind] = True 17 | vertices_vis = vertices_vis.astype(np.float32) #1 for visible and 0 for non-visible 18 | return vertices_vis 19 | 20 | def get_uv_mask(vertices_vis, triangles, uv_coords, h, w, resolution): 21 | triangles = triangles.T 22 | vertices_vis = vertices_vis.astype(np.float32) 23 | uv_mask = render_texture(uv_coords.T, vertices_vis[np.newaxis, :], triangles, resolution, resolution, 1) 24 | uv_mask = np.squeeze(uv_mask > 0) 25 | uv_mask = ndimage.binary_closing(uv_mask) 26 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 27 | uv_mask = ndimage.binary_closing(uv_mask) 28 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 29 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 30 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 31 | uv_mask = uv_mask.astype(np.float32) 32 | 33 | return np.squeeze(uv_mask) 34 | 35 | def get_depth_image(vertices, triangles, h, w, isShow = False): 36 | z = vertices[:, 2:] 37 | if isShow: 38 | z = z/max(z) 39 | depth_image = render_texture(vertices.T, z.T, triangles.T, h, w, 1) 40 | return np.squeeze(depth_image) -------------------------------------------------------------------------------- /codes/utils/rotate_vertices.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # import scipy.io as 4 | def frontalize(vertices): 5 | canonical_vertices = np.load('Data/uv-data/canonical_vertices.npy') 6 | 7 | vertices_homo = np.hstack((vertices, np.ones([vertices.shape[0],1]))) #n x 4 8 | P = np.linalg.lstsq(vertices_homo, canonical_vertices)[0].T # Affine matrix. 3 x 4 9 | front_vertices = vertices_homo.dot(P.T) 10 | 11 | return front_vertices 12 | -------------------------------------------------------------------------------- /codes/utils/write.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage.io import imsave 3 | import os 4 | 5 | def write_asc(path, vertices): 6 | ''' 7 | Args: 8 | vertices: shape = (nver, 3) 9 | ''' 10 | if path.split('.')[-1] == 'asc': 11 | np.savetxt(path, vertices) 12 | else: 13 | np.savetxt(path + '.asc', vertices) 14 | 15 | 16 | def write_obj_with_colors(obj_name, vertices, triangles, colors): 17 | ''' Save 3D face model with texture represented by colors. 18 | Args: 19 | obj_name: str 20 | vertices: shape = (nver, 3) 21 | colors: shape = (nver, 3) 22 | triangles: shape = (ntri, 3) 23 | ''' 24 | triangles = triangles.copy() 25 | triangles += 1 # meshlab start with 1 26 | 27 | if obj_name.split('.')[-1] != 'obj': 28 | obj_name = obj_name + '.obj' 29 | 30 | # write obj 31 | with open(obj_name, 'w') as f: 32 | 33 | # write vertices & colors 34 | for i in range(vertices.shape[0]): 35 | # s = 'v {} {} {} \n'.format(vertices[0,i], vertices[1,i], vertices[2,i]) 36 | s = 'v {} {} {} {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2], colors[i, 0], colors[i, 1], colors[i, 2]) 37 | f.write(s) 38 | 39 | # write f: ver ind/ uv ind 40 | [k, ntri] = triangles.shape 41 | for i in range(triangles.shape[0]): 42 | # s = 'f {} {} {}\n'.format(triangles[i, 0], triangles[i, 1], triangles[i, 2]) 43 | s = 'f {} {} {}\n'.format(triangles[i, 2], triangles[i, 1], triangles[i, 0]) 44 | f.write(s) 45 | 46 | 47 | def write_obj_with_texture(obj_name, vertices, triangles, texture, uv_coords): 48 | ''' Save 3D face model with texture represented by texture map. 49 | Ref: https://github.com/patrikhuber/eos/blob/bd00155ebae4b1a13b08bf5a991694d682abbada/include/eos/core/Mesh.hpp 50 | Args: 51 | obj_name: str 52 | vertices: shape = (nver, 3) 53 | triangles: shape = (ntri, 3) 54 | texture: shape = (256,256,3) 55 | uv_coords: shape = (nver, 3) max value<=1 56 | ''' 57 | if obj_name.split('.')[-1] != 'obj': 58 | obj_name = obj_name + '.obj' 59 | mtl_name = obj_name.replace('.obj', '.mtl') 60 | texture_name = obj_name.replace('.obj', '_texture.png') 61 | 62 | triangles = triangles.copy() 63 | triangles += 1 # mesh lab start with 1 64 | 65 | # write obj 66 | with open(obj_name, 'w') as f: 67 | # first line: write mtlib(material library) 68 | s = "mtllib {}\n".format(os.path.abspath(mtl_name)) 69 | f.write(s) 70 | 71 | # write vertices 72 | for i in range(vertices.shape[0]): 73 | s = 'v {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2]) 74 | f.write(s) 75 | 76 | # write uv coords 77 | for i in range(uv_coords.shape[0]): 78 | s = 'vt {} {}\n'.format(uv_coords[i,0], 1 - uv_coords[i,1]) 79 | f.write(s) 80 | 81 | f.write("usemtl FaceTexture\n") 82 | 83 | # write f: ver ind/ uv ind 84 | for i in range(triangles.shape[0]): 85 | # s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,0], triangles[i,0], triangles[i,1], triangles[i,1], triangles[i,2], triangles[i,2]) 86 | s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,2], triangles[i,2], triangles[i,1], triangles[i,1], triangles[i,0], triangles[i,0]) 87 | f.write(s) 88 | 89 | # write mtl 90 | with open(mtl_name, 'w') as f: 91 | f.write("newmtl FaceTexture\n") 92 | s = 'map_Kd {}\n'.format(os.path.abspath(texture_name)) # map to image 93 | f.write(s) 94 | 95 | # write texture as png 96 | imsave(texture_name, texture) 97 | 98 | 99 | def write_obj_with_colors_texture(obj_name, vertices, colors, triangles, texture, uv_coords): 100 | ''' Save 3D face model with texture. 101 | Ref: https://github.com/patrikhuber/eos/blob/bd00155ebae4b1a13b08bf5a991694d682abbada/include/eos/core/Mesh.hpp 102 | Args: 103 | obj_name: str 104 | vertices: shape = (nver, 3) 105 | colors: shape = (nver, 3) 106 | triangles: shape = (ntri, 3) 107 | texture: shape = (256,256,3) 108 | uv_coords: shape = (nver, 3) max value<=1 109 | ''' 110 | if obj_name.split('.')[-1] != 'obj': 111 | obj_name = obj_name + '.obj' 112 | mtl_name = obj_name.replace('.obj', '.mtl') 113 | texture_name = obj_name.replace('.obj', '_texture.png') 114 | 115 | triangles = triangles.copy() 116 | triangles += 1 # mesh lab start with 1 117 | 118 | # write obj 119 | with open(obj_name, 'w') as f: 120 | # first line: write mtlib(material library) 121 | s = "mtllib {}\n".format(os.path.abspath(mtl_name)) 122 | f.write(s) 123 | 124 | # write vertices 125 | for i in range(vertices.shape[0]): 126 | s = 'v {} {} {} {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2], colors[i, 0], colors[i, 1], colors[i, 2]) 127 | f.write(s) 128 | 129 | # write uv coords 130 | for i in range(uv_coords.shape[0]): 131 | s = 'vt {} {}\n'.format(uv_coords[i,0], 1 - uv_coords[i,1]) 132 | f.write(s) 133 | 134 | f.write("usemtl FaceTexture\n") 135 | 136 | # write f: ver ind/ uv ind 137 | for i in range(triangles.shape[0]): 138 | # s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,0], triangles[i,0], triangles[i,1], triangles[i,1], triangles[i,2], triangles[i,2]) 139 | s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,2], triangles[i,2], triangles[i,1], triangles[i,1], triangles[i,0], triangles[i,0]) 140 | f.write(s) 141 | 142 | # write mtl 143 | with open(mtl_name, 'w') as f: 144 | f.write("newmtl FaceTexture\n") 145 | s = 'map_Kd {}\n'.format(os.path.abspath(texture_name)) # map to image 146 | f.write(s) 147 | 148 | # write texture as png 149 | imsave(texture_name, texture) -------------------------------------------------------------------------------- /combined.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsouri/Face-Swap/308474c678255952a7040840ff3905cf809b8601/combined.png --------------------------------------------------------------------------------