├── Destination ├── Mark-vid-input.mp4 └── sontung.webp ├── ImageProcessing.py ├── Image_Face_Swapping.py ├── README.md ├── Realtime_Face_Swapping.py ├── Target ├── target.mp4 └── timothee.webp ├── Video_face_Swapping.py ├── canonical_face_model_uv_visualization.png ├── face_utils.py ├── result ├── result.mp4 └── sontungtimothee.jpg ├── samples.png └── utils.py /Destination/Mark-vid-input.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/Destination/Mark-vid-input.mp4 -------------------------------------------------------------------------------- /Destination/sontung.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/Destination/sontung.webp -------------------------------------------------------------------------------- /ImageProcessing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | def find_nearest_above(my_array, target): 4 | diff = my_array - target 5 | mask = np.ma.less_equal(diff, -1) 6 | # We need to mask the negative differences 7 | # since we are looking for values above 8 | if np.all(mask): 9 | c = np.abs(diff).argmin() 10 | return c # returns min index of the nearest if target is greater than any value 11 | masked_diff = np.ma.masked_array(diff, mask) 12 | return masked_diff.argmin() 13 | def hist_match(original, specified): 14 | oldshape = original.shape 15 | original = original.ravel() 16 | specified = specified.ravel() 17 | 18 | # get the set of unique pixel values and their corresponding indices and counts 19 | s_values, bin_idx, s_counts = np.unique(original, return_inverse=True,return_counts=True) 20 | t_values, t_counts = np.unique(specified, return_counts=True) 21 | 22 | # Calculate s_k for original image 23 | s_quantiles = np.cumsum(s_counts).astype(np.float64) 24 | s_quantiles /= s_quantiles[-1] 25 | 26 | # Calculate s_k for specified image 27 | t_quantiles = np.cumsum(t_counts).astype(np.float64) 28 | t_quantiles /= t_quantiles[-1] 29 | 30 | # Round the values 31 | sour = np.around(s_quantiles*255) 32 | temp = np.around(t_quantiles*255) 33 | 34 | # Map the rounded values 35 | b=[] 36 | for data in sour[:]: 37 | b.append(find_nearest_above(temp,data)) 38 | b= np.array(b,dtype='uint8') 39 | 40 | return b[bin_idx].reshape(oldshape) 41 | 42 | def blend_with_mask_matrix(src1, src2, mask): 43 | res_channels = [] 44 | for c in range(0, src1.shape[2]): 45 | a = src1[:, :, c] 46 | b = src2[:, :, c] 47 | m = mask[:, :] 48 | #Alpha blending 49 | res = cv2.add( 50 | cv2.multiply(b, cv2.divide(np.full_like(m, 255) - m, 255.0, dtype=cv2.CV_32F), dtype=cv2.CV_32F), 51 | cv2.multiply(a, cv2.divide(m, 255.0, dtype=cv2.CV_32F), dtype=cv2.CV_32F), 52 | dtype=cv2.CV_8U) 53 | res_channels += [res] 54 | res = cv2.merge(res_channels) 55 | return res -------------------------------------------------------------------------------- /Image_Face_Swapping.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | import face_utils 5 | import ImageProcessing 6 | 7 | def main(dest_img_path, target_img_path, result_path='result.jpg'): 8 | dest_img=cv2.imread(dest_img_path) 9 | target_img= cv2.imread(target_img_path) 10 | 11 | dest_xyz_landmark_points, dest_landmark_points= face_utils.get_face_landmark(dest_img) 12 | dest_convexhull= cv2.convexHull(np.array(dest_landmark_points)) 13 | 14 | target_img_hist_match=ImageProcessing.hist_match(target_img,dest_img) 15 | 16 | _, target_landmark_points= face_utils.get_face_landmark(target_img) 17 | target_convexhull= cv2.convexHull(np.array(target_landmark_points)) 18 | 19 | new_face, result= face_utils.face_swapping(dest_img, dest_landmark_points, dest_xyz_landmark_points, dest_convexhull, target_img, target_landmark_points, target_convexhull, return_face= True) 20 | 21 | height, width, _ = dest_img.shape 22 | h, w, _ = target_img.shape 23 | rate= width/w 24 | cv2.imshow("Destination image", dest_img) 25 | cv2.imshow("Target image", cv2.resize(target_img, (int(w * rate), int(h * rate)))) 26 | cv2.imshow("New face", new_face) 27 | cv2.imshow("Result", result) 28 | cv2.imwrite(result_path, result) 29 | cv2.waitKey(0) 30 | 31 | if __name__ == "__main__": 32 | dest_img_path='sontung.webp' 33 | target_img_path='timothee.webp' 34 | result_path='result/sontungtimothee.jpg' 35 | main(dest_img_path, target_img_path, result_path) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face Swapping 2 | ![alt text](https://github.com/htuannn/Face-Swapping/blob/f94eb91dd1be28bf02bd557fffed32cf1d2e918d/samples.png "Samples") 3 | 4 | Code for face swapping method preprocessing on image-to-image, video frames, and realtime camera. 5 | 6 | This app is written by Python3 and uses OpenCV, and MediaPipe Face Mesh and image blending to swap the face of a person in destination image/video or seen by camera with a face of a person in a provided target image/video. 7 | 8 | ## Introduction 9 | Face swapping has become a popular application in recent years, allowing users to swap their faces with others in images or videos. Using the Mediapipe Face Mesh, a facial landmark detection model, our application offers a seamless and realistic face swapping experience. 10 | 11 | By incorporating 3D facial angle estimation, the task of face swapping has become even more accurate and precise. With this advanced technology, we are able to map and manipulate facial features in three-dimensional space, resulting in a more realistic and seamless face swapping experience, providing an entertaining and engaging experience for users. 12 | 13 | ## Requirement 14 | To start the program you will have to run a Python file. You need to have Python3 and some additional libraries installed (Numpy, OpenCV-Python, Glob). 15 | 16 | You also have to install Mediapipe pakages from Google: https://google.github.io/mediapipe/getting_started/python.html 17 | 18 | Mediapipe can be installed with the following command: 19 | ``` 20 | pip install mediapipe 21 | ``` 22 | 23 | ## Usage 24 | Firstly, change the path to your own destination and target image/video in **_Face_Swapping.py source code file. 25 | 26 | For running the applications, run the following commands: 27 | - Image swapping: 28 | ``` 29 | python Image_Face_Swapping.py 30 | ``` 31 | - Video swapping: 32 | ``` 33 | python Video_Face_Swapping.py 34 | ``` 35 | - Realtime swapping: 36 | ``` 37 | python Realtime_Face_Swapping.py 38 | ``` 39 | 40 | ## Related 41 | - [Face Swapping with OpenCV](https://pysource.com/2019/05/28/face-swapping-explained-in-8-steps-opencv-with-python/) 42 | - [MediaPipe Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh.html) 43 | -------------------------------------------------------------------------------- /Realtime_Face_Swapping.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | import face_utils 5 | import utils 6 | import ImageProcessing 7 | import glob 8 | 9 | def main(): 10 | target_path="Target/target.mp4" 11 | #img_path="Target/Timothee.jpeg" 12 | #target_imgs= [cv2.imread(img_path)] 13 | 14 | #target_imgs= [cv2.imread(file) for file in glob.glob("Target/*.jpg")] 15 | 16 | target_imgs = utils.extract_frame_from_video(target_path) 17 | 18 | targets_xyz_landmark_points=[] 19 | targets_landmark_points=[] 20 | temp=[] 21 | for target_img in target_imgs: 22 | face_landmark=face_utils.get_face_landmark(target_img) 23 | if face_landmark is None: 24 | continue 25 | xyz_landmark_points, landmark_points = face_landmark 26 | targets_xyz_landmark_points.append(xyz_landmark_points) 27 | targets_landmark_points.append(landmark_points) 28 | temp.append(target_img) 29 | target_imgs= temp 30 | 31 | targets_facial_angle=[] 32 | for target_xyz_landmark_points in targets_xyz_landmark_points: 33 | target_left_iris=face_utils.get_iris_landmark(target_xyz_landmark_points, return_xyz=True) 34 | target_right_iris=face_utils.get_iris_landmark(target_xyz_landmark_points, return_xyz=True, location='Right') 35 | target_facial_angle=utils.AngleOfDepression(utils.getCenter_xyz(target_left_iris)[0], utils.getCenter_xyz(target_right_iris)[0]) 36 | targets_facial_angle.append(target_facial_angle) 37 | 38 | cap= cv2.VideoCapture(0) 39 | while (cap.isOpened()): 40 | ret, frame= cap.read() 41 | if ret==True: 42 | frame = cv2.flip(frame,1) 43 | dest_img = frame.copy() 44 | 45 | face_landmark= face_utils.get_face_landmark(dest_img) 46 | if face_landmark is None: 47 | cv2.imshow("Result", frame) 48 | if cv2.waitKey(1) & 0xFF == ord('q'): 49 | break 50 | continue 51 | dest_xyz_landmark_points, dest_landmark_points= face_landmark 52 | 53 | dest_left_iris=face_utils.get_iris_landmark(dest_xyz_landmark_points, return_xyz=True) 54 | dest_right_iris=face_utils.get_iris_landmark(dest_xyz_landmark_points, return_xyz=True, location='Right') 55 | 56 | dest_facial_angle=utils.AngleOfDepression(utils.getCenter_xyz(dest_left_iris)[0], utils.getCenter_xyz(dest_right_iris)[0]) 57 | most_simmilar_facial_angle= np.argmin(abs(targets_facial_angle - dest_facial_angle)) 58 | target_img=target_imgs[most_simmilar_facial_angle] 59 | 60 | if dest_landmark_points is None or len(dest_landmark_points) < 478: 61 | continue 62 | 63 | dest_convexhull= cv2.convexHull(np.array(dest_landmark_points)) 64 | 65 | #target_img=ImageProcessing.hist_match(target_img,dest_img) 66 | 67 | target_landmark_points= targets_landmark_points[most_simmilar_facial_angle] 68 | target_convexhull= cv2.convexHull(np.array(target_landmark_points)) 69 | 70 | new_face, result= face_utils.face_swapping(dest_img, dest_landmark_points, dest_xyz_landmark_points, dest_convexhull, target_img, target_landmark_points, target_convexhull, return_face= True) 71 | 72 | height, width, _ = frame.shape 73 | h, w, _ = target_img.shape 74 | rate= width/w 75 | cv2.imshow("Target image", cv2.resize(target_img, (int(w * rate), int(h * rate)))) 76 | cv2.imshow("New face", new_face) 77 | cv2.imshow("Result", result) 78 | 79 | if cv2.waitKey(1) & 0xFF == ord('q'): 80 | break 81 | cap.release() 82 | if __name__ == "__main__": 83 | main() -------------------------------------------------------------------------------- /Target/target.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/Target/target.mp4 -------------------------------------------------------------------------------- /Target/timothee.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/Target/timothee.webp -------------------------------------------------------------------------------- /Video_face_Swapping.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | import face_utils 5 | import utils 6 | import ImageProcessing 7 | import glob 8 | 9 | def main(): 10 | target_video_path="Target/target.mp4" 11 | #target_img_path="Target/Timothee.jpeg" 12 | #target_imgs= [cv2.imread(target_img_path)] 13 | 14 | #target_imgs= [cv2.imread(file) for file in glob.glob("Target/*.jpg")] 15 | 16 | target_imgs = utils.extract_frame_from_video(target_video_path) 17 | targets_xyz_landmark_points=[] 18 | targets_landmark_points=[] 19 | for target_img in target_imgs: 20 | xyz_landmark_points, landmark_points = face_utils.get_face_landmark(target_img) 21 | targets_xyz_landmark_points.append(xyz_landmark_points) 22 | targets_landmark_points.append(landmark_points) 23 | 24 | targets_facial_angle=[] 25 | for target_xyz_landmark_points in targets_xyz_landmark_points: 26 | target_left_iris=face_utils.get_iris_landmark(target_xyz_landmark_points, return_xyz=True) 27 | target_right_iris=face_utils.get_iris_landmark(target_xyz_landmark_points, return_xyz=True, location='Right') 28 | target_facial_angle=utils.AngleOfDepression(utils.getCenter_xyz(target_left_iris)[0], utils.getCenter_xyz(target_right_iris)[0]) 29 | targets_facial_angle.append(target_facial_angle) 30 | 31 | vid = cv2.VideoCapture("Destination/Mark-vid-input.mp4") 32 | ret = 1 33 | #video write 34 | frame_width = int(vid.get(3)) 35 | frame_height = int(vid.get(4)) 36 | 37 | size = (frame_width, frame_height) 38 | 39 | # Below VideoWriter object will create 40 | # a frame of above defined The output 41 | # is stored in 'filename.avi' file. 42 | #result = cv2.VideoWriter('result.mp4',cv2.VideoWriter_fourcc(*'XVID'),30, size) 43 | 44 | while (ret): 45 | ret,frame = vid.read() 46 | if ret==True: 47 | 48 | dest_img = frame.copy() 49 | face_landmark=face_utils.get_face_landmark(dest_img) 50 | if face_landmark is None: 51 | #result.write(frame) 52 | cv2.imshow("Result", frame) 53 | if cv2.waitKey(1) & 0xFF == ord('q'): 54 | break 55 | continue 56 | dest_xyz_landmark_points, dest_landmark_points = face_landmark 57 | dest_left_iris=face_utils.get_iris_landmark(dest_xyz_landmark_points, return_xyz=True) 58 | dest_right_iris=face_utils.get_iris_landmark(dest_xyz_landmark_points, return_xyz=True, location='Right') 59 | 60 | dest_facial_angle=utils.AngleOfDepression(utils.getCenter_xyz(dest_left_iris)[0], utils.getCenter_xyz(dest_right_iris)[0]) 61 | most_simmilar_facial_angle= np.argmin(abs(targets_facial_angle - dest_facial_angle)) 62 | target_img=target_imgs[most_simmilar_facial_angle] 63 | 64 | if dest_landmark_points is None or len(dest_landmark_points) < 478: 65 | continue 66 | 67 | dest_convexhull= cv2.convexHull(np.array(dest_landmark_points)) 68 | 69 | #target_img=ImageProcessing.hist_match(target_img,dest_img) 70 | 71 | target_landmark_points= targets_landmark_points[most_simmilar_facial_angle] 72 | target_convexhull= cv2.convexHull(np.array(target_landmark_points)) 73 | 74 | new_face, frame= face_utils.face_swapping(dest_img, dest_landmark_points, dest_xyz_landmark_points, dest_convexhull, target_img, target_landmark_points, target_convexhull, return_face= True) 75 | 76 | height, width, _ = frame.shape 77 | h, w, _ = target_img.shape 78 | rate= width/w 79 | #cv2.imshow("Target image", cv2.resize(target_img, (int(w * rate), int(h * rate)))) 80 | #cv2.imshow("New face", new_face) 81 | cv2.imshow("Result", frame) 82 | #result.write(frame) 83 | 84 | if cv2.waitKey(1) & 0xFF == ord('q'): 85 | break 86 | vid.release() 87 | #result.release() 88 | 89 | print("done!!") 90 | if __name__ == "__main__": 91 | main() 92 | -------------------------------------------------------------------------------- /canonical_face_model_uv_visualization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/canonical_face_model_uv_visualization.png -------------------------------------------------------------------------------- /face_utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | import utils 5 | import ImageProcessing 6 | mp_drawing = mp.solutions.drawing_utils 7 | mp_face_mesh = mp.solutions.face_mesh 8 | 9 | mouth_landmark_index=[13,312,311,310,415,308,324,318,402,317,14,87,178,88,95,78, 191, 80, 81, 82] 10 | 11 | 12 | def get_face_landmark(img): 13 | with mp_face_mesh.FaceMesh(static_image_mode = True, 14 | refine_landmarks=True, 15 | max_num_faces=1) as face_mesh: 16 | results= face_mesh.process(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) 17 | if results.multi_face_landmarks is None: 18 | return None 19 | if len(results.multi_face_landmarks) > 1: 20 | print("There are too much face") 21 | 22 | xyz_landmark_points=[] 23 | landmark_points= [] 24 | 25 | face_landmark= results.multi_face_landmarks[0].landmark 26 | for landmark in face_landmark: 27 | x = landmark.x 28 | y = landmark.y 29 | z = landmark.z 30 | relative_x=int(x * img.shape[1]) 31 | relative_y=int(y * img.shape[0]) 32 | #cv2.circle(img, (relative_x,relative_y) , 3, (255,255,255), -1) 33 | xyz_landmark_points.append((x, y, z)) 34 | landmark_points.append((relative_x,relative_y)) 35 | 36 | return xyz_landmark_points, landmark_points 37 | 38 | def get_iris_landmark(landmark_points, return_xyz=True, location= "Left"): 39 | points= [] 40 | if location == 'Left': 41 | iris_landmark_index= mp_face_mesh.FACEMESH_LEFT_IRIS 42 | else: 43 | iris_landmark_index= mp_face_mesh.FACEMESH_RIGHT_IRIS 44 | for idx, _ in iris_landmark_index: 45 | source = landmark_points[idx] 46 | if return_xyz: 47 | relative_source = [[source[0], source[1], source[2]]] 48 | else: 49 | relative_source = [[int(source[0]), int(source[1])]] 50 | points.append(relative_source) 51 | return np.array(points) 52 | 53 | def get_eye_landmark(landmark_points, location='Left'): 54 | points= [] 55 | if location == 'Left': 56 | eye_landmark_index= mp_face_mesh.FACEMESH_LEFT_EYE 57 | else: 58 | 59 | eye_landmark_index= mp_face_mesh.FACEMESH_RIGHT_EYE 60 | for idx, _ in eye_landmark_index: 61 | source = landmark_points[idx] 62 | relative_source = [[int(source[0]), int(source[1])]] 63 | points.append(relative_source) 64 | return np.array(points) 65 | 66 | def get_mouth_landmark(landmark_points): 67 | points= [] 68 | for idx in mouth_landmark_index: 69 | source = landmark_points[idx] 70 | relative_source = [[int(source[0]), int(source[1])]] 71 | points.append(relative_source) 72 | return np.array(points) 73 | 74 | def protected(mask, landmark_points): 75 | convexhull_landmark_points= cv2.convexHull(landmark_points) 76 | mask=cv2.fillConvexPoly(mask, np.array(convexhull_landmark_points), 0) 77 | return mask 78 | 79 | def create_face_mask(img, img_convexhull, img_landmark_points, protected_eyes= False, protected_mouth= False): 80 | face_mask = np.zeros(img.shape[:2]) 81 | face_mask = cv2.fillConvexPoly(face_mask, img_convexhull, 255) 82 | if protected_eyes: 83 | face_mask = protected(face_mask, get_eye_landmark(img_landmark_points)) 84 | face_mask = protected(face_mask, get_eye_landmark(img_landmark_points, location="Right")) 85 | if protected_mouth: 86 | face_mask = protected(face_mask, get_mouth_landmark(img_landmark_points)) 87 | return face_mask.astype(np.uint8) 88 | 89 | def face_swapping(dest_img, dest_landmark_points, dest_xyz_landmark_points, dest_convexhull, target_img, target_landmark_points, target_convexhull, return_face=False): 90 | new_face = np.zeros_like(dest_img, np.uint8) 91 | 92 | for triangle_index in utils.get_triangles(dest_convexhull, dest_landmark_points, dest_xyz_landmark_points): 93 | 94 | points_dest, _ ,cropped_triangle_mask_dest, rect =utils.triangulation(triangle_index, dest_landmark_points) 95 | 96 | 97 | points_target, cropped_triangle_target, cropped_triangle_mask_target, _ =utils.triangulation(triangle_index, target_landmark_points, target_img) 98 | 99 | #warp triangles 100 | warped_triangle = utils.warp_triangle(points_target, points_dest, cropped_triangle_target, cropped_triangle_mask_dest, rect) 101 | (x, y, w, h)= rect 102 | 103 | triangle_area= new_face[y: y + h, x: x + w] 104 | 105 | #remove the line between the triangles 106 | triangle_area_gray = cv2.cvtColor(triangle_area, cv2.COLOR_BGR2GRAY) 107 | _, mask_triangles_designed = cv2.threshold(triangle_area_gray, 1, 255, cv2.THRESH_BINARY_INV) 108 | warped_triangle = utils.apply_mask(warped_triangle, mask_triangles_designed) 109 | triangle_area= cv2.add(triangle_area, warped_triangle) 110 | 111 | new_face[y: y + h, x: x + w]= triangle_area 112 | 113 | dest_mask = create_face_mask(dest_img, dest_convexhull, dest_landmark_points, protected_eyes=True, protected_mouth=True) 114 | dest_without_face= utils.apply_mask(dest_img, cv2.bitwise_not(dest_mask)) 115 | 116 | #smoothing new face 117 | new_face = cv2.medianBlur(new_face, 3) 118 | 119 | new_face= utils.apply_mask(new_face, dest_mask) 120 | 121 | old_face= utils.apply_mask(dest_img, dest_mask) 122 | blending_mask= create_face_mask(dest_img, dest_convexhull, dest_landmark_points) 123 | 124 | cv2.GaussianBlur(blending_mask, (51, 51), 30, dst=blending_mask) 125 | blending_mask = utils.apply_mask(blending_mask, dest_mask) 126 | target_face= ImageProcessing.blend_with_mask_matrix(new_face, old_face, blending_mask) 127 | 128 | result = cv2.add(dest_without_face, target_face) 129 | result=cv2.seamlessClone(result, dest_img, blending_mask, utils.getCenter(dest_convexhull), cv2.NORMAL_CLONE) 130 | if return_face: 131 | return target_face, result 132 | return result -------------------------------------------------------------------------------- /result/result.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/result/result.mp4 -------------------------------------------------------------------------------- /result/sontungtimothee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/result/sontungtimothee.jpg -------------------------------------------------------------------------------- /samples.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/htuann2712/face_swapping/875de6c35a5f81dd41ca8e3182597c5339d9ccad/samples.png -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | import utils 5 | 6 | def extract_frame_from_video(path): 7 | """ 8 | :param path: path to the video 9 | return: list of numpy array video frame 10 | """ 11 | vid = cv2.VideoCapture(path) 12 | count = 0 13 | ret =1 14 | frames=[] 15 | while ret: 16 | ret,frame = vid.read() 17 | if frame is not None and len(frame)>0: 18 | frames.append(frame) 19 | count = count + 1 20 | print(f"Extract total {count} frames from video") 21 | return frames 22 | 23 | def apply_mask(img, mask): 24 | """ 25 | :param img: max 3 channel image 26 | :param mask: [0-255] values in mask 27 | return: image with mask apply 28 | """ 29 | masked_img= cv2.bitwise_and(img, img, mask=mask) 30 | return masked_img 31 | 32 | def get_point_index(point, landmark_points): 33 | # return: the index of the point in the list 34 | return landmark_points.index(point) 35 | 36 | def get_visuable_landmark(convexHull, landmark_points, xyz_landmark_points): 37 | """ 38 | :param convexHull: convexhull of points 39 | :param landmark_points: points in uv_dimensional 40 | :param xyz_landmark_points: points in xyz_dimensional 41 | return mask matrix 42 | """ 43 | #get z_coordination of convexHull 44 | z=[] 45 | for point in convexHull: 46 | indx=utils.get_point_index(tuple(point[0]), landmark_points) 47 | z.append(xyz_landmark_points[indx][2]) 48 | 49 | visuable= np.ones(len(xyz_landmark_points), dtype=bool) 50 | #filter the hidden landmark 51 | for idx, landmark in enumerate(xyz_landmark_points): 52 | if landmark[2] > np.max(z): 53 | visuable[idx]=False 54 | return visuable 55 | 56 | def get_triangles(convexhull, landmarks_points, xyz_landmark_points): 57 | rect= cv2.boundingRect(convexhull) 58 | subdiv= cv2.Subdiv2D(rect) 59 | visuable=get_visuable_landmark(convexhull, landmarks_points, xyz_landmark_points) 60 | facial_landmarks= [point for idx, point in enumerate(landmarks_points) if visuable[idx]==1] 61 | subdiv.insert(facial_landmarks) 62 | triangles= subdiv.getTriangleList() 63 | triangles= np.array(triangles, np.int32) 64 | index_points_triangles= [] 65 | 66 | for tri in triangles: 67 | pt1= (tri[0], tri[1]) 68 | pt2= (tri[2], tri[3]) 69 | pt3= (tri[4], tri[5]) 70 | 71 | index_pt1=get_point_index(pt1, landmarks_points) 72 | index_pt2=get_point_index(pt2, landmarks_points) 73 | index_pt3=get_point_index(pt3, landmarks_points) 74 | 75 | triangle = [index_pt1, index_pt2, index_pt3] 76 | index_points_triangles.append(triangle) 77 | return index_points_triangles 78 | 79 | def triangulation(triangle_point_index, landmark_points, img= None): 80 | #get triangluation point 81 | pt1= landmark_points[triangle_point_index[0]] 82 | pt2= landmark_points[triangle_point_index[1]] 83 | pt3= landmark_points[triangle_point_index[2]] 84 | 85 | triangle=np.array([pt1, pt2, pt3]) 86 | rect= cv2.boundingRect(triangle) 87 | 88 | (x, y, w, h) = rect 89 | 90 | cropped_triangle = None 91 | 92 | if img is not None: 93 | cropped_triangle= img[y:y+h, x:x+w] 94 | 95 | cropped_triangle_mask= np.zeros((h,w), np.uint8) 96 | 97 | points=np.array([[pt1[0] - x, pt1[1] - y], 98 | [pt2[0] - x, pt2[1] - y], 99 | [pt3[0] - x, pt3[1] - y]]) 100 | 101 | cv2.fillConvexPoly(cropped_triangle_mask, points, 255) 102 | 103 | return points, cropped_triangle, cropped_triangle_mask, rect 104 | 105 | def warp_triangle(points_target, points_dest, cropped_triangle_target, cropped_triangle_mask_dest, rect): 106 | 107 | (x, y, w, h) = rect 108 | M = cv2.getAffineTransform(np.float32(points_target),np.float32(points_dest)) 109 | warped_triangle= cv2.warpAffine(cropped_triangle_target, M, (w,h)) 110 | warped_triangle= apply_mask(warped_triangle, cropped_triangle_mask_dest) 111 | return warped_triangle.astype(np.uint8) 112 | 113 | def getCenter(convexHull_points): 114 | """ 115 | return: the centre point of convexHull 116 | """ 117 | x1, y1, w1, h1 = cv2.boundingRect(convexHull_points) 118 | center = ((x1 + int(w1 / 2), y1 + int(h1 / 2))) 119 | #bounding_rectangle = cv2.rectangle(face2.copy(), (x1, y1), (x1 + w1, y1 + h1), (0, 255, 0), 2) 120 | return center 121 | 122 | def getCenter_xyz(points): 123 | """ 124 | return: the centre point of each points in xyz_dimensional 125 | """ 126 | return np.mean(points,axis=0) 127 | 128 | def AngleOfDepression(pointA, pointB): 129 | """ 130 | :point: in xyz_dimensional 131 | return: angle of 2 point. Its real part is in [-pi/2, pi/2] 132 | """ 133 | (xA, yA, zA)= pointA 134 | (xB, yB, zB)= pointB 135 | horizontal_dist = np.linalg.norm(np.array(xA,yA) - np.array(xB, yB)) 136 | vertizontal_dist= zA- zB 137 | 138 | return np.arctan(vertizontal_dist/horizontal_dist) 139 | --------------------------------------------------------------------------------