├── pictures ├── jay.jpg ├── jay2.JPEG ├── jay3.JPEG └── image001.png ├── LICENSE ├── README.md └── pose_estimation.py /pictures/jay.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/HEAD/pictures/jay.jpg -------------------------------------------------------------------------------- /pictures/jay2.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/HEAD/pictures/jay2.JPEG -------------------------------------------------------------------------------- /pictures/jay3.JPEG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/HEAD/pictures/jay3.JPEG -------------------------------------------------------------------------------- /pictures/image001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/HEAD/pictures/image001.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 jerryhouuu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV 2 | 3 | ## Description 4 | This work is used for pose estimation(yaw, pitch and roll) by Face landmarks(left eye, right eye, nose, left mouth, right mouth and chin). 5 | Roll:+90°：-90°/Pitch:+90°：-90°/Yaw:+90°：-90°, like the picture below: 6 | 7 | ![Roll_Pitch_Yaw.png](https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/blob/master/pictures/image001.png?raw=true) 8 | 9 | The order of numbers is **ROLL**, **PITCH**, **YAW**： 10 | ![Jay_Result1.png](https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/blob/master/pictures/jay.jpg?raw=true) 11 | ![Jay_Result2.png](https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/blob/master/pictures/jay2.JPEG?raw=true) 12 | ![Jay_Result3.png](https://github.com/jerryhouuu/Face-Yaw-Roll-Pitch-from-Pose-Estimation-using-OpenCV/blob/master/pictures/jay3.JPEG?raw=true) 13 | 14 | ## Preprocessing 15 | * I fine-tune the MTCNN into the output of 6 landmark feature points, reference and make some adjustments in this article 'Head Pose Estimation using OpenCV and Dlib'. 16 | * Because the MTCNN's eyes are the middle of the position rather than the corner of the eye, so we modify the world coordinate(model point) from original to (-150.0, -150.0, -125.0)# Left Mouth corner/(150.0, -150.0, -125.0)# Right mouth corner 17 | * Modify the camera matrix's focal_length from original to img_size[1]/2 / np.tan(60/2 * np.pi / 180). 18 | 19 | ## Step 20 | 1. imgpts, jac = cv2.projectPoints(axis, rotation_vector, translation_vector, camera_matrix, dist_coeffs) 21 | 2. modelpts, jac2 = cv2.projectPoints(model_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs) 22 | 3. rvec_matrix = cv2.Rodrigues(rotation_vector)[0] 23 | 4. proj_matrix = np.hstack((rvec_matrix, translation_vector)) 24 | 5. eulerAngles = cv2.decomposeProjectionMatrix(proj_matrix)[6] 25 | 6. pitch, yaw, roll = [math.radians(_) for _ in eulerAngles] 26 | 7. pitch = math.degrees(math.asin(math.sin(pitch))) 27 | 8. roll = -math.degrees(math.asin(math.sin(roll))) 28 | 9. yaw = math.degrees(math.asin(math.sin(yaw))) 29 | 30 | ## References 31 | 1. [Head Pose Estimation using OpenCV and Dlib](https://www.learnopencv.com/head-pose-estimation-using-opencv-and-dlib/) 32 | 2. [MTCNN-tensorflow](https://github.com/AITTSMD/MTCNN-Tensorflow) 33 | -------------------------------------------------------------------------------- /pose_estimation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Jan 24 11:58:14 2018 4 | 5 | @author: jerry 6 | """ 7 | 8 | import cv2 9 | import math 10 | import numpy as np 11 | 12 | 13 | def face_orientation(frame, landmarks): 14 | size = frame.shape #(height, width, color_channel) 15 | 16 | image_points = np.array([ 17 | (landmarks[4], landmarks[5]), # Nose tip 18 | (landmarks[10], landmarks[11]), # Chin 19 | (landmarks[0], landmarks[1]), # Left eye left corner 20 | (landmarks[2], landmarks[3]), # Right eye right corne 21 | (landmarks[6], landmarks[7]), # Left Mouth corner 22 | (landmarks[8], landmarks[9]) # Right mouth corner 23 | ], dtype="double") 24 | 25 | 26 | 27 | 28 | model_points = np.array([ 29 | (0.0, 0.0, 0.0), # Nose tip 30 | (0.0, -330.0, -65.0), # Chin 31 | (-165.0, 170.0, -135.0), # Left eye left corner 32 | (165.0, 170.0, -135.0), # Right eye right corne 33 | (-150.0, -150.0, -125.0), # Left Mouth corner 34 | (150.0, -150.0, -125.0) # Right mouth corner 35 | ]) 36 | 37 | # Camera internals 38 | 39 | center = (size[1]/2, size[0]/2) 40 | focal_length = center[0] / np.tan(60/2 * np.pi / 180) 41 | camera_matrix = np.array( 42 | [[focal_length, 0, center[0]], 43 | [0, focal_length, center[1]], 44 | [0, 0, 1]], dtype = "double" 45 | ) 46 | 47 | dist_coeffs = np.zeros((4,1)) # Assuming no lens distortion 48 | (success, rotation_vector, translation_vector) = cv2.solvePnP(model_points, image_points, camera_matrix, dist_coeffs, flags=cv2.CV_ITERATIVE) 49 | 50 | 51 | axis = np.float32([[500,0,0], 52 | [0,500,0], 53 | [0,0,500]]) 54 | 55 | imgpts, jac = cv2.projectPoints(axis, rotation_vector, translation_vector, camera_matrix, dist_coeffs) 56 | modelpts, jac2 = cv2.projectPoints(model_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs) 57 | rvec_matrix = cv2.Rodrigues(rotation_vector)[0] 58 | 59 | proj_matrix = np.hstack((rvec_matrix, translation_vector)) 60 | eulerAngles = cv2.decomposeProjectionMatrix(proj_matrix)[6] 61 | 62 | 63 | pitch, yaw, roll = [math.radians(_) for _ in eulerAngles] 64 | 65 | 66 | pitch = math.degrees(math.asin(math.sin(pitch))) 67 | roll = -math.degrees(math.asin(math.sin(roll))) 68 | yaw = math.degrees(math.asin(math.sin(yaw))) 69 | 70 | return imgpts, modelpts, (str(int(roll)), str(int(pitch)), str(int(yaw))), (landmarks[4], landmarks[5]) 71 | 72 | f = open('/home/jerry/Documents/test/test/landmark.txt','r') 73 | for line in iter(f): 74 | img_info = line.split(' ') 75 | img_path = img_info[0] 76 | frame = cv2.imread(img_path) 77 | landmarks = map(int, img_info[1:]) 78 | 79 | print img_path 80 | imgpts, modelpts, rotate_degree, nose = face_orientation(frame, landmarks) 81 | 82 | cv2.line(frame, nose, tuple(imgpts[1].ravel()), (0,255,0), 3) #GREEN 83 | cv2.line(frame, nose, tuple(imgpts[0].ravel()), (255,0,), 3) #BLUE 84 | cv2.line(frame, nose, tuple(imgpts[2].ravel()), (0,0,255), 3) #RED 85 | 86 | remapping = [2,3,0,4,5,1] 87 | for index in range(len(landmarks)/2): 88 | random_color = tuple(np.random.random_integers(0,255,size=3)) 89 | 90 | cv2.circle(frame, (landmarks[index*2], landmarks[index*2+1]), 5, random_color, -1) 91 | cv2.circle(frame, tuple(modelpts[remapping[index]].ravel().astype(int)), 2, random_color, -1) 92 | 93 | 94 | # cv2.putText(frame, rotate_degree[0]+' '+rotate_degree[1]+' '+rotate_degree[2], (10, 30), 95 | # cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 96 | # thickness=2, lineType=2) 97 | 98 | for j in xrange(len(rotate_degree)): 99 | cv2.putText(frame, ('{:05.2f}').format(float(rotate_degree[j])), (10, 30 + (50 * j)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), thickness=2, lineType=2) 100 | 101 | cv2.imwrite('test2/'+img_path.split('/')[1], frame) 102 | 103 | f.close() 104 | --------------------------------------------------------------------------------