8 | you can down [dlib18.17](http://pan.baidu.com/s/1gey9Wd1)
9 | cd your dlib folder
10 | cd python_example
11 | ./compile_dlib_python_module.bat
12 | add dlib.so to the python path
13 | if using dlib18.18, you can follow the [official instruction](http://dlib.net/) 14 | - opencv
15 | 16 | ## Usage for images 17 | 18 | - Command : python landmarkPredict.py predictImage testList.txt
19 | (testList.txt is a file contain the path of the images.) 20 | 21 | 22 | ## Usage for usb camera 23 | This sctpt enables you to see intractive results for this face-landmark-localization. 24 | 25 | usage: landmarkPredict_video.py uvcID 26 | 27 | 28 | - Command : python landmarkPredict_video.py 0 29 | 30 | ## Model 31 | 32 | - You can download the pre-trained model from [here](http://pan.baidu.com/s/1c14aFyK) 33 | 34 | ## Train 35 | 36 | - add train.prototxt and train_solver.prototxt files, Training using the 300W data set 37 | 38 | ## Result 39 |  40 |  41 |  42 | 43 | --- 44 | ## class based rewrite for the landmarkPredict. 45 | facePos.py: FacePosePredictor class is defined. 46 | 47 | librect.py: helper functions for rectangles. 48 | 49 | landmarkPredict_video.py uses this class version. 50 | -------------------------------------------------------------------------------- /model/deploy.prototxt: -------------------------------------------------------------------------------- 1 | name: "dlib_vgg" 2 | layers { 3 | name: "data" 4 | type: MEMORY_DATA 5 | top: "data" 6 | top: "label" 7 | memory_data_param { 8 | batch_size: 1 #batch size, so how many prediction youu want to do at once. Best is "1", but higher number get better performance 9 | channels: 3 10 | height: 224 11 | width: 224 12 | 13 | } 14 | } 15 | 16 | 17 | layers { 18 | bottom: "data" 19 | top: "conv1" 20 | name: "conv1" 21 | type: CONVOLUTION 22 | convolution_param { 23 | num_output: 96 24 | kernel_size: 7 25 | stride: 2 26 | } 27 | } 28 | layers { 29 | bottom: "conv1" 30 | top: "conv1" 31 | name: "relu1" 32 | type: RELU 33 | } 34 | layers { 35 | bottom: "conv1" 36 | top: "norm1" 37 | name: "norm1" 38 | type: LRN 39 | lrn_param { 40 | local_size: 5 41 | alpha: 0.0005 42 | beta: 0.75 43 | k: 2 44 | } 45 | } 46 | layers { 47 | bottom: "norm1" 48 | top: "pool1" 49 | name: "pool1" 50 | type: POOLING 51 | pooling_param { 52 | pool: MAX 53 | kernel_size: 3 54 | stride: 3 55 | } 56 | } 57 | layers { 58 | bottom: "pool1" 59 | top: "conv2" 60 | name: "conv2" 61 | type: CONVOLUTION 62 | convolution_param { 63 | num_output: 256 64 | kernel_size: 5 65 | } 66 | } 67 | layers { 68 | bottom: "conv2" 69 | top: "conv2" 70 | name: "relu2" 71 | type: RELU 72 | } 73 | layers { 74 | bottom: "conv2" 75 | top: "pool2" 76 | name: "pool2" 77 | type: POOLING 78 | pooling_param { 79 | pool: MAX 80 | kernel_size: 2 81 | stride: 2 82 | } 83 | } 84 | layers { 85 | bottom: "pool2" 86 | top: "conv3" 87 | name: "conv3" 88 | type: CONVOLUTION 89 | convolution_param { 90 | num_output: 512 91 | pad: 1 92 | kernel_size: 3 93 | } 94 | } 95 | layers { 96 | bottom: "conv3" 97 | top: "conv3" 98 | name: "relu3" 99 | type: RELU 100 | } 101 | layers { 102 | bottom: "conv3" 103 | top: "conv4" 104 | name: "conv4" 105 | type: CONVOLUTION 106 | convolution_param { 107 | num_output: 512 108 | pad: 1 109 | kernel_size: 3 110 | } 111 | } 112 | layers { 113 | bottom: "conv4" 114 | top: "conv4" 115 | name: "relu4" 116 | type: RELU 117 | } 118 | layers { 119 | bottom: "conv4" 120 | top: "conv5" 121 | name: "conv5" 122 | type: CONVOLUTION 123 | convolution_param { 124 | num_output: 512 125 | pad: 1 126 | kernel_size: 3 127 | } 128 | } 129 | layers { 130 | bottom: "conv5" 131 | top: "conv5" 132 | name: "relu5" 133 | type: RELU 134 | } 135 | layers { 136 | bottom: "conv5" 137 | top: "pool5" 138 | name: "pool5" 139 | type: POOLING 140 | pooling_param { 141 | pool: MAX 142 | kernel_size: 3 143 | stride: 3 144 | } 145 | } 146 | layers { 147 | bottom: "pool5" 148 | top: "fc6" 149 | name: "fc6" 150 | type: INNER_PRODUCT 151 | inner_product_param { 152 | num_output: 4096 153 | } 154 | } 155 | layers { 156 | bottom: "fc6" 157 | top: "fc6" 158 | name: "relu6" 159 | type: RELU 160 | } 161 | layers { 162 | bottom: "fc6" 163 | top: "fc6" 164 | name: "drop6" 165 | type: DROPOUT 166 | dropout_param { 167 | dropout_ratio: 0.5 168 | } 169 | } 170 | layers { 171 | bottom: "fc6" 172 | top: "fc7" 173 | name: "fc7" 174 | type: INNER_PRODUCT 175 | inner_product_param { 176 | num_output: 4096 177 | } 178 | } 179 | layers { 180 | bottom: "fc7" 181 | top: "fc7" 182 | name: "relu7" 183 | type: RELU 184 | } 185 | layers { 186 | bottom: "fc7" 187 | top: "fc7" 188 | name: "drop7" 189 | type: DROPOUT 190 | dropout_param { 191 | dropout_ratio: 0.5 192 | } 193 | } 194 | layers { 195 | bottom: "fc7" 196 | top: "68point" 197 | name: "68point" 198 | type: INNER_PRODUCT 199 | inner_product_param { 200 | num_output: 136 201 | } 202 | } 203 | 204 | 205 | layers { 206 | bottom: "fc7" 207 | top: "poselayer" 208 | name: "poselayer" 209 | type: INNER_PRODUCT 210 | inner_product_param { 211 | num_output: 3 212 | } 213 | } 214 | 215 | 216 | 217 | -------------------------------------------------------------------------------- /model/deploy.prototxt~: -------------------------------------------------------------------------------- 1 | name: "dlib_vgg" 2 | layers { 3 | name: "data" 4 | type: MEMORY_DATA 5 | top: "data" 6 | top: "label" 7 | memory_data_param { 8 | batch_size: 3 #batch size, so how many prediction youu want to do at once. Best is "1", but higher number get better performance 9 | channels: 3 10 | height: 224 11 | width: 224 12 | 13 | } 14 | } 15 | 16 | 17 | layers { 18 | bottom: "data" 19 | top: "conv1" 20 | name: "conv1" 21 | type: CONVOLUTION 22 | convolution_param { 23 | num_output: 96 24 | kernel_size: 7 25 | stride: 2 26 | } 27 | } 28 | layers { 29 | bottom: "conv1" 30 | top: "conv1" 31 | name: "relu1" 32 | type: RELU 33 | } 34 | layers { 35 | bottom: "conv1" 36 | top: "norm1" 37 | name: "norm1" 38 | type: LRN 39 | lrn_param { 40 | local_size: 5 41 | alpha: 0.0005 42 | beta: 0.75 43 | k: 2 44 | } 45 | } 46 | layers { 47 | bottom: "norm1" 48 | top: "pool1" 49 | name: "pool1" 50 | type: POOLING 51 | pooling_param { 52 | pool: MAX 53 | kernel_size: 3 54 | stride: 3 55 | } 56 | } 57 | layers { 58 | bottom: "pool1" 59 | top: "conv2" 60 | name: "conv2" 61 | type: CONVOLUTION 62 | convolution_param { 63 | num_output: 256 64 | kernel_size: 5 65 | } 66 | } 67 | layers { 68 | bottom: "conv2" 69 | top: "conv2" 70 | name: "relu2" 71 | type: RELU 72 | } 73 | layers { 74 | bottom: "conv2" 75 | top: "pool2" 76 | name: "pool2" 77 | type: POOLING 78 | pooling_param { 79 | pool: MAX 80 | kernel_size: 2 81 | stride: 2 82 | } 83 | } 84 | layers { 85 | bottom: "pool2" 86 | top: "conv3" 87 | name: "conv3" 88 | type: CONVOLUTION 89 | convolution_param { 90 | num_output: 512 91 | pad: 1 92 | kernel_size: 3 93 | } 94 | } 95 | layers { 96 | bottom: "conv3" 97 | top: "conv3" 98 | name: "relu3" 99 | type: RELU 100 | } 101 | layers { 102 | bottom: "conv3" 103 | top: "conv4" 104 | name: "conv4" 105 | type: CONVOLUTION 106 | convolution_param { 107 | num_output: 512 108 | pad: 1 109 | kernel_size: 3 110 | } 111 | } 112 | layers { 113 | bottom: "conv4" 114 | top: "conv4" 115 | name: "relu4" 116 | type: RELU 117 | } 118 | layers { 119 | bottom: "conv4" 120 | top: "conv5" 121 | name: "conv5" 122 | type: CONVOLUTION 123 | convolution_param { 124 | num_output: 512 125 | pad: 1 126 | kernel_size: 3 127 | } 128 | } 129 | layers { 130 | bottom: "conv5" 131 | top: "conv5" 132 | name: "relu5" 133 | type: RELU 134 | } 135 | layers { 136 | bottom: "conv5" 137 | top: "pool5" 138 | name: "pool5" 139 | type: POOLING 140 | pooling_param { 141 | pool: MAX 142 | kernel_size: 3 143 | stride: 3 144 | } 145 | } 146 | layers { 147 | bottom: "pool5" 148 | top: "fc6" 149 | name: "fc6" 150 | type: INNER_PRODUCT 151 | inner_product_param { 152 | num_output: 4096 153 | } 154 | } 155 | layers { 156 | bottom: "fc6" 157 | top: "fc6" 158 | name: "relu6" 159 | type: RELU 160 | } 161 | layers { 162 | bottom: "fc6" 163 | top: "fc6" 164 | name: "drop6" 165 | type: DROPOUT 166 | dropout_param { 167 | dropout_ratio: 0.5 168 | } 169 | } 170 | layers { 171 | bottom: "fc6" 172 | top: "fc7" 173 | name: "fc7" 174 | type: INNER_PRODUCT 175 | inner_product_param { 176 | num_output: 4096 177 | } 178 | } 179 | layers { 180 | bottom: "fc7" 181 | top: "fc7" 182 | name: "relu7" 183 | type: RELU 184 | } 185 | layers { 186 | bottom: "fc7" 187 | top: "fc7" 188 | name: "drop7" 189 | type: DROPOUT 190 | dropout_param { 191 | dropout_ratio: 0.5 192 | } 193 | } 194 | layers { 195 | bottom: "fc7" 196 | top: "68point" 197 | name: "68point" 198 | type: INNER_PRODUCT 199 | inner_product_param { 200 | num_output: 136 201 | } 202 | } 203 | 204 | 205 | layers { 206 | bottom: "fc7" 207 | top: "poselayer" 208 | name: "poselayer" 209 | type: INNER_PRODUCT 210 | inner_product_param { 211 | num_output: 3 212 | } 213 | } 214 | 215 | 216 | 217 | -------------------------------------------------------------------------------- /librect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # pylint: disable=C0103 4 | import numpy as np 5 | import PIL.Image # https://github.com/python-pillow/Pillow 6 | 7 | def largestRect(rects): 8 | u"""retturn largest rect in rects 9 | rects: list of rect 10 | """ 11 | 12 | if len(rects) < 2: 13 | return rects 14 | 15 | largest = rects[0] 16 | for i in range(1, len(rects)): 17 | if rects[i][2] > largest[2]: 18 | largest = rects[i] 19 | 20 | return largest 21 | 22 | def overlapRange(lim1, lim2): 23 | """return overlapped lim 24 | lim1: 25 | lim2: 26 | """ 27 | 28 | start = max(lim1[0], lim2[0]) 29 | stop = min(lim1[1], lim2[1]) 30 | 31 | if start > stop: 32 | return [None, None] 33 | else: 34 | return [start, stop] 35 | 36 | def overlapRectArea(rect1, rect2): 37 | """return overlapped area 38 | rect1: 39 | rect2: 40 | """ 41 | 42 | left1, right1 = rect1[0], rect1[0]+rect1[2] 43 | top1, bottom1 = rect1[1], rect1[1]+rect1[3] 44 | 45 | 46 | left2, right2 = rect2[0], rect2[0]+rect2[2] 47 | top2, bottom2 = rect2[1], rect2[1]+rect2[3] 48 | 49 | [left3, right3] = overlapRange([left1, right1], [left2, right2]) 50 | [top3, bottom3] = overlapRange([top1, bottom1], [top2, bottom2]) 51 | 52 | if None in (left3, top3, right3, bottom3): 53 | return 0.0 54 | else: 55 | area = (right3-left3)*(bottom3-top3) 56 | area >= 0.0 57 | return area 58 | 59 | def getIoU(rect1, rect2): 60 | u""" 61 | return intersection over union 62 | """ 63 | 64 | area1 = rect1[2]*rect1[3] 65 | area2 = rect2[2]*rect2[3] 66 | intersection = overlapRectArea(rect1, rect2) 67 | assert intersection >= 0 68 | union = area1+area2 - intersection 69 | assert union >= 0 70 | 71 | IoU = intersection/float(union) 72 | assert IoU >= 0 73 | return IoU 74 | 75 | 76 | def rect2bbox(rect): 77 | """convert rect into bbox. 78 | tracker.init() need this data type. 79 | """ 80 | 81 | assert len(rect) == 4 82 | x, y, w, h = rect 83 | assert w > 0 84 | assert h > 0 85 | return (long(x), long(y), long(w), long(h)) 86 | 87 | 88 | def dets2rects(dets): 89 | """ 90 | convert dets type to rect type. 91 | left, top, width, height 92 | """ 93 | 94 | rects = [[d.left(), d.top(), d.right()-d.left(), d.bottom()-d.top()] for d in dets] 95 | return rects 96 | 97 | 98 | def getBestIoU(rects, states): 99 | u"""find best matched tracking for each rect. 100 | rects: detected rects 101 | states: tracking states 102 | 103 | """ 104 | 105 | asTrack = len(rects)*[None] 106 | alreadyFounds = len(rects)*[0.0] 107 | 108 | for j, rect in enumerate(rects):# 検出について 109 | for k, (_, bbox) in enumerate(states):#追跡について 110 | IoU = getIoU(bbox, rect) 111 | assert IoU >= 0.0 112 | assert len(rect) == 4 113 | assert rect[2] > 0 114 | assert rect[3] > 0 115 | if IoU > alreadyFounds[j]: 116 | alreadyFounds[j] = max(alreadyFounds[j], IoU) 117 | asTrack[j] = k 118 | return alreadyFounds, asTrack 119 | 120 | 121 | def expandRegion(rect, rate): 122 | """expand rectange x,y,w,h keeping center postion. 123 | rect: x,y,w,h 124 | rate 125 | """ 126 | 127 | x, y, w, h = rect 128 | xc, yc = x+w/2, y+w/2 129 | 130 | nw = int(rate*w) 131 | nh = int(rate*h) 132 | 133 | nx = xc - nw/2 134 | ny = yc - nh/2 135 | return [nx, ny, nw, nh] 136 | 137 | def sizedCrop(img, xyxy): 138 | u"""Returns a rectangular region from this alignedImg. 139 | The box is a 4-tuple defining the left, upper, right, and lower pixel coordinate. 140 | When outside of the image is specified, it is displayed as black without an error. 141 | """ 142 | 143 | pilImg = PIL.Image.fromarray(img) 144 | pilsubImg = pilImg.crop(xyxy) 145 | subImg = np.asarray(pilsubImg) 146 | return subImg 147 | 148 | 149 | def test_overlapRegion(): 150 | lim = overlapRange([0, 10], [0, 10]) 151 | assert lim == [0, 10] 152 | lim = overlapRange([0, 10], [0, 20]) 153 | assert lim == [0, 10] 154 | lim = overlapRange([0, 10], [-10, 20]) 155 | assert lim == [0, 10] 156 | 157 | 158 | lim = overlapRange([0, 10], [5, 10]) 159 | assert lim == [5, 10] 160 | 161 | lim = overlapRange([0, 10], [5, 20]) 162 | assert lim == [5, 10] 163 | 164 | lim = overlapRange([-10, 10], [5, 20]) 165 | assert lim == [5, 10] 166 | 167 | 168 | lim = overlapRange([5, 10], [5, 20]) 169 | assert lim == [5, 10] 170 | 171 | 172 | def test_getIoU(): 173 | IoU = getIoU([10, 20, 30, 40], [10, 20, 30, 40]) 174 | print IoU 175 | assert IoU == 1.0 176 | 177 | IoU = getIoU([10, 20, 30, 40], [10, 20, 30, 20]) 178 | print IoU 179 | assert IoU <= 0.5+0.01 180 | assert 0.5 - 0.01 <= IoU 181 | 182 | IoU = getIoU([10, 20, 30, 40], [10, 25, 30, 40]) 183 | print IoU 184 | assert IoU < 1.0 185 | assert IoU >= 0.0 186 | 187 | 188 | if __name__ == "__main__": 189 | test_overlapRegion() 190 | test_getIoU() -------------------------------------------------------------------------------- /landmarkPredict_video.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # pylint: disable=C0103 4 | # pylint: disable=E1101 5 | 6 | import os 7 | import sys 8 | import time 9 | import cv2 # OpenCV 10 | import dlib # http://dlib.net 11 | 12 | import librect # helper function for rectangles. 13 | import facePose # class based rewrite of landmarkPredict. 14 | 15 | """ 16 | In this module 17 | bbox = [left, right, top, bottom] 18 | """ 19 | 20 | pose_name = ['Pitch', 'Yaw', 'Roll'] # respect to ['head down','out of plane left','in plane right'] 21 | 22 | outDir = os.path.expanduser("~/output") 23 | cropDir = os.path.expanduser("~/crop") 24 | 25 | def show_image(img, landmarks, bboxs, headposes, enableSampling=False): 26 | u""" 27 | img: 28 | landmarks: landmark points 29 | bboxs: list of bounding box generated by dlib face detection 30 | headposes: 31 | headposes[0, :]: i'th face 's pitch, yaw, row 32 | When the value of pitch becomes large, it becomes an image with chin pulled down or an image looked down from above. 33 | When the value of yaw increases, the face direction becomes to face the left side of the image. 34 | When the value of roll becomes large, the face is inclined clockwise. 35 | enableSampling: If True, save croppped face image. 36 | """ 37 | 38 | orgImg = img+0 39 | 40 | 41 | system_height = 650 42 | system_width = 1280 43 | 44 | 45 | for faceNum in range(0, landmarks.shape[0]): 46 | cv2.rectangle(img, (int(bboxs[faceNum, 0]), int(bboxs[faceNum, 2])), (int(bboxs[faceNum, 1]), int(bboxs[faceNum, 3])), (0, 0, 255), 2) 47 | for p in range(0, 3): 48 | 49 | font = cv2.FONT_HERSHEY_SIMPLEX 50 | cv2.putText(img, '{:s} {:.2f}'.format(pose_name[p], headposes[faceNum, p]), (10, 400+25*p), font, 1, (255, 255, 255), 2, cv2.LINE_AA) 51 | cv2.putText(orgImg, '{:s} {:.2f}'.format(pose_name[p], headposes[faceNum, p]), (10, 400+25*p), font, 1, (255, 255, 255), 2, cv2.LINE_AA) 52 | 53 | for i in range(0, landmarks.shape[1]/2): 54 | cv2.circle(img, (int(round(landmarks[faceNum, i*2])), int(round(landmarks[faceNum, i*2+1]))), 1, (0, 255, 0), 2) 55 | 56 | pitch = headposes[faceNum, 0] 57 | yaw = headposes[faceNum, 1] 58 | roll = headposes[faceNum, 2] 59 | 60 | pyrStr = facePose.getPyrStr(pitch, yaw, roll) 61 | pyStr = facePose.getPyStr(pitch, yaw) 62 | cropPyDir = os.path.join(cropDir, pyStr) 63 | outPyDir = os.path.join(outDir, pyStr) 64 | for p in (cropPyDir, outPyDir): 65 | if not os.path.isdir(p): 66 | os.makedirs(p) 67 | 68 | left, right, top, bottom = bboxs[faceNum, :] 69 | rect = [left, top, right-left, bottom - top] 70 | nx, ny, nw, nh = librect.expandRegion(rect, rate=2.0) 71 | nleft, ntop, nright, nbottom = nx, ny, nx+nw, ny+nh 72 | assert ntop < nbottom 73 | assert nleft < nright 74 | 75 | if enableSampling: 76 | datetimeStr = time.strftime("%Y%m%d_%H%M%S", time.localtime()) 77 | 78 | subImg3 = librect.sizedCrop(orgImg, (nleft, ntop, nright, nbottom)) 79 | cropName3 = os.path.join(cropPyDir, "%s_%s_b.png" % (pyrStr, datetimeStr)) 80 | cv2.imwrite(cropName3, subImg3) 81 | 82 | 83 | pngname = os.path.join(outPyDir, "%s_%s.jpg" % (pyrStr, datetimeStr)) 84 | cv2.imwrite(pngname, orgImg) 85 | 86 | if landmarks.shape[0] < 1: 87 | pyrDir = "couldNotDetect" 88 | pyrDir = os.path.join(outDir, pyrDir) 89 | if not os.path.isdir(pyrDir): 90 | os.makedirs(pyrDir) 91 | 92 | datetimeStr = time.strftime("%Y%m%d_%H%M%S", time.localtime()) 93 | pngname = os.path.join(pyrDir, "%s.jpg" % datetimeStr) 94 | cv2.imwrite(pngname, orgImg) 95 | print pngname 96 | 97 | 98 | height, width = img.shape[:2] 99 | if height > system_height or width > system_width: 100 | height_radius = system_height*1.0/height 101 | width_radius = system_width*1.0/width 102 | radius = min(height_radius, width_radius) 103 | img = cv2.resize(img, (0, 0), fx=radius, fy=radius) 104 | 105 | cv2.imshow("headPose and landmark q:quit", img) 106 | 107 | 108 | 109 | def predictVideo(uvcID): 110 | """ 111 | uvcID: video camera ID 112 | """ 113 | 114 | detector = dlib.get_frontal_face_detector() 115 | posePredictor = facePose.FacePosePredictor() 116 | 117 | cap = cv2.VideoCapture(uvcID) 118 | cv2.namedWindow("headPose and landmark q:quit", cv2.WINDOW_NORMAL) 119 | 120 | while True: 121 | ok, colorImage = cap.read() 122 | if not ok: 123 | continue 124 | 125 | numUpSampling = 0 126 | dets, _, _ = detector.run(colorImage, numUpSampling) 127 | bboxs = facePose.dets2xxyys(dets) 128 | 129 | predictpoints, landmarks, predictpose = posePredictor.predict(colorImage, bboxs) 130 | 131 | show_image(colorImage, landmarks, bboxs, predictpose) 132 | 133 | k = cv2.waitKey(10) & 0xff 134 | if k == ord('q') or k == 27: 135 | break 136 | 137 | if __name__ == '__main__': 138 | if len(sys.argv) < 2: 139 | print """usage: %s uvcID 140 | """ % sys.argv[0] 141 | exit() 142 | 143 | uvcID = int(sys.argv[1]) 144 | predictVideo(uvcID) 145 | 146 | -------------------------------------------------------------------------------- /train.prototxt: -------------------------------------------------------------------------------- 1 | name: "sub2_vgg" 2 | layer { 3 | name: "MyData" 4 | type: "HDF5Data" 5 | top: "data" 6 | top: "label" 7 | top: "pose" 8 | hdf5_data_param { 9 | source: "/home/hkk/DATACENTER/hdf5/box_train_bgr_data_list.txt" 10 | batch_size: 70 11 | shuffle: true 12 | } 13 | include: { phase: TRAIN } 14 | } 15 | layer { 16 | name: "MyData" 17 | type: "HDF5Data" 18 | top: "data" 19 | top: "label" 20 | top: "pose" 21 | hdf5_data_param { 22 | source: "/home/hkk/DATACENTER/hdf5/box_train_bgr_data_list.txt" 23 | batch_size: 20 24 | } 25 | include: { phase: TEST } 26 | } 27 | 28 | layer { 29 | bottom: "data" 30 | top: "conv1" 31 | name: "conv1" 32 | type: "Convolution" 33 | convolution_param { 34 | num_output: 96 35 | kernel_size: 7 36 | stride: 2 37 | } 38 | } 39 | layer { 40 | bottom: "conv1" 41 | top: "conv1" 42 | name: "relu1" 43 | type: "ReLU" 44 | } 45 | layer { 46 | bottom: "conv1" 47 | top: "norm1" 48 | name: "norm1" 49 | type: "LRN" 50 | lrn_param { 51 | local_size: 5 52 | alpha: 0.0005 53 | beta: 0.75 54 | k: 2 55 | } 56 | } 57 | layer { 58 | bottom: "norm1" 59 | top: "pool1" 60 | name: "pool1" 61 | type: "Pooling" 62 | pooling_param { 63 | pool: MAX 64 | kernel_size: 3 65 | stride: 3 66 | } 67 | } 68 | layer { 69 | bottom: "pool1" 70 | top: "conv2" 71 | name: "conv2" 72 | type: "Convolution" 73 | convolution_param { 74 | num_output: 256 75 | kernel_size: 5 76 | } 77 | } 78 | layer { 79 | bottom: "conv2" 80 | top: "conv2" 81 | name: "relu2" 82 | type: "ReLU" 83 | } 84 | layer { 85 | bottom: "conv2" 86 | top: "pool2" 87 | name: "pool2" 88 | type: "Pooling" 89 | pooling_param { 90 | pool: MAX 91 | kernel_size: 2 92 | stride: 2 93 | } 94 | } 95 | layer { 96 | bottom: "pool2" 97 | top: "conv3" 98 | name: "conv3" 99 | type: "Convolution" 100 | convolution_param { 101 | num_output: 512 102 | pad: 1 103 | kernel_size: 3 104 | } 105 | } 106 | layer { 107 | bottom: "conv3" 108 | top: "conv3" 109 | name: "relu3" 110 | type: "ReLU" 111 | } 112 | layer { 113 | bottom: "conv3" 114 | top: "conv4" 115 | name: "conv4" 116 | type: "Convolution" 117 | convolution_param { 118 | num_output: 512 119 | pad: 1 120 | kernel_size: 3 121 | } 122 | } 123 | layer { 124 | bottom: "conv4" 125 | top: "conv4" 126 | name: "relu4" 127 | type: "ReLU" 128 | } 129 | 130 | layer { 131 | bottom: "conv4" 132 | top: "conv5" 133 | name: "conv5" 134 | type: "Convolution" 135 | 136 | convolution_param { 137 | num_output: 512 138 | pad: 1 139 | kernel_size: 3 140 | } 141 | } 142 | layer { 143 | bottom: "conv5" 144 | top: "conv5" 145 | name: "relu5" 146 | type: "ReLU" 147 | } 148 | layer { 149 | bottom: "conv5" 150 | top: "pool5" 151 | name: "pool5" 152 | type: "Pooling" 153 | pooling_param { 154 | pool: MAX 155 | kernel_size: 3 156 | stride: 3 157 | } 158 | } 159 | layer { 160 | bottom: "pool5" 161 | top: "fc6" 162 | name: "fc6" 163 | type: "InnerProduct" 164 | 165 | inner_product_param { 166 | num_output: 4096 167 | } 168 | } 169 | layer { 170 | bottom: "fc6" 171 | top: "fc6" 172 | name: "relu6" 173 | type: "ReLU" 174 | } 175 | layer { 176 | bottom: "fc6" 177 | top: "fc6" 178 | name: "drop6" 179 | type: "Dropout" 180 | dropout_param { 181 | dropout_ratio: 0.5 182 | } 183 | } 184 | layer { 185 | bottom: "fc6" 186 | top: "fc7" 187 | name: "fc7" 188 | type: "InnerProduct" 189 | 190 | inner_product_param { 191 | num_output: 4096 192 | } 193 | } 194 | layer { 195 | bottom: "fc7" 196 | top: "fc7" 197 | name: "relu7" 198 | type: "ReLU" 199 | } 200 | layer { 201 | bottom: "fc7" 202 | top: "fc7" 203 | name: "drop7" 204 | type: "Dropout" 205 | dropout_param { 206 | dropout_ratio: 0.5 207 | } 208 | } 209 | layer { 210 | bottom: "fc7" 211 | top: "68point" 212 | name: "68point" 213 | type: "InnerProduct" 214 | inner_product_param { 215 | num_output: 136 216 | } 217 | } 218 | layer { 219 | name: "loss" 220 | type: "EuclideanLoss" 221 | bottom: "68point" 222 | bottom: "label" 223 | top: "loss" 224 | loss_weight: 1 225 | } 226 | 227 | 228 | layer { 229 | bottom: "conv4" 230 | top: "conv5_b" 231 | name: "conv5_b" 232 | 233 | type: "Convolution" 234 | convolution_param { 235 | num_output: 512 236 | pad: 1 237 | kernel_size: 3 238 | } 239 | } 240 | layer { 241 | bottom: "conv5_b" 242 | top: "conv5_b" 243 | name: "relu5_b" 244 | type: "ReLU" 245 | } 246 | layer { 247 | bottom: "conv5_b" 248 | top: "pool5_b" 249 | name: "pool5_b" 250 | type: "Pooling" 251 | pooling_param { 252 | pool: MAX 253 | kernel_size: 3 254 | stride: 3 255 | } 256 | } 257 | layer { 258 | bottom: "pool5_b" 259 | top: "fc6_b" 260 | name: "fc6_b" 261 | type: "InnerProduct" 262 | 263 | inner_product_param { 264 | num_output: 4096 265 | } 266 | } 267 | layer { 268 | bottom: "fc6_b" 269 | top: "fc6_b" 270 | name: "relu6_b" 271 | type: "ReLU" 272 | } 273 | layer { 274 | bottom: "fc6_b" 275 | top: "fc6_b" 276 | name: "drop6_b" 277 | type: "Dropout" 278 | dropout_param { 279 | dropout_ratio: 0.5 280 | } 281 | } 282 | layer { 283 | bottom: "fc6_b" 284 | top: "fc7_b" 285 | name: "fc7_b" 286 | type: "InnerProduct" 287 | 288 | inner_product_param { 289 | num_output: 4096 290 | } 291 | } 292 | layer { 293 | bottom: "fc7_b" 294 | top: "fc7_b" 295 | name: "relu7_b" 296 | type: "ReLU" 297 | } 298 | layer { 299 | bottom: "fc7_b" 300 | top: "fc7_b" 301 | name: "drop7_b" 302 | type: "Dropout" 303 | dropout_param { 304 | dropout_ratio: 0.5 305 | } 306 | } 307 | layer { 308 | bottom: "fc7_b" 309 | top: "poselayer" 310 | name: "poselayer" 311 | type: "InnerProduct" 312 | inner_product_param { 313 | num_output: 3 314 | } 315 | } 316 | layer { 317 | name: "poseLoss" 318 | type: "EuclideanLoss" 319 | bottom: "poselayer" 320 | bottom: "pose" 321 | top: "poseLoss" 322 | loss_weight: 3 323 | } 324 | -------------------------------------------------------------------------------- /evaluate_48.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import matplotlib.patches as mpatches 5 | import sys 6 | import os 7 | from math import pow 8 | import skimage.io 9 | from skimage import transform as tf 10 | 11 | caffe_root = 'D:/caffe/caffe_python/caffe-windows-master/' 12 | sys.path.insert(0, caffe_root + 'python') 13 | import caffe 14 | 15 | from nms import nms_average,nms_max 16 | 17 | #caffe.set_device(1) 18 | #caffe.set_mode_gpu() 19 | 20 | 21 | #============ 22 | #Model related: 23 | model_path = 'D:/caffe/人脸检测，特征点标定caffe实现/DeepFace-master/FaceDetection/try1_4/' 24 | model_define= model_path+'deploy.prototxt' 25 | model_weight =model_path+'snapshot_iter_100000.caffemodel' 26 | model_define_fc =model_path+'deploy_fc.prototxt' 27 | model_weight_fc =model_path+'snapshot_iter_100000_fc.caffemodel' 28 | 29 | channel = 3 30 | raw_scale = 255.0 31 | face_w = 48 32 | stride = 16 33 | cellSize = face_w 34 | threshold = 0.95 35 | factor = 0.793700526 # 缩小因子 36 | 37 | map_idx = 0 38 | params = ['deepid', 'fc7'] 39 | params_fc = ['deepid-conv', 'fc7-conv'] 40 | 41 | 42 | def generateBoundingBox(featureMap, scale): 43 | ''' 44 | @brief: 生成窗口 45 | @param: featureMap,特征图，scale：尺度 46 | ''' 47 | boundingBox = [] 48 | for (x,y), prob in np.ndenumerate(featureMap): 49 | if(prob >= threshold): 50 | #映射到原始的图像中的大小 51 | x=x-1 52 | y=y-1 53 | boundingBox.append([float(stride * y)/scale, float(stride *x )/scale, 54 | float(stride * y + cellSize - 1)/scale, float(stride * x + cellSize - 1)/scale, prob]) 55 | #boundingBox.append([float(stride * y-cellSize/2.0)/scale, float(stride *x -cellSize/2.0)/scale, 56 | # float(stride * y + cellSize/2.0 - 1)/scale, float(stride * x + cellSize/2.0 - 1)/scale, prob]) 57 | return boundingBox 58 | 59 | def convert_full_conv(model_define,model_weight,model_define_fc,model_weight_fc): 60 | ''' 61 | @breif : 将原始网络转换为全卷积模型 62 | @param: model_define,二分类网络定义文件 63 | @param: model_weight，二分类网络训练好的参数 64 | @param: model_define_fc,生成的全卷积网络定义文件 65 | @param: model_weight_fc，转化好的全卷积网络的参数 66 | ''' 67 | net = caffe.Net(model_define, model_weight, caffe.TEST) 68 | fc_params = {pr: (net.params[pr][0].data, net.params[pr][1].data) for pr in params} 69 | net_fc = caffe.Net(model_define_fc, model_weight, caffe.TEST) 70 | conv_params = {pr: (net_fc.params[pr][0].data, net_fc.params[pr][1].data) for pr in params_fc} 71 | for pr, pr_conv in zip(params, params_fc): 72 | conv_params[pr_conv][0].flat = fc_params[pr][0].flat # flat unrolls the arrays 73 | conv_params[pr_conv][1][...] = fc_params[pr][1] 74 | net_fc.save(model_weight_fc) 75 | print 'convert done!' 76 | return net_fc 77 | 78 | def re_verify(net_vf, img): 79 | ''' 80 | @breif: 对检测到的目标框进行重新的验证 81 | ''' 82 | img= tf.resize(img,(face_w,face_w)) 83 | transformer = caffe.io.Transformer({'data': net_vf.blobs['data'].data.shape}) 84 | transformer.set_transpose('data', (2,0,1)) 85 | transformer.set_channel_swap('data', (2,1,0)) 86 | transformer.set_raw_scale('data', raw_scale) 87 | out = net_vf.forward_all(data=np.asarray([transformer.preprocess('data', img)])) 88 | #print out['prob'] 89 | if out['prob'][0,map_idx] > threshold: 90 | return True 91 | else: 92 | return False 93 | 94 | def face_detection_image(net,net_vf,image_name): 95 | ''' 96 | @检测单张人脸图像 97 | ''' 98 | scales = [] 99 | imgs = skimage.io.imread(image_name) 100 | if imgs.ndim==3: 101 | rows,cols,ch = imgs.shape 102 | else: 103 | rows,cols = imgs.shape 104 | #计算需要的检测的尺度因子 105 | min = rows if rows<=cols else cols 106 | max = rows if rows>=cols else cols 107 | # 放大的尺度 108 | delim = 2500/max 109 | while (delim >= 1): 110 | scales.append(delim) 111 | delim=delim-0.5 112 | #缩小的尺度 113 | min = min * factor 114 | factor_count = 1 115 | while(min >= face_w): 116 | scale = pow(factor, factor_count) 117 | scales.append(scale) 118 | min = min * factor 119 | factor_count += 1 120 | #========================= 121 | #scales.append(1) 122 | total_boxes = [] 123 | ###显示热图用 124 | num_scale = len(scales) 125 | s1=int(np.sqrt(num_scale))+1 126 | tt=1 127 | plt.subplot(s1, s1+1, tt) 128 | plt.axis('off') 129 | plt.title("Input Image") 130 | im=caffe.io.load_image(image_name) 131 | plt.imshow(im) 132 | #============ 133 | for scale in scales: 134 | w,h = int(rows* scale),int(cols* scale) 135 | scale_img= tf.resize(imgs,(w,h)) 136 | #更改网络输入data图像的大小 137 | net.blobs['data'].reshape(1,channel,w,h) 138 | #转换结构 139 | transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) 140 | #transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) 141 | transformer.set_transpose('data', (2,0,1)) 142 | transformer.set_channel_swap('data', (2,1,0)) 143 | transformer.set_raw_scale('data', raw_scale) 144 | #前馈一次 145 | out = net.forward_all(data=np.asarray([transformer.preprocess('data', scale_img)])) 146 | ###显示热图用 147 | tt=tt+1 148 | plt.subplot(s1, s1+1, tt) 149 | plt.axis('off') 150 | plt.title("sacle: "+ "%.2f" %scale) 151 | plt.imshow(out['prob'][0,map_idx]) 152 | #=========== 153 | boxes = generateBoundingBox(out['prob'][0,map_idx], scale) 154 | if(boxes): 155 | total_boxes.extend(boxes) 156 | #非极大值抑制 157 | boxes_nms = np.array(total_boxes) 158 | true_boxes1 = nms_max(boxes_nms, overlapThresh=0.3) 159 | true_boxes = nms_average(np.array(true_boxes1), overlapThresh=0.07) 160 | #=================== 161 | plt.savefig('heatmap/'+image_name.split('/')[-1]) 162 | #在图像中画出检测到的人脸框 163 | fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(6, 6)) 164 | ax.imshow(imgs) 165 | for box in true_boxes: 166 | im_crop = im[box[0]:box[2],box[1]:box[3],:] 167 | if im_crop.shape[0] == 0 or im_crop.shape[1] == 0: 168 | continue 169 | if re_verify(net_vf, im_crop) == True: 170 | rect = mpatches.Rectangle((box[0], box[1]), box[2]-box[0], box[3]-box[1], 171 | fill=False, edgecolor='red', linewidth=1) 172 | ax.text(box[0], box[1]+20,"{0:.3f}".format(box[4]),color='white', fontsize=6) 173 | ax.add_patch(rect) 174 | plt.savefig('result/'+image_name.split('/')[-1]) 175 | plt.close() 176 | return out['prob'][0,map_idx] 177 | 178 | 179 | if __name__ == "__main__": 180 | if not os.path.isfile(model_weight_fc): 181 | net_fc = convert_full_conv(model_define,model_weight,model_define_fc,model_weight_fc) 182 | else: 183 | net_fc = caffe.Net(model_define_fc, model_weight_fc, caffe.TEST) 184 | net_vf = caffe.Net(model_define, model_weight, caffe.TEST) 185 | for i in range(210): 186 | image_name = 'D:/caffe/人脸检测，特征点标定caffe实现/DeepFace-master/FaceDetection/baseline/'+str(i+1)+'.jpg' 187 | print i 188 | fm = face_detection_image(net_fc,net_vf,image_name) 189 | plt.close('all') 190 | -------------------------------------------------------------------------------- /landmarkPredict.py: -------------------------------------------------------------------------------- 1 | 2 | #usage :python landmarkPredict.py predictImage testList.txt 3 | 4 | import os 5 | import sys 6 | import numpy as np 7 | import cv2 8 | 9 | caffe_root = 'D:/caffe/caffe-windows-master/' 10 | sys.path.insert(0, caffe_root + 'python') 11 | import caffe 12 | 13 | import dlib 14 | import matplotlib.pyplot as plt 15 | 16 | system_height = 650 17 | system_width = 1280 18 | channels = 1 19 | test_num = 1 20 | pointNum = 68 21 | 22 | S0_width = 60 23 | S0_height = 60 24 | vgg_height = 224 25 | vgg_width = 224 26 | M_left = -0.15 27 | M_right = +1.15 28 | M_top = -0.10 29 | M_bottom = +1.25 30 | pose_name = ['Pitch', 'Yaw', 'Roll'] # respect to ['head down','out of plane left','in plane right'] 31 | count = 0; 32 | 33 | def recover_coordinate(largetBBox, facepoint, width, height): 34 | point = np.zeros(np.shape(facepoint)) 35 | cut_width = largetBBox[1] - largetBBox[0] 36 | cut_height = largetBBox[3] - largetBBox[2] 37 | scale_x = cut_width*1.0/width; 38 | scale_y = cut_height*1.0/height; 39 | point[0::2]=[float(j * scale_x + largetBBox[0]) for j in facepoint[0::2]] 40 | point[1::2]=[float(j * scale_y + largetBBox[2]) for j in facepoint[1::2]] 41 | return point 42 | 43 | def show_image(img, facepoint, bboxs, headpose): 44 | plt.figure(figsize=(20,10)) 45 | for faceNum in range(0,facepoint.shape[0]): 46 | cv2.rectangle(img, (int(bboxs[faceNum,0]), int(bboxs[faceNum,2])), (int(bboxs[faceNum,1]), int(bboxs[faceNum,3])), (0,0,255), 2) 47 | for p in range(0,3): 48 | plt.text(int(bboxs[faceNum,0]), int(bboxs[faceNum,2])-p*30, 49 | '{:s} {:.2f}'.format(pose_name[p], headpose[faceNum,p]), 50 | bbox=dict(facecolor='blue', alpha=0.5), 51 | fontsize=12, color='white') 52 | for i in range(0,facepoint.shape[1]/2): 53 | cv2.circle(img,(int(round(facepoint[faceNum,i*2])),int(round(facepoint[faceNum,i*2+1]))),1,(0,255,0),2) 54 | height = img.shape[0] 55 | width = img.shape[1] 56 | if height > system_height or width > system_width: 57 | height_radius = system_height*1.0/height 58 | width_radius = system_width*1.0/width 59 | radius = min(height_radius,width_radius) 60 | img = cv2.resize(img, (0,0), fx=radius, fy=radius) 61 | 62 | img = img[:,:,[2,1,0]] 63 | plt.imshow(img) 64 | plt.show() 65 | 66 | 67 | def recoverPart(point,bbox,left,right,top,bottom,img_height,img_width,height,width): 68 | largeBBox = getCutSize(bbox,left,right,top,bottom) 69 | retiBBox = retifyBBoxSize(img_height,img_width,largeBBox) 70 | recover = recover_coordinate(retiBBox,point,height,width) 71 | recover=recover.astype('float32') 72 | return recover 73 | 74 | 75 | def getRGBTestPart(bbox,left,right,top,bottom,img,height,width): 76 | largeBBox = getCutSize(bbox,left,right,top,bottom) 77 | retiBBox = retifyBBox(img,largeBBox) 78 | # cv2.rectangle(img, (int(retiBBox[0]), int(retiBBox[2])), (int(retiBBox[1]), int(retiBBox[3])), (0,0,255), 2) 79 | # cv2.imshow('f',img) 80 | # cv2.waitKey(0) 81 | retiBBox = [int(round(x)) for x in retiBBox] 82 | face=img[retiBBox[2]:retiBBox[3],retiBBox[0]:retiBBox[1],:] 83 | #cv2.imshow('f', face) 84 | #cv2.waitkey(0) 85 | face = cv2.resize(face,(height,width),interpolation = cv2.INTER_AREA) 86 | face=face.astype('float32') 87 | return face 88 | 89 | def batchRecoverPart(predictPoint,totalBBox,totalSize,left,right,top,bottom,height,width): 90 | recoverPoint = np.zeros(predictPoint.shape) 91 | for i in range(0,predictPoint.shape[0]): 92 | recoverPoint[i] = recoverPart(predictPoint[i],totalBBox[i],left,right,top,bottom,totalSize[i,0],totalSize[i,1],height,width) 93 | return recoverPoint 94 | 95 | 96 | 97 | def retifyBBox(img,bbox): 98 | img_height = np.shape(img)[0] - 1 99 | img_width = np.shape(img)[1] - 1 100 | if bbox[0] <0: 101 | bbox[0] = 0 102 | if bbox[1] <0: 103 | bbox[1] = 0 104 | if bbox[2] <0: 105 | bbox[2] = 0 106 | if bbox[3] <0: 107 | bbox[3] = 0 108 | if bbox[0] > img_width: 109 | bbox[0] = img_width 110 | if bbox[1] > img_width: 111 | bbox[1] = img_width 112 | if bbox[2] > img_height: 113 | bbox[2] = img_height 114 | if bbox[3] > img_height: 115 | bbox[3] = img_height 116 | return bbox 117 | 118 | def retifyBBoxSize(img_height,img_width,bbox): 119 | if bbox[0] <0: 120 | bbox[0] = 0 121 | if bbox[1] <0: 122 | bbox[1] = 0 123 | if bbox[2] <0: 124 | bbox[2] = 0 125 | if bbox[3] <0: 126 | bbox[3] = 0 127 | if bbox[0] > img_width: 128 | bbox[0] = img_width 129 | if bbox[1] > img_width: 130 | bbox[1] = img_width 131 | if bbox[2] > img_height: 132 | bbox[2] = img_height 133 | if bbox[3] > img_height: 134 | bbox[3] = img_height 135 | return bbox 136 | 137 | def getCutSize(bbox,left,right,top,bottom): #left, right, top, and bottom 138 | 139 | box_width = bbox[1] - bbox[0] 140 | box_height = bbox[3] - bbox[2] 141 | cut_size=np.zeros((4)) 142 | cut_size[0] = bbox[0] + left * box_width 143 | cut_size[1] = bbox[1] + (right - 1) * box_width 144 | cut_size[2] = bbox[2] + top * box_height 145 | cut_size[3] = bbox[3] + (bottom-1) * box_height 146 | return cut_size 147 | 148 | 149 | def getFaceImage(image,bboxs,left,right,top,bottom,height,width): 150 | num = bboxs.shape[0] 151 | faces = np.zeros((num,channels,height,width)) 152 | for i in range (0,num): 153 | faces[i] = getTestPart(bboxs[i],left,right,top,bottom,image,height,width)/255.0 154 | print faces[i].shape 155 | # cv2.imshow('f',faces[i][0]) 156 | # cv2.waitKey(0) 157 | return faces 158 | 159 | def detectFace(img): 160 | detector = dlib.get_frontal_face_detector() 161 | dets = detector(img,1) 162 | bboxs = np.zeros((len(dets),4)) 163 | for i, d in enumerate(dets): 164 | bboxs[i,0] = d.left(); 165 | bboxs[i,1] = d.right(); 166 | bboxs[i,2] = d.top(); 167 | bboxs[i,3] = d.bottom(); 168 | return bboxs; 169 | 170 | 171 | def predictImage(filename): 172 | vgg_point_MODEL_FILE = 'model/deploy.prototxt' 173 | vgg_point_PRETRAINED = 'model/68point_dlib_with_pose.caffemodel' 174 | mean_filename='model/VGG_mean.binaryproto' 175 | vgg_point_net=caffe.Net(vgg_point_MODEL_FILE,vgg_point_PRETRAINED,caffe.TEST) 176 | caffe.set_mode_cpu() 177 | # caffe.set_mode_gpu() 178 | # caffe.set_device(0) 179 | f = open(filename) 180 | line = f.readline() 181 | index = 0 182 | proto_data = open(mean_filename, "rb").read() 183 | a = caffe.io.caffe_pb2.BlobProto.FromString(proto_data) 184 | mean = caffe.io.blobproto_to_array(a)[0] 185 | 186 | while line: 187 | print index 188 | line = line.strip() 189 | info = line.split(' ') 190 | imgPath = info[0] 191 | print imgPath 192 | num = 1 193 | colorImage = cv2.imread(imgPath) 194 | bboxs = detectFace(colorImage) 195 | faceNum = bboxs.shape[0] 196 | faces = np.zeros((1,3,vgg_height,vgg_width)) 197 | predictpoints = np.zeros((faceNum,pointNum*2)) 198 | predictpose = np.zeros((faceNum,3)) 199 | imgsize = np.zeros((2)) 200 | imgsize[0] = colorImage.shape[0]-1 201 | imgsize[1] = colorImage.shape[1]-1 202 | TotalSize = np.zeros((faceNum,2)) 203 | for i in range(0,faceNum): 204 | TotalSize[i] = imgsize 205 | for i in range(0,faceNum): 206 | bbox = bboxs[i] 207 | colorface = getRGBTestPart(bbox,M_left,M_right,M_top,M_bottom,colorImage,vgg_height,vgg_width) 208 | normalface = np.zeros(mean.shape) 209 | normalface[0] = colorface[:,:,0] 210 | normalface[1] = colorface[:,:,1] 211 | normalface[2] = colorface[:,:,2] 212 | normalface = normalface - mean 213 | faces[0] = normalface 214 | 215 | blobName = '68point' 216 | data4DL = np.zeros([faces.shape[0],1,1,1]) 217 | vgg_point_net.set_input_arrays(faces.astype(np.float32),data4DL.astype(np.float32)) 218 | vgg_point_net.forward() 219 | predictpoints[i] = vgg_point_net.blobs[blobName].data[0] 220 | 221 | blobName = 'poselayer' 222 | pose_prediction = vgg_point_net.blobs[blobName].data 223 | predictpose[i] = pose_prediction * 50 224 | 225 | predictpoints = predictpoints * vgg_height/2 + vgg_width/2 226 | level1Point = batchRecoverPart(predictpoints,bboxs,TotalSize,M_left,M_right,M_top,M_bottom,vgg_height,vgg_width) 227 | 228 | show_image(colorImage, level1Point, bboxs, predictpose) 229 | line = f.readline() 230 | index = index + 1 231 | 232 | 233 | if __name__ == '__main__': 234 | if len(sys.argv) < 2: 235 | print(__doc__) 236 | else: 237 | func = globals()[sys.argv[1]] 238 | func(*sys.argv[2:]) 239 | -------------------------------------------------------------------------------- /facePose.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # pylint: disable=C0103 4 | # pylint: disable=E1101 5 | 6 | import os 7 | import numpy as np 8 | import cv2 9 | import caffe 10 | 11 | def retifyxxyy(img, xxyy): 12 | """ 13 | let xxyy within image size 14 | img: image 15 | xxyy: left, right, top, bottom 16 | return modified xxyy 17 | """ 18 | 19 | img_height, img_width = img.shape[:2] 20 | xxyy = retifyxxyysize(img_height, img_width, xxyy) 21 | return xxyy 22 | 23 | def retifyxxyysize(img_height, img_width, xxyy): 24 | """return xxyy within image region 25 | img_height: 26 | img_width: 27 | xxyy: 28 | return xxyy 29 | """ 30 | 31 | xxyy[0] = max(xxyy[0], 0) 32 | xxyy[1] = max(xxyy[1], 0) 33 | xxyy[2] = max(xxyy[2], 0) 34 | xxyy[3] = max(xxyy[3], 0) 35 | xxyy[0] = min(xxyy[0], img_width) 36 | xxyy[1] = min(xxyy[1], img_width) 37 | xxyy[2] = min(xxyy[2], img_height) 38 | xxyy[3] = min(xxyy[3], img_height) 39 | return xxyy 40 | 41 | def getCutSize(xxyy, left, right, top, bottom): #left, right, top, and bottom 42 | u""" 43 | xxyy: 44 | left: 45 | right: 46 | top: 47 | bottom: 48 | left, right, top, bottom are ratio. 49 | The return value is a region with a margin. 50 | """ 51 | 52 | box_width = xxyy[1] - xxyy[0] 53 | box_height = xxyy[3] - xxyy[2] 54 | cut_size = np.zeros((4)) 55 | cut_size[0] = xxyy[0] + left * box_width 56 | cut_size[1] = xxyy[1] + (right - 1) * box_width 57 | cut_size[2] = xxyy[2] + top * box_height 58 | cut_size[3] = xxyy[3] + (bottom-1) * box_height 59 | return cut_size 60 | 61 | 62 | def dets2xxyys(dets): 63 | """ 64 | In this module 65 | xxyy = [left, right, top, bottom] 66 | """ 67 | xxyys = np.zeros((len(dets), 4)) 68 | for i, d in enumerate(dets): 69 | xxyys[i, 0] = d.left() 70 | xxyys[i, 1] = d.right() 71 | xxyys[i, 2] = d.top() 72 | xxyys[i, 3] = d.bottom() 73 | return xxyys 74 | 75 | 76 | class FacePosePredictor(object): 77 | """ 78 | A face Pose Predcitor using pre-trained caffe model. 79 | 80 | The orignal code was modified to class version. 81 | 82 | https://github.com/guozhongluo/head-pose-estimation-and-face-landmark 83 | 84 | Example: 85 | 86 | posePredictor = facePose.FacePosePredictor() 87 | predictpoints, landmarks, headposes = posePredictor.predict(frameCopy, np.array([[left, right, top, bottom]])) 88 | 89 | 90 | """ 91 | 92 | def __init__(self): 93 | self.M_left = -0.15 94 | self.M_right = +1.15 95 | self.M_top = -0.10 96 | self.M_bottom = +1.25 97 | 98 | self.vgg_height = 224 99 | self.vgg_width = 224 100 | 101 | vgg_point_MODEL_FILE = 'model/deploy.prototxt' 102 | vgg_point_PRETRAINED = 'model/68point_dlib_with_pose.caffemodel' 103 | mean_filename = 'model/VGG_mean.binaryproto' 104 | self.vgg_point_net = caffe.Net(vgg_point_MODEL_FILE, vgg_point_PRETRAINED, caffe.TEST) 105 | caffe.set_mode_cpu() 106 | # caffe.set_mode_gpu() 107 | # caffe.set_device(0) 108 | proto_data = open(mean_filename, "rb").read() 109 | a = caffe.io.caffe_pb2.BlobProto.FromString(proto_data) 110 | self.mean = caffe.io.blobproto_to_array(a)[0] 111 | 112 | 113 | def predict(self, colorImage, xxyys): 114 | """ 115 | predcit pitch yaw, roll for each rectangle. 116 | colorImage: 117 | xxyys: list of rectangle 118 | 119 | return 120 | predictpoints: 68 point 121 | landmarks: 122 | predictposes: pitch yaw roll 123 | """ 124 | 125 | 126 | def getRGBTestPart(img, xxyy, left, right, top, bottom, asHeight, asWidth): 127 | """return face image as float32 128 | returned image size width, height 129 | 130 | """ 131 | 132 | largexxyy = getCutSize(xxyy, left, right, top, bottom) 133 | retixxyy = retifyxxyy(img, largexxyy) 134 | retixxyy = [int(round(x)) for x in retixxyy] 135 | face = img[retixxyy[2]:retixxyy[3], retixxyy[0]:retixxyy[1], :] 136 | face = cv2.resize(face, (asHeight, asWidth), interpolation=cv2.INTER_AREA) 137 | face = face.astype('float32') 138 | return face 139 | 140 | 141 | pointNum = 68 142 | 143 | faceNum = xxyys.shape[0] 144 | faces = np.zeros((1, 3, self.vgg_height, self.vgg_width)) 145 | predictpoints = np.zeros((faceNum, pointNum*2)) 146 | predictposes = np.zeros((faceNum, 3)) 147 | imgsize = colorImage.shape[:2] 148 | TotalSize = np.zeros((faceNum, 2)) 149 | normalface = np.zeros(self.mean.shape) 150 | 151 | for i in range(0, faceNum): 152 | TotalSize[i] = imgsize 153 | colorface = getRGBTestPart(colorImage, xxyys[i], self.M_left, self.M_right, self.M_top, self.M_bottom, self.vgg_height, self.vgg_width) 154 | 155 | normalface[0] = colorface[:, :, 0] 156 | normalface[1] = colorface[:, :, 1] 157 | normalface[2] = colorface[:, :, 2] 158 | normalface = normalface - self.mean 159 | faces[0] = normalface 160 | 161 | data4DL = np.zeros([faces.shape[0], 1, 1, 1]) 162 | self.vgg_point_net.set_input_arrays(faces.astype(np.float32), data4DL.astype(np.float32)) 163 | self.vgg_point_net.forward() 164 | predictpoints[i] = self.vgg_point_net.blobs['68point'].data[0] 165 | 166 | predictposes[i] = 50 * self.vgg_point_net.blobs['poselayer'].data 167 | 168 | predictpoints = predictpoints * self.vgg_height/2 + self.vgg_width/2 169 | landmarks = self.batchRecoverPart(predictpoints, xxyys, TotalSize, self.M_left, self.M_right, self.M_top, self.M_bottom, self.vgg_height, self.vgg_width) 170 | 171 | return predictpoints, landmarks, predictposes 172 | 173 | def batchRecoverPart(self, predictPoint, totalxxyy, totalSize, left, right, top, bottom, height, width): 174 | 175 | def recover_coordinate(largetxxyy, landmarks, width, height): 176 | point = np.zeros(np.shape(landmarks)) 177 | cut_width = largetxxyy[1] - largetxxyy[0] 178 | cut_height = largetxxyy[3] - largetxxyy[2] 179 | scale_x = cut_width*1.0/width 180 | scale_y = cut_height*1.0/height 181 | point[0::2] = [float(j * scale_x + largetxxyy[0]) for j in landmarks[0::2]] 182 | point[1::2] = [float(j * scale_y + largetxxyy[2]) for j in landmarks[1::2]] 183 | return point 184 | 185 | def recoverPart(point, xxyy, left, right, top, bottom, img_height, img_width, height, width): 186 | largexxyy = getCutSize(xxyy, left, right, top, bottom) 187 | retixxyy = retifyxxyysize(img_height, img_width, largexxyy) 188 | recover = recover_coordinate(retixxyy, point, height, width) 189 | recover = recover.astype('float32') 190 | return recover 191 | 192 | recoverPoint = np.zeros(predictPoint.shape) 193 | for i in range(0, predictPoint.shape[0]): 194 | recoverPoint[i] = recoverPart(predictPoint[i], totalxxyy[i], left, right, top, bottom, totalSize[i, 0], totalSize[i, 1], height, width) 195 | return recoverPoint 196 | 197 | 198 | def predict1(self, colorImage, xxyy): 199 | """ 200 | predcit pitch yaw, roll for single rectangle. 201 | colorImage: 202 | xxyy: single rectangle 203 | 204 | return value 205 | predictposes[0, :] : pitch, yaw, roll 206 | 207 | """ 208 | predictpoints, landmarks, predictposes = self.predict(colorImage, np.array([xxyy])) 209 | 210 | return predictpoints[0], landmarks[0], predictposes[0, :] 211 | 212 | 213 | def roundByD(angle, delta): 214 | """round angle by delta 215 | angle: 216 | delta: 217 | >>> roundByD(8, 10) 218 | 10.0 219 | >>> roundByD(-9.5, 10) 220 | -10.0 221 | """ 222 | return delta*round(angle/float(delta)) 223 | 224 | 225 | def getPyrStr(pitch, yaw, roll): 226 | """ 227 | pitch: 228 | yaw: 229 | roll: 230 | """ 231 | 232 | pitchDelta = 5 233 | yawDelta = 5 234 | rollDelta = 10 235 | 236 | pyrDir = "P_%+03d_Y_%+03d_R_%+03d" % (roundByD(pitch, pitchDelta), roundByD(yaw, yawDelta), roundByD(roll, rollDelta)) 237 | return pyrDir 238 | 239 | def getPyStr(pitch, yaw): 240 | """ 241 | pitch: 242 | yaw: 243 | """ 244 | 245 | pitchDelta = 5 246 | yawDelta = 5 247 | rollDelta = 10 248 | 249 | pyrDir = "P_%+03d_Y_%+03d" % (roundByD(pitch, pitchDelta), roundByD(yaw, yawDelta)) 250 | return pyrDir 251 | 252 | def getPyrDir(outDir, pitch, yaw, roll): 253 | """ 254 | pitch: 255 | yaw: 256 | roll: 257 | """ 258 | 259 | pyrDir = os.path.join(outDir, getPyrStr(pitch, yaw, roll)) 260 | if not os.path.isdir(pyrDir): 261 | os.makedirs(pyrDir) 262 | return pyrDir 263 | 264 | def getPyDir(outDir, pitch, yaw): 265 | """ 266 | pitch: 267 | yaw: 268 | roll: 269 | """ 270 | 271 | pitchDelta = 5 272 | yawDelta = 5 273 | rollDelta = 10 274 | 275 | pyrDir = "P_%+03d_Y_%+03d" % (roundByD(pitch, pitchDelta), roundByD(yaw, yawDelta)) 276 | pyrDir = os.path.join(outDir, pyrDir) 277 | if not os.path.isdir(pyrDir): 278 | os.makedirs(pyrDir) 279 | return pyrDir 280 | --------------------------------------------------------------------------------