├── README.md
├── face_landmark.cpp
├── face_landmark.py
├── model
    ├── VanFace.caffemodel
    └── landmark_deploy.prototxt
├── result
    ├── 1.jpg
    ├── 1_result.jpg
    ├── 2.jpg
    ├── 2_result.jpg
    ├── 4.jpg
    ├── 4_result.jpg
    ├── 7.jpg
    ├── 7_result.jpg
    ├── 8.jpg
    └── 8_result.jpg
├── rotation_image.cpp
└── train
    ├── DataRow.py
    ├── Demo.py
    ├── NormlizedMSE.py
    ├── adam_solver.prototxt
    ├── landmark_train.prototxt
    └── mainloop.py


/README.md:
--------------------------------------------------------------------------------
 1 | # face-landmark-localization
 2 | 
 3 | This is a project predict face landmarks (68 points).
 4 | - Model Size : 3.3MB.
 5 | - Speed : 5ms per face on i5-2.7GHz CPU.
 6 | 
 7 | ## Install
 8 | - [caffe](https://github.com/BVLC/caffe)
 9 | - [dlib face detector](http://dlib.net/)
10 | - add libdlib.a to your project lib path
11 | - add face_lardmark.cpp to your caffe project example folder
12 | - opencv<p>
13 | 
14 | ## Usage
15 | 
16 | - CPP Demo : ./face_lardmark 
17 | - Python Demo: python face_landmark.py (created by [jiangwqcooler](https://github.com/jiangwqcooler))
18 | 
19 | ## notice
20 | - This model was trained on dlib face detector. Other face detector may not get landmark result as good as dlib
21 | 
22 | ## Result
23 | ![](result/7_result.jpg)
24 | ![](result/8_result.jpg)
25 | 


--------------------------------------------------------------------------------
/face_landmark.cpp:
--------------------------------------------------------------------------------
  1 | #include <caffe/caffe.hpp>
  2 | #include <opencv2/core/core.hpp>
  3 | #include <opencv2/highgui/highgui.hpp>
  4 | #include <opencv2/imgproc/imgproc.hpp>
  5 | 
  6 | #include <algorithm>
  7 | #include <iosfwd>
  8 | #include <memory>
  9 | #include <string>
 10 | #include <utility>
 11 | #include <vector>
 12 | 
 13 | #include "dlib/image_processing/frontal_face_detector.h"
 14 | #include "dlib/image_io.h"
 15 | 
 16 | #include <time.h>
 17 | 
 18 | using namespace caffe;
 19 | using namespace std;
 20 | using namespace cv;
 21 | using namespace dlib;
 22 | 
 23 | int main()
 24 | {
 25 |     string network = "/Users/CaffeMac/CaffeMac/Models/landmark_deploy.prototxt";
 26 |     string weights = "/Users/CaffeMac/CaffeMac/Models/landmark.caffemodel";
 27 |     string img = "/Users/CaffeMac/CaffeMac/test.jpg";
 28 |     Net<float> *net = new Net<float>(network,TEST);
 29 |     
 30 |     net->CopyTrainedLayersFrom(weights);
 31 |     
 32 |     Caffe::set_mode(Caffe::CPU);
 33 |     
 34 |     Mat image = imread(img);
 35 |     
 36 |     frontal_face_detector detector = get_frontal_face_detector();
 37 |     
 38 |     
 39 |     array2d< bgr_pixel> arrimg(image.rows, image.cols);
 40 |     for(int i=0; i < image.rows; i++)
 41 |     {
 42 |         
 43 |         for(int j=0; j < image.cols; j++)
 44 |         {
 45 |             
 46 |             arrimg[i][j].blue = image.at< cv::Vec3b>(i,j)[0];
 47 |             arrimg[i][j].green=image.at< cv::Vec3b>(i,j)[1];
 48 |             arrimg[i][j].red = image.at< cv::Vec3b>(i,j)[2];
 49 |         }
 50 |     }
 51 |     
 52 |     //开始检测，返回一系列的边界框
 53 |     clock_t start;
 54 |     int cnt = 0;
 55 |     start = clock();
 56 |     std::vector<dlib::rectangle> dets = detector(arrimg);
 57 |     start = clock() -start;
 58 |     printf("detection time is: %f ms\n", (double(start))/CLOCKS_PER_SEC*1000);
 59 |     //cout<<"time is  "<<start/10e3<<endl;
 60 |     //cout << cnt++ <<endl;
 61 |     for(int i = 0;i < dets.size();i++)
 62 |     {
 63 |         //Bbox tmp = DetBox[i];
 64 |         dlib::rectangle tmp = dets[i];
 65 |         //cv::rectangle(image, Point(tmp.y1, tmp.x1), Point(tmp.y2, tmp.x2), Scalar(0,0,255), 1,4,0);
 66 |         //Mat srcROI(image, Rect(tmp.y1,tmp.x1,tmp.y2 - tmp.y1,tmp.x2-tmp.x1));
 67 |         cv::rectangle(image, Point(tmp.left(), tmp.top()), Point(tmp.right(), tmp.bottom()), Scalar(0,0,255), 1,4,0);
 68 |         Mat srcROI(image, Rect(tmp.left(),tmp.top(),tmp.right()-tmp.left(),tmp.bottom() - tmp.top()));
 69 |         
 70 |         Mat img2;
 71 |         cvtColor(srcROI,img2,CV_RGB2GRAY);
 72 |         
 73 |         img2.convertTo(img2, CV_32FC1);
 74 |         Size dsize = Size(60,60);
 75 |         Mat img3 = Mat(dsize, CV_32FC1);
 76 |         resize(img2, img3, dsize, 0,0,INTER_CUBIC);
 77 |         
 78 |         Mat tmp_m, tmp_sd;
 79 |         double m = 0, sd = 0;
 80 |         meanStdDev(img3, tmp_m, tmp_sd);
 81 |         m = tmp_m.at<double>(0,0);
 82 |         sd = tmp_sd.at<double>(0,0);
 83 |         
 84 |         img3 = (img3 - m)/(0.000001 + sd);
 85 |         
 86 |         if (img3.channels() * img3.rows * img3.cols != net->input_blobs()[0]->count())
 87 |             LOG(FATAL) << "Incorrect " << image << ", resize to correct dimensions.\n";
 88 |         // prepare data into array
 89 |         float *data = (float*)malloc( img3.rows * img3.cols * sizeof(float));
 90 |         
 91 |         int pix_count = 0;
 92 |         
 93 |         for (int i = 0; i < img3.rows; ++i) {
 94 |             for (int j = 0; j < img3.cols; ++j) {
 95 |                 float pix = img3.at<float>(i, j);
 96 |                 float* p = (float*)(data);
 97 |                 p[pix_count] = pix;
 98 |                 ++pix_count;
 99 |             }
100 |         }
101 |         
102 |         std::vector<Blob<float>*> in_blobs = net->input_blobs();
103 |         in_blobs[0]->Reshape(1, 1, img3.rows, img3.cols);
104 |         net->Reshape();
105 |         
106 |         in_blobs[0]->set_cpu_data((float*)data);
107 |         Timer total_timer;
108 |         total_timer.Start();
109 |         
110 |         net->Forward();
111 |         cout << " total time = " << total_timer.MicroSeconds() / 1000 <<endl;
112 |         const boost::shared_ptr<Blob<float> > feature_blob = net->blob_by_name("Dense3");//获取该层特征
113 |         
114 |         float feat_dim = feature_blob->count() / feature_blob->num();//计算特征维度
115 |         cout << feat_dim << endl;
116 |         const float* data_ptr = (const float *)feature_blob->cpu_data();//特征块数据
117 |         
118 |         
119 |         std::vector<float> feat2;
120 |         
121 |         for (int i = 0; i < feat_dim; i++)
122 |         {
123 |             feat2.push_back(*data_ptr);
124 |             if (i < feat_dim - 1)
125 |                 data_ptr++;
126 |         }
127 |         
128 |         for(int i = 0;i < feat_dim/2;i++)
129 |         {
130 |             Point x = Point(int(feat2[2*i]*(tmp.right() - tmp.left()) + tmp.left()),int(feat2[2*i + 1]*(tmp.bottom() - tmp.top()) + tmp.top()));
131 |             cv::circle(image, x, 0.1, Scalar(0, 0, 255), 4, 8, 0);
132 |         }
133 |         free(data);    
134 |     }
135 |     imshow("result", image);
136 |     imwrite("/Users/CaffeMac/CaffeMac/result.jpg", image);
137 |     free(net);
138 |     
139 |     image.release();
140 |     return 0;
141 |     
142 | }
143 | 


--------------------------------------------------------------------------------
/face_landmark.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import os
  3 | 
  4 | import time
  5 | import pprint
  6 | 
  7 | import caffe
  8 | import dlib
  9 | import cv2
 10 | import numpy as np
 11 | 
 12 | def file_list_fn(path):
 13 | 
 14 |     file_list = []
 15 |     files = os.listdir(path)
 16 |     for f in files:
 17 |         file_list.append(f)
 18 |     return file_list
 19 | 
 20 | net_work_path = '/home/code/face-landmark/model/landmark_deploy.prototxt'
 21 | weight_path = '/home/code/face-landmark/model/VanFace.caffemodel'
 22 | images_dir = '/home/code/face-landmark/images/'
 23 | result_dir = '/home/code/face-landmark/results/'
 24 | 
 25 | image_list = file_list_fn(images_dir)
 26 | caffe.set_mode_cpu()
 27 | net = caffe.Net(net_work_path, weight_path, caffe.TEST)
 28 | net.name = 'FaceThink_face_landmark_test'
 29 | 
 30 | detector = dlib.get_frontal_face_detector()
 31 | 
 32 | total_detecting_time = 0.0
 33 | total_landmark_time = 0.0
 34 | face_total = 0.0
 35 | for image in image_list:
 36 |     print("Processing file: {}".format(image))
 37 |     img = cv2.imread(images_dir + image)
 38 | 
 39 |     # The 1 in the second argument indicates that we should upsample the image
 40 |     # 1 time.  This will make everything bigger and allow us to detect more
 41 |     # faces.
 42 |     det_start_time = time.time()
 43 |     dets = detector(img, 1)
 44 |     det_end_time = time.time()
 45 |     det_time = det_end_time - det_start_time
 46 |     total_detecting_time += det_time
 47 |     print "Detecting time is {}".format(det_time)
 48 |     print "Number of faces detected: {}".format(len(dets))
 49 |     for i, d in enumerate(dets):
 50 |             print "Detection {}: Left: {} Top: {} Right: {} Bottom: {}".format(
 51 |                 i, d.left(), d.top(), d.right(), d.bottom())
 52 | 
 53 |     for index, det in enumerate(dets):
 54 |         face_total += 1
 55 |         x1 = det.left()
 56 |         y1 = det.top()
 57 |         x2 = det.right()
 58 |         y2 = det.bottom()
 59 |         if x1 < 0: x1 = 0
 60 |         if y1 < 0: y1 = 0
 61 |         if x2 > img.shape[1]: x2 = img.shape[1]
 62 |         if y2 > img.shape[0]: y2 = img.shape[0]
 63 |         cv2.rectangle(img, (x1, y1), (x2, y2), (0, 0, 255), 2)
 64 |         roi = img[y1:y2 + 1, x1:x2 + 1, ]
 65 |         gary_img = cv2.cvtColor(roi, cv2.COLOR_RGB2GRAY)
 66 |         w = 60
 67 |         h = 60
 68 | 
 69 |         print image
 70 |         res = cv2.resize(gary_img, (w, h), 0.0, 0.0, interpolation=cv2.INTER_CUBIC)
 71 |         resize_mat = np.float32(res)
 72 | 
 73 |         m = np.zeros((w, h))
 74 |         sd = np.zeros((w, h))
 75 |         mean, std_dev = cv2.meanStdDev(resize_mat, m, sd)
 76 |         new_m = mean[0][0]
 77 |         new_sd = std_dev[0][0]
 78 |         new_img = (resize_mat - new_m) / (0.000001 + new_sd)
 79 | 
 80 |         if new_img.shape[0] != net.blobs['data'].data[0].shape or new_img.shape[1] != net.blobs['data'].data[1].shape:
 81 |             print "Incorrect " + image + ", resize to correct dimensions."
 82 | 
 83 |         net.blobs['data'].data[...] = new_img
 84 |         landmark_time_start = time.time()
 85 |         out = net.forward()
 86 |         landmark_time_end = time.time()
 87 |         landmark_time = landmark_time_end - landmark_time_start
 88 |         total_landmark_time += landmark_time
 89 |         print "landmark time is {}".format(landmark_time)
 90 |         points = net.blobs['Dense3'].data[0].flatten()
 91 | 
 92 |         point_pair_l = len(points)
 93 |         for i in range(point_pair_l / 2):
 94 |             x = points[2*i] * (x2 - x1) + x1
 95 |             y = points[2*i+1] * (y2 - y1) + y1
 96 |             cv2.circle(img, (int(x), int(y)), 1, (0, 0, 255), 2)
 97 | 
 98 |     cv2.imwrite(result_dir + image, img)
 99 | 
100 | print total_detecting_time
101 | print total_landmark_time
102 | print face_total
103 | per_face_det_time = total_detecting_time / face_total
104 | per_face_landmark_time = total_landmark_time / face_total
105 | 
106 | per_image_det_time = total_detecting_time / len(image_list)
107 | per_image_landmark_time = total_landmark_time / len(image_list)
108 | 
109 | print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
110 | print "per face detecting time is {}".format(per_face_det_time)
111 | print "per face landmark time is {}".format(per_face_landmark_time)
112 | print "per image detecting time is {}".format(per_image_det_time)
113 | print "per image detecting time is {}".format(per_image_landmark_time)
114 | 
115 | 
116 | 
117 | 


--------------------------------------------------------------------------------
/model/VanFace.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/model/VanFace.caffemodel


--------------------------------------------------------------------------------
/model/landmark_deploy.prototxt:
--------------------------------------------------------------------------------
  1 | name: "landmark"
  2 | input: "data"
  3 | input_dim: 1
  4 | input_dim: 1
  5 | input_dim: 60
  6 | input_dim: 60
  7 | ########################################
  8 | # the actual net
  9 | # layer 1
 10 | layer {
 11 |     name: "Conv1"
 12 |     type: "Convolution"
 13 |     bottom: "data"
 14 |     top: "Conv1"
 15 |     param {
 16 |         lr_mult: 1
 17 |         decay_mult: 1
 18 |     }
 19 |     param {
 20 |         lr_mult: 2
 21 |         decay_mult: 0
 22 |     }
 23 |     convolution_param {
 24 |         num_output: 20
 25 |         pad: 2
 26 |         kernel_size: 5
 27 |         stride: 1
 28 |         weight_filler {
 29 |             type: "xavier"
 30 |             std: 0.1
 31 |         }
 32 |         bias_filler {
 33 |             type: "constant"
 34 |             value: 0.2
 35 |         }
 36 |     }
 37 | }
 38 | 
 39 | layer {
 40 |   name: "ActivationTangH1"
 41 |   bottom: "Conv1"
 42 |   top: "ActivationTangH1"
 43 |   type: "TanH"
 44 | }
 45 | 
 46 | layer {
 47 |   name: "ActivationAbs1"
 48 |   bottom: "ActivationTangH1"
 49 |   top: "Abs1"
 50 |   type: "AbsVal"
 51 | }
 52 | 
 53 | layer {
 54 |     name: "Pool1"
 55 |     type: "Pooling"
 56 |     bottom: "Abs1"
 57 |     top: "Pool1"
 58 |     pooling_param {
 59 |         pool: MAX
 60 |         kernel_size: 2
 61 |         stride: 2
 62 |     }
 63 | }
 64 | 
 65 | layer {
 66 |     name: "Conv2"
 67 |     type: "Convolution"
 68 |     bottom: "Pool1"
 69 |     top: "Conv2"
 70 |     param {
 71 |         lr_mult: 1
 72 |         decay_mult: 1
 73 |     }
 74 |     param {
 75 |         lr_mult: 2
 76 |         decay_mult: 0
 77 |     }
 78 |     convolution_param {
 79 |         num_output: 48
 80 |         pad: 2
 81 |         kernel_size: 5
 82 |         stride: 1
 83 |         weight_filler {
 84 |         type: "xavier"
 85 |         std: 0.1
 86 |     }
 87 |     bias_filler {
 88 |         type: "constant"
 89 |         value: 0.2
 90 |         }
 91 |     }
 92 | }
 93 | 
 94 | layer {
 95 |   name: "ActivationTangH2"
 96 |   bottom: "Conv2"
 97 |   top: "ActivationTangH2"
 98 |   type: "TanH"
 99 | }
100 | 
101 | layer {
102 |   name: "ActivationAbs2"
103 |   bottom: "ActivationTangH2"
104 |   top: "Abs2"
105 |   type: "AbsVal"
106 | }
107 | 
108 | 
109 | layer {
110 |     name: "Pool2"
111 |     type: "Pooling"
112 |     bottom: "Abs2"
113 |     top: "Pool2"
114 |     pooling_param {
115 |         pool: MAX
116 |         kernel_size: 2
117 |         stride: 2
118 |     }
119 | }
120 | 
121 | # layer 3
122 | layer {
123 |     name: "Conv3"
124 |     type: "Convolution"
125 |     bottom: "Pool2"
126 |     top: "Conv3"
127 |     param {
128 |         lr_mult: 1
129 |         decay_mult: 1
130 |     }
131 |     param {
132 |         lr_mult: 2
133 |         decay_mult: 0
134 |     }
135 |     convolution_param {
136 |         num_output: 64
137 |         pad: 0
138 |         kernel_size: 3
139 |         stride: 1
140 |         weight_filler {
141 |             type: "xavier"
142 |             std: 0.1
143 |         }
144 |         bias_filler {
145 |             type: "constant"
146 |             value: 0.2
147 |         }
148 |     }
149 | }
150 | 
151 | 
152 | layer {
153 |   name: "ActivationTangH3"
154 |   bottom: "Conv3"
155 |   top: "ActivationTangH3"
156 |   type: "TanH"
157 | }
158 | 
159 | layer {
160 |   name: "ActivationAbs3"
161 |   bottom: "ActivationTangH3"
162 |   top: "Abs3"
163 |   type: "AbsVal"
164 | }
165 | 
166 | layer {
167 |     name: "Pool3"
168 |     type: "Pooling"
169 |     bottom: "Abs3"
170 |     top: "Pool3"
171 |     pooling_param {
172 |         pool: MAX
173 |         kernel_size: 3
174 |         stride: 2
175 |     }
176 | }
177 | 
178 | # layer 4
179 | layer {
180 |     name: "Conv4"
181 |     type: "Convolution"
182 |     bottom: "Pool3"
183 |     top: "Conv4"
184 |     param {
185 |         lr_mult: 1
186 |         decay_mult: 1
187 |     }
188 |     param {
189 |         lr_mult: 2
190 |         decay_mult: 0
191 |     }
192 |     convolution_param {
193 |         num_output: 80
194 |         pad: 0
195 |         kernel_size: 3
196 |         stride: 1
197 |         weight_filler {
198 |             type: "xavier"
199 |             std: 0.1
200 |         }
201 |         bias_filler {
202 |             type: "constant"
203 |             value: 0.2
204 |         }
205 |     }
206 | }
207 | 
208 | 
209 | layer {
210 |   name: "ActivationTangH4"
211 |   bottom: "Conv4"
212 |   top: "ActivationTangH4"
213 |   type: "TanH"
214 | }
215 | 
216 | layer {
217 |   name: "ActivationAbs4"
218 |   bottom: "ActivationTangH4"
219 |   top: "Abs4"
220 |   type: "AbsVal"
221 | }
222 | 
223 | 
224 | ########################################
225 | 
226 | layer {
227 |     name: "Dense1"
228 |     type: "InnerProduct"
229 |     bottom: "Abs4"
230 |     top: "Dense1"
231 |     param {
232 |         lr_mult: 1
233 |         decay_mult: 1
234 |     }
235 |     param {
236 |         lr_mult: 2
237 |         decay_mult: 0
238 |     }
239 |     inner_product_param {
240 |         num_output: 512
241 |         weight_filler {
242 |             type: "xavier"
243 |         }
244 |         bias_filler {
245 |             type: "constant"
246 |             value: 0
247 |         }
248 |     }
249 | }
250 | 
251 | 
252 | layer {
253 |   name: "ActivationTangH5"
254 |   bottom: "Dense1"
255 |   top: "ActivationTangH5"
256 |   type: "TanH"
257 | }
258 | 
259 | layer {
260 |   name: "ActivationAbs5"
261 |   bottom: "ActivationTangH5"
262 |   top: "Abs5"
263 |   type: "AbsVal"
264 | }
265 | 
266 | 
267 | layer {
268 |     name: "Dense3"
269 |     type: "InnerProduct"
270 |     bottom: "Abs5"
271 |     top: "Dense3"
272 |     param {
273 |         lr_mult: 1
274 |         decay_mult: 1
275 |     }
276 |     param {
277 |         lr_mult: 2
278 |         decay_mult: 0
279 |     }
280 |     inner_product_param {
281 |         num_output: 136
282 |         weight_filler {
283 |             type: "xavier"
284 |         }
285 |         bias_filler {
286 |             type: "constant"
287 |             value: 0
288 |         }
289 |     }
290 | }
291 | 


--------------------------------------------------------------------------------
/result/1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/1.jpg


--------------------------------------------------------------------------------
/result/1_result.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/1_result.jpg


--------------------------------------------------------------------------------
/result/2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/2.jpg


--------------------------------------------------------------------------------
/result/2_result.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/2_result.jpg


--------------------------------------------------------------------------------
/result/4.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/4.jpg


--------------------------------------------------------------------------------
/result/4_result.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/4_result.jpg


--------------------------------------------------------------------------------
/result/7.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/7.jpg


--------------------------------------------------------------------------------
/result/7_result.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/7_result.jpg


--------------------------------------------------------------------------------
/result/8.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/8.jpg


--------------------------------------------------------------------------------
/result/8_result.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/panda-lab/face-landmark/34c8a893fee03eb3b6e5b90576fb335bbfc99571/result/8_result.jpg


--------------------------------------------------------------------------------
/rotation_image.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <opencv2/opencv.hpp>
  3 | #include <opencv2/highgui/highgui.hpp>
  4 | #include <opencv/cvaux.hpp>
  5 | #include <fstream>
  6 | #include <dirent.h>
  7 | using namespace std;
  8 | using namespace cv;
  9 | #define BYTE unsigned char
 10 | 
 11 | Mat ImageRotate(Mat & src, const CvPoint &_center, double angle)
 12 | {
 13 |     CvPoint2D32f center;
 14 |     center.x = float(_center.x);
 15 |     center.y = float(_center.y);
 16 |     
 17 |     //计算二维旋转的仿射变换矩阵
 18 |     Mat M = getRotationMatrix2D(center, angle, 1);
 19 |     
 20 |     // rotate
 21 |     Mat dst;
 22 |     warpAffine(src, dst, M, cvSize(src.cols, src.rows), CV_INTER_LINEAR);
 23 |     return dst;
 24 | }
 25 | 
 26 | // 获取指定像素点放射变换后的新的坐标位置
 27 | CvPoint getPointAffinedPos(const CvPoint &src, const CvPoint center, double angle)
 28 | {
 29 |     CvPoint dst;
 30 |     int x = src.x - center.x;
 31 |     int y = src.y - center.y;
 32 |     
 33 |     dst.x = cvRound(x * cos(angle) + y * sin(angle) + center.x);
 34 |     dst.y = cvRound(-x * sin(angle) + y * cos(angle) + center.y);
 35 |     return dst;
 36 | }
 37 | 
 38 | 
 39 | 
 40 | 
 41 | int main(int argc, const char * argv[])
 42 | {
 43 |     struct dirent *ptr;
 44 |     DIR *dir;
 45 |     string PATH = "/home/workspace/Dataset/68/helen/trainset/";
 46 |     dir=opendir(PATH.c_str());
 47 |     vector<string> files;
 48 |     cout << "文件列表: "<< endl;
 49 |     while((ptr=readdir(dir))!=NULL)
 50 |     {
 51 |         
 52 |         //跳过'.'和'..'两个目录
 53 |         if(ptr->d_name[0] == '.')
 54 |             continue;
 55 |         //cout << ptr->d_name << endl;
 56 |         string tmp = ptr->d_name;
 57 |         long k = tmp.find(".png",0);
 58 |         long m = tmp.find(".jpg",0);
 59 |         if(k > 0 || m > 0)
 60 |         {
 61 |             files.push_back(ptr->d_name);
 62 |         }
 63 |     }
 64 |     closedir(dir);
 65 |     
 66 |     for (int i = 0; i < files.size(); ++i)
 67 |     {
 68 |         cout << files[i] << endl;
 69 |         string filename = "/home/workspace/Dataset/68/helen/trainset/" + files[i];
 70 |         Mat img = imread(filename);
 71 |         cout << filename << endl;
 72 |         CvPoint center;
 73 |         center.x = img.cols / 2;
 74 |         center.y = img.rows / 2;
 75 |     
 76 |     
 77 |         double angle = 9;
 78 |     
 79 |         Mat dst = ImageRotate(img, center, angle);
 80 |     
 81 |     
 82 |         //读取pts文件
 83 |         long idx = filename.find(".", 0);
 84 | 	printf("---- %ld ----\n",filename.size());
 85 |         string pre_path = filename.erase(idx);
 86 |         pre_path.insert(idx, ".pts");
 87 |         ifstream ifs(pre_path);
 88 |  
 89 |     
 90 |         string rotate_pts = "/home/workspace/Dataset/train(helen+ibug+lfpw+afw)/img_09/" + files[i];
 91 | 	long idx2 = rotate_pts.find(".", 0);
 92 |         rotate_pts.erase(idx2);
 93 | 
 94 |         string out_pts = rotate_pts.insert(idx2, "_09.pts");
 95 |         ofstream outfile(out_pts,ios::out);
 96 |     
 97 |         float x;
 98 |         float y;
 99 |         char drop;
100 |         int cnt = 0;
101 |         string temp;
102 |         getline(ifs,temp);  //跳过前三行
103 |         outfile << temp << endl;
104 |         getline(ifs,temp);
105 |         outfile << temp << endl;
106 |         getline(ifs,temp);
107 |         outfile << temp << endl;
108 |     
109 |         while (!ifs.eof())
110 |         {
111 |             if (ifs>>x>>y)
112 |             {
113 |                 CvPoint tmp;
114 |                 tmp.x = x;
115 |                 tmp.y = y;
116 |             
117 |                 CvPoint l2 = getPointAffinedPos(tmp, center, angle * CV_PI / 180);
118 |                 //circle(dst,l2,1,CV_RGB(0,255,255),2);
119 |                 outfile.precision(3);
120 |                 outfile.setf(ios::fixed);
121 |                 outfile << l2.x << " " << l2.y << endl;
122 |             }
123 |             else
124 |             {
125 |                 ifs.clear();
126 |                 ifs.sync();
127 |                 ifs>>drop;
128 |             }
129 |         
130 |             cnt++;
131 |             if(cnt >= 68)
132 |             {
133 |                 cnt = 0;
134 |                 break;
135 |             }
136 |         }
137 |     
138 |         outfile.close();
139 |         //imshow("src", dst);
140 |         string rotate_img = "/home/workspace/Dataset/train(helen+ibug+lfpw+afw)/img_09/" + files[i];
141 |         rotate_img.erase(idx2);
142 |         string out_img = rotate_img.insert(idx2, "_09.jpg");
143 | 
144 |         imwrite(out_img,dst);
145 |     
146 |         img.release();
147 |         dst.release();
148 | 
149 |         //waitKey(0);
150 | 
151 |     }
152 |     
153 |   
154 |     return 0;
155 | }
156 | 


--------------------------------------------------------------------------------
/train/DataRow.py:
--------------------------------------------------------------------------------
   1 | # -*- coding: utf-8 -*-
   2 | """
   3 | Created on Fri Dec  4 22:30:29 2015
   4 | 
   5 | @author: Ishay Tubi
   6 | """
   7 | 
   8 | import os
   9 | import cv2
  10 | import numpy as np
  11 | import sys
  12 | import csv
  13 | import time
  14 | def getGitRepFolder():
  15 | #    import subprocess
  16 | #    return subprocess.Popen(['git', 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE).communicate()[0].rstrip()
  17 |     return '/home/ly/workspace/Vanilla'
  18 | 
  19 | def mse_normlized(groundTruth, pred):
  20 |     delX = groundTruth[78]-groundTruth[84] 
  21 |     delY = groundTruth[79]-groundTruth[85] 
  22 |     interOc = (1e-6+(delX*delX + delY*delY))**0.5  # Euclidain distance
  23 |     diff = (pred-groundTruth)**2
  24 |     sumPairs = (diff[0::2]+diff[1::2])**0.5  # Euclidian distance 
  25 |     return (sumPairs / interOc)  # normlized 
  26 | 
  27 | 
  28 | 
  29 | 
  30 | class RetVal:
  31 |     pass  ## A generic class to return multiple values without a need for a dictionary.
  32 | 
  33 | def createDataRowsFromCSV(csvFilePath, csvParseFunc, DATA_PATH, limit = sys.maxint):
  34 |     ''' Returns a list of DataRow from CSV files parsed by csvParseFunc, 
  35 |         DATA_PATH is the prefix to add to the csv file names,
  36 |         limit can be used to parse only partial file rows.
  37 |     ''' 
  38 |     data = []  # the array we build
  39 |     validObjectsCounter = 0 
  40 |     
  41 |     with open(csvFilePath, 'r') as csvfile:
  42 | 
  43 |         reader = csv.reader(csvfile, delimiter=' ')
  44 |         for row in reader:
  45 |             d = csvParseFunc(row, DATA_PATH)
  46 |             if d is not None:
  47 |                 data.append(d)
  48 |                 validObjectsCounter += 1
  49 |                 if (validObjectsCounter > limit ):  # Stop if reached to limit
  50 |                     return data 
  51 |     return data
  52 | 
  53 | def createBoxRowsFromCSV(csvFilePath, DATA_PATH, limit = sys.maxint):
  54 |     ''' Returns a list of DataRow from CSV files parsed by csvParseFunc, 
  55 |         DATA_PATH is the prefix to add to the csv file names,
  56 |         limit can be used to parse only partial file rows.
  57 |     ''' 
  58 |     data = []  # the array we build
  59 |     validObjectsCounter = 0 
  60 |     with open(csvFilePath, 'r') as csvfile:
  61 | 
  62 |         reader = csv.reader(csvfile, delimiter=' ')
  63 |         for row in reader:
  64 |             # print "row: ", row
  65 |             box = BBox()
  66 |             box.path = os.path.join(DATA_PATH, row[0]).replace("\\", "/")
  67 |             #print box.path
  68 |             box.left = row[1]
  69 |             box.top = row[2]
  70 |             box.right = row[3]
  71 |             box.bottom = row[4]
  72 |             if  box.right>0 and  box.bottom>0 :
  73 |                 data.append(box)
  74 |                 validObjectsCounter += 1
  75 |                 if (validObjectsCounter > limit ):  # Stop if reached to limit
  76 |                     return data 
  77 |     return data
  78 | 
  79 | def getValidWithBBox(dataRows, boxRows=[]):
  80 |     ''' Returns a list of valid DataRow of a given list of dataRows 
  81 |     '''
  82 |     import dlib
  83 |     import random
  84 |     R=RetVal()
  85 |     
  86 |     R.outsideLandmarks = 0 
  87 |     R.noImages = 0 
  88 |     R.noFacesAtAll = 0 
  89 |     R.couldNotMatch = 0
  90 |     detector=dlib.get_frontal_face_detector()
  91 | 
  92 |     validRow=[]
  93 |     for rowIdx, dataRow in enumerate(dataRows):
  94 |         if dataRow.image is None or len(dataRow.image)==0:
  95 |             R.noImages += 1
  96 |             continue
  97 |         #print 'landmarks ', dataRow.landmarks()
  98 |         lmd_xy = dataRow.landmarks().reshape([-1,2])
  99 | 
 100 |         #left,  top = lmd_xy.min( axis=0 )
 101 |         #right, bot = lmd_xy.max( axis=0 )
 102 |         #left,  top = lmd_xy[37]
 103 |         #right, bot = lmd_xy[46]
 104 |         left,  top0 = lmd_xy[36]
 105 |         right, top1 = lmd_xy[45]
 106 |         mid, bot = lmd_xy[57]
 107 | 
 108 |         top = max([top0, top1])
 109 |         left = min([left, mid])
 110 |         right = max([right, mid])
 111 |         #left,  top = lmd_xy.min(axis=0)
 112 |         #right, bot = lmd_xy.max(axis=0)
 113 |         #print "landmark box:  ", left, top, right , bot
 114 | 
 115 | 
 116 |         if not boxRows == []:
 117 |             dets = [boxRows[rowIdx]]
 118 |             #print "box:  ",boxRows[rowIdx].left, boxRows[rowIdx].top, boxRows[rowIdx].right, boxRows[rowIdx].bottom
 119 |         else:
 120 |             dets = detector( np.array(dataRow.image, dtype = 'uint8' ), 1);
 121 |             #dets = [BBox.BBoxFromLTRB(left, top, right, bot)]
 122 | 
 123 | 
 124 |         det_bbox = None  # the valid bbox if found 
 125 |         #print R.couldNotMatch
 126 |         for det in dets:
 127 |             if not boxRows == []:
 128 |                 det_box = BBox.BBoxFromLTRB(float(det.left), float(det.top), float(det.right), float(det.bottom))
 129 |                 # print "det_box:  ",det_box.left, det_box.top, det_box.right, det_box.bottom
 130 |             else:
 131 |                 det_box = BBox.BBoxFromLTRB(det.left(), det.top(), det.right(), det.bottom())
 132 |                 #print det.left(), det.top(), det.right(), det.bottom()
 133 |                 #det_box = BBox.BBoxFromLTRB(det[0], det[1], det[0]+det[2], det[1]+det[3])
 134 |                 #det_box = det
 135 | 
 136 |             # Does all landmarks fit into this box?
 137 |             if top >= det_box.top and bot<= det_box.bottom and left>=det_box.left and right<=det_box.right:
 138 |                     # det_bbox = det_box
 139 |                     # height = det_box.bottom - det_box.top
 140 | 
 141 |                     # det_bbox.top = det_box.top - height * 0.1
 142 |                     # det_bbox.bottom = det_box.bottom + height*0.25
 143 |                     # weight = det_box.right - det_box.left
 144 |                     # det_bbox.left = det_box.left - weight*0.15 
 145 |                     # det_bbox.right = det_box.right + weight*0.15
 146 | 
 147 |                     # center random shift
 148 |                     tx = random.uniform(-0.04,0.04)
 149 |                     ty = random.uniform(-0.04,0.04)
 150 |                     det_box.offset(tx*det_box.width(), ty*det_box.height())
 151 | 
 152 |                     # scale random
 153 |                     s = random.uniform(-0.04,0.04)
 154 |                     #s = 0.1#random.uniform(-0.05,0.1)
 155 |                     #s = random.uniform(0.0,0.03)
 156 |                     det_bbox = det_box
 157 |                     height = det_box.bottom - det_box.top
 158 |                     det_bbox.top = det_box.top - height * s#0.1
 159 |                     det_bbox.bottom = det_box.bottom + height*s#0.1
 160 |                     weight = det_box.right - det_box.left
 161 |                     det_bbox.left = det_box.left - weight*s#0.1
 162 |                     det_bbox.right = det_box.right + weight*s#0.1
 163 |                     # print "det_bbox:  ",det_bbox.left, det_bbox.top, det_bbox.right, det_bbox.bottom
 164 |                     
 165 |         if det_bbox is None:
 166 |             if len(dets)>0:
 167 |                 R.couldNotMatch += 1  # For statistics, dlib found faces but they did not match our landmarks.
 168 |             else:
 169 |                 R.noFacesAtAll += 1  # dlib found 0 faces.
 170 |         else:
 171 |             dataRow.fbbox = det_bbox  # Save the bbox to the data row
 172 |             #if det_bbox.left<0 or det_bbox.top<0 or det_bbox.right>dataRow.image.shape[0] or det_bbox.bottom>dataRow.image.shape[1]:
 173 |             if det_bbox.left<0 or det_bbox.top<0 or det_bbox.right>dataRow.image.shape[1] or det_bbox.bottom>dataRow.image.shape[0]:
 174 |                 #print 'det_bbox: ', det_bbox
 175 |                 #print 'detb: ', detb
 176 |                 #print 'image.shape: ', dataRow.image.shape
 177 |                 R.outsideLandmarks += 1  # Saftey check, make sure nothing goes out of bound.
 178 |             else:
 179 |                 validRow.append(dataRow)  
 180 |     
 181 |     
 182 |     return validRow,R 
 183 |         
 184 | def writeHD5(dataRows, outputPath, setTxtFilePATH, meanTrainSet, stdTrainSet , IMAGE_SIZE=60, mirror=False):
 185 |     ''' Create HD5 data set for caffe from given valid data rows.
 186 |     if mirror is True, duplicate data by mirroring. 
 187 |     ''' 
 188 |     from numpy import zeros
 189 |     import h5py
 190 |     
 191 |     if mirror:
 192 |         BATCH_SIZE = len(dataRows) *2
 193 |     else:
 194 |         BATCH_SIZE = len(dataRows) 
 195 | 
 196 |     #HD5Images = zeros([BATCH_SIZE, 3, IMAGE_SIZE, IMAGE_SIZE], dtype='float32')
 197 |     HD5Images = zeros([BATCH_SIZE, 1, IMAGE_SIZE, IMAGE_SIZE], dtype='float32')
 198 |     HD5Landmarks = zeros([BATCH_SIZE, 136], dtype='float32')
 199 |     #prefix  = os.path.join(ROOT, 'caffeData', 'hd5', 'train')
 200 |     setTxtFile = open(setTxtFilePATH, 'w')
 201 | 
 202 |         
 203 |     i = 0 
 204 |     
 205 |     for dataRowOrig in dataRows:
 206 |         if i % 1000 == 0 or i >= BATCH_SIZE-1:
 207 |             print "Processing row %d " % (i+1) 
 208 |             
 209 |         if not hasattr(dataRowOrig, 'fbbox'):
 210 |             print "Warning, no fbbox"
 211 |             continue
 212 |         
 213 |         dataRow = dataRowOrig.copyCroppedByBBox(dataRowOrig.fbbox)  # Get a cropped scale copy of the data row
 214 |         scaledLM = dataRow.landmarksScaledMinus05_plus05() 
 215 |         image = dataRow.image.astype('f4')
 216 |         #image = (image-meanTrainSet)/(1.e-6 + stdTrainSet)
 217 |         image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(1,IMAGE_SIZE,IMAGE_SIZE)
 218 |         m, s = cv2.meanStdDev(image)
 219 |         image = (image-m)/(1.e-6 + s)
 220 |         
 221 |         #HD5Images[i, :] = cv2.split(image)  # split interleaved (40,40,3) to (3,40,40)
 222 |         HD5Images[i, :] = image
 223 |         HD5Landmarks[i,:] = scaledLM
 224 |         i+=1
 225 |         
 226 |         if mirror:
 227 |             dataRow = dataRowOrig.copyCroppedByBBox(dataRowOrig.fbbox).copyMirrored()  # Get a cropped scale copy of the data row
 228 |             scaledLM = dataRow.landmarksScaledMinus05_plus05() 
 229 |             image = dataRow.image.astype('f4')
 230 |             #image = (image-meanTrainSet)/(1.e-6 + stdTrainSet)
 231 |             image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(1,IMAGE_SIZE,IMAGE_SIZE)
 232 |             m, s = cv2.meanStdDev(image)
 233 |             image = (image-m)/(1.e-6 + s)
 234 |             
 235 |             #HD5Images[i, :] = cv2.split(image)  # split interleaved (40,40,3) to (3,40,40)
 236 |             HD5Images[i, :] = image
 237 |             HD5Landmarks[i,:] = scaledLM
 238 |             i+=1
 239 |         
 240 |         
 241 |     with h5py.File(outputPath, 'w') as T:
 242 |         T.create_dataset("X", data=HD5Images)
 243 |         T.create_dataset("landmarks", data=HD5Landmarks)
 244 | 
 245 |     setTxtFile.write(outputPath+"\n")
 246 |     setTxtFile.flush()
 247 |     setTxtFile.close()
 248 |     
 249 |     
 250 |     
 251 |   
 252 | 
 253 | 
 254 | 
 255 | class ErrorAcum:  # Used to count error per landmark
 256 |     def __init__(self):
 257 |         self.errorPerLandmark = np.zeros(68, dtype ='f4')
 258 |         self.itemsCounter = 0
 259 |         self.failureCounter = 0
 260 |         
 261 |     def __repr__(self):
 262 |         return '%f mean error, %d items, %d failures  %f precent' % (self.meanError().mean()*100, self.itemsCounter, self.failureCounter, float(self.failureCounter)/self.itemsCounter if self.itemsCounter>0 else 0)
 263 |         
 264 |         
 265 |     def add(self, groundTruth, pred):
 266 |         normlized = mse_normlized(groundTruth, pred)
 267 |         self.errorPerLandmark += normlized
 268 |         self.itemsCounter +=1
 269 |         if normlized.mean() > 0.1: 
 270 |             # Count error above 10% as failure
 271 |             self.failureCounter +=1
 272 | 
 273 |     def meanError(self):
 274 |         if self.itemsCounter > 0:
 275 |             return self.errorPerLandmark/self.itemsCounter
 276 |         else:
 277 |             return self.errorPerLandmark
 278 | 
 279 |     def __add__(self, x):
 280 |         ret = ErrorAcum()
 281 |         ret.errorPerLandmark = self.errorPerLandmark + x.errorPerLandmark
 282 |         ret.itemsCounter    = self.itemsCounter + x.itemsCounter
 283 |         ret.failureCounter  = self.failureCounter + x.failureCounter        
 284 |         return ret
 285 |         
 286 |     def plot(self):
 287 |         from matplotlib.pylab import show, plot, stem
 288 |         pass
 289 | 
 290 | 
 291 | class BBox:  # Bounding box
 292 |     
 293 |     @staticmethod
 294 |     def BBoxFromLTRB(l, t, r, b):
 295 |         return BBox(l, t, r, b)
 296 |     
 297 |     @staticmethod
 298 |     def BBoxFromXYWH_array(xywh):
 299 |         return BBox(xywh[0], xywh[1], +xywh[0]+xywh[2], xywh[1]+xywh[3])
 300 |     
 301 |     @staticmethod
 302 |     def BBoxFromXYWH(x,y,w,h):
 303 |         return BBox(x,y, x+w, y+h)
 304 |     
 305 |     def top_left(self):
 306 |         return (self.top, self.left)
 307 |     
 308 |     def left_top(self):
 309 |         return (self.left, self.top)
 310 | 
 311 |     def bottom_right(self):
 312 |         return (self.bottom, self.right)
 313 | 
 314 |     def right_bottom(self):
 315 |         return (self.right, self.bottom)
 316 |     
 317 |     def right_top(self):
 318 |         return (self.right, self.top)
 319 |     
 320 |     def relaxed(self, clip ,relax=3):  #@Unused
 321 |         from numpy import array
 322 |         _A = array
 323 |         maxWidth, maxHeight =  clip[0], clip[1]
 324 |         
 325 |         nw, nh = self.size()*(1+relax)*.5       
 326 |         center = self.center()
 327 |         offset=_A([nw,nh])
 328 |         lefttop = center - offset
 329 |         rightbot= center + offset 
 330 |          
 331 |         self.left, self.top  = int( max( 0, lefttop[0] ) ), int( max( 0, lefttop[1]) )
 332 |         self.right, self.bottom = int( min( rightbot[0], maxWidth ) ), int( min( rightbot[1], maxHeight ) )
 333 |         return self
 334 | 
 335 |     def clip(self, maxRight, maxBottom):
 336 |         self.left = max(self.left, 0)
 337 |         self.top = max(self.top, 0)
 338 |         self.right = min(self.right, maxRight)
 339 |         self.bottom = min(self.bottom, maxBottom)
 340 |         
 341 |     def size(self):
 342 |         from numpy import  array
 343 |         return array([self.width(), self.height()])
 344 |      
 345 |     def center(self):
 346 |         from numpy import  array
 347 |         return array([(self.left+self.right)/2, (self.top+self.bottom)/2])
 348 |                 
 349 |     def __init__(self,left=0, top=0, right=0, bottom=0, path=''):
 350 |         self.left = left
 351 |         self.top = top
 352 |         self.right = right
 353 |         self.bottom = bottom
 354 |         self.path = path
 355 |         
 356 |     def width(self):
 357 |         return self.right - self.left
 358 |         
 359 |     def height(self):
 360 |         return self.bottom - self.top
 361 |         
 362 |     def xywh(self):
 363 |         return self.left, self.top, self.width(), self.height()
 364 |         
 365 |     def offset(self, x, y):
 366 |         self.left += x 
 367 |         self.right += x
 368 |         self.top += y 
 369 |         self.bottom += y
 370 |          
 371 |     def scale(self, rx, ry):
 372 |         self.left *= rx 
 373 |         self.right *= rx
 374 |         self.top *= ry 
 375 |         self.bottom *= ry
 376 |                         
 377 |     def __repr__(self):
 378 |         return 'left(%.1f), top(%.1f), right(%.1f), bottom(%.1f) w(%d) h(%d)' % (self.left, self.top, self.right, self.bottom,self.width(), self.height())
 379 | 
 380 |     def makeInt(self):
 381 |         self.left    = int(self.left)
 382 |         self.top     = int(self.top)
 383 |         self.right   = int(self.right)
 384 |         self.bottom  = int(self.bottom)
 385 |         return self
 386 | 
 387 | 
 388 | 
 389 | class DataRow:
 390 |     global TrainSetMean
 391 |     global TrainSetSTD
 392 |     
 393 |     IMAGE_SIZE = 60
 394 |     def __init__(self, path='', p1=(0, 0, ), p2=(0, 0), p3=(0, 0), p4=(0, 0), p5=(0, 0),p6=(0, 0, ), p7=(0, 0), p8=(0, 0), p9=(0, 0), p10=(0, 0), 
 395 | p11=(0, 0, ), p12=(0, 0), p13=(0, 0), p14=(0, 0), p15=(0, 0),p16=(0, 0, ), p17=(0, 0), p18=(0, 0), p19=(0, 0), p20=(0, 0),
 396 | p21=(0, 0, ), p22=(0, 0), p23=(0, 0), p24=(0, 0), p25=(0, 0),p26=(0, 0, ), p27=(0, 0), p28=(0, 0), p29=(0, 0), p30=(0, 0),
 397 | p31=(0, 0, ), p32=(0, 0), p33=(0, 0), p34=(0, 0), p35=(0, 0),p36=(0, 0, ), p37=(0, 0), p38=(0, 0), p39=(0, 0), p40=(0, 0),
 398 | p41=(0, 0, ), p42=(0, 0), p43=(0, 0), p44=(0, 0), p45=(0, 0),p46=(0, 0, ), p47=(0, 0), p48=(0, 0), p49=(0, 0), p50=(0, 0),
 399 | p51=(0, 0, ), p52=(0, 0), p53=(0, 0), p54=(0, 0), p55=(0, 0),p56=(0, 0, ), p57=(0, 0), p58=(0, 0), p59=(0, 0), p60=(0, 0),
 400 | p61=(0, 0, ), p62=(0, 0), p63=(0, 0), p64=(0, 0), p65=(0, 0),p66=(0, 0, ), p67=(0, 0), p68=(0, 0)):
 401 |         self.image = cv2.imread(path)
 402 |         self.p1 = p1
 403 |         self.p2 = p2
 404 |         self.p3 = p3
 405 |         self.p4 = p4
 406 |         self.p5 = p5
 407 |         self.p6 = p6
 408 |         self.p7 = p7
 409 |         self.p8 = p8
 410 |         self.p9 = p9
 411 |         self.p10 = p10
 412 |         self.p11 = p11
 413 |         self.p12 = p12
 414 |         self.p13 = p13
 415 |         self.p14 = p14
 416 |         self.p15 = p15
 417 |         self.p16 = p16
 418 |         self.p17 = p17
 419 |         self.p18 = p18
 420 |         self.p19 = p19
 421 |         self.p20 = p20
 422 |         self.p21 = p21
 423 |         self.p22 = p22
 424 |         self.p23 = p23
 425 |         self.p24 = p24
 426 |         self.p25 = p25
 427 |         self.p26 = p26
 428 |         self.p27 = p27
 429 |         self.p28 = p28
 430 |         self.p29 = p29
 431 |         self.p30 = p30
 432 |         self.p31 = p31
 433 |         self.p32 = p32
 434 |         self.p33 = p33
 435 |         self.p34 = p34
 436 |         self.p35 = p35
 437 |         self.p36 = p36
 438 |         self.p37 = p37
 439 |         self.p38 = p38
 440 |         self.p39 = p39
 441 |         self.p40 = p40
 442 |         self.p41 = p41
 443 |         self.p42 = p42
 444 |         self.p43 = p43
 445 |         self.p44 = p44
 446 |         self.p45 = p45
 447 |         self.p46 = p46
 448 |         self.p47 = p47
 449 |         self.p48 = p48
 450 |         self.p49 = p49
 451 |         self.p50 = p50
 452 |         self.p51 = p51
 453 |         self.p52 = p52
 454 |         self.p53 = p53
 455 |         self.p54 = p54
 456 |         self.p55 = p55
 457 |         self.p56 = p56
 458 |         self.p57 = p57
 459 |         self.p58 = p58
 460 |         self.p59 = p59
 461 |         self.p60 = p60
 462 |         self.p61 = p61
 463 |         self.p62 = p62
 464 |         self.p63 = p63
 465 |         self.p64 = p64
 466 |         self.p65 = p65
 467 |         self.p66 = p66
 468 |         self.p67 = p67
 469 |         self.p68 = p68
 470 | 
 471 |         self.name = os.path.split(path)[-1]
 472 |         self.sx = 1.
 473 |         self.sy = 1.
 474 |         self.offsetX = 0.
 475 |         self.offsetY = 0.
 476 | 
 477 |     def __repr__(self):
 478 |         return '{} le:{},{} re:{},{} nose:{},{}, lm:{},{} rm:{},{}'.format(
 479 |             self.name,
 480 |             self.p1[0], self.p1[1],
 481 |             self.p2[0], self.p2[1],
 482 |             self.p3[0], self.p3[1],
 483 |             self.p4[0], self.p4[1],
 484 |             self.p5[0], self.p5[1],
 485 |             self.p6[0], self.p6[1],
 486 |             self.p7[0], self.p7[1],
 487 |             self.p8[0], self.p8[1],
 488 |             self.p9[0], self.p9[1],
 489 |             self.p10[0], self.p10[1],
 490 |             self.p11[0], self.p11[1],
 491 |             self.p12[0], self.p12[1],
 492 |             self.p13[0], self.p13[1],
 493 |             self.p14[0], self.p14[1],
 494 |             self.p15[0], self.p15[1],
 495 |             self.p16[0], self.p16[1],
 496 |             self.p17[0], self.p17[1],
 497 |             self.p18[0], self.p18[1],
 498 |             self.p19[0], self.p19[1],
 499 |             self.p20[0], self.p20[1],
 500 |             self.p21[0], self.p21[1],
 501 |             self.p22[0], self.p22[1],
 502 |             self.p23[0], self.p23[1],
 503 |             self.p24[0], self.p24[1],
 504 |             self.p25[0], self.p25[1],
 505 |             self.p26[0], self.p26[1],
 506 |             self.p27[0], self.p27[1],
 507 |             self.p28[0], self.p28[1],
 508 |             self.p29[0], self.p29[1],
 509 |             self.p30[0], self.p20[1],
 510 |             self.p31[0], self.p31[1],
 511 |             self.p32[0], self.p32[1],
 512 |             self.p33[0], self.p33[1],
 513 |             self.p34[0], self.p34[1],
 514 |             self.p35[0], self.p35[1],
 515 |             self.p36[0], self.p36[1],
 516 |             self.p37[0], self.p37[1],
 517 |             self.p38[0], self.p38[1],
 518 |             self.p39[0], self.p39[1],
 519 |             self.p40[0], self.p40[1],
 520 |             self.p51[0], self.p51[1],
 521 |             self.p52[0], self.p52[1],
 522 |             self.p53[0], self.p53[1],
 523 |             self.p54[0], self.p54[1],
 524 |             self.p55[0], self.p55[1],
 525 |             self.p56[0], self.p56[1],
 526 |             self.p57[0], self.p57[1],
 527 |             self.p58[0], self.p58[1],
 528 |             self.p59[0], self.p59[1],
 529 |             self.p60[0], self.p60[1],
 530 |             self.p61[0], self.p61[1],
 531 |             self.p62[0], self.p62[1],
 532 |             self.p63[0], self.p63[1],
 533 |             self.p64[0], self.p64[1],
 534 |             self.p65[0], self.p65[1],
 535 |             self.p66[0], self.p66[1],
 536 |             self.p67[0], self.p67[1],
 537 |             self.p68[0], self.p68[1]
 538 |             )
 539 | 
 540 |     def setLandmarks(self,landMarks):
 541 |         """
 542 |         @landMarks : np.array
 543 |         set the landmarks from array
 544 |         """
 545 |         self.p1 = landMarks[0:2]
 546 |         self.p2 = landMarks[2:4]
 547 |         self.p3 = landMarks[4:6]
 548 |         self.p4 = landMarks[6:8]
 549 |         self.p5 = landMarks[8:10]
 550 |         self.p6 = landMarks[10:12]
 551 |         self.p7 = landMarks[12:14]
 552 |         self.p8 = landMarks[14:16]
 553 |         self.p9 = landMarks[16:18]
 554 |         self.p10 = landMarks[18:20]
 555 |         self.p11 = landMarks[20:22]
 556 |         self.p12 = landMarks[22:24]
 557 |         self.p13 = landMarks[24:26]
 558 |         self.p14 = landMarks[26:28]
 559 |         self.p15 = landMarks[28:30]
 560 |         self.p16 = landMarks[30:32]
 561 |         self.p17 = landMarks[32:34]
 562 |         self.p18 = landMarks[34:36]
 563 |         self.p19 = landMarks[36:38]
 564 |         self.p20 = landMarks[38:40]
 565 |         self.p21 = landMarks[40:42]
 566 |         self.p22 = landMarks[42:44]
 567 |         self.p23 = landMarks[44:46]
 568 |         self.p24 = landMarks[46:48]
 569 |         self.p25 = landMarks[48:50]
 570 |         self.p26 = landMarks[50:52]
 571 |         self.p27 = landMarks[52:54]
 572 |         self.p28 = landMarks[54:56]
 573 |         self.p29 = landMarks[56:58]
 574 |         self.p30 = landMarks[58:60]
 575 |         self.p31 = landMarks[60:62]
 576 |         self.p32 = landMarks[62:64]
 577 |         self.p33 = landMarks[64:66]
 578 |         self.p34 = landMarks[66:68]
 579 |         self.p35 = landMarks[68:70]
 580 |         self.p36 = landMarks[70:72]
 581 |         self.p37 = landMarks[72:74]
 582 |         self.p38 = landMarks[74:76]
 583 |         self.p39 = landMarks[76:78]
 584 |         self.p40 = landMarks[78:80]        
 585 |         self.p41 = landMarks[80:82]
 586 |         self.p42 = landMarks[82:84]
 587 |         self.p43 = landMarks[84:86]
 588 |         self.p44 = landMarks[86:88]
 589 |         self.p45 = landMarks[88:90]
 590 |         self.p46 = landMarks[90:92]
 591 |         self.p47 = landMarks[92:94]
 592 |         self.p48 = landMarks[94:96]
 593 |         self.p49 = landMarks[96:98]
 594 |         self.p50 = landMarks[98:100]
 595 |         self.p51 = landMarks[100:102]
 596 |         self.p52 = landMarks[102:104]
 597 |         self.p53 = landMarks[104:106]
 598 |         self.p54 = landMarks[106:108]
 599 |         self.p55 = landMarks[108:110]
 600 |         self.p56 = landMarks[110:112]
 601 |         self.p57 = landMarks[112:114]
 602 |         self.p58 = landMarks[114:116]
 603 |         self.p59 = landMarks[116:118]
 604 |         self.p60 = landMarks[118:120]
 605 |         self.p61 = landMarks[120:122]
 606 |         self.p62 = landMarks[122:124]
 607 |         self.p63 = landMarks[124:126]
 608 |         self.p64 = landMarks[126:128]
 609 |         self.p65 = landMarks[128:130]
 610 |         self.p66 = landMarks[130:132]
 611 |         self.p67 = landMarks[132:134]
 612 |         self.p68 = landMarks[134:136]       
 613 |     def landmarks(self):
 614 |         # return numpy float array with ordered values
 615 |         stright = [
 616 |             self.p1[0],self.p1[1],
 617 |             self.p2[0],self.p2[1],
 618 |             self.p3[0],self.p3[1],
 619 |             self.p4[0],self.p4[1],
 620 |             self.p5[0],self.p5[1],
 621 |             self.p6[0],self.p6[1],
 622 |             self.p7[0],self.p7[1],
 623 |             self.p8[0],self.p8[1],
 624 |             self.p9[0],self.p9[1],
 625 |             self.p10[0],self.p10[1],
 626 |             self.p11[0],self.p11[1],
 627 |             self.p12[0],self.p12[1],
 628 |             self.p13[0],self.p13[1],
 629 |             self.p14[0],self.p14[1],
 630 |             self.p15[0],self.p15[1],
 631 |             self.p16[0],self.p16[1],
 632 |             self.p17[0],self.p17[1],
 633 |             self.p18[0],self.p18[1],
 634 |             self.p19[0],self.p19[1],
 635 |             self.p20[0],self.p20[1],
 636 |             self.p21[0],self.p21[1],
 637 |             self.p22[0],self.p22[1],
 638 |             self.p23[0],self.p23[1],
 639 |             self.p24[0],self.p24[1],
 640 |             self.p25[0],self.p25[1],
 641 |             self.p26[0],self.p26[1],
 642 |             self.p27[0],self.p27[1],
 643 |             self.p28[0],self.p28[1],
 644 |             self.p29[0],self.p29[1],
 645 |             self.p30[0],self.p30[1],
 646 |             self.p31[0],self.p31[1],
 647 |             self.p32[0],self.p32[1],
 648 |             self.p33[0],self.p33[1],
 649 |             self.p34[0],self.p34[1],
 650 |             self.p35[0],self.p35[1],
 651 |             self.p36[0],self.p36[1],
 652 |             self.p37[0],self.p37[1],
 653 |             self.p38[0],self.p38[1],
 654 |             self.p39[0],self.p39[1],
 655 |             self.p40[0],self.p40[1],
 656 |             self.p41[0],self.p41[1],
 657 |             self.p42[0],self.p42[1],
 658 |             self.p43[0],self.p43[1],
 659 |             self.p44[0],self.p44[1],
 660 |             self.p45[0],self.p45[1],
 661 |             self.p46[0],self.p46[1],
 662 |             self.p47[0],self.p47[1],
 663 |             self.p48[0],self.p48[1],
 664 |             self.p49[0],self.p49[1],
 665 |             self.p50[0],self.p50[1],
 666 |             self.p51[0],self.p51[1],
 667 |             self.p52[0],self.p52[1],
 668 |             self.p53[0],self.p53[1],
 669 |             self.p54[0],self.p54[1],
 670 |             self.p55[0],self.p55[1],
 671 |             self.p56[0],self.p56[1],
 672 |             self.p57[0],self.p57[1],
 673 |             self.p58[0],self.p58[1],
 674 |             self.p59[0],self.p59[1],
 675 |             self.p60[0],self.p60[1],
 676 |             self.p61[0],self.p61[1],
 677 |             self.p62[0],self.p62[1],
 678 |             self.p63[0],self.p63[1],
 679 |             self.p64[0],self.p64[1],
 680 |             self.p65[0],self.p65[1],
 681 |             self.p66[0],self.p66[1],
 682 |             self.p67[0],self.p67[1],
 683 |             self.p68[0],self.p68[1]]
 684 | 
 685 |         return np.array(stright, dtype='f4')
 686 | 
 687 |     def landmarksScaledMinus05_plus05(self):
 688 |         # return numpy float array with ordered values
 689 |         #return self.landmarks().astype('f4')/40. - 0.5
 690 |         return self.landmarks().astype('f4')/60.
 691 |         
 692 |     def scale(self, sx, sy):
 693 |         self.sx *= sx
 694 |         self.sy *= sy
 695 | 
 696 |         self.p1 = (self.p1[0]*sx, self.p1[1]*sy)
 697 |         self.p2 = (self.p2[0]*sx, self.p2[1]*sy)
 698 |         self.p3 = (self.p3[0]*sx, self.p3[1]*sy)
 699 |         self.p4 = (self.p4[0]*sx, self.p4[1]*sy)
 700 |         self.p5 = (self.p5[0]*sx, self.p5[1]*sy)
 701 |         self.p6 = (self.p6[0]*sx, self.p6[1]*sy)
 702 |         self.p7 = (self.p7[0]*sx, self.p7[1]*sy)
 703 |         self.p8 = (self.p8[0]*sx, self.p8[1]*sy)
 704 |         self.p9 = (self.p9[0]*sx, self.p9[1]*sy)
 705 |         self.p10 = (self.p10[0]*sx, self.p10[1]*sy)
 706 |         self.p11 = (self.p11[0]*sx, self.p11[1]*sy)
 707 |         self.p12 = (self.p12[0]*sx, self.p12[1]*sy)
 708 |         self.p13 = (self.p13[0]*sx, self.p13[1]*sy)
 709 |         self.p14 = (self.p14[0]*sx, self.p14[1]*sy)
 710 |         self.p15 = (self.p15[0]*sx, self.p15[1]*sy)
 711 |         self.p16 = (self.p16[0]*sx, self.p16[1]*sy)
 712 |         self.p17 = (self.p17[0]*sx, self.p17[1]*sy)
 713 |         self.p18 = (self.p18[0]*sx, self.p18[1]*sy)
 714 |         self.p19 = (self.p19[0]*sx, self.p19[1]*sy)
 715 |         self.p20 = (self.p20[0]*sx, self.p20[1]*sy)
 716 |         self.p21 = (self.p21[0]*sx, self.p21[1]*sy)
 717 |         self.p22 = (self.p22[0]*sx, self.p22[1]*sy)
 718 |         self.p23 = (self.p23[0]*sx, self.p23[1]*sy)
 719 |         self.p24 = (self.p24[0]*sx, self.p24[1]*sy)
 720 |         self.p25 = (self.p25[0]*sx, self.p25[1]*sy)
 721 |         self.p26 = (self.p26[0]*sx, self.p26[1]*sy)
 722 |         self.p27 = (self.p27[0]*sx, self.p27[1]*sy)
 723 |         self.p28 = (self.p28[0]*sx, self.p28[1]*sy)
 724 |         self.p29 = (self.p29[0]*sx, self.p29[1]*sy)
 725 |         self.p30 = (self.p30[0]*sx, self.p30[1]*sy)
 726 |         self.p31 = (self.p31[0]*sx, self.p31[1]*sy)
 727 |         self.p32 = (self.p32[0]*sx, self.p32[1]*sy)
 728 |         self.p33 = (self.p33[0]*sx, self.p33[1]*sy)
 729 |         self.p34 = (self.p34[0]*sx, self.p34[1]*sy)
 730 |         self.p35 = (self.p35[0]*sx, self.p35[1]*sy)
 731 |         self.p36 = (self.p36[0]*sx, self.p36[1]*sy)
 732 |         self.p37 = (self.p37[0]*sx, self.p37[1]*sy)
 733 |         self.p38 = (self.p38[0]*sx, self.p38[1]*sy)
 734 |         self.p39 = (self.p39[0]*sx, self.p39[1]*sy)
 735 |         self.p40 = (self.p40[0]*sx, self.p40[1]*sy)
 736 |         self.p41 = (self.p41[0]*sx, self.p41[1]*sy)
 737 |         self.p42 = (self.p42[0]*sx, self.p42[1]*sy)
 738 |         self.p43 = (self.p43[0]*sx, self.p43[1]*sy)
 739 |         self.p44 = (self.p44[0]*sx, self.p44[1]*sy)
 740 |         self.p45 = (self.p45[0]*sx, self.p45[1]*sy)
 741 |         self.p46 = (self.p46[0]*sx, self.p46[1]*sy)
 742 |         self.p47 = (self.p47[0]*sx, self.p47[1]*sy)
 743 |         self.p48 = (self.p48[0]*sx, self.p48[1]*sy)
 744 |         self.p49 = (self.p49[0]*sx, self.p49[1]*sy)
 745 |         self.p50 = (self.p50[0]*sx, self.p50[1]*sy)
 746 |         self.p51 = (self.p51[0]*sx, self.p51[1]*sy)
 747 |         self.p52 = (self.p52[0]*sx, self.p52[1]*sy)
 748 |         self.p53 = (self.p53[0]*sx, self.p53[1]*sy)
 749 |         self.p54 = (self.p54[0]*sx, self.p54[1]*sy)
 750 |         self.p55 = (self.p55[0]*sx, self.p55[1]*sy)
 751 |         self.p56 = (self.p56[0]*sx, self.p56[1]*sy)
 752 |         self.p57 = (self.p57[0]*sx, self.p57[1]*sy)
 753 |         self.p58 = (self.p58[0]*sx, self.p58[1]*sy)
 754 |         self.p59 = (self.p59[0]*sx, self.p59[1]*sy)
 755 |         self.p60 = (self.p60[0]*sx, self.p60[1]*sy)
 756 |         self.p61 = (self.p61[0]*sx, self.p61[1]*sy)
 757 |         self.p62 = (self.p62[0]*sx, self.p62[1]*sy)
 758 |         self.p63 = (self.p63[0]*sx, self.p63[1]*sy)
 759 |         self.p64 = (self.p64[0]*sx, self.p64[1]*sy)
 760 |         self.p65 = (self.p65[0]*sx, self.p65[1]*sy)
 761 |         self.p66 = (self.p66[0]*sx, self.p66[1]*sy)
 762 |         self.p67 = (self.p67[0]*sx, self.p67[1]*sy)
 763 |         self.p68 = (self.p68[0]*sx, self.p68[1]*sy)
 764 | 
 765 |         
 766 |         if hasattr(self, 'prediction'):
 767 |             self.prediction = self.prediction.reshape(-1, 2)*[sx, sy]
 768 | 
 769 |         return self
 770 | 
 771 |     def offsetCropped(self, offset=(0., 0.)):
 772 |         """ given the cropped values - offset the positions by offset
 773 |         """
 774 |         self.offsetX -= offset[0]
 775 |         self.offsetY -= offset[1]
 776 | 
 777 |         if hasattr(self, 'prediction'):
 778 |             self.prediction = self.prediction.reshape(-1,2)-offset
 779 | 
 780 | 
 781 |         self.p1 = (self.p1[0]-offset[0], self.p1[1]-offset[1])
 782 |         self.p2 = (self.p2[0]-offset[0], self.p2[1]-offset[1])
 783 |         self.p3 = (self.p3[0]-offset[0], self.p3[1]-offset[1])
 784 |         self.p4 = (self.p4[0]-offset[0], self.p4[1]-offset[1])
 785 |         self.p5 = (self.p5[0]-offset[0], self.p5[1]-offset[1])
 786 |         self.p6 = (self.p6[0]-offset[0], self.p6[1]-offset[1])
 787 |         self.p7 = (self.p7[0]-offset[0], self.p7[1]-offset[1])
 788 |         self.p8 = (self.p8[0]-offset[0], self.p8[1]-offset[1])
 789 |         self.p9 = (self.p9[0]-offset[0], self.p9[1]-offset[1])
 790 |         self.p10 = (self.p10[0]-offset[0], self.p10[1]-offset[1])
 791 |         self.p11 = (self.p11[0]-offset[0], self.p11[1]-offset[1])
 792 |         self.p12 = (self.p12[0]-offset[0], self.p12[1]-offset[1])
 793 |         self.p13 = (self.p13[0]-offset[0], self.p13[1]-offset[1])
 794 |         self.p14 = (self.p14[0]-offset[0], self.p14[1]-offset[1])
 795 |         self.p15 = (self.p15[0]-offset[0], self.p15[1]-offset[1])
 796 |         self.p16 = (self.p16[0]-offset[0], self.p16[1]-offset[1])
 797 |         self.p17 = (self.p17[0]-offset[0], self.p17[1]-offset[1])
 798 |         self.p18 = (self.p18[0]-offset[0], self.p18[1]-offset[1])
 799 |         self.p19 = (self.p19[0]-offset[0], self.p19[1]-offset[1])
 800 |         self.p20 = (self.p20[0]-offset[0], self.p20[1]-offset[1])
 801 |         self.p21 = (self.p21[0]-offset[0], self.p21[1]-offset[1])
 802 |         self.p22 = (self.p22[0]-offset[0], self.p22[1]-offset[1])
 803 |         self.p23 = (self.p23[0]-offset[0], self.p23[1]-offset[1])
 804 |         self.p24 = (self.p24[0]-offset[0], self.p24[1]-offset[1])
 805 |         self.p25 = (self.p25[0]-offset[0], self.p25[1]-offset[1])
 806 |         self.p26 = (self.p26[0]-offset[0], self.p26[1]-offset[1])
 807 |         self.p27 = (self.p27[0]-offset[0], self.p27[1]-offset[1])
 808 |         self.p28 = (self.p28[0]-offset[0], self.p28[1]-offset[1])
 809 |         self.p29 = (self.p29[0]-offset[0], self.p29[1]-offset[1])
 810 |         self.p30 = (self.p30[0]-offset[0], self.p30[1]-offset[1])
 811 |         self.p31 = (self.p31[0]-offset[0], self.p31[1]-offset[1])
 812 |         self.p32 = (self.p32[0]-offset[0], self.p32[1]-offset[1])
 813 |         self.p33 = (self.p33[0]-offset[0], self.p33[1]-offset[1])
 814 |         self.p34 = (self.p34[0]-offset[0], self.p34[1]-offset[1])
 815 |         self.p35 = (self.p35[0]-offset[0], self.p35[1]-offset[1])
 816 |         self.p36 = (self.p36[0]-offset[0], self.p36[1]-offset[1])
 817 |         self.p37 = (self.p37[0]-offset[0], self.p37[1]-offset[1])
 818 |         self.p38 = (self.p38[0]-offset[0], self.p38[1]-offset[1])
 819 |         self.p39 = (self.p39[0]-offset[0], self.p39[1]-offset[1])
 820 |         self.p40 = (self.p40[0]-offset[0], self.p40[1]-offset[1])
 821 |         self.p41 = (self.p41[0]-offset[0], self.p41[1]-offset[1])
 822 |         self.p42 = (self.p42[0]-offset[0], self.p42[1]-offset[1])
 823 |         self.p43 = (self.p43[0]-offset[0], self.p43[1]-offset[1])
 824 |         self.p44 = (self.p44[0]-offset[0], self.p44[1]-offset[1])
 825 |         self.p45 = (self.p45[0]-offset[0], self.p45[1]-offset[1])
 826 |         self.p46 = (self.p46[0]-offset[0], self.p46[1]-offset[1])
 827 |         self.p47 = (self.p47[0]-offset[0], self.p47[1]-offset[1])
 828 |         self.p48 = (self.p48[0]-offset[0], self.p48[1]-offset[1])
 829 |         self.p49 = (self.p49[0]-offset[0], self.p49[1]-offset[1])
 830 |         self.p50 = (self.p50[0]-offset[0], self.p50[1]-offset[1])
 831 |         self.p51 = (self.p51[0]-offset[0], self.p51[1]-offset[1])
 832 |         self.p52 = (self.p52[0]-offset[0], self.p52[1]-offset[1])
 833 |         self.p53 = (self.p53[0]-offset[0], self.p53[1]-offset[1])
 834 |         self.p54 = (self.p54[0]-offset[0], self.p54[1]-offset[1])
 835 |         self.p55 = (self.p55[0]-offset[0], self.p55[1]-offset[1])
 836 |         self.p56 = (self.p56[0]-offset[0], self.p56[1]-offset[1])
 837 |         self.p57 = (self.p57[0]-offset[0], self.p57[1]-offset[1])
 838 |         self.p58 = (self.p58[0]-offset[0], self.p58[1]-offset[1])
 839 |         self.p59 = (self.p59[0]-offset[0], self.p59[1]-offset[1])
 840 |         self.p60 = (self.p60[0]-offset[0], self.p60[1]-offset[1])
 841 |         self.p61 = (self.p61[0]-offset[0], self.p61[1]-offset[1])
 842 |         self.p62 = (self.p62[0]-offset[0], self.p62[1]-offset[1])
 843 |         self.p63 = (self.p63[0]-offset[0], self.p63[1]-offset[1])
 844 |         self.p64 = (self.p64[0]-offset[0], self.p64[1]-offset[1])
 845 |         self.p65 = (self.p65[0]-offset[0], self.p65[1]-offset[1])
 846 |         self.p66 = (self.p66[0]-offset[0], self.p66[1]-offset[1])
 847 |         self.p67 = (self.p67[0]-offset[0], self.p67[1]-offset[1])
 848 |         self.p68 = (self.p68[0]-offset[0], self.p68[1]-offset[1])
 849 | 
 850 |         return self
 851 | 
 852 |     def inverseScaleAndOffset(self, landmarks):
 853 |         """ computes the inverse scale and offset of input data according to the inverse scale factor and inverse offset factor
 854 |         """
 855 |         from numpy import array; _A = array ; # Shothand 
 856 |         
 857 |         ret = _A(landmarks.reshape(-1,2)) *_A([1./self.sx, 1./self.sy])
 858 |         ret += _A([-self.offsetX, -self.offsetY])
 859 |         return ret
 860 | 
 861 |     @staticmethod
 862 |     def DataRowFromNameBoxInterlaved(row, root=''):  # lfw_5590 + net_7876 (interleaved) 
 863 |         '''
 864 |         name , bounding box(w,h), left eye (x,y) ,right eye (x,y)..nose..left mouth,..right mouth
 865 |         '''
 866 |         d = DataRow()
 867 |         #print 'row: ', row
 868 |         d.path = os.path.join(root, row[0]).replace("\\", "/")
 869 |         print d.path
 870 |         d.name = os.path.split(d.path)[-1]
 871 | 
 872 |         d.image = cv2.imread(d.path)
 873 | 
 874 |         d.p1 = (float(row[1]), float(row[2]))
 875 |         d.p2 = (float(row[3]), float(row[4]))
 876 |         d.p3 = (float(row[5]), float(row[6]))
 877 |         d.p4 = (float(row[7]), float(row[8]))
 878 |         d.p5 = (float(row[9]), float(row[10]))
 879 |         d.p6 = (float(row[11]), float(row[12]))
 880 |         d.p7 = (float(row[13]), float(row[14]))
 881 |         d.p8 = (float(row[15]), float(row[16]))
 882 |         d.p9 = (float(row[17]), float(row[18]))
 883 |         d.p10 = (float(row[19]), float(row[20]))
 884 |         d.p11 = (float(row[21]), float(row[22]))
 885 |         d.p12 = (float(row[23]), float(row[24]))
 886 |         d.p13 = (float(row[25]), float(row[26]))
 887 |         d.p14 = (float(row[27]), float(row[28]))
 888 |         d.p15 = (float(row[29]), float(row[30]))
 889 |         d.p16 = (float(row[31]), float(row[32]))
 890 |         d.p17 = (float(row[33]), float(row[34]))
 891 |         d.p18 = (float(row[35]), float(row[36]))
 892 |         d.p19 = (float(row[37]), float(row[38]))
 893 |         d.p20 = (float(row[39]), float(row[40])) 
 894 |         d.p21 = (float(row[41]), float(row[42]))
 895 |         d.p22 = (float(row[43]), float(row[44]))
 896 |         d.p23 = (float(row[45]), float(row[46]))
 897 |         d.p24 = (float(row[47]), float(row[48]))
 898 |         d.p25 = (float(row[49]), float(row[50]))
 899 |         d.p26 = (float(row[51]), float(row[52]))
 900 |         d.p27 = (float(row[53]), float(row[54]))
 901 |         d.p28 = (float(row[55]), float(row[56]))
 902 |         d.p29 = (float(row[57]), float(row[58]))
 903 |         d.p30 = (float(row[59]), float(row[60])) 
 904 |         d.p31 = (float(row[61]), float(row[62]))
 905 |         d.p32 = (float(row[63]), float(row[64]))
 906 |         d.p33 = (float(row[65]), float(row[66]))
 907 |         d.p34 = (float(row[67]), float(row[68]))
 908 |         d.p35 = (float(row[69]), float(row[70]))
 909 |         d.p36 = (float(row[71]), float(row[72]))
 910 |         d.p37 = (float(row[73]), float(row[74]))
 911 |         d.p38 = (float(row[75]), float(row[76]))
 912 |         d.p39 = (float(row[77]), float(row[78]))
 913 |         d.p40 = (float(row[79]), float(row[80])) 
 914 |         d.p41 = (float(row[81]), float(row[82]))
 915 |         d.p42 = (float(row[83]), float(row[84]))
 916 |         d.p43 = (float(row[85]), float(row[86]))
 917 |         d.p44 = (float(row[87]), float(row[88]))
 918 |         d.p45 = (float(row[89]), float(row[90]))
 919 |         d.p46 = (float(row[91]), float(row[92]))
 920 |         d.p47 = (float(row[93]), float(row[94]))
 921 |         d.p48 = (float(row[95]), float(row[96]))
 922 |         d.p49 = (float(row[97]), float(row[98]))
 923 |         d.p50 = (float(row[99]), float(row[100]))
 924 |         d.p51 = (float(row[101]), float(row[102]))
 925 |         d.p52 = (float(row[103]), float(row[104]))
 926 |         d.p53 = (float(row[105]), float(row[106]))
 927 |         d.p54 = (float(row[107]), float(row[108]))
 928 |         d.p55 = (float(row[109]), float(row[110]))
 929 |         d.p56 = (float(row[111]), float(row[112]))
 930 |         d.p57 = (float(row[113]), float(row[114]))
 931 |         d.p58 = (float(row[115]), float(row[116]))
 932 |         d.p59 = (float(row[117]), float(row[118]))
 933 |         d.p60 = (float(row[119]), float(row[120]))
 934 |         d.p61 = (float(row[121]), float(row[122]))
 935 |         d.p62 = (float(row[123]), float(row[124]))
 936 |         d.p63 = (float(row[125]), float(row[126]))
 937 |         d.p64 = (float(row[127]), float(row[128]))
 938 |         d.p65 = (float(row[129]), float(row[130]))
 939 |         d.p66 = (float(row[131]), float(row[132]))
 940 |         d.p67 = (float(row[133]), float(row[134]))
 941 |         d.p68 = (float(row[135]), float(row[136]))
 942 | 
 943 |         return d
 944 | 
 945 |     @staticmethod
 946 |     def DataRowFromMTFL(row, root=''):
 947 |         '''
 948 |         --x1...x5,y1...y5: the locations for left eye, right eye, nose, left mouth corner, right mouth corner.
 949 |         '''
 950 |         d = DataRow()
 951 |         if len(row[0]) <= 1:
 952 |             # bug in the files, it has spaces seperating them, skip it
 953 |             row=row[1:]
 954 |             
 955 |         if len(row)<10:
 956 |             print 'error parsing ', row
 957 |             return None
 958 |         
 959 |         d.path = os.path.join(root, row[0]).replace("\\", "/")
 960 |         d.name = os.path.split(d.path)[-1]
 961 |         d.image = cv2.imread(d.path)
 962 |         
 963 |         if d.image is None:
 964 |             print 'Error reading image', d.path
 965 |             return None
 966 |         
 967 |         d.p1 = (float(row[1]), float(row[2]))
 968 |         d.p2 = (float(row[3]), float(row[4]))
 969 |         d.p3 = (float(row[5]), float(row[6]))
 970 |         d.p4 = (float(row[7]), float(row[8]))
 971 |         d.p5 = (float(row[9]), float(row[10]))
 972 |         d.p6 = (float(row[11]), float(row[12]))
 973 |         d.p7 = (float(row[13]), float(row[14]))
 974 |         d.p8 = (float(row[15]), float(row[16]))
 975 |         d.p9 = (float(row[17]), float(row[18]))
 976 |         d.p10 = (float(row[19]), float(row[20]))
 977 |         d.p11 = (float(row[21]), float(row[22]))
 978 |         d.p12 = (float(row[23]), float(row[24]))
 979 |         d.p13 = (float(row[25]), float(row[26]))
 980 |         d.p14 = (float(row[27]), float(row[28]))
 981 |         d.p15 = (float(row[29]), float(row[30]))
 982 |         d.p16 = (float(row[31]), float(row[32]))
 983 |         d.p17 = (float(row[33]), float(row[34]))
 984 |         d.p18 = (float(row[35]), float(row[36]))
 985 |         d.p19 = (float(row[37]), float(row[38]))
 986 |         d.p20 = (float(row[39]), float(row[40])) 
 987 |         d.p21 = (float(row[41]), float(row[42]))
 988 |         d.p22 = (float(row[43]), float(row[44]))
 989 |         d.p23 = (float(row[45]), float(row[46]))
 990 |         d.p24 = (float(row[47]), float(row[48]))
 991 |         d.p25 = (float(row[49]), float(row[50]))
 992 |         d.p26 = (float(row[51]), float(row[52]))
 993 |         d.p27 = (float(row[53]), float(row[54]))
 994 |         d.p28 = (float(row[55]), float(row[56]))
 995 |         d.p29 = (float(row[57]), float(row[58]))
 996 |         d.p30 = (float(row[59]), float(row[60])) 
 997 |         d.p31 = (float(row[61]), float(row[62]))
 998 |         d.p32 = (float(row[63]), float(row[64]))
 999 |         d.p33 = (float(row[65]), float(row[66]))
1000 |         d.p34 = (float(row[67]), float(row[68]))
1001 |         d.p35 = (float(row[69]), float(row[70]))
1002 |         d.p36 = (float(row[71]), float(row[72]))
1003 |         d.p37 = (float(row[73]), float(row[74]))
1004 |         d.p38 = (float(row[75]), float(row[76]))
1005 |         d.p39 = (float(row[77]), float(row[78]))
1006 |         d.p40 = (float(row[79]), float(row[80])) 
1007 |         d.p41 = (float(row[81]), float(row[82]))
1008 |         d.p42 = (float(row[83]), float(row[84]))
1009 |         d.p43 = (float(row[85]), float(row[86]))
1010 |         d.p44 = (float(row[87]), float(row[88]))
1011 |         d.p45 = (float(row[89]), float(row[90]))
1012 |         d.p46 = (float(row[91]), float(row[92]))
1013 |         d.p47 = (float(row[93]), float(row[94]))
1014 |         d.p48 = (float(row[95]), float(row[96]))
1015 |         d.p49 = (float(row[97]), float(row[98]))
1016 |         d.p50 = (float(row[99]), float(row[100]))
1017 |         d.p51 = (float(row[101]), float(row[102]))
1018 |         d.p52 = (float(row[103]), float(row[104]))
1019 |         d.p53 = (float(row[105]), float(row[106]))
1020 |         d.p54 = (float(row[107]), float(row[108]))
1021 |         d.p55 = (float(row[109]), float(row[110]))
1022 |         d.p56 = (float(row[111]), float(row[112]))
1023 |         d.p57 = (float(row[113]), float(row[114]))
1024 |         d.p58 = (float(row[115]), float(row[116]))
1025 |         d.p59 = (float(row[117]), float(row[118]))
1026 |         d.p60 = (float(row[119]), float(row[120]))
1027 |         d.p61 = (float(row[121]), float(row[122]))
1028 |         d.p62 = (float(row[123]), float(row[124]))
1029 |         d.p63 = (float(row[125]), float(row[126]))
1030 |         d.p64 = (float(row[127]), float(row[128]))
1031 |         d.p65 = (float(row[129]), float(row[130]))
1032 |         d.p66 = (float(row[131]), float(row[132]))
1033 |         d.p67 = (float(row[133]), float(row[134]))
1034 |         d.p68 = (float(row[135]), float(row[136]))
1035 | 
1036 |   
1037 | 
1038 |         return d
1039 | 
1040 |     @staticmethod
1041 |     def DataRowFromAFW(anno, root=''): # Assume data comming from parsed anno-v7.mat file.
1042 |         name = str(anno[0][0])
1043 |         bbox = anno[1][0][0]
1044 | #        yaw, pitch, roll = anno[2][0][0][0]
1045 |         lm = anno[3][0][0]  # 6 landmarks
1046 | 
1047 |         if np.isnan(lm).any():
1048 |             return None  # Fail
1049 | 
1050 |         d = DataRow()
1051 |         d.path = os.path.join(root, name).replace("\\", "/")
1052 |         d.name = os.path.split(d.path)[-1]
1053 |         d.image = cv2.imread(d.path)
1054 | 
1055 | 
1056 |         d.leftEye = (float(lm[0][0]), float(lm[0][1]))
1057 |         d.rightEye = (float(lm[1][0]), float(lm[1][1]))
1058 |         d.middle = (float(lm[2][0]), float(lm[2][1]))
1059 |         d.leftMouth = (float(lm[3][0]), float(lm[3][1]))
1060 |         # skip point 4 middle mouth - We take 0 left eye, 1 right eye, 2 nose, 3 left mouth, 5 right mouth
1061 |         d.rightMouth = (float(lm[5][0]), float(lm[5][1]))
1062 | 
1063 | 
1064 |         return d
1065 | 
1066 |     @staticmethod
1067 |     def DataRowFromPrediction(p, path='', image=None):
1068 |         d = DataRow(path)        
1069 |         p = (p+0.5)*60.  # scale from -0.5..+0.5 to 0..40
1070 |         
1071 |         d.p1 = (p[0], p[1])
1072 |         d.p2 = (p[2], p[3])
1073 |         d.p3 = (p[4], p[5])
1074 |         d.p4 = (p[6], p[7])
1075 |         d.p5 = (p[8], p[9])
1076 | 
1077 |         return d
1078 | 
1079 |     def drawLandmarks(self, r=3, color=255, other=None, title=None):
1080 |         M = self.image
1081 |         if hasattr(self, 'prediction'):
1082 |             for x,y in self.prediction.reshape(-1,2):
1083 |                 cv2.circle(M, (int(x), int(y)), r, (0,200,0), -1)            
1084 | 
1085 |         cv2.circle(M, (int(self.p1[0]), int(self.p1[1])), r, color, -1)
1086 |         cv2.circle(M, (int(self.p2[0]), int(self.p2[1])), r, color, -1)
1087 |         cv2.circle(M, (int(self.p3[0]), int(self.p3[1])), r, color, -1)
1088 |         cv2.circle(M, (int(self.p4[0]), int(self.p4[1])), r, color, -1)
1089 |         cv2.circle(M, (int(self.p5[0]), int(self.p5[1])), r, color, -1)
1090 |         cv2.circle(M, (int(self.p6[0]), int(self.p6[1])), r, color, -1)
1091 |         cv2.circle(M, (int(self.p7[0]), int(self.p7[1])), r, color, -1)
1092 |         cv2.circle(M, (int(self.p8[0]), int(self.p8[1])), r, color, -1)
1093 |         cv2.circle(M, (int(self.p9[0]), int(self.p9[1])), r, color, -1)
1094 |         cv2.circle(M, (int(self.p10[0]), int(self.p10[1])), r, color, -1)
1095 |         cv2.circle(M, (int(self.p11[0]), int(self.p11[1])), r, color, -1)
1096 |         cv2.circle(M, (int(self.p12[0]), int(self.p12[1])), r, color, -1)
1097 |         cv2.circle(M, (int(self.p13[0]), int(self.p13[1])), r, color, -1)
1098 |         cv2.circle(M, (int(self.p14[0]), int(self.p14[1])), r, color, -1)
1099 |         cv2.circle(M, (int(self.p15[0]), int(self.p15[1])), r, color, -1)
1100 |         cv2.circle(M, (int(self.p16[0]), int(self.p16[1])), r, color, -1)
1101 |         cv2.circle(M, (int(self.p17[0]), int(self.p17[1])), r, color, -1)
1102 |         cv2.circle(M, (int(self.p18[0]), int(self.p18[1])), r, color, -1)
1103 |         cv2.circle(M, (int(self.p19[0]), int(self.p19[1])), r, color, -1)
1104 |         if hasattr(self, 'fbbox'):
1105 |             #cv2.rectangle(M, self.fbbox.top_left(), self.fbbox.bottom_right(), color)
1106 | #            cv2.rectangle(M, self.fbbox.left_top(), self.fbbox.right_bottom(), color)
1107 |             cv2.rectangle(M, (int(self.fbbox.left), int(self.fbbox.top)), (int(self.fbbox.right), int(self.fbbox.bottom)), color, 2)
1108 | 
1109 | 
1110 |             det_bbox = self.fbbox
1111 |             det_box = self.fbbox
1112 |             #height = (det_box.bottom - det_box.top)/1.35
1113 |             #det_bbox.top = det_box.top + height * 0.1
1114 |             #det_bbox.bottom = det_box.bottom - height*0.25
1115 |             #weight = (det_box.right - det_box.left)/1.3
1116 |             #det_bbox.left = det_box.left + weight*0.15
1117 |             #det_bbox.right = det_box.right - weight*0.15
1118 |             height = (det_box.bottom - det_box.top)/1.2
1119 |             det_bbox.top = det_box.top + height * 0.1
1120 |             det_bbox.bottom = det_box.bottom - height*0.1
1121 |             weight = (det_box.right - det_box.left)/1.2
1122 |             det_bbox.left = det_box.left + weight*0.1
1123 |             det_bbox.right = det_box.right - weight*0.1
1124 |             
1125 |             cv2.rectangle(M, (int(det_bbox.left), int(det_bbox.top)), (int(det_bbox.right), int(det_bbox.bottom)), (0,200,0), 2)
1126 |         return M
1127 | 
1128 |     def show(self, r=3, color=255, other=None, title=None):
1129 |         M = self.drawLandmarks(r, color, other, title)
1130 |         if title is None:
1131 |             title = self.name
1132 |         # my debug
1133 |         #cv2.imshow(title, M)
1134 | 
1135 |         return M
1136 |         
1137 |     def makeInt(self):
1138 |         self.p1 = (int(self.p1[0]), int(self.p1[1]))
1139 |         self.p2 = (int(self.p2[0]), int(self.p2[1]))
1140 |         self.p3 = (int(self.p3[0]), int(self.p3[1]))
1141 |         self.p4 = (int(self.p4[0]), int(self.p4[1]))
1142 |         self.p5 = (int(self.p5[0]), int(self.p5[1]))
1143 |         self.p6 = (int(self.p6[0]), int(self.p6[1]))
1144 |         self.p7 = (int(self.p7[0]), int(self.p7[1]))
1145 |         self.p8 = (int(self.p8[0]), int(self.p8[1]))
1146 |         self.p9 = (int(self.p9[0]), int(self.p9[1]))
1147 |         self.p10 = (int(self.p10[0]), int(self.p10[1]))
1148 |         self.p11 = (int(self.p11[0]), int(self.p11[1]))
1149 |         self.p12 = (int(self.p12[0]), int(self.p12[1]))
1150 |         self.p13 = (int(self.p13[0]), int(self.p13[1]))
1151 |         self.p14 = (int(self.p14[0]), int(self.p14[1]))
1152 |         self.p15 = (int(self.p15[0]), int(self.p15[1]))
1153 |         self.p16 = (int(self.p16[0]), int(self.p16[1]))
1154 |         self.p17 = (int(self.p17[0]), int(self.p17[1]))
1155 |         self.p18 = (int(self.p18[0]), int(self.p18[1]))
1156 |         self.p19 = (int(self.p19[0]), int(self.p19[1]))
1157 |         self.p20 = (int(self.p20[0]), int(self.p20[1]))
1158 |         self.p21 = (int(self.p21[0]), int(self.p21[1]))
1159 |         self.p22 = (int(self.p22[0]), int(self.p22[1]))
1160 |         self.p23 = (int(self.p23[0]), int(self.p23[1]))
1161 |         self.p24 = (int(self.p24[0]), int(self.p24[1]))
1162 |         self.p25 = (int(self.p25[0]), int(self.p25[1]))
1163 |         self.p26 = (int(self.p26[0]), int(self.p26[1]))
1164 |         self.p27 = (int(self.p27[0]), int(self.p27[1]))
1165 |         self.p28 = (int(self.p28[0]), int(self.p28[1]))
1166 |         self.p29 = (int(self.p29[0]), int(self.p29[1]))
1167 |         self.p30 = (int(self.p30[0]), int(self.p30[1]))
1168 |         self.p31 = (int(self.p31[0]), int(self.p31[1]))
1169 |         self.p32 = (int(self.p32[0]), int(self.p32[1]))
1170 |         self.p33 = (int(self.p33[0]), int(self.p33[1]))
1171 |         self.p34 = (int(self.p34[0]), int(self.p34[1]))
1172 |         self.p35 = (int(self.p35[0]), int(self.p35[1]))
1173 |         self.p36 = (int(self.p36[0]), int(self.p36[1]))
1174 |         self.p37 = (int(self.p37[0]), int(self.p37[1]))
1175 |         self.p38 = (int(self.p38[0]), int(self.p38[1]))
1176 |         self.p39 = (int(self.p39[0]), int(self.p39[1]))
1177 |         self.p40 = (int(self.p40[0]), int(self.p40[1]))
1178 |         self.p41 = (int(self.p41[0]), int(self.p41[1]))
1179 |         self.p42 = (int(self.p42[0]), int(self.p42[1]))
1180 |         self.p43 = (int(self.p43[0]), int(self.p43[1]))
1181 |         self.p44 = (int(self.p44[0]), int(self.p44[1]))
1182 |         self.p45 = (int(self.p45[0]), int(self.p45[1]))
1183 |         self.p46 = (int(self.p46[0]), int(self.p46[1]))
1184 |         self.p47 = (int(self.p47[0]), int(self.p47[1]))
1185 |         self.p48 = (int(self.p48[0]), int(self.p48[1]))
1186 |         self.p49 = (int(self.p49[0]), int(self.p49[1]))
1187 |         self.p50 = (int(self.p50[0]), int(self.p50[1]))
1188 |         self.p51 = (int(self.p51[0]), int(self.p51[1]))
1189 |         self.p52 = (int(self.p52[0]), int(self.p52[1]))
1190 |         self.p53 = (int(self.p53[0]), int(self.p53[1]))
1191 |         self.p54 = (int(self.p54[0]), int(self.p54[1]))
1192 |         self.p55 = (int(self.p55[0]), int(self.p55[1]))
1193 |         self.p56 = (int(self.p56[0]), int(self.p56[1]))
1194 |         self.p57 = (int(self.p57[0]), int(self.p57[1]))
1195 |         self.p58 = (int(self.p58[0]), int(self.p58[1]))
1196 |         self.p59 = (int(self.p59[0]), int(self.p59[1]))
1197 |         self.p60 = (int(self.p60[0]), int(self.p60[1]))
1198 |         self.p61 = (int(self.p61[0]), int(self.p61[1]))
1199 |         self.p62 = (int(self.p62[0]), int(self.p62[1]))
1200 |         self.p63 = (int(self.p63[0]), int(self.p63[1]))
1201 |         self.p64 = (int(self.p64[0]), int(self.p64[1]))
1202 |         self.p65 = (int(self.p65[0]), int(self.p65[1]))
1203 |         self.p66 = (int(self.p66[0]), int(self.p66[1]))
1204 |         self.p67 = (int(self.p67[0]), int(self.p67[1]))
1205 |         self.p68 = (int(self.p68[0]), int(self.p68[1]))
1206 | 
1207 |         return self        
1208 |          
1209 |     def copyCroppedByBBox(self,fbbox, siz=np.array([60.,60.])):
1210 |         """
1211 |         @ fbbox : BBox
1212 |         Returns a copy with cropped, scaled to size
1213 |         """        
1214 |         fbbox.makeInt() # assume BBox class
1215 |         if fbbox.width()<10 or fbbox.height()<10:
1216 |             print "Invalid bbox size:",fbbox
1217 |             return None
1218 |         #print "fbbox: ", fbbox
1219 |         faceOnly = self.image[fbbox.top : fbbox.bottom, fbbox.left:fbbox.right, :]
1220 |         scaled = DataRow() 
1221 |         scaled.image = cv2.resize(faceOnly, (int(siz[0]), int(siz[1])))                
1222 |         scaled.setLandmarks(self.landmarks())        
1223 |         """ @scaled: DataRow """
1224 |         scaled.offsetCropped(fbbox.left_top()) # offset the landmarks
1225 |         ry, rx = siz/faceOnly.shape[:2]
1226 |         scaled.scale(rx, ry)
1227 |         
1228 |         return scaled        
1229 |         
1230 |     def copyMirrored(self):
1231 |         '''
1232 |         Return a copy with mirrored data (and mirrored landmarks).
1233 |         '''
1234 |         import numpy
1235 |         _A=numpy.array
1236 |         ret = DataRow() 
1237 |         ret.image=cv2.flip(self.image.copy(),1)
1238 |         # Now we mirror the landmarks and swap left and right
1239 |         width = ret.image.shape[0] 
1240 |         ret.p1 = _A([width-self.p17[0], self.p17[1]]) # Toggle left\right eyes position and mirror x axis only
1241 |         ret.p2 = _A([width-self.p16[0], self.p16[1]])
1242 |         ret.p3 = _A([width-self.p15[0], self.p15[1]])        
1243 |         ret.p4 = _A([width-self.p14[0], self.p14[1]]) # Toggle mouth positions and mirror x axis only
1244 |         ret.p5 = _A([width-self.p13[0], self.p13[1]])
1245 |         ret.p6 = _A([width-self.p12[0], self.p12[1]])
1246 |         ret.p7 = _A([width-self.p11[0], self.p11[1]])
1247 |         ret.p8 = _A([width-self.p10[0], self.p10[1]])        
1248 |         ret.p9 = _A([width-self.p9[0], self.p9[1]])
1249 |         ret.p10 = _A([width-self.p8[0], self.p8[1]])
1250 |         ret.p11 = _A([width-self.p7[0], self.p7[1]]) # Toggle left\right eyes position and mirror x axis only
1251 |         ret.p12 = _A([width-self.p6[0], self.p6[1]])
1252 |         ret.p13 = _A([width-self.p5[0], self.p5[1]])        
1253 |         ret.p14 = _A([width-self.p4[0], self.p4[1]]) # Toggle mouth positions and mirror x axis only
1254 |         ret.p15 = _A([width-self.p3[0], self.p3[1]])
1255 |         ret.p16 = _A([width-self.p2[0], self.p2[1]])
1256 |         ret.p17 = _A([width-self.p1[0], self.p1[1]])
1257 |         ret.p18 = _A([width-self.p27[0], self.p27[1]])        
1258 |         ret.p19 = _A([width-self.p26[0], self.p26[1]])
1259 |         ret.p20 = _A([width-self.p25[0], self.p25[1]])
1260 |         ret.p21 = _A([width-self.p24[0], self.p24[1]])
1261 |         ret.p22 = _A([width-self.p23[0], self.p23[1]])
1262 |         ret.p23 = _A([width-self.p22[0], self.p22[1]])        
1263 |         ret.p24 = _A([width-self.p21[0], self.p21[1]])
1264 |         ret.p25 = _A([width-self.p20[0], self.p20[1]])
1265 |         ret.p26 = _A([width-self.p19[0], self.p19[1]])
1266 |         ret.p27 = _A([width-self.p18[0], self.p18[1]])
1267 |         ret.p28 = _A([width-self.p28[0], self.p28[1]])        
1268 |         ret.p29 = _A([width-self.p29[0], self.p29[1]])
1269 |         ret.p30 = _A([width-self.p30[0], self.p30[1]])
1270 |         ret.p31 = _A([width-self.p31[0], self.p31[1]]) # Toggle left\right eyes position and mirror x axis only
1271 |         ret.p32 = _A([width-self.p36[0], self.p36[1]])
1272 |         ret.p33 = _A([width-self.p35[0], self.p35[1]])        
1273 |         ret.p34 = _A([width-self.p34[0], self.p34[1]]) # Toggle mouth positions and mirror x axis only
1274 |         ret.p35 = _A([width-self.p33[0], self.p33[1]])
1275 |         ret.p36 = _A([width-self.p32[0], self.p32[1]])
1276 |         ret.p37 = _A([width-self.p46[0], self.p46[1]])
1277 |         ret.p38 = _A([width-self.p45[0], self.p45[1]])        
1278 |         ret.p39 = _A([width-self.p44[0], self.p44[1]])
1279 |         ret.p40 = _A([width-self.p43[0], self.p43[1]])
1280 |         ret.p41 = _A([width-self.p48[0], self.p48[1]]) # Toggle left\right eyes position and mirror x axis only
1281 |         ret.p42 = _A([width-self.p47[0], self.p47[1]])
1282 |         ret.p43 = _A([width-self.p40[0], self.p40[1]])        
1283 |         ret.p44 = _A([width-self.p39[0], self.p39[1]]) # Toggle mouth positions and mirror x axis only
1284 |         ret.p45 = _A([width-self.p38[0], self.p38[1]])
1285 |         ret.p46 = _A([width-self.p37[0], self.p37[1]])
1286 |         ret.p47 = _A([width-self.p42[0], self.p42[1]])
1287 |         ret.p48 = _A([width-self.p41[0], self.p41[1]])        
1288 |         ret.p49 = _A([width-self.p55[0], self.p55[1]])
1289 |         ret.p50 = _A([width-self.p54[0], self.p54[1]])
1290 |         ret.p51 = _A([width-self.p53[0], self.p53[1]]) # Toggle left\right eyes position and mirror x axis only
1291 |         ret.p52 = _A([width-self.p52[0], self.p52[1]])
1292 |         ret.p53 = _A([width-self.p51[0], self.p51[1]])        
1293 |         ret.p54 = _A([width-self.p50[0], self.p50[1]]) # Toggle mouth positions and mirror x axis only
1294 |         ret.p55 = _A([width-self.p49[0], self.p49[1]])
1295 |         ret.p56 = _A([width-self.p60[0], self.p60[1]])
1296 |         ret.p57 = _A([width-self.p59[0], self.p59[1]])
1297 |         ret.p58 = _A([width-self.p58[0], self.p58[1]])        
1298 |         ret.p59 = _A([width-self.p57[0], self.p57[1]])
1299 |         ret.p60 = _A([width-self.p56[0], self.p56[1]])
1300 |         ret.p61 = _A([width-self.p65[0], self.p65[1]]) # Toggle left\right eyes position and mirror x axis only
1301 |         ret.p62 = _A([width-self.p64[0], self.p64[1]])
1302 |         ret.p63 = _A([width-self.p63[0], self.p63[1]])        
1303 |         ret.p64 = _A([width-self.p62[0], self.p62[1]]) # Toggle mouth positions and mirror x axis only
1304 |         ret.p65 = _A([width-self.p61[0], self.p61[1]])
1305 |         ret.p66 = _A([width-self.p68[0], self.p68[1]])
1306 |         ret.p67 = _A([width-self.p67[0], self.p67[1]])
1307 |         ret.p68 = _A([width-self.p66[0], self.p66[1]])        
1308 | 
1309 |         return ret
1310 | 
1311 |     @staticmethod
1312 |     def dummyDataRow():
1313 |         ''' Returns a dummy dataRow object to play with
1314 |         '''
1315 |         return DataRow('/Users/ishay/Dev/VanilaCNN/data/train/lfw_5590/Abbas_Kiarostami_0001.jpg',
1316 |                      leftEye=(106.75, 108.25),
1317 |                      rightEye=(143.75,108.75) ,
1318 |                      middle = (131.25, 127.25),
1319 |                      leftMouth = (106.25, 155.25),
1320 |                      rightMouth =(142.75,155.25)
1321 |                      )    
1322 |         
1323 |   
1324 |             
1325 | class Predictor:
1326 |     ROOT = getGitRepFolder() 
1327 |     
1328 |     def preprocess(self, resized, landmarks):
1329 |         #ret = resized.astype('f4')
1330 |         #ret -= self.mean
1331 |         #ret /= (1.e-6+ self.std)
1332 |         #return  ret, (landmarks/40.)-0.5
1333 |         grayImg = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY).astype('f4')
1334 |         m, s = cv2.meanStdDev(grayImg)
1335 |         grayImg = (grayImg-m)/(1.e-6 + s)
1336 |         return  grayImg, landmarks/60.
1337 |     
1338 |     def predict(self, resized):
1339 |         """
1340 |         @resized: image 40,40 already pre processed 
1341 |         """         
1342 |         #self.net.blobs['data'].data[...] = cv2.split(resized)
1343 |         self.net.blobs['data'].data[...] = resized.reshape(1,1,60,60)
1344 |         prediction = self.net.forward()['Dense3'][0]
1345 |         return prediction
1346 |         
1347 |     def __init__(self, protoTXTPath, weightsPath):
1348 |         import caffe
1349 |         caffe.set_mode_cpu()
1350 |         self.net = caffe.Net(protoTXTPath, weightsPath, caffe.TEST)
1351 |         self.mean = cv2.imread(os.path.join(Predictor.ROOT, 'trainMean.png')).astype('float')
1352 |         self.mean = cv2.resize(self.mean, (60,60), interpolation=cv2.INTER_CUBIC)
1353 |         self.std  = cv2.imread(os.path.join(Predictor.ROOT,'trainSTD.png')).astype('float')
1354 |         self.std = cv2.resize(self.std, (60,60), interpolation=cv2.INTER_CUBIC)
1355 | 
1356 |     
1357 | 


--------------------------------------------------------------------------------
/train/Demo.py:
--------------------------------------------------------------------------------
 1 | 
 2 | import numpy as np
 3 | _A = np.array  # A shortcut to creating arrays in command line 
 4 | import os
 5 | import cv2
 6 | import sys
 7 | 
 8 | sys.path.append(os.path.join('/home/ly/workspace/Vanilla-40', 'python'))  # Assume git root directory
 9 | 
10 | ###########################    PATHS TO SET   ####################
11 | # Either define CAFFE_ROOT in your enviroment variables or set it here
12 | CAFFE_ROOT = os.environ.get('CAFFE_ROOT','~/caffe/distribute')
13 | sys.path.append(CAFFE_ROOT+'/python')  # Add caffe python path so we can import it
14 | import caffe
15 | 
16 | #Import helper functions
17 | from DataRow import DataRow, BBox, Predictor
18 | 
19 | 
20 | PATH_TO_WEIGHTS  = os.path.join('/home/ly/workspace/Vanilla-40', 'caffeData', 'snapshots_60_medium_ibug_0.04c_0.04s', 'snap_iter_80000.caffemodel')
21 | #PATH_TO_WEIGHTS  = os.path.join('/home/ly/workspace/Vanilla-40', 'caffeData', 'snapshots-40-unit-cvt1-inner', 'snap_iter_670000.caffemodel')
22 | PATH_TO_DEPLOY_TXT = os.path.join('/home/ly/workspace/Vanilla-40', 'ZOO', 'vanilla_deploy_60_medium.prototxt')
23 | predictor = Predictor(protoTXTPath=PATH_TO_DEPLOY_TXT, weightsPath=PATH_TO_WEIGHTS)
24 | 
25 | # Make sure dlib python path exists on PYTHONPATH else "pip install dlib" if needed.
26 | import dlib
27 | detector=dlib.get_frontal_face_detector() # Load dlib's face detector
28 | 
29 | 
30 | img_path = os.path.join('/home/ly/workspace/Vanilla-40/TestSet','helen_3139620200_1.jpg')
31 | 
32 | 
33 | img = cv2.imread(img_path)
34 | #dets = detector( np.array(img, dtype = 'uint8' ), 1)
35 | dets = [[1]]
36 | 
37 | for det in dets:
38 | 
39 |     det_box = BBox.BBoxFromLTRB(51, 238, 545, 656) #helen_3139620200_1
40 | 
41 | 
42 |     print "det_box:  ",det_box.left, det_box.top, det_box.right, det_box.bottom
43 | 
44 |     #det_box = det_bbox
45 |     #det_box.offset(-0.1*det_box.width(), 0.0*det_box.height())
46 |     det_bbox = det_box
47 | 
48 | 
49 |     scaledImage = DataRow()
50 |     scaledImage.image = img
51 |     scaledImage.fbbox = det_bbox
52 |     dataRow60 = scaledImage.copyCroppedByBBox(det_bbox)
53 |     image, lm = predictor.preprocess(dataRow60.image, dataRow60.landmarks())
54 |     prediction = predictor.predict(image)
55 |     #dataRow60.prediction = (prediction+0.5)*60.  # Scale -0.5..+0.5 to 0..60
56 |     dataRow60.prediction = prediction*60.  # Scale -0.5..+0.5 to 0..60
57 |     scaledImage.prediction = dataRow60.inverseScaleAndOffset(dataRow60.prediction) # Scale up to the original image scale
58 |     ALIGNED_DATA_PATH = os.path.join('/home/ly/workspace/Vanilla-40/TestSet', os.path.basename(img_path)[:-4]+'_reuslt.jpg')
59 |     aligned_im = scaledImage.show()
60 |     print ALIGNED_DATA_PATH
61 |     print aligned_im.shape
62 |     cv2.imwrite(ALIGNED_DATA_PATH, aligned_im)
63 | 
64 | 


--------------------------------------------------------------------------------
/train/NormlizedMSE.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | """
  3 | Created on Fri Dec  4 22:57:25 2015
  4 | 
  5 | @author:Ishay Tubi
  6 | """
  7 | 
  8 | import caffe
  9 | import numpy as np
 10 | 
 11 | 
 12 | class NormlizedMSE(caffe.Layer):
 13 |     """
 14 |     Compute the normlized MSE Loss 
 15 |     """
 16 |     def setup(self, bottom, top):
 17 |         # check input pair
 18 |         if len(bottom) != 2:
 19 |             raise Exception("Need two inputs to compute distance.")
 20 | 
 21 |     def reshape(self, bottom, top):
 22 |         # check input dimensions match
 23 | 
 24 |         if bottom[0].count != bottom[1].count:
 25 |             raise Exception("Inputs must have the same dimension.")
 26 |         # difference is shape of inputs
 27 |         self.diff = np.zeros_like(bottom[0].data, dtype=np.float32)
 28 |  #       self.diff = np.zeros(bottom[0].count,dtype='float32')
 29 |         # loss output is scalar
 30 |         top[0].reshape(1)
 31 |       #  print('NormlizedMSE bottom shape ',bottom[0].shape[0],bottom[0].shape[1])
 32 |                 
 33 |     def forward(self,bottom, top):
 34 |         '''Mean square error of landmark regression normalized w.r.t.                                                
 35 |         inter-ocular distance                                                                                        
 36 |         '''
 37 |          # Lets assume batch size is 16 in this example remarks
 38 |         # input size is (16,10) 
 39 |         y_true = bottom[1].data # Assume second entry is the labled data
 40 |         y_pred = bottom[0].data
 41 |         
 42 |         #eye_indices = left eye x, left eye y, right eye x, right eye y, nose x, left mouth, right mouth
 43 |         #delX = y_true[:,78]-y_true[:,84] # del X size 16
 44 |         #delY = y_true[:,79]-y_true[:,85] # del y size 16
 45 |         delX = y_true[:,72]-y_true[:,90] # del X size 16
 46 |         delY = y_true[:,73]-y_true[:,91] # del y size 16
 47 | 
 48 | 
 49 |         #distXLeft = y_true[:,78]-y_true[:,54]
 50 |         #distYLeft = y_true[:,79]-y_true[:,55]
 51 |         #distXRight = y_true[:,84]-y_true[:,54]
 52 |         #distYRight = y_true[:,85]-y_true[:,55]
 53 |         #distLeft = (1e-6+(distXLeft*distXLeft + distYLeft*distYLeft)**0.5)
 54 |         #distRight = (1e-6+(distXRight*distXRight + distYRight*distYRight)**0.5)
 55 |         #dist = np.vstack((distLeft,distRight))
 56 |         #maxDist = dist.max(axis=0)
 57 |         #minDist = dist.min(axis=0)
 58 |         #ratio = (minDist/maxDist)
 59 | 
 60 |         self.interOc = (1e-6+(delX*delX + delY*delY)**0.5).T # Euclidain distance
 61 |         #self.interOc = self.interOc + (1e-6+(delX1*delX1 + delY1*delY1)**0.5).T # Euclidain distance
 62 |         #self.interOc = self.interOc/2.0
 63 |         #self.interOc = 1.0
 64 |         #self.interOc = (ratio*(1e-6+(delX*delX + delY*delY)**0.5)).T # Euclidain distance
 65 |         #self.interOc = ratio
 66 | 
 67 |         # Cannot multiply shape (16,10) by (16,1) so we transpose to (10,16) and (1,16) 
 68 |         diff = (y_pred-y_true).T # Transpose so we can divide a (16,10) array by (16,1)
 69 |         
 70 |         self.diff[...]  = (diff/self.interOc).T # We transpose back to (16,10)
 71 |         top[0].data[...] = np.sum(self.diff**2) / bottom[0].num / 2. # Loss is scalar
 72 | 
 73 |         # loss = np.zeros((bottom[0].num), dtype=np.float32)
 74 |         # for k in xrange(0,68):
 75 |         #     delX = y_pred[:,k*2]-y_true[:,k*2] # del X size 16
 76 |         #     delY = y_pred[:,k*2+1]-y_true[:,k*2+1] # del y size 16
 77 |         #     loss = loss + (delX*delX + delY*delY)**0.5
 78 |         # loss = loss.T
 79 |         # top[0].data[...] = np.sum(loss/self.interOc) / bottom[0].num/68
 80 | 
 81 |     def backward(self, top, propagate_down, bottom):
 82 |         
 83 |         for i in range(2):
 84 |             if not propagate_down[i]:
 85 |                 continue
 86 |             if i == 0:
 87 |                 sign = 1
 88 |             else:
 89 |                 sign = -1
 90 |             bottom[i].diff[...] = sign * self.diff / bottom[i].num
 91 | #            print(bottom[i].diff[...])
 92 | 
 93 | ##################################
 94 | 
 95 | class EuclideanLossLayer(caffe.Layer):
 96 |     #ORIGINAL EXAMPLE
 97 |     def setup(self, bottom, top):
 98 |         # check input pair
 99 |         if len(bottom) != 2:
100 |             raise Exception("Need two inputs to compute distance.")
101 | 
102 |     def reshape(self, bottom, top):
103 |         # check input dimensions match
104 |         if bottom[0].count != bottom[1].count:
105 |             raise Exception("Inputs must have the same dimension.")
106 |         # difference is shape of inputs
107 |         self.diff = np.zeros_like(bottom[0].data, dtype=np.float32)
108 |         # loss output is scalar
109 |         top[0].reshape(1)
110 | 
111 |     def forward(self, bottom, top):
112 |         self.diff[...] = bottom[0].data - bottom[1].data
113 |         top[0].data[...] = np.sum(self.diff**2) / bottom[0].num / 2.
114 | 
115 |     def backward(self, top, propagate_down, bottom):
116 |         for i in range(2):
117 |             if not propagate_down[i]:
118 |                 continue
119 |             if i == 0:
120 |                 sign = 1
121 |             else:
122 |                 sign = -1
123 |             bottom[i].diff[...] = sign * self.diff / bottom[i].num
124 |             
125 |             
126 |             
127 | if __name__ =="__main__":
128 |     print '---------------------------'
129 |     net=caffe.Net('/home/ly/workspace/Vanilla-40/ZOO/vanilla_train.prototxt', '/home/ly/workspace/Vanilla-40/ZOO/vanilaCNN.caffemodel',caffe.TRAIN)
130 |     prediction = net.forward()['loss'][0]
131 |     print 'lose ', prediction
132 |      
133 | 


--------------------------------------------------------------------------------
/train/adam_solver.prototxt:
--------------------------------------------------------------------------------
 1 | net: "ZOO/train_60_medium.prototxt"
 2 | type: "Adam"
 3 | test_iter: 30
 4 | test_interval: 5000
 5 | base_lr: 0.000001
 6 | momentum: 0.9
 7 | momentum2: 0.999
 8 | lr_policy: "fixed"
 9 | gamma:0.8
10 | stepsize:100000
11 | display: 2500
12 | max_iter: 1500000
13 | snapshot: 5000
14 | snapshot_prefix: "caffeData/snapshots/snap"
15 | solver_mode: GPU
16 | 


--------------------------------------------------------------------------------
/train/landmark_train.prototxt:
--------------------------------------------------------------------------------
  1 | name: "vanila"
  2 | 
  3 | layer {
  4 |    type: "HDF5Data"
  5 |    top: "X" # same name as given in create_dataset!
  6 |    top: "landmarks"
  7 |    hdf5_data_param {
  8 |      source: "/home/ly/workspace/Vanilla-40/caffeData/train.txt" # do not give the h5 files directly, but the list.
  9 |      batch_size: 30
 10 |    }
 11 |    include { phase:TRAIN }
 12 |  }
 13 | 
 14 | layer {
 15 |    type: "HDF5Data"
 16 |    top: "X" # same name as given in create_dataset!
 17 |    top: "landmarks"
 18 |    hdf5_data_param {
 19 |      source: "/home/ly/workspace/Vanilla-40/caffeData/test.txt" # do not give the h5 files directly, but the list.
 20 |      batch_size: 10
 21 |    }
 22 |    include { phase:TEST }
 23 | }
 24 | 
 25 | layer {
 26 |     name: "Conv1"
 27 |     type: "Convolution"
 28 |     bottom: "X"
 29 |     top: "Conv1"
 30 |     param {
 31 |         lr_mult: 1
 32 |         decay_mult: 1
 33 |     }
 34 |     param {
 35 |         lr_mult: 2
 36 |         decay_mult: 0
 37 |     }
 38 |     convolution_param {
 39 |         num_output: 20
 40 |         pad: 2
 41 |         kernel_size: 5
 42 |         stride: 1
 43 |         weight_filler {
 44 |             type: "xavier"
 45 |             std: 0.1
 46 |         }
 47 |         bias_filler {
 48 |             type: "constant"
 49 |             value: 0.2
 50 |         }
 51 |     }
 52 | }
 53 | 
 54 | layer {
 55 |   name: "ActivationTangH1"
 56 |   bottom: "Conv1"
 57 |   top: "ActivationTangH1"
 58 |   type: "TanH"
 59 | }
 60 | 
 61 | layer {
 62 |   name: "ActivationAbs1"
 63 |   bottom: "ActivationTangH1"
 64 |   top: "Abs1"
 65 |   type: "AbsVal"
 66 | }
 67 | 
 68 | layer {
 69 |     name: "Pool1"
 70 |     type: "Pooling"
 71 |     bottom: "Abs1"
 72 |     top: "Pool1"
 73 |     pooling_param {
 74 |         pool: MAX
 75 |         kernel_size: 2
 76 |         stride: 2
 77 |     }
 78 | }
 79 | 
 80 | layer {
 81 |     name: "Conv2"
 82 |     type: "Convolution"
 83 |     bottom: "Pool1"
 84 |     top: "Conv2"
 85 |     param {
 86 |         lr_mult: 1
 87 |         decay_mult: 1
 88 |     }
 89 |     param {
 90 |         lr_mult: 2
 91 |         decay_mult: 0
 92 |     }
 93 |     convolution_param {
 94 |         num_output: 48
 95 |         pad: 2
 96 |         kernel_size: 5
 97 |         stride: 1
 98 |         weight_filler {
 99 |         type: "xavier"
100 |         std: 0.1
101 |     }
102 |     bias_filler {
103 |         type: "constant"
104 |         value: 0.2
105 |         }
106 |     }
107 | }
108 | 
109 | layer {
110 |   name: "ActivationTangH2"
111 |   bottom: "Conv2"
112 |   top: "ActivationTangH2"
113 |   type: "TanH"
114 | }
115 | 
116 | layer {
117 |   name: "ActivationAbs2"
118 |   bottom: "ActivationTangH2"
119 |   top: "Abs2"
120 |   type: "AbsVal"
121 | }
122 | 
123 | 
124 | layer {
125 |     name: "Pool2"
126 |     type: "Pooling"
127 |     bottom: "Abs2"
128 |     top: "Pool2"
129 |     pooling_param {
130 |         pool: MAX
131 |         kernel_size: 2
132 |         stride: 2
133 |     }
134 | }
135 | 
136 | # layer 3
137 | layer {
138 |     name: "Conv3"
139 |     type: "Convolution"
140 |     bottom: "Pool2"
141 |     top: "Conv3"
142 |     param {
143 |         lr_mult: 1
144 |         decay_mult: 1
145 |     }
146 |     param {
147 |         lr_mult: 2
148 |         decay_mult: 0
149 |     }
150 |     convolution_param {
151 |         num_output: 64
152 |         pad: 0
153 |         kernel_size: 3
154 |         stride: 1
155 |         weight_filler {
156 |             type: "xavier"
157 |             std: 0.1
158 |         }
159 |         bias_filler {
160 |             type: "constant"
161 |             value: 0.2
162 |         }
163 |     }
164 | }
165 | 
166 | 
167 | layer {
168 |   name: "ActivationTangH3"
169 |   bottom: "Conv3"
170 |   top: "ActivationTangH3"
171 |   type: "TanH"
172 | }
173 | 
174 | layer {
175 |   name: "ActivationAbs3"
176 |   bottom: "ActivationTangH3"
177 |   top: "Abs3"
178 |   type: "AbsVal"
179 | }
180 | 
181 | layer {
182 |     name: "Pool3"
183 |     type: "Pooling"
184 |     bottom: "Abs3"
185 |     top: "Pool3"
186 |     pooling_param {
187 |         pool: MAX
188 |         kernel_size: 3
189 |         stride: 2
190 |     }
191 | }
192 | 
193 | # layer 4
194 | layer {
195 |     name: "Conv4"
196 |     type: "Convolution"
197 |     bottom: "Pool3"
198 |     top: "Conv4"
199 |     param {
200 |         lr_mult: 1
201 |         decay_mult: 1
202 |     }
203 |     param {
204 |         lr_mult: 2
205 |         decay_mult: 0
206 |     }
207 |     convolution_param {
208 |         num_output: 80
209 |         pad: 0
210 |         kernel_size: 3
211 |         stride: 1
212 |         weight_filler {
213 |             type: "xavier"
214 |             std: 0.1
215 |         }
216 |         bias_filler {
217 |             type: "constant"
218 |             value: 0.2
219 |         }
220 |     }
221 | }
222 | 
223 | 
224 | layer {
225 |   name: "ActivationTangH4"
226 |   bottom: "Conv4"
227 |   top: "ActivationTangH4"
228 |   type: "TanH"
229 | }
230 | 
231 | layer {
232 |   name: "ActivationAbs4"
233 |   bottom: "ActivationTangH4"
234 |   top: "Abs4"
235 |   type: "AbsVal"
236 | }
237 | 
238 | 
239 | ########################################
240 | 
241 | layer {
242 |     name: "Dense1"
243 |     type: "InnerProduct"
244 |     bottom: "Abs4"
245 |     top: "Dense1"
246 |     param {
247 |         lr_mult: 1
248 |         decay_mult: 1
249 |     }
250 |     param {
251 |         lr_mult: 2
252 |         decay_mult: 0
253 |     }
254 |     inner_product_param {
255 |         num_output: 512
256 |         weight_filler {
257 |             type: "xavier"
258 |         }
259 |         bias_filler {
260 |             type: "constant"
261 |             value: 0
262 |         }
263 |     }
264 | }
265 | 
266 | 
267 | layer {
268 |   name: "ActivationTangH5"
269 |   bottom: "Dense1"
270 |   top: "ActivationTangH5"
271 |   type: "TanH"
272 | }
273 | 
274 | layer {
275 |   name: "ActivationAbs5"
276 |   bottom: "ActivationTangH5"
277 |   top: "Abs5"
278 |   type: "AbsVal"
279 | }
280 | 
281 | 
282 | layer {
283 |     name: "Dense3"
284 |     type: "InnerProduct"
285 |     bottom: "Abs5"
286 |     top: "Dense3"
287 |     param {
288 |         lr_mult: 1
289 |         decay_mult: 1
290 |     }
291 |     param {
292 |         lr_mult: 2
293 |         decay_mult: 0
294 |     }
295 |     inner_product_param {
296 |         num_output: 136
297 |         weight_filler {
298 |             type: "xavier"
299 |         }
300 |         bias_filler {
301 |             type: "constant"
302 |             value: 0
303 |         }
304 |     }
305 | }
306 | 
307 | layer {
308 |   type: 'Python'
309 |   name: 'loss'
310 |   top: 'loss'
311 |   bottom: 'Dense3'
312 |   bottom: 'landmarks'
313 |   python_param {
314 |     # the module name -- usually the filename -- that needs to be in $PYTHONPATH
315 |     module: 'python.NormlizedMSE'
316 |     # the layer name -- the class name in the module
317 |     layer: 'NormlizedMSE'
318 |   }
319 |   # set loss weight so Caffe knows this is a loss layer.
320 |   # since PythonLayer inherits directly from Layer, this isn't automatically
321 |   # known to Caffe
322 |   loss_weight: 1
323 | }
324 | 


--------------------------------------------------------------------------------
/train/mainloop.py:
--------------------------------------------------------------------------------
  1 | 
  2 | import numpy as np
  3 | _A = np.array  # A shortcut to creating arrays in command line 
  4 | import os
  5 | import cv2
  6 | import sys
  7 | from pickle import load, dump
  8 | from zipfile import ZipFile
  9 | from urllib import urlretrieve
 10 | 
 11 | #Import helper functions
 12 | from DataRow import DataRow, ErrorAcum, Predictor, getGitRepFolder, createDataRowsFromCSV, getValidWithBBox, writeHD5
 13 | 
 14 | ###########################    PATHS TO SET   ####################
 15 | # Either define CAFFE_ROOT in your enviroment variables or set it here
 16 | CAFFE_ROOT = os.environ.get('CAFFE_ROOT','~/caffe/distribute')
 17 | sys.path.append(CAFFE_ROOT+'/python')  # Add caffe python path so we can import it
 18 | import caffe
 19 | 
 20 | ROOT = getGitRepFolder()  # ROOT is the git root folder .
 21 | sys.path.append(os.path.join(ROOT, 'python'))  # Assume git root directory
 22 | #DATA_PATH = os.path.join(ROOT, 'data')
 23 | CSV_TEST  = os.path.join(ROOT, 'data', 'mtflTestImageList.txt')
 24 | 
 25 | 
 26 | # my debug celeba
 27 | DATA_PATH = ''
 28 | 
 29 | 
 30 | MEAN_TRAIN_SET = cv2.imread(os.path.join(ROOT, 'trainMean.png')).astype('f4')
 31 | MEAN_TRAIN_SET = cv2.resize(MEAN_TRAIN_SET,(60,60),interpolation=cv2.INTER_CUBIC)
 32 | STD_TRAIN_SET  = cv2.imread(os.path.join(ROOT, 'trainSTD.png')).astype('f4')
 33 | STD_TRAIN_SET = cv2.resize(STD_TRAIN_SET,(60,60),interpolation=cv2.INTER_CUBIC)
 34 | 
 35 | AFW_DATA_PATH = os.path.join(ROOT, 'data', 'afwTestImages')
 36 | AFW_MAT_PATH = os.path.join(ROOT, 'data', 'anno-v7.mat')
 37 | 
 38 | 
 39 | PATH_TO_WEIGHTS  = os.path.join('/home/ly/workspace/Vanilla-40', 'caffeData', 'snapshots', 'snap_iter_5910000.caffemodel')
 40 | PATH_TO_DEPLOY_TXT = os.path.join('/home/ly/workspace/Vanilla-40', 'ZOO', 'vanilla_deploy_large.prototxt')
 41 | 
 42 | ###########################    STEPS TO RUN       ####################
 43 | AFW_STEPS =['downloadAFW', 
 44 |             'createAFW_TestSet',
 45 |             'testAFW_TestSet'] # Steps needed for AFW
 46 | 
 47 | AFLW_STEPS=['downloadAFLW', 
 48 |             'testSetHD5', 
 49 |             'testSetPickle', 
 50 |             'trainSetHD5', 
 51 |             'calcTrainSetMean', # depends on trainSetHD5
 52 |             'createAFLW_TestSet', 
 53 |             'testAFLW_TestSet', 
 54 |             'testErrorMini'] # Steps needed for AFLW
 55 | 
 56 | #STEPS =['trainSetHD5', 'calcTrainSetMean'] # AFLW_STEPS+AFW_STEPS # Run AFLW and AFW steps
 57 | 
 58 | #STEPS = ['createAFW_TestSet']
 59 | #STEPS = ['testAFW_TestSet']
 60 | 
 61 | #STEPS = ['createAFLW_TestSet']
 62 | #STEPS = ['createAFLW_TestSet']
 63 | STEPS = ['testAFLW_TestSet']
 64 | 
 65 | ##########################################    SCRIPT STEPS       ##################################################
 66 | 
 67 | if 'downloadAFW' in STEPS:
 68 |     theurl = 'https://www.ics.uci.edu/~xzhu/face/AFW.zip'
 69 |     filename = ROOT+'/AFW.zip'
 70 |     if os.path.isfile(filename):
 71 |         print "AFW.zip already downloaded"
 72 |     else:
 73 |         print "Downloading "+theurl + " ....."
 74 |         name, hdrs = urlretrieve(theurl, filename)
 75 |         print "Finished downloading AFW....."
 76 |         
 77 |     print "Extracting AFW zip file......"
 78 |     folderPATH = ROOT+'/data'
 79 |     with ZipFile(filename) as theOpenedFile:
 80 |         theOpenedFile.extractall(folderPATH)
 81 |         theOpenedFile.close()
 82 | 
 83 | if 'downloadAFLW' in STEPS:
 84 |     theurl='http://mmlab.ie.cuhk.edu.hk/archive/CNN/data/train.zip'
 85 |     filename = ROOT+'/AFLW.zip'
 86 |     if os.path.isfile(filename):
 87 |         print "AFLW.zip training data already downloaded"
 88 |     else:
 89 |         print "Downloading "+theurl + " ....."
 90 |         name, hdrs = urlretrieve(theurl, filename)
 91 |         print "Finished downloading AFLW....."
 92 | 
 93 |     print 'Extracting zip data...'
 94 |     folderPATH = ROOT+'/data'
 95 |     with ZipFile(filename) as theOpenedFile:
 96 |         theOpenedFile.extractall(folderPATH)
 97 |         theOpenedFile.close()
 98 |     print "Done extracting AFW zip folder"
 99 | 
100 | if 'testSetHD5' in STEPS or 'testSetPickle' in STEPS:
101 |     print "Creating test set....."
102 |     dataRowsTest_CSV  = createDataRowsFromCSV(CSV_TEST , DataRow.DataRowFromNameBoxInterlaved, DATA_PATH)
103 |     print "Finished reading %d rows from test" % len(dataRowsTest_CSV)
104 |     dataRowsTestValid,R = getValidWithBBox(dataRowsTest_CSV)
105 |     print "Original test:",len(dataRowsTest_CSV), "Valid Rows:", len(dataRowsTestValid), " noFacesAtAll", R.noFacesAtAll, " outside:", R.outsideLandmarks, " couldNotMatch:", R.couldNotMatch
106 |     if 'testSetPickle' in STEPS:
107 |         with open('testSet.pickle','w') as f:
108 |             dump(dataRowsTestValid,f)
109 |         print "Finished dumping to testSet.pickle"
110 |         # Also save mini test set for debug
111 |         with open('testSetMini.pickle','w') as f:
112 |             dump(dataRowsTestValid[0:10],f)
113 |         print "Finished dumping to testSetMini.pickle"
114 |         
115 |         
116 |     if 'testSetHD5' in STEPS:
117 |         writeHD5(dataRowsTestValid, ROOT+'/caffeData/hd5/test.hd5', ROOT+'/caffeData/test.txt', MEAN_TRAIN_SET, STD_TRAIN_SET)
118 |         print "Finished writing test to caffeData/test.txt"
119 | 
120 |  
121 | DEBUG = True
122 | if 'testErrorMini' in STEPS:
123 |     with open('testSetMini.pickle','r') as f:
124 |         dataRowsTrainValid = load(f)
125 |         
126 |     testErrorMini=ErrorAcum()
127 |     predictor = Predictor(protoTXTPath=PATH_TO_DEPLOY_TXT, weightsPath=PATH_TO_WEIGHTS)
128 |     for i, dataRow in enumerate(dataRowsTrainValid):
129 |         dataRow40 = dataRow.copyCroppedByBBox(dataRow.fbbox).copyMirrored()
130 |         image, lm_0_5 = predictor.preprocess(dataRow40.image, dataRow40.landmarks())
131 |         prediction = predictor.predict(image)
132 |         testErrorMini.add(lm_0_5, prediction)
133 |         dataRow40.prediction = (prediction+0.5)*40.  # Scale -0.5..+0.5 to 0..40
134 |         
135 |         if DEBUG:
136 |             dataRow.prediction = dataRow40.inverseScaleAndOffset(dataRow40.prediction) # Scale up to the original image scale
137 |             dataRow.show()
138 |             if i>40:
139 |                 print "Debug breaked after %d rows:" % i
140 |             
141 | 
142 |     print "Test Error mini:", testErrorMini
143 | 
144 |   
145 |   
146 | if 'trainSetHD5' in STEPS:
147 |     dataRowsTrain_CSV = createDataRowsFromCSV(CSV_TRAIN, DataRow.DataRowFromNameBoxInterlaved, DATA_PATH)
148 |     print "Finished reading %d rows from training data. Parsing BBox...." % len(dataRowsTrain_CSV)
149 |     dataRowsTrainValid,R = getValidWithBBox(dataRowsTrain_CSV)
150 |     print "Original train:",len(dataRowsTrain_CSV), "Valid Rows:", len(dataRowsTrainValid), " noFacesAtAll", R.noFacesAtAll, " outside:", R.outsideLandmarks, " couldNotMatch:", R.couldNotMatch
151 |     dataRowsTrain_CSV=[]  # remove from memory
152 |     
153 |     writeHD5(dataRowsTrainValid, ROOT+'/caffeData/hd5/train.hd5', ROOT+'/caffeData/train.txt', MEAN_TRAIN_SET, STD_TRAIN_SET ,mirror=True)
154 |     print "Finished writing train to caffeData/train.txt"
155 |     
156 | 
157 | #%% Calculate train mean image - Assume data was read to dataRowsTrainValid
158 | if 'calcTrainSetMean' in STEPS:
159 |     print ('Calculating train data mean value')
160 |     meanTrainSet = np.zeros([40,40,3], dtype='double')
161 |     for dataRow in dataRowsTrainValid:
162 |         meanTrainSet += dataRow.copyCroppedByBBox(dataRow.fbbox).image.astype('double')
163 |     
164 |     MEAN_TRAIN_SET = meanTrainSet / len(dataRowsTrainValid)
165 |     cv2.imwrite(os.path.join(ROOT, 'trainMean.png'), (MEAN_TRAIN_SET).astype('uint8'))
166 |     print ('Finished Calculating train data mean value to file trainMean.png', MEAN_TRAIN_SET.mean())
167 | 
168 |     print ('Calculating train data std value')
169 | 
170 |     stdTrainSet = np.zeros([40,40,3], dtype='double')
171 |     for dataRow in dataRowsTrainValid:
172 |         diff = dataRow.copyCroppedByBBox(dataRow.fbbox).image.astype('double') - MEAN_TRAIN_SET
173 |         stdTrainSet += diff*diff
174 |         
175 |     stdTrainSet /= len(dataRowsTrainValid)
176 |     STD_TRAIN_SET = stdTrainSet**0.5
177 |     cv2.imwrite(os.path.join(ROOT, 'trainSTD.png'), (STD_TRAIN_SET).astype('uint8'))
178 |     print 'Finished Calculating train data std value to file trainSTD.png with mean', STD_TRAIN_SET.mean()
179 | else:
180 |     MEAN_TRAIN_SET = cv2.imread(os.path.join(ROOT, 'trainMean.png')).astype('f4')
181 |     STD_TRAIN_SET  = cv2.imread(os.path.join(ROOT, 'trainSTD.png')).astype('f4')
182 | 
183 | 
184 | # Run the same caffe test set using python
185 | DEBUG = False  # Set this to true if you wish to plot the images
186 | if 'testError' in STEPS:
187 |     with open('testSet.pickle','r') as f:
188 |         dataRowsTrainValid = load(f)
189 |         
190 |     testError=ErrorAcum()
191 |     predictor = Predictor(protoTXTPath=PATH_TO_DEPLOY_TXT, weightsPath=PATH_TO_WEIGHTS)
192 |     for i, dataRow in enumerate(dataRowsTrainValid):
193 |         dataRow40 = dataRow.copyCroppedByBBox(dataRow.fbbox)
194 |         image, lm40 = predictor.preprocess(dataRow40.image, dataRow40.landmarks())
195 |         prediction = predictor.predict(image)
196 |         testError.add(lm40, prediction)
197 |         dataRow40.prediction = (prediction+0.5)*40.
198 |         
199 |         if DEBUG and i%40 ==0:
200 |             dataRow.prediction = dataRow40.inverseScaleAndOffset(dataRow40.prediction)
201 |             dataRow.show()
202 |             break
203 |         
204 |             
205 |     print "Test Error:", testError
206 | 
207 |   
208 |     
209 | # AFW test - Make the pickle data set
210 | if 'createAFW_TestSet' in STEPS:
211 |     print "Parsing AFW anno-v7.mat ....."
212 |     from scipy.io import loadmat
213 |     annotaions = loadmat(AFW_MAT_PATH)['anno']
214 |     dataRowsAFW = []
215 |         
216 |     for anno in annotaions:
217 |         dataRow = DataRow.DataRowFromAFW(anno, AFW_DATA_PATH)
218 |         if dataRow is not None:
219 |             dataRowsAFW.append(dataRow)
220 |     print "Finished parsing anno-v7.mat with total rows:", len(dataRowsAFW)
221 |     annotaions = None  # remove from memory
222 |     
223 |     dataRowsAFW_Valid, R=getValidWithBBox(dataRowsAFW)
224 |     print "Original AFW:",len(dataRowsAFW), "Valid Rows:", len(dataRowsAFW_Valid), " No faces at all", R.noFacesAtAll, " illegal landmarks:", R.outsideLandmarks, " Could not match:", R.couldNotMatch
225 |     dataRowsAFW = None  # remove from Memory
226 |     with open('afwTestSet.pickle','w') as f:
227 |         dump(dataRowsAFW_Valid, f)
228 |         print "Data saved to afwTestSet.pickle"
229 |     
230 |     
231 | DEBUG = True
232 | if 'testAFW_TestSet' in STEPS:
233 |     with open('afwTestSet.pickle','r') as f:
234 |         dataRowsAFW_Valid = load(f)
235 | 
236 |     testErrorAFW=ErrorAcum()
237 |     predictor = Predictor(protoTXTPath=PATH_TO_DEPLOY_TXT, weightsPath=PATH_TO_WEIGHTS)
238 |     for i, dataRow in enumerate(dataRowsAFW_Valid):
239 |         dataRow40 = dataRow.copyCroppedByBBox(dataRow.fbbox)
240 |         # my debug
241 |         #CROP_DATA_PATH = os.path.join(ROOT, 'data', 'afwTestCropImages', str(i) + '.jpg')
242 |         #cv2.imwrite(CROP_DATA_PATH, dataRow40.image)
243 | 
244 |         image, lm_0_5 = predictor.preprocess(dataRow40.image, dataRow40.landmarks())
245 |         prediction = predictor.predict(image)
246 |         testErrorAFW.add(lm_0_5, prediction)
247 |         dataRow40.prediction = (prediction+0.5)*40.  # Scale -0.5..+0.5 to 0..40
248 |         
249 |         if DEBUG:
250 |             dataRow.prediction = dataRow40.inverseScaleAndOffset(dataRow40.prediction) # Scale up to the original image scale
251 |             # my debug
252 |             #ALIGNED_DATA_PATH = os.path.join(ROOT, 'data', 'mineAFWAlignedTestImages', str(i) + '.jpg')
253 |             ALIGNED_DATA_PATH = os.path.join('/home/ly/workspace/Vanilla-40', 'data', 'mineAFWAlignedTestImages', str(i) + '.jpg')
254 |             aligned_im = dataRow.show()
255 |             print ALIGNED_DATA_PATH
256 |             print aligned_im.shape
257 |             cv2.imwrite(ALIGNED_DATA_PATH, aligned_im)
258 |             
259 | 
260 |     print "Test Error AFW:", testErrorAFW
261 | 
262 |     
263 | 
264 | # Create the MTFL benchmark
265 | if 'createAFLW_TestSet' in STEPS:  
266 |     #MTFL_LINK = 'http://mmlab.ie.cuhk.edu.hk/projects/TCDCN/data/MTFL.zip'
267 |     #MTFL_ZIP = ROOT+"/MTFL.zip"
268 |     #if os.path.isfile(MTFL_ZIP):
269 |     #    print "MTFL.zip already downloaded"
270 |     #else:
271 |     #    print "Downloading:"+MTFL_ZIP+" from url:"+MTFL_LINK+"....."
272 |     #    urlretrieve(MTFL_LINK, MTFL_ZIP)
273 |     #    print "Finished download. Extracting file....."
274 |     #    with ZipFile(MTFL_ZIP) as f:
275 |     #        f.extractall(ROOT+'/data')
276 |     #        print "Done extracting MTFL"
277 |     #        f.close()
278 |             
279 | 
280 |     #dataRowsMTFL_CSV  = createDataRowsFromCSV(CSV_MTFL , DataRow.DataRowFromMTFL, AFLW_PATH)
281 | 
282 |     # my debug
283 |     #dataRowsMTFL_CSV  = createDataRowsFromCSV(CSV_TRAIN , DataRow.DataRowFromNameBoxInterlaved, DATA_PATH)
284 |     dataRowsMTFL_CSV  = createDataRowsFromCSV(CSV_MTFL , DataRow.DataRowFromNameBoxInterlaved, DATA_PATH)
285 |     
286 |     print "Finished reading %d rows from test" % len(dataRowsMTFL_CSV)
287 |     dataRowsMTFLValid,R = getValidWithBBox(dataRowsMTFL_CSV)
288 |     print "Original test:",len(dataRowsMTFL_CSV), "Valid Rows:", len(dataRowsMTFLValid), " No faces at all", R.noFacesAtAll, " Illegal landmarks:", R.outsideLandmarks, " Could not match:", R.couldNotMatch
289 |     with open('testSetMTFL.pickle','w') as f:
290 |         dump(dataRowsMTFLValid,f)
291 |     print "Finished dumping to testSetMTFL.pickle"        
292 | 
293 | 
294 | # Run AFLW benchmark
295 | DEBUG = True
296 | if 'testAFLW_TestSet' in STEPS:
297 |     print "Running AFLW benchmark........."
298 |     with open('testSetMTFL-det1.pickle','r') as f:
299 |         dataRowsAFW_Valid = load(f)
300 |     print "%d rows in AFLW benchmark ....." % len(dataRowsAFW_Valid)
301 |     testErrorAFLW=ErrorAcum()
302 |     predictor = Predictor(protoTXTPath=PATH_TO_DEPLOY_TXT, weightsPath=PATH_TO_WEIGHTS)
303 |     for i, dataRow in enumerate(dataRowsAFW_Valid):
304 |         dataRow40 = dataRow.copyCroppedByBBox(dataRow.fbbox)
305 |         image, lm_0_5 = predictor.preprocess(dataRow40.image, dataRow40.landmarks())
306 |         prediction = predictor.predict(image)
307 |         testErrorAFLW.add(lm_0_5, prediction)
308 |         dataRow40.prediction = (prediction+0.5)*60.  # Scale -0.5..+0.5 to 0..40
309 |         
310 | #        if DEBUG and i%40 == 0:
311 |         if DEBUG:
312 |             dataRow.prediction = dataRow40.inverseScaleAndOffset(dataRow40.prediction) # Scale up to the original image scale
313 |             # my debug
314 |             #ALIGNED_DATA_PATH = os.path.join(ROOT, 'data', 'mineAFLWAlignedTestImages', str(i) + '.jpg')
315 |             ALIGNED_DATA_PATH = os.path.join('/home/ly/workspace/Vanilla-40', 'data', 'mineAFLWAlignedTestImages', str(i) + '.jpg')
316 |             aligned_im = dataRow.show()
317 |             print ALIGNED_DATA_PATH
318 |             print aligned_im.shape
319 |             cv2.imwrite(ALIGNED_DATA_PATH, aligned_im)
320 | 
321 | 
322 |     print "Test Error AFLW:", testErrorAFLW
323 | 
324 | 
325 | if 'findSnapshotWithMinError' in STEPS:
326 |     pass
327 | 


--------------------------------------------------------------------------------