├── ESR ├── ESRTesting.m ├── ESRTraining.m ├── Initialization.m ├── ApplyStageRegressor.m ├── LearnStageRegressor.m ├── CorrelationBasedFeature.m ├── GenerateShapeIndexedFeatures.m ├── init_rectangle.m ├── generate.m └── test_train.m ├── LBF ├── train_rfs.m ├── Initialization.m ├── GenerateShapeIndexedFeatures.m ├── test_rfs.m └── LBFTraining.m ├── LBF_cpp ├── main.cpp ├── utils.h └── train_rfs.cpp ├── result ├── esr_1000.png ├── lbf_1000.png ├── sample1.png ├── sample2.png ├── sample3.png ├── sample4.png ├── sample5.png ├── esr_landmark.png └── lbf_landmark.png └── README.md /ESR/ESRTesting.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/ESRTesting.m -------------------------------------------------------------------------------- /LBF/train_rfs.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/LBF/train_rfs.m -------------------------------------------------------------------------------- /LBF_cpp/main.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/LBF_cpp/main.cpp -------------------------------------------------------------------------------- /ESR/ESRTraining.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/ESRTraining.m -------------------------------------------------------------------------------- /result/esr_1000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/esr_1000.png -------------------------------------------------------------------------------- /result/lbf_1000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/lbf_1000.png -------------------------------------------------------------------------------- /result/sample1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/sample1.png -------------------------------------------------------------------------------- /result/sample2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/sample2.png -------------------------------------------------------------------------------- /result/sample3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/sample3.png -------------------------------------------------------------------------------- /result/sample4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/sample4.png -------------------------------------------------------------------------------- /result/sample5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/sample5.png -------------------------------------------------------------------------------- /ESR/Initialization.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/Initialization.m -------------------------------------------------------------------------------- /LBF/Initialization.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/LBF/Initialization.m -------------------------------------------------------------------------------- /result/esr_landmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/esr_landmark.png -------------------------------------------------------------------------------- /result/lbf_landmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/result/lbf_landmark.png -------------------------------------------------------------------------------- /ESR/ApplyStageRegressor.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/ApplyStageRegressor.m -------------------------------------------------------------------------------- /ESR/LearnStageRegressor.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/LearnStageRegressor.m -------------------------------------------------------------------------------- /ESR/CorrelationBasedFeature.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/CorrelationBasedFeature.m -------------------------------------------------------------------------------- /ESR/GenerateShapeIndexedFeatures.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/ESR/GenerateShapeIndexedFeatures.m -------------------------------------------------------------------------------- /LBF/GenerateShapeIndexedFeatures.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GentleZhu/Face_Alignment/HEAD/LBF/GenerateShapeIndexedFeatures.m -------------------------------------------------------------------------------- /ESR/init_rectangle.m: -------------------------------------------------------------------------------- 1 | face_init=face_init*2.5; 2 | w=face_init(:,3); 3 | h=1.1*face_init(:,4); 4 | center=[face_init(:,1),face_init(:,2)]; 5 | face_init=[center,w,h]; -------------------------------------------------------------------------------- /LBF/test_rfs.m: -------------------------------------------------------------------------------- 1 | function [ y_b ] = test_rfs( node ,X,num_trees,max_depth ) 2 | %test_rfs Get feature vector 3 | N=size(X,1); 4 | %index=1; 5 | y_b=zeros(N,2,'single'); 6 | y_temp=zeros(num_trees,2,'single'); 7 | for i=1:N 8 | for k=1:num_trees 9 | index=1; 10 | for d=1:max_depth-1 11 | num_feat=node{k,index}.split_point; 12 | if X(i,num_feat)>node{k,index}.threshold 13 | index=2*index+1; 14 | else 15 | index=2*index; 16 | end 17 | end 18 | y_temp(k,:)=node{k,index}; 19 | end 20 | y_b(i,:)=mean(y_temp,1); 21 | end 22 | 23 | -------------------------------------------------------------------------------- /ESR/generate.m: -------------------------------------------------------------------------------- 1 | %load 'LFW.mat' 2 | all=[t_label;add_label]; 3 | [N,d,~]=size(all); 4 | meanshape=double(reshape(all(randi([1 N]),:,:),[74 2])); 5 | normal_shape=zeros(N,74,2); 6 | for k=1:5 7 | %diff=0; 8 | for i=1:N 9 | X=double(reshape(all(i,:,:),[74,2])); 10 | [~,normal_shape(i,:,:)] = procrustes(meanshape,X); 11 | %norm(meanshape-reshape(normal_shape(i,:,:),[74 2]),'fro') 12 | %d 13 | %w(i,:)=struct2cell(tmp)'; 14 | %diff=diff+d; 15 | end 16 | %break; 17 | meanshape_2=reshape(mean(normal_shape,1),[74 2]); 18 | %meanshape_2=meanshape_2-repmat(mean(meanshape_2,2),1,size(meanshape_2,2)); 19 | %meanshape_2=meanshape_2/norm(meanshape_2,'fro'); 20 | diff=norm(meanshape-meanshape_2,'fro') 21 | %diff 22 | %if diff<0.00001 23 | % break; 24 | %end 25 | meanshape=meanshape_2; 26 | end 27 | %test_procrustes 28 | %for alpha=1:P -------------------------------------------------------------------------------- /ESR/test_train.m: -------------------------------------------------------------------------------- 1 | %imgs = cell(10,1) 2 | 3 | for n=1:84 4 | I=reshape(test2_feat(n,:,:),[250,250]); 5 | % I=reshape(add_feat(n,:,:),[250,250]); 6 | for i=1:2 7 | %subplot(2,5,i); 8 | 9 | %markerInserter1 = vision.MarkerInserter('Shape','Plus','BorderColor','black'); 10 | %markerInserter = vision.MarkerInserter('Shape','Plus','BorderColor','white'); 11 | %Pts = reshape(int16(t{i,1}{n,1}),[74,2]); 12 | Pts = reshape(int16(t(n,i*5,:)),[74,2]); 13 | %Pts=uint8(T); 14 | %J = step(markerInserter, I, Pts); 15 | %imgs(i)=J; 16 | imshow(I); 17 | hold on; 18 | plot(Pts(:,1),Pts(:,2), 'g.'); 19 | input(''); 20 | %subimage(J); 21 | %h = imshow(J, 'InitialMag',1000, 'Border','loose'); 22 | %imshow(J); 23 | end 24 | % markerInserter = vision.MarkerInserter('Shape','Plus','BorderColor','white'); 25 | %Pts = reshape(label(n,:,:),[74,2]); 26 | %J = step(markerInserter, I, Pts); 27 | %imshow(J); 28 | %input(''); 29 | end 30 | -------------------------------------------------------------------------------- /LBF/LBFTraining.m: -------------------------------------------------------------------------------- 1 | N=1000; 2 | Nfp=74; 3 | I=t_feat(1:N,:,:); 4 | S0=t_label(1:N,1:Nfp,:); 5 | meanshape_x=meanshape(1:Nfp,:); 6 | N_aug=1; 7 | 8 | init_scope=face_init(1:N,:,:); 9 | init_set=t_label(1:N,:,:); 10 | P=100; 11 | %T=5; 12 | T=1; 13 | S_t=cell(T,1); 14 | 15 | S_t{1}=Initialization(N,N_aug,Nfp,init_scope,init_set); 16 | trans_matrix=cell(N*N_aug,3); 17 | Y=zeros(N*N_aug,Nfp,2,'single'); 18 | y_b=zeros(N*N_aug,Nfp,2,'single'); 19 | S0=single(S0); 20 | %F=5; 21 | K=500; 22 | out=zeros(T+1,1); 23 | radius=[20 15 10 5 1]; 24 | %encoder=[1 2 4 8 16]; 25 | %fern=cell(T,K); 26 | for t=1:T 27 | tic; 28 | for n=1:N 29 | idx=n; 30 | %for k=1:N_aug 31 | %idx=(n-1)*N_aug+k; 32 | [~,Ytmp,trans] = procrustes(meanshape_x,reshape(S_t{t}{n,1},[Nfp,2])); 33 | %size(S_t) 34 | %[~,Ytmp,trans] = procrustes(meanshape,reshape(S_t{n,k},[74,2])); 35 | trans_matrix(idx,:)=struct2cell(trans)'; 36 | Y(idx,:,:)=trans_matrix{idx,2}*reshape(S0(n,:,:),[Nfp,2])*trans_matrix{idx,1}+trans_matrix{idx,3}-Ytmp;%,[1,148]); 37 | %end 38 | end 39 | out(t)=norm(reshape(Y(:,1,:),[N*N_aug,2]),'fro'); 40 | toc 41 | M_t=trans_matrix; 42 | ShapeIndexedFeatures=GenerateShapeIndexedFeatures(I,S_t{t},M_t,1,Nfp,P,radius(t)); 43 | %delta_s=zeros(N,Nfp*2); 44 | tic; 45 | for l=1:1 46 | fprintf('landmark %d\n',l); 47 | yy=reshape(Y(:,l,:),[N 2]); 48 | %out(1,1)=norm(yy,'fro'); 49 | xx=reshape(ShapeIndexedFeatures(l,:,:),[N,P*P]); 50 | nodes= train_rfs( xx,yy,10,5,500); 51 | y_b(:,l,:)= reshape(test_rfs( nodes ,xx,10,5 ),[N,1,2]); 52 | %out(2,1)=norm(yy-y_b,'fro'); 53 | end 54 | toc 55 | out(t+1)=norm(yy-reshape(y_b(:,1,:),[N,2]),'fro'); 56 | %Y=Y-y_b; 57 | fprintf('Update!\n'); 58 | for n=1:N 59 | xxx=reshape(y_b(n,:,:),[Nfp,2])/(trans_matrix{n,1}*trans_matrix{n,2}); 60 | S_t{t+1}{n,1}=S_t{t}{n,1}+reshape(xxx,[1,2*Nfp]); 61 | % Y(n,:)=Y(n,:)-regressor(bin_num(n),:); 62 | % delta_s(n,:)=delta_s(n,:)+regressor(bin_num(n),:); 63 | end 64 | end -------------------------------------------------------------------------------- /LBF_cpp/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_H 2 | #define UTILS_H 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | //#include "Eigen" 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "liblinear-2.1\linear.h" 19 | const int Nfp = 74; 20 | const int Nimg =10858; 21 | const int Nadd = 7258; 22 | const int Ntest = 9888; 23 | //const int Nimg = 100; 24 | const int N_aug = 10; 25 | const int observation_bin = 3; 26 | const int stage = 1; 27 | const double radius[5] = { 0.4, 0.25, 0.2, 0.16, 0.1 }; 28 | typedef Eigen::Matrix landmark; 29 | typedef Eigen::Matrix transformer; 30 | typedef Eigen::Matrix regression_err; 31 | //typedef double *regression_errx; 32 | //typedef double *regression_erry; 33 | typedef Eigen::Matrix delta_y; 34 | typedef struct trans_matrix{ 35 | double sint; 36 | double cost; 37 | Eigen::Vector2d translation; 38 | double scale; 39 | //landmark normal; 40 | }; 41 | typedef struct refershape{ 42 | landmark shape; 43 | Eigen::Vector2d translation; 44 | double scale; 45 | }; 46 | typedef struct treenode{ 47 | int split_point; 48 | std::vector left_child; 49 | std::vector right_child; 50 | double threshold; 51 | Eigen::Vector2d output; 52 | } rfs; 53 | typedef struct boundingbox{ 54 | int centerx; 55 | int centery; 56 | int width; 57 | int height; 58 | }; 59 | typedef std::vector randomtree; 60 | typedef std::vector randomforest; 61 | 62 | rfs split_node(const Eigen::MatrixXd& feat, const regression_err& Y, const rfs* parent, const std::vector& selected_feature, int min_num); 63 | randomforest train_rfs(const Eigen::MatrixXd& feat, const regression_err&Y, const int num_trees, const int max_depth, const int num_feat, const int num_selected_sample, const int num_selected_feature); 64 | void test_rfs(const int Nsample,const randomforest & r, const Eigen::MatrixXd& feat, struct feature_node **global_binary_features, const int num_landmark, const int num_trees, const int max_depth, const int offset=0); 65 | #endif // !UTILS_H 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face_Alignment 2 | This repository is part of my recent project, including implementation of two face alighment paper: 3 | 4 | 1.Cao X, Wei Y, Wen F, et al. Face alignment by explicit shape regression[J]. International Journal of Computer Vision, 2014, 107(2): 177-190. 5 | 6 | 2.Ren S, Cao X, Wei Y, et al. Face alignment at 3000 fps via regressing local binary features[C]//Computer Vision and Pattern Recognition (CVPR), 2014 IEEE Conference on. IEEE, 2014: 1685-1692. 7 | 8 | ## Introduction 9 | ### Explicit Shape Regression(ESR) 10 | * Cascade regressor(boosting) 11 | * Random Fern 12 | 13 | ### Local Binary Features(LBF) 14 | * Cascade feature extractor 15 | * Random forest(Bootstrap) 16 | * Global regression 17 | 18 | ## Implementation 19 | 1. ESR is easy to be vectorized. Quick matlab implementation in the above folder. The training/testing data should be cropped into same size 20 | 2. LBF is implemented in C++ with a prototype in matlab. Random Forest training and testng are parealleled by Openmp. Overall landmark detection speed is 300fps(740 trees in each stage, tree depth is 5). 21 | 22 | ## Result 23 | ### Face alignment on images 24 | 25 | 26 | 27 | ### Real time face alignment on video 28 | 29 | 30 | 31 | ## Evaluation 32 | The average point-to-point Euclidean error normalized by the inter-ocular distance (measured as the Euclidean distance between the outer corners of the eyes) will be used as the error measure. Each method is trained on a subset of LFW dataset(one thousand images with 74 landmarks) The face detection in ESR is pre-computed using Faceplusplus public API, so ESR seems to have a better performance. Opencv's Haarcascade face detector used in LBF has a high false-positive detection rate. 33 | ### Explicit Shape Regression(ESR) 34 | 35 | 36 | ### Local Binary Features(LBF) 37 | 38 | 39 | 40 | Both regression methods yield good accuracy on most images, however, they suffer large error on face contour(See #landmark1-15 in the right figures) 41 | 42 | -------------------------------------------------------------------------------- /LBF_cpp/train_rfs.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | 3 | using namespace std; 4 | using namespace cv; 5 | int zero_count = 0; 6 | 7 | //std::random_device rd; 8 | inline double calculateVariance(const Eigen::MatrixXd& feat, const regression_err& Y, const vector& sample, double threshold, int num_feat, int ind){ 9 | deque temp; 10 | int cnt = 0; 11 | Eigen::Vector2d mean_l(0, 0), mean_r(0,0); 12 | double var1 = 0, var2 = 0, tt = 0; 13 | //cout << "num_feat"<0){ 42 | mean_r /= temp.size() - cnt; 43 | for (int i = ind; i < temp.size(); ++i){ 44 | var2 += Y.row(temp[i]).squaredNorm(); 45 | } 46 | var2 -= mean_r.squaredNorm()*(temp.size() - cnt); 47 | return var1 + var2; 48 | //} 49 | //else 50 | // return var1; 51 | } 52 | void permulation(int num_feat, int k, vector&selected_feature){ 53 | int index; 54 | //short * bin = new short[num_feat]; 55 | selected_feature.clear(); 56 | //cout < num_feat - k; --i) { 68 | index = rand() % i; 69 | //t = a[index]; 70 | a[index] = a[i]; 71 | selected_feature.push_back(index); 72 | //a[i] = t; 73 | } 74 | delete[]a; 75 | } 76 | inline void boostrap_sample(int num_feat,vector&selected_feature){ 77 | selected_feature.clear(); 78 | for (int i = 0; i < int(0.8*num_feat); ++i) 79 | selected_feature.push_back(rand() % num_feat); 80 | } 81 | rfs split_node(const Eigen::MatrixXd& feat, const regression_err& Y, const vector&sample, const vector& selected_feature, int height){ 82 | rfs new_node; 83 | int num_sample, ind; 84 | double candidate[2], can_th[2]; 85 | double varLR; 86 | num_sample = sample.size(); 87 | int max_i = -1; 88 | double min_loss = 0; 89 | double max_threshold = 0, threshold; 90 | //cout << "split_node" << endl; 91 | if (num_sample>observation_bin&&height>0){ 92 | for (vector::const_iterator fiter = selected_feature.begin(); fiter != selected_feature.end(); fiter++){ 93 | ind = rand() % num_sample; 94 | threshold = feat(sample[ind], *fiter); 95 | varLR = calculateVariance(feat, Y, sample, threshold, *fiter, ind); 96 | if (varLR < min_loss||min_loss==0){ 97 | min_loss = varLR; 98 | max_threshold = threshold; 99 | max_i = *fiter; 100 | //cout << "max_i" << max_i; 101 | } 102 | } 103 | for (vector::const_iterator siter = sample.begin(); siter != sample.end(); siter++){ 104 | if (feat(*siter, max_i) < max_threshold) 105 | new_node.left_child.push_back(*siter); 106 | else 107 | new_node.right_child.push_back(*siter); 108 | } 109 | new_node.threshold = max_threshold; 110 | new_node.split_point = max_i; 111 | 112 | } 113 | else if (height>0){ 114 | new_node.split_point = -1; 115 | Eigen::Vector2d output(0, 0); 116 | for (int i = 0; i < num_sample; ++i) 117 | output += Y.row(sample[i]); 118 | new_node.output = output / num_sample; 119 | } 120 | else if (num_sample){ 121 | Eigen::Vector2d output(0, 0); 122 | for (int i = 0; i < num_sample; ++i) 123 | output += Y.row(sample[i]); 124 | new_node.output = output /num_sample; 125 | new_node.split_point = num_sample; 126 | } 127 | return new_node; 128 | } 129 | randomforest train_rfs(const Eigen::MatrixXd& feat, const regression_err&Y, const int num_trees, const int max_depth, const int num_feat, const int num_selected_sample, const int num_selected_feature) 130 | { 131 | 132 | randomforest r; 133 | randomtree rf; 134 | rfs tmp; 135 | int parent; 136 | vector selected_feature; 137 | vector selected_sample; 138 | for (int i = 0; i < num_trees; ++i){ 139 | int depth = 0; 140 | boostrap_sample(Nimg*N_aug, selected_sample); 141 | permulation(num_feat, num_selected_feature, selected_feature); 142 | tmp=split_node(feat, Y, selected_sample, selected_feature, max_depth - depth); 143 | rf.push_back(tmp); 144 | for (int j = 1; j < pow(2, max_depth + 1) - 1; ++j){ 145 | if (j >= pow(2, depth + 1) - 1) 146 | ++depth; 147 | 148 | permulation(num_feat, num_selected_feature, selected_feature); 149 | parent = floor((double)(j + 1) / 2) - 1; 150 | if (!rf[parent].split_point){ 151 | rfs node; 152 | node.split_point = -1; 153 | rf.push_back(node); 154 | continue; 155 | } 156 | //cout << "parent" << parent << endl; 157 | if (j == parent * 2 + 1) 158 | rf.push_back(split_node(feat, Y, rf[parent].left_child, selected_feature, max_depth - depth)); 159 | else 160 | rf.push_back(split_node(feat, Y, rf[parent].right_child, selected_feature, max_depth - depth)); 161 | } 162 | r.push_back(rf); 163 | rf.clear(); 164 | } 165 | return r; 166 | } 167 | void test_rfs(const int Nsample, const randomforest & r, const Eigen::MatrixXd& feat, struct feature_node **global_binary_features, const int num_landmark, const int num_trees, const int max_depth, const int offset){ 168 | //Eigen::MatrixXd y_temp; 169 | int num_feat; 170 | //y_temp.resize(num_trees, 2); 171 | int index; 172 | //cout << "Node number:"<= r[k][index].threshold) 181 | index = 2 * index + 2; 182 | else 183 | index = 2 * index + 1; 184 | } 185 | 186 | global_binary_features[i][num_landmark*num_trees + k].index = 16 * (num_landmark*num_trees + k) + index - 14; 187 | global_binary_features[i][num_landmark*num_trees + k].value = 1.0; 188 | //cout << "index"<