├── .github └── FUNDING.yml ├── .gitignore ├── Example_FindFace.m ├── Example_buildMuctModel.m ├── Faces └── Face.jpg ├── LICENSE ├── Landmarks ├── Example_FindFace_Landmarks_standard.mat └── Example_FindFace_landmarks_MUCT.mat ├── Media ├── ADIP_ActiveShapeModels_FinalReport.pdf ├── Faces_MultiResolution_horizontal.png ├── Faces_PC_Variations.png ├── FoundFaceExample_cropped.png └── Video │ ├── ASM_FaceDetection_24-Jul-2017_MUCT.gif │ └── ASM_FindFaceExample.mp4 ├── README.md ├── SavedModels ├── grayModel_MUCT.mat └── grayModel_standard.mat ├── Utilities ├── FS.m ├── dif2.m ├── parseKeyValuePairs.m └── xylim.m ├── Visualization ├── guiPrinComps.m ├── plotLandmarks.m ├── plotPrinComp.m └── viewGrayProfile.m ├── alignShapes.m ├── buildGrayLevelModel.m ├── buildShapeModel.m ├── estimateFaceLocation.m ├── findFace.m ├── getFaceRegions.m ├── placeLandmarks.m └── placeShape.m /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: johnwmillr 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Specific files 2 | shapeModel_CootesTaylor.m 3 | 4 | # Face images & videos 5 | *.jpg 6 | *.Jpg 7 | *.png 8 | *.ai 9 | *.mp4 10 | 11 | # Ignore directories 12 | /Faces 13 | /Media 14 | /Old 15 | /Landmarks 16 | /scratch 17 | /SavedModels 18 | /Evaluation 19 | 20 | # Matlab files 21 | *_v*.m 22 | *.m~ 23 | *.mat 24 | *.asv 25 | 26 | # Dropbox settings and caches 27 | *.dropbox 28 | *.dropbox.attr 29 | *.dropbox.cache 30 | 31 | # OS X 32 | *.DS_Store 33 | -------------------------------------------------------------------------------- /Example_FindFace.m: -------------------------------------------------------------------------------- 1 | % EXAMPLE_FINDFACE steps the user through the process of detecting a face in a single 2 | % example image. The models are built from 50 training images from the link below. 3 | % 4 | % Run this script for an example of how to use the active shape model (ASM) code. 5 | % This script needs to be within the main ActiveShapeModels directory in order to 6 | % load all of the paths properly. 7 | % 8 | % Link to training images: 9 | % http://robotics.csie.ncku.edu.tw/Databases/FaceDetect_PoseEstimate.htm#Our_Database_ 10 | % 11 | % Shape analysis techniques based on this paper: 12 | % Cootes, T. F., Taylor, C. J., Cooper, D. H., & Graham, J. (1995). 13 | % "Active Shape Models-Their Training and Application. Computer Vision and 14 | % Image Understanding." 15 | % 16 | % 2D gray-level profiles inspired by: 17 | % Milborrow, S. (2016). "Multiview Active Shape Models with SIFT Descriptors." 18 | % 19 | % See also BUILDSHAPEMODEL, BUILDGRAYLEVELMODEL, FINDFACE, EXAMPLE_BUILDMUCTMODEL 20 | % 21 | % John W. Miller 22 | % Electrical & Computer Engineering 23 | % University of Iowa 24 | % 15-Dec-2017 25 | 26 | close all; clear all; clc 27 | %% Add necessary paths 28 | % Make sure you run this example script while within its containing folder 29 | project_dir = pwd; 30 | addpath(project_dir,fullfile(project_dir,'Utilities'),fullfile(project_dir,'Visualization')) 31 | 32 | %% Load image landmarks from disk 33 | landmark_style = 'MUCT'; 34 | load(fullfile(project_dir,'Landmarks','Example_FindFace_Landmarks_MUCT')) 35 | 36 | % View the landmarks we just loaded (they are not aligned, we will align them) 37 | plotLandmarks(allLandmarks), pause(1), close 38 | 39 | %% Create the shape model from the unaligned shapes 40 | shapeModel = buildShapeModel(allLandmarks); 41 | 42 | %% Explore the shape model 43 | view_gui = 0; 44 | if view_gui 45 | guiPrinComps(shapeModel,'layout',landmark_style) % Effect of PC weights on shape (GUI) 46 | end 47 | 48 | %% Create the gray-level 2D profile model (or load it from disk more likely) 49 | create_new_gray_model = false; 50 | if create_new_gray_model 51 | faceFiles = dir(fullfile(pathToImages,'*.jpg')); 52 | faceFiles = faceFiles(~cellfun(@isempty,regexp({faceFiles(:).name}','i\d{3}qa-[fm]n'))); 53 | shapeModel.trainingImages = fullfile(project_dir,'Faces','MUCT','muct_images'); 54 | grayModel = buildGrayLevelModel(faceFiles,shapeModel); % This takes about 30 seconds 55 | else 56 | % Load the 'grayModel' struct into the workspace. 57 | % grayModel is a struct ([n_resolutions x 1]) containing the gray-level 58 | % gradient information for each of the training images at each resolution for 59 | % each landmark making up the face shape. 60 | fprintf('\nLoading the gray-level model...') 61 | load(fullfile(project_dir,'SavedModels','grayModel_MUCT')); fprintf(' Done loading.\n') 62 | end 63 | 64 | %% Explore the gray model 65 | n_landmark = 67; 66 | viewGrayProfile(grayModel,1,n_landmark) 67 | 68 | %% Find a face! 69 | disp('Finding a face...') 70 | im = imread(fullfile(project_dir,'Faces','Face.jpg')); close all 71 | findFace(im,shapeModel,grayModel,'visualize',1,'facefinder','click','layout','muct') 72 | 73 | -------------------------------------------------------------------------------- /Example_buildMuctModel.m: -------------------------------------------------------------------------------- 1 | % Build MUCT model from scratch 2 | % 3 | % John W. Miller 4 | % 15-Dec-2017 5 | 6 | % Download the faces dataset from the link below 7 | % http://www.milbo.org/muct/index.html 8 | project_dir = pwd; 9 | addpath(project_dir,fullfile(project_dir,'Utilities'),fullfile(project_dir,'Visualization')), cd(project_dir) 10 | 11 | % Load landmarks and images 12 | pathToLandmarks = fullfile(project_dir,'Faces','MUCT','muct_landmarks'); 13 | pathToImages = fullfile(project_dir,'Faces','MUCT','muct_images'); 14 | faceFiles = dir(fullfile(pathToImages,'*.jpg')); 15 | 16 | % Load landmarks 17 | C = importdata(fullfile(pathToLandmarks,'muct76.csv')); 18 | landmarks = C.data(1:end,2:end); 19 | subj_labels = C.textdata(2:end,1); 20 | 21 | % Extract the landmarks and images we want 22 | expression = 'i\d{3}[qrs]a-[fm]n'; 23 | mask_landmarks = ~cellfun(@isempty,regexpi(subj_labels,expression)); 24 | mask_images = ~cellfun(@isempty,regexp({faceFiles(:).name}',expression)); 25 | 26 | faceFiles = faceFiles(mask_images); 27 | allLandmarks = landmarks(mask_landmarks,:)'; 28 | im = rgb2gray(imread(fullfile(pathToImages,faceFiles(1).name))); 29 | allLandmarks(1:2:end,:) = allLandmarks(1:2:end,:) + size(im,2)/2; % Adjust for coordinate differences 30 | allLandmarks(2:2:end,:) = -1*allLandmarks(2:2:end,:) + size(im,1)/2; 31 | % save('Example_FindFace_landmarks_MUCT','allLandmarks') 32 | 33 | %% View an image with landmarks 34 | n_im = 2; 35 | im = rgb2gray(imread(fullfile(pathToImages,faceFiles(n_im).name))); 36 | figure, imshow(im,[]), hold on 37 | plot(allLandmarks(1:2:end,n_im),allLandmarks(2:2:end,n_im),'ro'), hold off 38 | 39 | %% Create the shape model from the unaligned shapes 40 | shapeModel = buildShapeModel(allLandmarks,pathToImages); 41 | plotLandmarks(shapeModel.alignedShapes,'layout','muct') 42 | 43 | %% Explore the shape model 44 | guiPrinComps(shapeModel,'layout','muct') % Effect of PC weights on shape (GUI) 45 | plotPrinComp(shapeModel,1,'muct') % Plot variations of the different PCs 46 | 47 | %% Create the gray-level 2D profile model 48 | grayModel = buildGrayLevelModel(faceFiles,shapeModel,'resolutions',[16 6 2 1],'region_size',[15 3]); 49 | 50 | %% Explore the gray model 51 | n_resolution = 1; 52 | n_landmark = 67; 53 | viewGrayProfile(grayModel,n_resolution,n_landmark) 54 | 55 | %% Find a face! 56 | n_im = 30; 57 | im = rgb2gray(imread(fullfile(pathToImages,faceFiles(n_im).name))); 58 | findFace(im,shapeModel,grayModel,'layout','muct','evolutions',3) 59 | -------------------------------------------------------------------------------- /Faces/Face.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Faces/Face.jpg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 John W. Miller 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Landmarks/Example_FindFace_Landmarks_standard.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Landmarks/Example_FindFace_Landmarks_standard.mat -------------------------------------------------------------------------------- /Landmarks/Example_FindFace_landmarks_MUCT.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Landmarks/Example_FindFace_landmarks_MUCT.mat -------------------------------------------------------------------------------- /Media/ADIP_ActiveShapeModels_FinalReport.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Media/ADIP_ActiveShapeModels_FinalReport.pdf -------------------------------------------------------------------------------- /Media/Faces_MultiResolution_horizontal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Media/Faces_MultiResolution_horizontal.png -------------------------------------------------------------------------------- /Media/Faces_PC_Variations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Media/Faces_PC_Variations.png -------------------------------------------------------------------------------- /Media/FoundFaceExample_cropped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Media/FoundFaceExample_cropped.png -------------------------------------------------------------------------------- /Media/Video/ASM_FaceDetection_24-Jul-2017_MUCT.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Media/Video/ASM_FaceDetection_24-Jul-2017_MUCT.gif -------------------------------------------------------------------------------- /Media/Video/ASM_FindFaceExample.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/Media/Video/ASM_FindFaceExample.mp4 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Active Shape Models for face detection 2 | ![](/Media/Faces_MultiResolution_horizontal.png "Variations in the gray-level model") 3 | 4 | This project was part of my work for *Advanced Digital Image Processing* at the University of Iowa during spring 2017. You can read my [final report for the class here](/Media/ADIP_ActiveShapeModels_FinalReport.pdf). The report was a tad rushed, my apologies! 5 | 6 | ![](/Media/Video/ASM_FaceDetection_24-Jul-2017_MUCT.gif "Finding a face using the MUCT layout") 7 | 8 | ## Usage ## 9 | After cloning this repository, run the [Example_FindFace](Example_FindFace.m) script for a walkthrough demonstration of how to use this ASM code for locating a face in an example image. 10 | 11 | ## More than just faces ## 12 | It's probably worth pointing out that the ASM technique (and this implementation) is *not* limited to face detection. The models can be trained to detect whatever class of shapes the user chooses. So if you have a set of labeled images of hands (or whatever), you can train a model using the `buildShapeModel.m` and `buildGrayLevelModel.m` functions to search for hands (or whatever). 13 | 14 | ## Background ## 15 | Here is the original [Cootes et al. paper.](http://www.sciencedirect.com/science/article/pii/S1077314285710041) PDFs of the paper are available elsewhere online if you don't have access to the journal. Here is a link to the [faces training set](http://robotics.csie.ncku.edu.tw/Databases/FaceDetect_PoseEstimate.htm#Our_Database_) I annotated to train my model. 16 | 17 | Manipulating the weights on the 1st and 2nd principal components deforms the face shape within an allowable range of variation. 18 | 19 | 20 | -------------------------------------------------------------------------------- /SavedModels/grayModel_MUCT.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/SavedModels/grayModel_MUCT.mat -------------------------------------------------------------------------------- /SavedModels/grayModel_standard.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnwmillr/ActiveShapeModels/c69529c38d110967c39bba77329db77c0acc9c65/SavedModels/grayModel_standard.mat -------------------------------------------------------------------------------- /Utilities/FS.m: -------------------------------------------------------------------------------- 1 | function defaultFontSize = FS() 2 | % FS Sets the default font size (in a janky way) 3 | % 4 | % INPUT 5 | % 6 | % OUTPUT 7 | % 8 | % John W. Miller 9 | % 2015-06-02 10 | % 11 | 12 | defaultFontSize = 14; 13 | 14 | end -------------------------------------------------------------------------------- /Utilities/dif2.m: -------------------------------------------------------------------------------- 1 | function fd=dif2(f) 2 | %dif2 Computes two point diference estimate of velocity. 3 | % 4 | % DIFFERENCE = dif2(SIGNAL) 5 | % 6 | % Computes the difference at a point as being the average of slopes on 7 | % either side of the point, returns a vector DIFFERENCE, the same size as 8 | % in the input SIGNAL 9 | % 10 | % If SIGNAL is a matrix it takes the difference of the rows in each column 11 | % 12 | % To convert the output to dif2 to the proper units multiply by fs 13 | % 14 | % tags: math, calculus, slope 15 | [r c] = size(f); 16 | 17 | %May 04, 2010 : JAH, modified to keep dimensions of single vector 18 | 19 | if isempty(f) 20 | fd = f; 21 | return 22 | end 23 | 24 | if r == 1 && c == 1 25 | error('Dif2 is undefined for a single point') 26 | elseif r >1 && c > 1 27 | fd=zeros(r,c); 28 | fd(1,:)=f(2,:)-f(1,:); 29 | fd(r,:)=f(r,:)-f(r-1,:); 30 | df=diff(f); 31 | fd(2:r-1,:)=.5*(df(2:r-1,:)+df(1:r-2,:)); 32 | else %The single vector case 33 | fd=zeros(r,c); 34 | fd(1)=f(2)-f(1); 35 | fd(end)=f(end)-f(end-1); 36 | df=diff(f); 37 | fd(2:end-1)=.5*(df(2:end)+df(1:end-1)); 38 | end 39 | end -------------------------------------------------------------------------------- /Utilities/parseKeyValuePairs.m: -------------------------------------------------------------------------------- 1 | function varargout = parseKeyValuePairs(userKeyNamePairs,valid_keys,default_values) 2 | % PARSEKEYVALUEPAIRS accepts a calling function's 'varargin' cell and interprets its key-value pairs. 3 | % 4 | % INPUT 5 | % userKeyNamePairs: The 'varargin' cell from another function 6 | % valid_keyNames: A cell array of valid key names. Passed in from calling function. 7 | % default_values: A cell array of the default values for each key name. 8 | % 9 | % OUTPUT 10 | % varargout: Values of the key-name pairs, in the order specified by 'valid_keyNames'. 11 | % 12 | % TODO 13 | % Combine the 'valid_keyNames' and 'default_values' into a single variable 14 | % 15 | % John W. Miller 16 | % 2015-07-31 17 | 18 | % Check for the proper dimensions 19 | if mod(length(userKeyNamePairs),2) 20 | error('Variable names and their values must come in pairs.') 21 | end 22 | valid_keys = lower(valid_keys); 23 | 24 | % Extract the key names and values 25 | keyNames_FromUser = lower(userKeyNamePairs(1:2:end)); 26 | values_FromUser = userKeyNamePairs(2:2:end); 27 | 28 | % Check that the user-supplied key names are valid (Excessive, I know...) 29 | name_check = ismember(keyNames_FromUser,valid_keys); 30 | if ~all(name_check) 31 | errmsg = sprintf('Invalid key names: %s', print_cell(keyNames_FromUser(~name_check),', ')); 32 | error('%s\nValid key names:\n%s',errmsg,print_cell(valid_keys)) 33 | end 34 | 35 | % Replace the default values with user-supplied values 36 | processed_vars = containers.Map(valid_keys,default_values,'UniformValues',false); 37 | n_vars_FromUser = length(keyNames_FromUser); 38 | for n_var_FromUser = 1:n_vars_FromUser 39 | processed_vars(keyNames_FromUser{n_var_FromUser}) = values_FromUser{n_var_FromUser}; 40 | end 41 | 42 | % Varargout 43 | varargout = values(processed_vars,valid_keys); % Must specify the order of keys to pull out 44 | 45 | end % End of main 46 | 47 | function varargout = print_cell(cell_to_print,delim) 48 | % PRINTCELL 49 | % 50 | % INPUT 51 | % 52 | % 53 | % 54 | % OUTPUT 55 | % 56 | % 57 | % John W. Miller 58 | % 02-Jul-2016 59 | 60 | if nargin < 2 61 | delim = sprintf('\n'); 62 | else 63 | delim = sprintf(delim); 64 | end 65 | 66 | clear cell_as_string 67 | cell_as_string = cell_to_print{1}; 68 | for i = 2:length(cell_to_print) 69 | cell_as_string = [cell_as_string sprintf('%s%s',delim,cell_to_print{i})]; 70 | end 71 | 72 | if nargout > 0 73 | varargout{1} = cell_as_string; 74 | else 75 | disp(cell_as_string) 76 | end 77 | end % End of main -------------------------------------------------------------------------------- /Utilities/xylim.m: -------------------------------------------------------------------------------- 1 | function varargout = xylim(xy) 2 | % XYLIM sets the x and y axis to equal limits 3 | % 4 | % John W. Miller 5 | % 01-May-2017 6 | 7 | ax = gca; 8 | if nargin ==0 9 | 10 | xLim = xlim(ax); yLim = ylim(ax); 11 | xy = [min([xLim(1) yLim(1)]) max([xLim(2) yLim(2)])]; 12 | end 13 | xlim(xy), ylim(xy) 14 | 15 | if nargout == 1 16 | varargout{1} = xy; 17 | end 18 | 19 | end % End of main -------------------------------------------------------------------------------- /Visualization/guiPrinComps.m: -------------------------------------------------------------------------------- 1 | function guiPrinComps(shapeModel,varargin) 2 | % GUIPRINCOMPS 3 | % 4 | % INPUT 5 | % xBar: Mean shape [2*n_landmarks x 1] 6 | % V: Principal components (eigenvectors) 7 | % D: Shape weights (eigenvalues) 8 | % OPTIONAL 9 | % show_image: (bool) Display a face image in the background if true 10 | % 11 | % John W. Miller 12 | % 17-Mar-2017 13 | 14 | % Varargin 15 | keys = {'show_image','layout'}; default_values = {0,'muct'}; 16 | [show_image,face_layout] = parseKeyValuePairs(varargin,keys,default_values); 17 | 18 | % Extract stuff from model struct 19 | [xBar, V, D] = deal(shapeModel.meanShape,shapeModel.eVectors,shapeModel.eValues); 20 | 21 | % Initialization 22 | n_pcs = 3; 23 | slider_values = zeros(n_pcs,1); 24 | faceRegions = getFaceRegions(face_layout); % Connecting dots around the face 25 | 26 | % Examine variations from individual PCs 27 | n_variations = 11; % Must be an odd number 28 | if mod(n_variations,2) == 0 29 | n_variations = n_variations + 1; 30 | end 31 | var_step = (n_variations-1)/2; 32 | weights = zeros(var_step,n_variations); 33 | for n = 1:n_pcs 34 | weights(n,:) = sqrt(D(n))*(-var_step:var_step); 35 | end 36 | 37 | % Create some shape variations 38 | P = V(:,1:n_pcs); 39 | shapeVariations = repmat(xBar,1,n_variations) + P*weights(1:n_pcs,:); 40 | 41 | % Generate an initial shape 42 | % firstShape = generateShape(); 43 | 44 | %% Initial visualization 45 | f = figure(); 46 | if show_image 47 | imFile = 'Face.jpg'; 48 | im = imread(fullfile('Faces',imFile)); 49 | xBar = placeShape(im,xBar); 50 | end 51 | 52 | % Determine mean shape and plot dimensions 53 | mew(:,1) = xBar(1:2:end); 54 | mew(:,2) = xBar(2:2:end); 55 | xLim = [0.8 0; 0 1.1]*[min(min(shapeVariations(1:2:end,:))) max(max(shapeVariations(1:2:end,:)))]'; 56 | yLim = [0.8 0; 0 1.3]*[min(min(shapeVariations(2:2:end,:))) max(max(shapeVariations(2:2:end,:)))]'; 57 | 58 | updatePlot(generateShape()); % Initial plot 59 | 60 | %% Create the GUI sliders 61 | [min_val,max_val] = deal(-(n_variations-1)/2,(n_variations-1)/2); 62 | stepsize = (1/range([min_val max_val]))*[0.5 1]; % min, max 63 | 64 | b1 = uicontrol('Parent',f,'Style','slider','Position',[81,75,419,23],... 65 | 'value',0, 'min',min_val, 'max',max_val, 'callback', @b_callback_pc_slider,... 66 | 'SliderStep', stepsize); 67 | 68 | b2 = uicontrol('Parent',f,'Style','slider','Position',[81,50,419,23],... 69 | 'value',0, 'min',min_val, 'max',max_val, 'callback', @b_callback_pc_slider,... 70 | 'SliderStep', stepsize); 71 | 72 | b3 = uicontrol('Parent',f,'Style','slider','Position',[81,25,419,23],... 73 | 'value',0, 'min',min_val, 'max',max_val, 'callback', @b_callback_pc_slider,... 74 | 'SliderStep', stepsize); 75 | 76 | %% Callback function 77 | function b_callback_pc_slider(source,event) 78 | switch get(source,'position')*[0 1 0 0]' 79 | case 75 % PC 1 80 | slider_values(1) = get(source,'value'); 81 | case 50 % PC 2 82 | slider_values(2) = get(source,'value'); 83 | case 25 % PC 3 84 | slider_values(3) = get(source,'value'); 85 | end 86 | updatePlot(generateShape()) 87 | end 88 | 89 | %% Generate and plot new shapes 90 | 91 | function newShape = generateShape() 92 | % Calculate the weights for the updated shape 93 | b = zeros(n_pcs,1); 94 | for n_pc = 1:n_pcs 95 | b(n_pc,1) = sqrt(D(n_pc))*(slider_values(n_pc)); 96 | end 97 | 98 | % Generate the shape 99 | x = xBar + P*b; 100 | newShape(:,1) = x(1:2:end); 101 | newShape(:,2) = x(2:2:end); 102 | end 103 | 104 | function updatePlot(newShape) 105 | figure(f), hold off 106 | if show_image imshow(im), hold on, end 107 | for i = 1:length(faceRegions) 108 | plot(newShape(faceRegions{i},1),newShape(faceRegions{i},2), '.-','linewidth',2,'color','g'), hold on 109 | end 110 | plot(mew(:,1),mew(:,2),'.','color','k','linewidth',2) 111 | set(gca,'xtick',[],'ytick',[]) 112 | set(gca,'xlim',xLim,'ylim',yLim,'ydir','reverse') 113 | title('Move the sliders to change the weights on the first 3 PCs','fontsize',FS) 114 | end 115 | 116 | end % End of main -------------------------------------------------------------------------------- /Visualization/plotLandmarks.m: -------------------------------------------------------------------------------- 1 | function varargout = plotLandmarks(landmarks,varargin) 2 | % PLOTLANDMARKS plots all of the aligned landmarks from an active shapes model 3 | % 4 | % INPUT 5 | % landmarks: The aligned landmarks from multiple images 6 | % [2*n_landmarks x n_shapes] 7 | % OPTIONAL 8 | % show_lines: 9 | % hold: 10 | % 11 | % See also PLACELANDMARKS, ALIGNSHAPES 12 | % 13 | % John W. Miller 14 | % 14-Mar-2017 15 | 16 | % Key-value pair varargin 17 | keys = {'show_lines','hold','color','linewidth','linestyle','layout'}; default_values = {1,0,'m',1,'-','muct'}; 18 | [show_lines,hold_on,line_color,lw,ls,face_layout] = parseKeyValuePairs(varargin,keys,default_values); 19 | 20 | % Plot the landmarks for each shape 21 | n_shapes = size(landmarks,2); 22 | if hold_on 23 | h = gcf; 24 | else 25 | h = figure; 26 | end 27 | hold on 28 | try colors = parula(n_shapes); 29 | catch 30 | colors = hsv(n_shapes); 31 | end 32 | for n_shape = 1:n_shapes 33 | iShape = [landmarks(1:2:end,n_shape) landmarks(2:2:end,n_shape)]; 34 | % plot(iShape(:,1),iShape(:,2),'o','color', colors(n_shape,:),... 35 | % 'linewidth',2,'markersize',2,'markerfacecolor',colors(n_shape,:)) 36 | plot(iShape(:,1),iShape(:,2),'+','color', 'k',... 37 | 'linewidth',2,'markersize',7,'markerfacecolor',colors(n_shape,:)) 38 | end 39 | 40 | % Add mean shape to the plot 41 | if n_shapes > 1 42 | meanShape = mean(landmarks,2); % x1, y1, x2, y2, ..., x20, y20 43 | else 44 | meanShape = landmarks; 45 | end 46 | ax = plot(meanShape(1:2:end),meanShape(2:2:end),'ro',... 47 | 'markersize',5,'linewidth',1,'markerfacecolor','k'); 48 | 49 | % Connect dots on the face (optional) 50 | if show_lines 51 | faceLabels = getFaceRegions(face_layout); 52 | 53 | mew = [meanShape(1:2:end) meanShape(2:2:end)]; 54 | for i = 1:length(faceLabels) 55 | mew_handle = plot(mew(faceLabels{i},1), mew(faceLabels{i},2),... 56 | 'linestyle',ls,'color',line_color,'linewidth',lw); 57 | end 58 | end 59 | 60 | % Touch up the plot 61 | if ~hold_on 62 | legend(ax,{'Mean shape'},'fontsize',FS,'location','southeast') 63 | text(0.05,0.5,sprintf('n=%d',n_shapes),'units','normalized','fontsize',FS,'fontweight','bold') 64 | end 65 | set(h,'Units','Inches'); 66 | pos = get(h,'Position'); 67 | set(h,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) 68 | axis off 69 | set(gca,'YDir','reverse'); 70 | drawnow 71 | 72 | % Varargout 73 | if nargout > 0 74 | varargout{1} = mew_handle; 75 | end 76 | 77 | end % End of main -------------------------------------------------------------------------------- /Visualization/plotPrinComp.m: -------------------------------------------------------------------------------- 1 | function plotPrinComp(shapeModel,n_pc,layout) 2 | % PLOTPRINCOMP displays the variation along a single principal component from an 3 | % active shapes model 4 | % 5 | % INPUT 6 | % V: Principal components (eigenvectors) 7 | % D: Shape weights (eigenvalues) 8 | % xBar: Mean shape [2*n_landmarks x 1] 9 | % n_pc: Which PC do you want to display variations along? 10 | % 11 | % The shape variations are determined by the equation x = xBar + P*b, 12 | % where P is a single principal component and b is a vector of weights. 13 | % 14 | % See also PLACELANDMARKS, PLOTLANDMARKS, ALIGNSHAPES 15 | % 16 | % TODO: 17 | % If n_pc is a vector, plot a single subplot containing multiple PCs 18 | % 19 | % John W. Miller 20 | % 16-Mar-2017 21 | 22 | if nargin < 3 23 | layout = 'standard'; 24 | end 25 | 26 | % Extract stuff from model struct 27 | [xBar, V, D] = deal(shapeModel.meanShape,shapeModel.eVectors,shapeModel.eValues); 28 | 29 | % Examine variations from individual PCs 30 | b = sqrt(D(n_pc))*(-3:3); 31 | n_vars = length(b); 32 | 33 | % Create some shape variations 34 | P = V(:,n_pc); 35 | shapeVariations = repmat(xBar,1,n_vars) + P*b; 36 | xLim = [floor(min(min(shapeVariations(1:2:end,:)))) ceil(max(max(shapeVariations(1:2:end))))]; 37 | yLim = [floor(min(min(shapeVariations(2:2:end,:)))) ceil(max(max(shapeVariations(2:2:end))))]; 38 | 39 | % Change plot color for different weights of the selected PC 40 | faceRegions = getFaceRegions(layout); 41 | mew(:,1) = xBar(1:2:end); 42 | mew(:,2) = xBar(2:2:end); 43 | figure, hold on 44 | colors = hsv(n_vars); 45 | for n = 1:n_vars 46 | 47 | iVar = zeros(size(shapeVariations,1)/2,2); 48 | iVar(:,1) = shapeVariations(1:2:end,n); 49 | iVar(:,2) = shapeVariations(2:2:end,n); 50 | 51 | % Plot the PC variations 52 | plot(iVar(:,1),iVar(:,2),'o','color',colors(n,:)) 53 | 54 | % Connect the dots 55 | for i = 1:length(faceRegions) 56 | plot(mew(faceRegions{i},1), mew(faceRegions{i},2), 'k-','linewidth',1) 57 | plot(iVar(faceRegions{i},1),iVar(faceRegions{i},2), '-','linewidth',1,'color',colors(n,:)) 58 | end 59 | 60 | end 61 | plot(xBar(1:2:end),xBar(2:2:end),'k.','linewidth',3) 62 | set(gca,'ydir','reverse'), axis square 63 | xlim(xLim), ylim(yLim) 64 | title(sprintf('Variation of PC #%d',n_pc),'fontsize',20) 65 | 66 | 67 | end % End of main -------------------------------------------------------------------------------- /Visualization/viewGrayProfile.m: -------------------------------------------------------------------------------- 1 | function viewGrayProfile(grayModel,nr,nl) 2 | % VIEWGRAYPROFILE 3 | % 4 | % INPUT 5 | % grayModel: Comes from BUILDGRAYLEVELMODEL. 6 | % nr: resolution number 7 | % nl: landmark number 8 | % 9 | % See also BUILDGRAYLEVELMODEL 10 | % 11 | % John W. Miller 12 | % 15-May-2017 13 | 14 | % Reshape from vector to square 15 | graySquare = reshape(grayModel(nr).meanProfile{nl},grayModel(nr).Info.rc_squaresize-1,grayModel(nr).Info.rc_squaresize-1); 16 | 17 | % View the square region 18 | figure, imshow(imresize(graySquare,round(200/length(graySquare))),[]) 19 | 20 | end % End of main -------------------------------------------------------------------------------- /alignShapes.m: -------------------------------------------------------------------------------- 1 | function alignedShapes = alignShapes(allShapes, scaling) 2 | % ALIGNSHAPES uses Procrustes analysis to align a set of shapes (with or without 3 | % scaling). 4 | % 5 | % INPUT 6 | % allShapes: [2*n_landmarks x n_shapes], n_shapes is basically n_subjects 7 | % Collected from the HD scan semi-automated segmentation 8 | % for each row: 20 points (40 elements): x1, y1, x2, y2, ..., x20, y20% % 9 | % scaling: (bool) Do you want to scale your images or not? (default = 0) 10 | % 11 | % OUTPUT 12 | % alignedShapes: The realigned shapes. Same shape as totalShapes 13 | % 14 | % Shape analysis techniques based on this paper: 15 | % Cootes, T. F., Taylor, C. J., Cooper, D. H., & Graham, J. (1995). 16 | % "Active Shape Models-Their Training and Application. Computer Vision and 17 | % Image Understanding." 18 | % 19 | % See also PROCRUSTES, PLACELANDMARKS, PLOTLANDMARKS 20 | % 21 | % John W. Miller 22 | % 15-Mar-2017 23 | 24 | % Pre-allocate 25 | n_shapes = size(allShapes,2); 26 | alignedShapes = zeros(size(allShapes)); 27 | 28 | % Mean shape across all subjects (assuming each shape in totalShapes is a different subj) 29 | meanShape = mean(allShapes,2); % x1, y1, x2, y2, ..., x20, y20 30 | meanShape = [meanShape(1:2:end) meanShape(2:2:end)]; % Reshape for Procrustes 31 | 32 | %% Loop thru each shape in totalShapes and transform via Procrustes analysis 33 | for n_shape = 1:n_shapes 34 | % Landmarks shape for the current subject (if 1 shape per subject) 35 | iShape = [allShapes(1:2:end,n_shape) allShapes(2:2:end,n_shape)]; 36 | 37 | % Do the Procrustes alignment 38 | [~, iShapeAligned] = procrustes(meanShape,iShape,'scaling',scaling,'reflection','best'); 39 | 40 | % Store the aligned shape in a similar manner as how totalShapes is passed in 41 | alignedShapes(1:2:end,n_shape) = iShapeAligned(:,1)'; 42 | alignedShapes(2:2:end,n_shape) = iShapeAligned(:,2)'; 43 | end 44 | 45 | end % End of main -------------------------------------------------------------------------------- /buildGrayLevelModel.m: -------------------------------------------------------------------------------- 1 | function grayModel = buildGrayLevelModel(pathsToImages,shapeModel,varargin) 2 | % BUILDGRAYLEVELMODEL 3 | % 4 | % INPUT 5 | % pathsToImages: Struct containing path to each image. 6 | % Generate it like this: dir(fullfile(pathToImages,'*.jpg')); 7 | % shapeModel: Comes from the BUILDSHAPEMODEL function. 8 | % 9 | % OUTPUT 10 | % grayProfileModel: (struct) Contains all ya need for the model. 11 | % 12 | % 13 | % TODO: Make the process for getting gray-level info its own function, so it can be 14 | % called during the search process for new images (right now I'm repeating code) 15 | % 16 | % See also BUILDSHAPEMODEL 17 | % 18 | % John W. Miller 19 | % 15-Dec-2017 20 | tic 21 | 22 | % Key-value pair varargin 23 | keys = {'save_model','resolutions','region_size','view_model'}; default_values = {0,6:-1:3,[20 5],0}; 24 | [save_model, downsample_factors, r_size, view_model] = parseKeyValuePairs(varargin,keys,default_values); 25 | if save_model 26 | fprintf('\n\n') 27 | save_name = ['grayModel_' datestr(date,'yyyy-mm-dd') '_' input('Save name? grayModel_','s') '.mat']; 28 | 29 | save_dir = fullfile(uigetdir); 30 | disp('Saving file...') 31 | end, n_images = length(pathsToImages); 32 | 33 | % Multi-resolution 34 | n_resolutions = numel(downsample_factors); 35 | 36 | % Change parameters based on resolution 37 | interp_step_sizes = 1*ones(n_resolutions,1); % Probably just leave this as 1 38 | 39 | % These model parameters are very important. 40 | filter_sigs = linspace(0.8,0.35,n_resolutions); % Should go from about 0.8 to 0.3 41 | region_size = round(linspace(r_size(1),r_size(2),n_resolutions)); % Should go from about 20 to 5 42 | 43 | % Build a 2D gray-level profile for each landmark at each resolution 44 | for n_resolution = 1:n_resolutions 45 | downsample_factor = downsample_factors(n_resolution); 46 | % Pre-allocation and stuff 47 | % Downsample the image and shape coordinates 48 | x_bar = shapeModel.meanShape./downsample_factor; 49 | imageLandmarks = shapeModel.unalignedShapes./downsample_factor; 50 | 51 | % Make sure we're working with just the images we want (for MUCT) 52 | msg = sprintf('More images found in image directory than in the shape model.\nTry using the "regexp" key-pair input.'); 53 | assert(n_images == shapeModel.n_shapes,msg) 54 | 55 | % Sample a square region of pixels around each landmark 56 | rc = region_size(n_resolution); % Size of square (rc+1) 57 | if mod(rc,2)~=0;rc=rc+1;end 58 | 59 | % Convolution mask (for gradient of the square regions) 60 | kernel = [0 -1 0; -1 2 0; 0 0 0]; 61 | C = 5; % Sigmoid equalization 62 | 63 | % Pre-allocate to store square profiles at each landmark 64 | n_landmarks = length(x_bar)/2; 65 | im_profiles = cell(n_landmarks,1); % One profile per landmark 66 | 67 | % Filter for smoothing before measuring profile 68 | [A,mew,sig] = deal(1,0,filter_sigs(n_resolution)); % Increase sig for more smoothing 69 | x_vals = (-3*sig+mew):0.1:(3*sig+mew); 70 | h_filt = A*exp(-((x_vals-mew).^2)./(2*sig^2)); 71 | 72 | %% Start loopin' 73 | % For each image in the training set, calculate the image gradient (kinda) in a 74 | % square region of pixels around each landmark 75 | for n_image = 1:n_images 76 | im = rgb2gray(imread(fullfile(shapeModel.trainingImages,pathsToImages(n_image).name))); 77 | 78 | % Smooth and downsample the image 79 | im = conv2(h_filt,h_filt,im,'same'); 80 | im = imresize(im,1./downsample_factor); 81 | for n_landmark = 1:n_landmarks 82 | iPixel = imageLandmarks(2*n_landmark+[-1 0],n_image); % Coordinates of current landmark 83 | 84 | % Crop a square region around the pixel 85 | im_region = imcrop(im,[iPixel(1)-rc/2 iPixel(2)-rc/2 rc rc]); 86 | sz = size(im_region); 87 | if any(sz < rc+1) 88 | im_region = padarray(im_region,max(rc-sz+1,0),'replicate','post'); 89 | end 90 | if sz(1) > rc+1 91 | im_region = im_region(1:rc,:); 92 | end 93 | if sz(2) > rc+1 94 | im_region = im_region(:,1:rc); 95 | end 96 | 97 | % Calculate the gradient for this region 98 | im_region_filt = conv2(im_region,kernel,'valid'); 99 | abs_sum = sum(abs(im_region_filt(:))); 100 | if abs_sum ~= 0 101 | im_region_filt = im_region_filt./abs_sum; 102 | end 103 | 104 | % Sigmoid equalization 105 | im_region_filt = im_region_filt./(abs(im_region_filt)+C); 106 | 107 | % Store as 1D vector 108 | im_profile = reshape(im_region_filt,size(im_region_filt,1).^2,1); 109 | im_profiles{n_landmark}(:,n_image) = im_profile; 110 | end % Looping through landmarks 111 | end % Looping through images 112 | 113 | %% Build gray-level model for each landmark (using PCA) from all images 114 | % Covariance matrices (one per landmark) 115 | [gBar,S,eVectors,eValues] = deal(cell(n_landmarks,1)); % This deal might be slow 116 | for n_landmark = 1:n_landmarks 117 | gBar{n_landmark} = mean(im_profiles{n_landmark},2); 118 | S{n_landmark} = cov(im_profiles{n_landmark}'); % Must transpose here 119 | [V,D] = eig(S{n_landmark}); 120 | D = sort(diag(D),'descend'); V = fliplr(V); 121 | eVectors{n_landmark} = V; 122 | eValues{n_landmark} = D; 123 | end 124 | 125 | %% Visualize the model (optional) 126 | if view_model 127 | n_landmark = 14; 128 | % Mean profile at a specific landmark 129 | figure,imshow(reshape(gBar{n_landmark},rc-1,rc-1),[]) 130 | end 131 | 132 | %% Store in a struct 133 | if n_resolution == 1 134 | grayModel(n_resolution,1) = struct(); %#ok<*AGROW> 135 | else 136 | grayModel(n_resolution,1) = cell2struct(cell(length(fields(grayModel)),1),fields(grayModel),1); 137 | end 138 | 139 | % Populate the struct 140 | grayModel(n_resolution).meanProfile = gBar; 141 | grayModel(n_resolution).covMatrix = S; 142 | grayModel(n_resolution).eVectors = eVectors; 143 | grayModel(n_resolution).eValues = eValues; 144 | grayModel(n_resolution).Info.n_images = n_images; 145 | grayModel(n_resolution).Info.imageDir = shapeModel.trainingImages; 146 | grayModel(n_resolution).Info.downsampleFactor = downsample_factor; 147 | grayModel(n_resolution).Info.rc_squaresize = rc; 148 | grayModel(n_resolution).Info.SigmoidEQ = C; 149 | grayModel(n_resolution).Info.SmoothingFilter = h_filt; 150 | grayModel(n_resolution).Info.EdgeKernel = kernel; 151 | grayModel(n_resolution).Info.interp_step_size = interp_step_sizes(n_resolution); 152 | 153 | fprintf('\nResolution scale: 1/%d. %d remaining. ',downsample_factor,numel(downsample_factors)-n_resolution) 154 | end, toc % End looping through downsampling factors 155 | 156 | % Save the model (optional) 157 | if save_model 158 | save(fullfile(save_dir,save_name),'grayModel'); 159 | end 160 | 161 | fprintf('\nAll done. Have a nice day!\n') 162 | end % End of main -------------------------------------------------------------------------------- /buildShapeModel.m: -------------------------------------------------------------------------------- 1 | function shapeModel = buildShapeModel(unalignedShapes,pathToTrainingImages) 2 | % BUILDSHAPEMODEL performs PCA on a set of aligned shapes to create an active shape 3 | % model. 4 | % 5 | % INPUT 6 | % unalignedLandmarks: Unaligned shapes, placed on training images [2*n_landmarks x n_shapes] 7 | % 8 | % OUTPUT 9 | % ShapeModel (struct) 10 | % xBar: Mean shape [2*n_landmarks x 1] 11 | % V: Eigenvectors (decreasing energy) 12 | % D: Eigenvalues (decreasing energy) 13 | % 14 | % TODO: Do I need to account for resolution somehow with the eigenvectors and values? 15 | % 16 | % John W. Miller 17 | % 25-Apr-2017 18 | 19 | % Align shapes from training images using Procrustes 20 | scaling = 0; % Almost always best to set scaling to 0 21 | x = alignShapes(unalignedShapes,scaling); 22 | 23 | % Use PCA to create model 24 | xBar = mean(x,2); % Mean shape 25 | S = cov(x'); % Covariance matrix 26 | [V,D] = eig(S); % Eigenvectors 27 | D = sort(diag(D),'descend'); 28 | V = fliplr(V); 29 | 30 | % Store model as a struct 31 | shapeModel = struct(); 32 | shapeModel.meanShape = xBar; 33 | shapeModel.eVectors = V; 34 | shapeModel.eValues = D; 35 | shapeModel.alignedShapes = x; 36 | shapeModel.unalignedShapes = unalignedShapes; 37 | if nargin > 1 38 | shapeModel.trainingImages = pathToTrainingImages; 39 | end 40 | shapeModel.n_shapes = size(x,2); 41 | 42 | end % End of main -------------------------------------------------------------------------------- /estimateFaceLocation.m: -------------------------------------------------------------------------------- 1 | function [x_aligned, f] = estimateFaceLocation(im_original,xBar,h_filt,visualize) 2 | % ESTIMATEFACELOCATION 3 | % 4 | % INPUT 5 | % 6 | % 7 | % OUTPUT 8 | % 9 | % John W. Miller 10 | % 12-Apr-2017 11 | 12 | % Filter (smooth image) 13 | if nargin < 3 14 | A = 1; 15 | sig = .3; % Need to play w/ these parameters 16 | mew = 1; 17 | x = (-3*sig+mew):0.1:(3*sig+mew); 18 | h_filt = A*exp(-((x-mew).^2)./(2*sig^2)); 19 | end 20 | im_filt = conv2(h_filt,h_filt,im_original,'same'); 21 | 22 | if nargin < 4, visualize = 1; end 23 | 24 | % Downsample 25 | n = 1; 26 | im = double(im_filt(1:n:end,1:n:end)); 27 | 28 | % Convolve with head shape 29 | hair_shape = ones(round(size(im).*[1/5 1/3])); 30 | hair_response = conv2(1./im,hair_shape,'same'); 31 | rc = 8; % rows and columns to cut off the edge off the image (Kludge) 32 | hair_response(:,1:rc) = 0; 33 | hair_response(:,end-rc:end) = 0; 34 | hair_response(1:rc,:) = 0; 35 | hair_response(end-rc:end,:) = 0; 36 | [~,idx] = max(hair_response(:)); 37 | [I,J] = ind2sub(size(im),idx); 38 | 39 | % Determine where to place mean shape 40 | I = I + round(size(hair_shape,2)/2.5); % KLUDGE (try to move from hair to eyes) 41 | 42 | [x1,y1] = deal(xBar(17),xBar(18)); 43 | [x2,y2] = deal(xBar(19),xBar(20)); 44 | [xM, yM] = deal(mean([x1 x2]),mean([y1 y2])); % Top middle of face 45 | subMean = zeros(size(xBar)); 46 | subMean(1:2:end) = xM; 47 | subMean(2:2:end) = yM; 48 | x_aligned = xBar-subMean; 49 | x_aligned(1:2:end) = x_aligned(1:2:end) + J*n; % Scale back up the detected region and shift 50 | x_aligned(2:2:end) = x_aligned(2:2:end) + I*n; 51 | 52 | % Add mean shape to image 53 | if visualize 54 | f = figure('units','normalized','outerposition',[0.1 0.1 0.9 0.9]); 55 | hold on, imshow(im_original,[],'InitialMagnification','fit') 56 | plotLandmarks(x_aligned,'show_lines',1,'hold',1); 57 | else 58 | f = []; 59 | end 60 | 61 | end % End of main -------------------------------------------------------------------------------- /findFace.m: -------------------------------------------------------------------------------- 1 | function varargout = findFace(im_original,shapeModel,grayModel,varargin) 2 | % FINDFACE uses an active shape model to locate the facial features in the supplied 3 | % image. 4 | % 5 | % INPUT 6 | % im_original: Image file, roughly 480x640. Grayscale. 7 | % shapeModel: Active shape model, comes from BUILDSHAPEMODEL 8 | % grayModel: Gray-level 2D profiles, comes from BUILDGRAYMODEL 9 | % OPTIONAL (key-value pairs) 10 | % save_video: (bool) Do you want to save a video? 11 | % visualize: (bool) Do you want to watch the iterations? 12 | % facefinder: 'click' or 'auto', method for initial face localization, default: 'click' 13 | % dist_metric: 'pca' or 'maha', method for measuring accuracy of shifted profiles, default: 'pca' 14 | % evolutions: Number of shape evolutions at each resolution level, default: 4 15 | % 16 | % OUTPUT 17 | % x_final: Final estimate for face shape position [2*n_landmarks x 1] 18 | % b: Final model weights for the PCA shape model 19 | % 20 | % John W. Miller 21 | % 25-Apr-2017 22 | 23 | % Key-value pair varargin 24 | keys = {'save_video','save_gif','visualize','facefinder','dist_metric','evolutions','vis_grid','layout'}; 25 | default_values = {0,0,1,'click','pca',4,0,'muct'}; 26 | [save_video, save_gif, vis, face_find_method, dist_metric, evolutions, vis_grid, layout] =... 27 | parseKeyValuePairs(varargin,keys,default_values); 28 | 29 | % Save a video? A GIF? 30 | if save_video || save_gif, close all, vis=1; 31 | if save_gif, videoFileName = [pwd filesep sprintf('ASM_FaceDetection_%s.gif',date)]; end 32 | end 33 | if save_video 34 | videoFileName = [pwd filesep sprintf('ASM_FaceDetection_%s',date)]; 35 | global vidObj %#ok<*TLEV> 36 | vidObj = VideoWriter(videoFileName,'MPEG-4'); 37 | vidObj.FrameRate = 50; 38 | open(vidObj); disp('Recording video...') 39 | end 40 | 41 | % Visualization stuff 42 | if vis_grid, vis=1; end; 43 | 44 | % Inverse of covariance matrix sometimes badly scaled 45 | warning('off','MATLAB:nearlySingularMatrix'); 46 | 47 | % Shape model and constraints on new shapes 48 | [~,V,D] = deal(shapeModel.meanShape,shapeModel.eVectors,shapeModel.eValues); 49 | 50 | % How many resolution levels are in the gray model? 51 | n_resolutions = length(grayModel); 52 | search_sizes = round(linspace(8,3,n_resolutions)); 53 | if ~isscalar(evolutions) 54 | evolutions = round(linspace(evolutions(1),evolutions(2),n_resolutions)); 55 | else 56 | evolutions = repmat(evolutions,n_resolutions,1); 57 | end 58 | 59 | % Perform ASM search at each resolution level 60 | for n_resolution = 1:n_resolutions 61 | GM = grayModel(n_resolution); 62 | 63 | % Smooth the image (Anti-aliasing?) 64 | h_filt = GM.Info.SmoothingFilter; 65 | im = conv2(h_filt,h_filt,im_original,'same'); 66 | 67 | % Downsample everything that needs to be downsampled 68 | downsampleFactor = GM.Info.downsampleFactor; 69 | im = imresize(im,1/downsampleFactor); % Resize image 70 | x_mean = shapeModel.meanShape./downsampleFactor; % Resize mean shape 71 | 72 | % Gray-level profiles model 73 | [g_mean, g_S, g_eVecs, g_eVals] = deal(GM.meanProfile, GM.covMatrix, GM.eVectors, GM.eValues); 74 | n_pcs = 5; 75 | if n_pcs >= size(g_eVecs{1},1) 76 | n_pcs = round(0.8*size(g_eVecs{1},1)); 77 | end 78 | 79 | % Shape model constraints 80 | P = V(:,1:n_pcs); 81 | maxb=3*sqrt(D(1:n_pcs)); 82 | 83 | % Sample a square region of pixels around each landmark 84 | rc = GM.Info.rc_squaresize; % Size of square (rc+1) 85 | n_landmarks = length(x_mean)/2; 86 | 87 | % Convolution mask (for gradient of the square regions) 88 | kernel = GM.Info.EdgeKernel; % Basically the image gradient 89 | C = GM.Info.SigmoidEQ; % Sigmoid equalization 90 | 91 | % Place mean shape over face (Can be automatic or user clicks on image) 92 | if n_resolution == 1 93 | if strcmpi(face_find_method,'auto') 94 | [x_original_estimate, h_im] = estimateFaceLocation(im,x_mean,GM.Info.SmoothingFilter,vis); 95 | else 96 | [x_original_estimate, h_im] = placeShape(im,x_mean,layout); 97 | end, tic, drawnow 98 | [x_aligned,x_current] = deal(x_original_estimate); 99 | 100 | if save_video && ishandle(h_im) 101 | for n = 1:5 102 | savevideo(h_im) 103 | end 104 | elseif save_gif && ishandle(h_im) 105 | savegif(h_im, videoFileName, 1) 106 | for n = 1:2 % Start with multiple frames at beginning of GIF 107 | savegif(h_im, videoFileName, 0) 108 | end 109 | end 110 | else 111 | resolutionScale = grayModel(n_resolution-1).Info.downsampleFactor/downsampleFactor; 112 | x_original_estimate = x_original_estimate*resolutionScale; % For final display 113 | x_current = x_current*resolutionScale; 114 | x_aligned = x_current; % Use the position determined from the lower resolution 115 | if vis, figure(h_im), hold off 116 | imshow(im,[]),plotLandmarks(x_aligned,'hold',1,'layout',layout), end 117 | end 118 | 119 | % Evolve estimate of face location, adjusting landmarks w/in model space 120 | if isscalar(evolutions), n_evolutions = evolutions; 121 | else n_evolutions = evolutions(n_resolution); end; 122 | for n_evolution = 1:n_evolutions; 123 | if vis, title(sprintf('Downsample: 1/%d. %d remaining.\nEvolution %d/%d',... 124 | downsampleFactor,n_resolutions-n_resolution,n_evolution,n_evolutions),'fontsize',FS), drawnow('expose'), end 125 | x_suggested = zeros(size(x_aligned)); 126 | 127 | for n_landmark = 1:n_landmarks 128 | %% Calculate 2D profiles near iPixel 129 | % Think of this as moving a square grid around iPixel and calculating a profile 130 | % for the grid in each shifted location 131 | search_size = search_sizes(n_resolution); % Shift amount in x,y directions (xy^2 locations in total) 132 | n_shift = 0; 133 | dist_min = []; 134 | iPixel_startingPosition = x_current(2*n_landmark+[-1 0]); % Starting position of current pixel for current evolution 135 | 136 | % Shift the 2D profile around the current landmark 137 | for c = -(search_size/2):(search_size/2) 138 | for r = -(search_size/2):(search_size/2) 139 | n_shift = n_shift+1; 140 | iPixel = iPixel_startingPosition+[r c]'; % Coordinates of current pixel 141 | if vis_grid, plot(iPixel(1),iPixel(2),'bs'), hold on, end 142 | 143 | im_region = imcrop(im,[iPixel(1)-rc/2 iPixel(2)-rc/2 rc rc]); 144 | sz = size(im_region); 145 | if any(sz < rc+1) 146 | im_region = padarray(im_region,max(rc-sz+1,0),'replicate','post'); 147 | end 148 | if sz(1) > rc+1 149 | im_region = im_region(1:rc,:); 150 | end 151 | if sz(2) > rc+1 152 | im_region = im_region(:,1:rc); 153 | end 154 | 155 | % Calculate the gradient for this region 156 | im_region_filt = conv2(im_region,kernel,'valid'); 157 | abs_sum = sum(abs(im_region_filt(:))); 158 | if abs_sum ~= 0 159 | im_region_filt = im_region_filt./abs_sum; 160 | end 161 | 162 | % Sigmoid equalization 163 | im_region_filt = im_region_filt./(abs(im_region_filt)+C); 164 | 165 | % Store the current 2D profile as a 1D vector 166 | g_new = reshape(im_region_filt,size(im_region_filt,1).^2,1); 167 | 168 | % Compute the 'distance' from the current profile to the mean profile 169 | g_bar = g_mean{n_landmark}; 170 | if strcmpi(dist_metric,'pca') 171 | % Approximate current gray profile in the gray model space 172 | g_P = g_eVecs{n_landmark}(:,1:n_pcs); eVals = g_eVals{n_landmark}(1:n_pcs); 173 | g_new_b = g_P'*(g_new-g_bar); 174 | 175 | % How well does the model approximation fit the mean profile? 176 | R = (g_new-g_bar)'*(g_new-g_bar) - g_new_b'*g_new_b; % This is actually R^2 177 | F = sum((g_new_b.^2)./eVals) + 2*(R./eVals(end)); % Not sure if the second term should be in the sum 178 | dist = F; 179 | elseif strcmpi(dist_metric,'maha') 180 | % Measure Mahalanobis distance for this shift 181 | % (This is the distance between this image's profile at the current 182 | % shift, compared to the mean profile for this landmark from the 183 | % gray-level model that has been passed into this function) 184 | md = (g_new-g_bar)'*inv(g_S{n_landmark})*(g_new-g_bar); 185 | 186 | % Should I be doing the abs(md) ? 187 | md = abs(md); 188 | dist = md; 189 | end 190 | 191 | % Keep track of best shift 192 | if isempty(dist_min) 193 | dist_min = dist; 194 | best_pixel = iPixel'; 195 | elseif dist < dist_min 196 | dist_min = dist; 197 | best_pixel = iPixel'; 198 | end 199 | end 200 | end % End shifting square grid around iPixel 201 | 202 | % TODO: Move the distance calculations outside of the loops 203 | x_suggested(2.*n_landmark+[-1 0]) = best_pixel; 204 | 205 | % Visualize & save video (optional) 206 | if vis_grid, plot(best_pixel(1),best_pixel(2),'y.'), drawnow(), end 207 | if save_video && ishandle(h_im) 208 | savevideo(h_im) 209 | end 210 | end % Looping through landmarks 211 | 212 | % Update pose parameters towards suggested shape 213 | [~,x_posed] = procrustes(x_suggested,x_current); 214 | 215 | % Deform shape towards suggested points 216 | b_suggested = P'*(x_posed-x_mean); 217 | b=max(min(b_suggested,maxb),-maxb); % Keep adjustments within model limits 218 | 219 | % Generate new shape (within model space) 220 | x_new = x_mean + P*b; 221 | 222 | % Transfer x_new to image space (for some reason we need to change the array shape) 223 | xn = [x_new(1:2:end) x_new(2:2:end)]; xs = [x_suggested(1:2:end) x_suggested(2:2:end)]; 224 | [~,xn] = procrustes(xs,xn); 225 | x_new = zeros(size(x_mean)); 226 | x_new(1:2:end) = xn(:,1); x_new(2:2:end) = xn(:,2); 227 | x_current = x_new; 228 | 229 | if vis % View the current evolution 230 | imshow(im,[]), plotLandmarks(x_current,'hold',1,'linewidth',3,'layout',layout), hold on 231 | plot(x_suggested(1:2:end),x_suggested(2:2:end),'bo','linewidth',4), drawnow() 232 | 233 | if save_video && ishandle(h_im) 234 | savevideo(h_im) 235 | elseif save_gif && ishandle(h_im) 236 | savegif(h_im, videoFileName) 237 | end 238 | end 239 | end % End looping through evolutions 240 | end % End looping through resolution levels 241 | 242 | % Scale the final shape estimation to original image size 243 | x_final = x_new*downsampleFactor; 244 | x_original_estimate = x_original_estimate*downsampleFactor; % For final display 245 | 246 | % Compare the original estimate with the final evolution 247 | if vis 248 | figure(gcf), hold off, imshow(im_original,[]) 249 | h_orig = plotLandmarks(x_original_estimate,'hold',1,'linestyle','--','layout',layout); 250 | h_final = plotLandmarks(x_final,'hold',1,'color','g','linewidth',2,'layout',layout); 251 | legend([h_orig,h_final],{'Original','Final'},'location','nw','fontsize',FS) 252 | title('Final shape','fontsize',FS) 253 | if save_video 254 | for n = 1:30 % Add extra copies of the final image 255 | savevideo(h_im) 256 | end 257 | close(vidObj), disp('Video closed.') % Close video 258 | elseif save_gif && ishandle(h_im) 259 | for n = 1:6 260 | savegif(h_im, videoFileName) 261 | end 262 | end 263 | end 264 | 265 | % Output final shape and model parameters 266 | if nargout == 1 267 | varargout{1} = x_final; 268 | elseif nargout == 2 269 | varargout{1} = x_final; 270 | varargout{2} = b; 271 | end, toc 272 | 273 | end % End of main 274 | 275 | %% -------------------------------------------------------------------- 276 | % Internal functions 277 | % -------------------------------------------------------------------- 278 | 279 | function savevideo(im_handle) 280 | % SAVEVIDEO saves a single frame to the global video object. 281 | global vidObj 282 | 283 | drawnow() 284 | try frame = getframe(im_handle); 285 | writeVideo(vidObj,frame); 286 | catch 287 | close(vidObj) 288 | end 289 | end 290 | 291 | function savegif(im_handle, filename, first_frame) 292 | % SAVEGIF saves a single frame to the GIF file. 293 | if nargin < 3 294 | first_frame = false; 295 | end 296 | 297 | drawnow() 298 | gif_im = frame2im(getframe(im_handle)); 299 | [imind,cm] = rgb2ind(gif_im,256); 300 | if first_frame 301 | imwrite(imind,cm,filename,'gif','LoopCount',inf); 302 | else 303 | imwrite(imind,cm,filename,'gif','WriteMode','append','delaytime',0.5); 304 | end 305 | end 306 | -------------------------------------------------------------------------------- /getFaceRegions.m: -------------------------------------------------------------------------------- 1 | function faceRegions = getFaceRegions(layout) 2 | % GETFACEREGIONS returns a cell array grouping the different regions of a face shape 3 | % (e.g. left eye, right eye, nose, etc.) 4 | % 5 | % OUTPUT 6 | % faceRegions: (cell) Arrays corresponding to different regions of the face 7 | % 8 | % 9 | % John W. Miller 10 | % 19-Mar-2017 11 | 12 | if nargin == 0 13 | layout = 'standard'; 14 | end 15 | 16 | switch lower(layout) 17 | case 'standard' 18 | % Connect dots around the face 19 | faceRegions = cell(7,1); 20 | faceRegions{1} = 1:3; % Left eye 21 | faceRegions{2} = 4:6; % Right eye 22 | faceRegions{3} = 7:9; % Left eyebrow 23 | faceRegions{4} = 10:12; % Right eyebrow 24 | faceRegions{5} = 13:15; % Nose 25 | faceRegions{6} = [16:19 16]; % Mouth 26 | faceRegions{7} = 20; % Chin 27 | case 'nobrows' 28 | % Connect dots around the face 29 | faceRegions = cell(9,1); 30 | faceRegions{1} = 1:3; % Left eye 31 | faceRegions{2} = 4:6; % Right eye 32 | faceRegions{3} = 7; % Left side of head 33 | faceRegions{4} = 8:9; % Left eyebrow 34 | faceRegions{5} = 10:11; % Right eyebrow 35 | faceRegions{6} = 12; % Right side of head 36 | faceRegions{7} = 13:15; 37 | faceRegions{8} = [16:19 16]; 38 | faceRegions{9} = 20; 39 | case 'muct' 40 | faceRegions{1} = 1:76; 41 | end 42 | 43 | 44 | end % End of main -------------------------------------------------------------------------------- /placeLandmarks.m: -------------------------------------------------------------------------------- 1 | function allLandmarks = placeLandmarks(pathToImages,n_landmarks,n_scans_to_label,varargin) 2 | % PLACELANDMARKS allows user to interactively place landmarks on an image. 3 | % 4 | % INPUT 5 | % pathToImages: Directory containing images (string) 6 | % n_landmarks: Number of landmarks to assign to each image 7 | % n_scans_to_label: Number of scans to label from the directory 8 | % OPTIONAL (key-value pairs) 9 | % image_idxs: 10 | % save_landmarks: 11 | % 12 | % OUTPUT 13 | % allLandmarks: Matrix containing the assigned landmarks for each image 14 | % [2*n_landmarks x n_images] 15 | % 16 | % See also PLOTLANDMARKS, ALIGNSHAPES 17 | % 18 | % John W. Miller 19 | % 14-Feb-2017 20 | 21 | % Key-value pair varargin 22 | keys = {'image_idxs','save_landmarks','file_ext'}; default_values = {[],1,'.jpg'}; 23 | [image_idxs,save_landmarks,file_ext] = parseKeyValuePairs(varargin,keys,default_values); 24 | 25 | % Get paths for every single patient scan in directory 26 | files = dir(fullfile(pathToImages,['*' file_ext])); 27 | pathsToAllImages = strcat([pathToImages filesep],{files.name}'); 28 | 29 | % Label all images in directory, unless scan indices were specified 30 | if isempty(image_idxs) 31 | imagesToLabel = pathsToAllImages; 32 | else 33 | imagesToLabel = pathsToAllImages(image_idxs); 34 | end 35 | allLandmarks = zeros(2*n_landmarks,n_scans_to_label); 36 | landmarkInfo = struct(); 37 | 38 | %% Loop through each scan, user placing landmarks on each 39 | tic 40 | for n_scan = 1:n_scans_to_label 41 | % Load and view the image 42 | im = imread(imagesToLabel{n_scan}); 43 | if n_scan==1 44 | figure(1) 45 | else 46 | figure(2) 47 | end 48 | imshow(im,[]), hold on 49 | text(0.01,0.1, sprintf('Place %d landmarks',n_landmarks),'units','normalized','color','r','fontsize',14) 50 | text(0.01,0.95,sprintf('Scan %d / %d',n_scan,n_scans_to_label),'units','normalized','color','r','fontsize',14) 51 | 52 | % Let user click on the image 53 | [x,y] = deal(zeros(n_landmarks,1)); 54 | for n = 1:n_landmarks 55 | [x(n),y(n)] = ginput(1); 56 | plot(x(n),y(n),'rs','markerfacecolor','r','linewidth',3) 57 | text(x(n),y(n),sprintf('%d',n),'color','g','fontsize',18) 58 | end 59 | 60 | % Store the coordinates in the landmarks vector 61 | landmarks = zeros(2*n_landmarks,1); 62 | landmarks(1:2:end,:) = x; 63 | landmarks(2:2:end,:) = y; 64 | allLandmarks(:,n_scan) = landmarks; 65 | 66 | % Store info about the landmarks and image 67 | landmarkInfo(n_scan).path = imagesToLabel{n_scan}; 68 | landmarkInfo(n_scan).landmarks = landmarks; 69 | pause(0.3) 70 | end 71 | close, toc 72 | 73 | % Save the landmarks? 74 | if save_landmarks 75 | save_name = ['landmarks_' input('Save name? landmarks_','s') '.mat']; 76 | save(save_name,'allLandmarks','image_idxs','landmarkInfo'); 77 | end 78 | 79 | end % End of main 80 | -------------------------------------------------------------------------------- /placeShape.m: -------------------------------------------------------------------------------- 1 | function [x_aligned, f] = placeShape(im,x,layout) 2 | % PLACESHAPE 3 | % 4 | % INPUT 5 | % 6 | % 7 | % 8 | % OUTPUT 9 | % 10 | % 11 | % John W. Miller 12 | % 21-Apr-2017 13 | 14 | if nargin < 3 15 | layout = 'standard'; 16 | end 17 | 18 | % View the image 19 | % f = figure('units','normalized','outerposition',[0.1 0.1 0.9 0.9]); 20 | f = figure('units','normalized','outerposition',[.25 0.4 .3 .55]); 21 | hold on, imshow(im,[],'InitialMagnification','fit') 22 | text(0.07,0.95,'Click on center of nose. Or close by. Test your luck.','fontsize',FS,'units','normalized','color','r') 23 | 24 | % Get input from user 25 | [I,J] = ginput(1); 26 | 27 | % Center shape on user's point 28 | x_aligned = x; 29 | 30 | switch lower(layout) 31 | case 'standard' 32 | x_aligned(1:2:end) = x_aligned(1:2:end)-x(27); % Center on middle of nose 33 | x_aligned(2:2:end) = x_aligned(2:2:end)-x(28); 34 | case 'muct' 35 | x_aligned(1:2:end) = x_aligned(1:2:end)-x(135); % Center on middle of nose 36 | x_aligned(2:2:end) = x_aligned(2:2:end)-x(136); 37 | end 38 | x_aligned(1:2:end) = x_aligned(1:2:end)+I; 39 | x_aligned(2:2:end) = x_aligned(2:2:end)+J; 40 | 41 | % Display centered shape 42 | plotLandmarks(x_aligned,'show_lines',1,'hold',1,'layout',layout) 43 | 44 | % Varargout 45 | if nargout == 2 % User wants the figure handle 46 | return 47 | else 48 | pause(1), close(f) 49 | end 50 | 51 | end % End of main --------------------------------------------------------------------------------