├── MEXTest.mp4 ├── GenerateVidFrames.m ├── README.md ├── oflo.m ├── faceDetector.m ├── opticalFlow.m └── mainScript.m /MEXTest.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kamilste/Micro-expression-Detection-System/HEAD/MEXTest.mp4 -------------------------------------------------------------------------------- /GenerateVidFrames.m: -------------------------------------------------------------------------------- 1 | %Author: Kamil Stepien 2 | %Date: 08 April 2016 3 | 4 | fd=VideoReader('normalPro.avi'); %read Video 5 | for img = 1:fd.NumberOfFrames; %get all frames in video 6 | filename=strcat('frames', num2str(img), '.jpg'); %number of to string 7 | fdr = read(fd, img); %read the frames 8 | imwrite(fdr, filename); %write or save the frame 9 | end %end loop -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Micro-expression-Detection-System 2 | 3 | Author - Kamil Stepien 4 | 5 | Date – 21 April 2016 6 | Version – 10.0 7 | 8 | Notes 9 | - Before running this code make sure that you have either have MatLab R2015 or MatLab R2016 installed on your machine. 10 | - You will also need to have Image Processing Toolbox and Computer Vision System Toolbox installed. 11 | 12 | How to run 13 | 1.Open ‘mainScript.m’ 14 | 2.Specify the video and its type in line 9. 15 | 3.Press F5 (make sure the cursor is placed in editor window) or press Run within the program. 16 | 17 | For additional files 18 | - All the processes specified in “how to run” apply to other scripts such as ‘faceDetectror.m’ and ‘opticalFlow.m’. 19 | -------------------------------------------------------------------------------- /oflo.m: -------------------------------------------------------------------------------- 1 | %Author: Kamil Stepien 2 | %Date: 08 April 2016 3 | 4 | %Function to generate the coordinate points of optical flow lines. 5 | function valLines = oflo(valLines, motionVecGain) 6 | 7 | %Initialize variables used in plotting motion vectors. 8 | firstTime = true; 9 | borderOffset = 15; 10 | decimFactorRow = 5; 11 | decimFactorCol = 5; 12 | 13 | %Generate coordinates for plotting motion vectors. 14 | if firstTime 15 | [R C] = size(valLines); %height and width in pixels 16 | RV = borderOffset:decimFactorRow:(R-borderOffset); 17 | CV = borderOffset:decimFactorCol:(C-borderOffset); 18 | [Y X] = meshgrid(CV,RV); 19 | firstTime = false; 20 | end 21 | 22 | %Calculate and draw the motion vectors. 23 | tmp = valLines(RV,CV) .* motionVecGain; 24 | valLines = [Y(:), X(:), Y(:) + real(tmp(:)), X(:) + imag(tmp(:))]; -------------------------------------------------------------------------------- /faceDetector.m: -------------------------------------------------------------------------------- 1 | %Author: Kamil Stepien 2 | %Date: 08 April 2016 3 | 4 | close all %close all windows 5 | clear %clear all variables 6 | clc %clear command window 7 | 8 | file = 'MEXTest.mp4'; %define file to be processed 9 | %Create video object with some properties, 10 | %read video frames from video file 11 | vid = vision.VideoFileReader(file,'ImageColorSpace','RGB',... 12 | 'VideoOutputDataType', 'single'); 13 | 14 | %Create Face Detectors 15 | detectFace = vision.CascadeObjectDetector(); 16 | 17 | figH = figure; %assign figure to a variable 18 | 19 | while ~isDone(vid) %run video until done 20 | colorFrame = step(vid); %set individual colour frames 21 | colorFrameRes = imresize(colorFrame,0.3); %resize colour frame for faster computation 22 | 23 | sharpColorFrame = imsharpen(colorFrameRes); %sharpen the colour face frame 24 | adjustColorFrame = imadjust(sharpColorFrame,[.1 .1 0; .6 .7 1],[]); %adjust frame intensity values/colours 25 | 26 | faceBbox = step(detectFace,adjustColorFrame); %create bounding box(bbox) around the face 27 | addfbbox = insertObjectAnnotation(adjustColorFrame,'rectangle',... 28 | faceBbox,'Face','Color','Blue','TextColor','White',... 29 | 'LineWidth',2); %add bbox to the frame 30 | imshow(addfbbox,'border','tight');... 31 | title('Detected Face'); %show the above 32 | 33 | if ~ishghandle(figH) %if the program is closed 34 | close all %close all windows 35 | break %stop the process of the system 36 | end %end of the statement 37 | end %end while loop 38 | release(vid); %release video -------------------------------------------------------------------------------- /opticalFlow.m: -------------------------------------------------------------------------------- 1 | %Author: Kamil Stepien 2 | %Date: 08 April 2016 3 | 4 | close all %close all windows 5 | clear %clear all variables 6 | clc %clear command window 7 | 8 | file = 'vid41.avi'; %define file to be processed 9 | 10 | %Create video object with some properties, 11 | %read video frames from video file 12 | vid = vision.VideoFileReader(file,'ImageColorSpace','RGB',... 13 | 'VideoOutputDataType', 'single'); 14 | 15 | %Estimate object velocities / Optical Flow 16 | %This need to be changed for future releases of Matlab 17 | optFlo2 = vision.OpticalFlow('OutputValue',... 18 | 'Horizontal and vertical components in complex form',... 19 | 'ReferenceFrameDelay',1, 'DiscardIllConditionedEstimates',true); 20 | 21 | %Draw rectangles, lines, polygons, or circles on an image 22 | %Create shape inserters for lines on optical flow 23 | %This need to be changed for future releases of Matlab 24 | shapeInsert = vision.ShapeInserter('Shape','Lines',... 25 | 'BorderColor','Custom','CustomBorderColor',[255 255 0]); 26 | 27 | numFrames = 0; %initial number of frames 28 | frameList = {}; %initial array of frame list 29 | hasLines = zeros(10, 1, 'uint8'); %create array of all zeros 30 | 31 | figH = figure; %assign figure to a variable 32 | 33 | while ~isDone(vid) %run video until done 34 | colorFrame = step(vid); %set individual colour frames 35 | colorFrameRes = imresize(colorFrame,0.3); %resize colour frame for faster computation 36 | grayFrame = rgb2gray(colorFrameRes); %convert resized colour frame to grey 37 | 38 | optFloVectors = step(optFlo2, grayFrame); %set optical flow on grey frame 39 | 40 | lines = oflo(optFloVectors,50); %use the external function to draw lines for optical flow 41 | motionVectors = step(shapeInsert, colorFrameRes, lines); %set the lines of optical flow on the main colour frame 42 | imshow(motionVectors); title('Optical Flow on Frame'); %display optical flow on colour frame 43 | 44 | %hasPoints stores a flag equal to ~isempty(points) for each of the 45 | %last 10 frames. If all(hasPoints) is true, than all of the last 10 46 | %frames detected a feature 47 | notEmpty = ~isempty(lines); 48 | if numel(notEmpty) ~= 1, notEmpty = 1; end 49 | hasLines = [hasLines(2:end); notEmpty]; 50 | numFrames = numFrames + 1; 51 | 52 | %Stashing the frames, a circular buffer to track last 10 frames 53 | if numFrames >= 10 54 | frameList = [frameList(2:end) colorFrame]; 55 | else 56 | frameList = [frameList colorFrame]; 57 | end 58 | 59 | if numFrames >= 10 && all(hasLines) %if all frames had lines 60 | disp('Micro-Expression Detected') %micro-expression detected 61 | else %if not 62 | disp('Not detected') %display not detected 63 | end %end of the statement 64 | 65 | if ~ishghandle(figH) %if the program is closed 66 | close all %close all windows 67 | break %stop the process of the system 68 | end %end of the statement 69 | end %end while loop 70 | release(vid); %release video -------------------------------------------------------------------------------- /mainScript.m: -------------------------------------------------------------------------------- 1 | %Author: Kamil Stepien 2 | %Date: 08 April 2016 3 | 4 | close all %close all windows 5 | clear %clear all variables 6 | clc %clear command window 7 | tic; %start measuring time 8 | 9 | file = 'MEXTest.mp4'; %define file to be processed 10 | 11 | %Create video object with some properties, 12 | %read video frames from video file 13 | vid = vision.VideoFileReader(file,'ImageColorSpace','RGB',... 14 | 'VideoOutputDataType','single'); 15 | 16 | %Create object Detectors(Eyes,Nose,Mouth) 17 | detectFace = vision.CascadeObjectDetector(); 18 | 19 | %Create object Detectors(Eyes,Nose,Mouth) 20 | detectEyes = vision.CascadeObjectDetector('EyePairBig'); 21 | detectNose = vision.CascadeObjectDetector('Nose','MergeThreshold',100); 22 | detectMouth = vision.CascadeObjectDetector('Mouth','MergeThreshold',200); 23 | 24 | %Create face detector properties 25 | detectFace.MaxSize = [220 220]; 26 | detectFace.MinSize = [120 120]; 27 | detectFace.ScaleFactor = size(detectFace)/(size(detectFace)-0.01); 28 | detectFace.MergeThreshold = 100; %munimises multple faces detected 29 | 30 | %Estimate object velocities / Optical Flow 31 | %This need to be changed for future releases of Matlab 32 | optFlo = vision.OpticalFlow('OutputValue',... 33 | 'Horizontal and vertical components in complex form',... 34 | 'ReferenceFrameDelay',3); 35 | optFlo2 = vision.OpticalFlow('OutputValue',... 36 | 'Horizontal and vertical components in complex form',... 37 | 'ReferenceFrameDelay',3); 38 | 39 | %Draw rectangles, lines, polygons, or circles on an image 40 | %Create shape inserters for lines on optical flow 41 | %This need to be changed for future releases of Matlab 42 | shapeInsert = vision.ShapeInserter('Shape','Lines',... 43 | 'BorderColor','Custom','CustomBorderColor',[255 255 0]); 44 | 45 | %Find mean value of input or sequence of inputs 46 | mean1 = vision.Mean; 47 | mean2 = vision.Mean('RunningMean',true); 48 | %2D median filtering 49 | medianFilter = vision.MedianFilter; 50 | %Perform morphological closing on image 51 | morphClose = vision.MorphologicalClose('Neighborhood',strel('line',5,45)); 52 | %Perform morphological erosion on an image 53 | morphErode = vision.MorphologicalErode('Neighborhood',strel('square',2)); 54 | 55 | numFrames = 0; %initial number of frames 56 | frameList = {}; %initial array of frame list 57 | hasLines = zeros(10, 1, 'uint8'); %create array of all zeros 58 | 59 | figH = figure; %assign figure to a variable 60 | 61 | while ~isDone(vid) %run video until done 62 | colorFrame = step(vid); %set individual colour frames 63 | colorFrameRes = imresize(colorFrame,0.3); %resize colour frame for faster computation 64 | grayFrame = rgb2gray(colorFrameRes); %convert resized colour frame to grey 65 | 66 | faceBbox = step(detectFace,colorFrameRes); %create bounding box(bbox) around the face 67 | addfbbox = insertObjectAnnotation(colorFrameRes,'rectangle',... 68 | faceBbox,'Face', 'Color','Blue','TextColor','White',... 69 | 'LineWidth',2); %add bbox to the frame 70 | subplot(3,3,1); imshow(addfbbox,'border','tight');... 71 | title('Detected Face'); %show the above 72 | 73 | if size(faceBbox,1) == 0 %if no face detected 74 | disp('Face not Detected'); %display this 75 | else if size(faceBbox,1) > 1 %if more than 1 face detected 76 | disp('Too many faced detected'); %display this 77 | else %for 1 face progress with the code 78 | colorFace = imcrop(colorFrameRes,faceBbox); %crop colour frame and face 79 | grayFace = imcrop(grayFrame,faceBbox); %crop grey frame and face 80 | sharpGrayFace = imsharpen(grayFace); %sharpen the grey frame 81 | sharpColorFace = imsharpen(colorFace); %sharpen the colour face frame 82 | adjustColorFace = imadjust(sharpColorFace,... 83 | [.1 .1 0; .6 .7 1],[]); %adjust frame intensity values/colours 84 | subplot(3,3,2); imshow(adjustColorFace);... 85 | title('Face cropped & enhanced'); %show enhanced colour image 86 | 87 | featuresDetected = detectHarrisFeatures(sharpGrayFace); %detect Harris feature points 88 | subplot(3,3,3); imshow(sharpGrayFace), hold on,... 89 | plot(featuresDetected); title('Harris Feature Points'); %display Harris features 90 | 91 | eyesBbox = step(detectEyes,colorFace); %create bounding box around the eyes 92 | noseBbox = step(detectNose,colorFace); %create bounding box around the nose 93 | mouthBbox = step(detectMouth,colorFace); %create bounding box around the mouth 94 | 95 | if size(eyesBbox,1) == 0 %if no eyes detected 96 | disp('Eyes not detected'); %display this 97 | else if size(eyesBbox,1) > 1 %if more than 1 pair of eyes detected 98 | disp('Too many pair of eyes detected'); %display this 99 | else %for 1 pair of eyes progress with the code 100 | eyes = imcrop(adjustColorFace,eyesBbox); %crop the frame with eyes detected 101 | subplot(3,3,4); imshow(eyes); title('Detected Eyes'); %show pair of eyes 102 | end %end of the statement 103 | end %end of the statement 104 | 105 | if size(noseBbox,1) == 0 %if no nose detected 106 | disp('Nose not detected'); %display this 107 | else if size(eyesBbox,1) > 1 %if more than 1 nose detected 108 | disp('Too many noses detected'); %display this 109 | else %for 1 nose progress with the code 110 | nose = imcrop(adjustColorFace,noseBbox); %crop the frame with nose detected 111 | subplot(3,3,5); imshow(nose); title('Detected Nose'); %show nose 112 | end %end of the statement 113 | end %end of the statement 114 | 115 | if size(mouthBbox,1) == 0 %if no mouth detected 116 | disp('Mouth not detected'); %display this 117 | else if size(mouthBbox,1) > 1 %if more than 1 mouth detected 118 | disp('Too many mouths detected'); %display this 119 | else %for 1 mouth progress with the code 120 | mouth = imcrop(adjustColorFace,mouthBbox); %crop the frame with mouth detected 121 | subplot(3,3,6); imshow(mouth);... 122 | title('Detected Mouth'); %show mouth 123 | end %end of the statement 124 | end %end of the statement 125 | 126 | optFloVect = step(optFlo,sharpGrayFace); %set optical flow on converted face frame 127 | lines = oflo(optFloVect,2); %use the external function to draw lines for optical flow 128 | motionVect = step(shapeInsert,colorFace,lines); %set the lines of optical flow on the face frame 129 | subplot(3,3,7); imshow(motionVect);... 130 | title('Optical Flow on Face'); %display optical flow on face 131 | end %end of the statement 132 | end %end of the statement 133 | 134 | release(optFlo); %release optical flow for next frame 135 | optFloVectors = step(optFlo2, grayFrame); %set optical flow on grey frame 136 | lines2 = oflo(optFloVectors,20); %use the external function to draw lines for optical flow 137 | motionVectors = step(shapeInsert, colorFrameRes, lines2); %set the lines of optical flow on the main colour frame 138 | subplot(3,3,8); imshow(motionVectors); title('Optical Flow on Frame'); %display optical flow on colour frame 139 | 140 | %The optical flow vectors are stored as complex numbers. 141 | %Compute their magnitude squared which will be used for thresholding. 142 | magnitudeSqr = optFloVectors .* conj(optFloVectors); 143 | velocityThreshold = 0.6 * step(mean2, step(mean1, magnitudeSqr)); 144 | %Threshold the image and then filter it to remove speckle noise. 145 | segObj = step(medianFilter, magnitudeSqr >= velocityThreshold); 146 | %Thin-out the parts of the face and fill holes in the blobs. 147 | segObj = step(morphClose, step(morphErode, segObj)); 148 | %show blob analysis based on optical flow 149 | subplot(3,3,9); imshow(segObj); title('Blob Analysis') 150 | 151 | %hasPoints stores a flag equal to ~isempty(points) for each of the 152 | %last 10 frames. If all(hasPoints) is true, than all of the last 10 153 | %frames detected a feature 154 | notEmpty = ~isempty(lines2); 155 | if numel(notEmpty) ~= 1, notEmpty = 1; end 156 | hasLines = [hasLines(2:end); notEmpty]; 157 | numFrames = numFrames + 1; 158 | 159 | %Stashing the frames, a circular buffer to track last 10 frames 160 | if numFrames >= 10 161 | frameList = [frameList(2:end) colorFrame]; 162 | else 163 | frameList = [frameList colorFrame]; 164 | end 165 | 166 | if numFrames >= 10 && all(hasLines) %if all frames had lines 167 | disp('Micro-Expression Detected') %micro-expression detected 168 | else %if not 169 | disp('Not detected') %display not detected 170 | end %end of the statement 171 | 172 | if ~ishghandle(figH) %if the program is closed 173 | close all %close all windows 174 | break %stop the process of the system 175 | end %end of the statement 176 | 177 | end %end while loop 178 | release(vid); %release video 179 | toc; %stop counting time --------------------------------------------------------------------------------