├── .gitignore ├── CMakeLists.txt ├── README.md ├── assets └── .gitkeep ├── build └── .gitkeep ├── doc ├── .gitignore └── doxyconfig ├── report ├── .gitignore ├── approach.tex ├── architecture.tex ├── bib │ └── bib.bib ├── goal.tex ├── images │ ├── db_anger.png │ ├── db_contempt.png │ ├── db_fear.png │ ├── db_surprise.png │ ├── exampl_happy2.png │ ├── example_anger.png │ ├── example_happy1.png │ ├── example_happy3.png │ ├── example_sad.png │ ├── example_surprise.png │ ├── gabor.png │ └── haarfeatures.png ├── main.tex ├── organization.tex ├── parameters.tex ├── results.tex └── workflow.tex ├── resources ├── haarcascade_eye.xml ├── haarcascade_frontalface_alt2.xml ├── haarcascade_frontalface_cbcl1.xml └── haarcascade_frontalface_default.xml ├── src ├── CMakeLists.txt ├── dataset │ ├── CMakeLists.txt │ ├── cleandataset.sh │ ├── datasetConfigParser.py │ ├── datasetCropFaces.py │ ├── datasetFeatures.py │ ├── datasetFillCK.py │ ├── datasetInit.py │ ├── datasetPrepTrain.py │ ├── datasetTrain.py │ ├── datasetVerifyPrediction.py │ ├── example_dataset.cfg │ ├── gui.py │ ├── train_models.py │ └── train_models.sh ├── detector │ ├── BoostEmoDetector.cpp │ ├── BoostEmoDetector.h │ ├── CMakeLists.txt │ ├── EmoDetector.cpp │ ├── EmoDetector.h │ ├── FacePreProcessor.cpp │ ├── FacePreProcessor.h │ ├── SVMEmoDetector.cpp │ ├── SVMEmoDetector.h │ └── emo_detector_cli.cpp ├── facedetector │ ├── CMakeLists.txt │ ├── FaceDetector.cpp │ ├── FaceDetector.h │ └── facecrop_cli.cpp ├── gaborbank │ ├── CMakeLists.txt │ ├── GaborBank.cpp │ ├── GaborBank.h │ ├── GaborKernel.h │ └── gaborbank_cli.cpp ├── gui │ ├── ACapture.cpp │ ├── ACapture.h │ ├── AGui.cpp │ ├── AGui.h │ ├── CMakeLists.txt │ ├── DebugGui.cpp │ ├── DebugGui.h │ ├── EmotimeGui.cpp │ ├── EmotimeGui.h │ ├── GaborGui.hpp │ ├── ImageCapture.cpp │ ├── ImageCapture.h │ ├── VideoCapture.cpp │ ├── VideoCapture.h │ ├── WebcamCapture.h │ ├── emotimegui_cli.cpp │ ├── emotimevideo_cli.cpp │ ├── emotimevideodebug_cli.cpp │ └── gaborgui_cli.cpp ├── training │ ├── AdaBoostClassifier.cpp │ ├── AdaBoostClassifier.h │ ├── CMakeLists.txt │ ├── Classifier.cpp │ ├── Classifier.h │ ├── SVMClassifier.cpp │ ├── SVMClassifier.h │ ├── TrainingParameters.h │ └── train_cli.cpp └── utils │ ├── CMakeLists.txt │ ├── matrix_io.cpp │ ├── matrix_io.h │ ├── string_utils.cpp │ └── string_utils.h └── test ├── adaboost_train_error.sh ├── boost_emodetector.sh ├── clahe.py ├── face_test.py ├── gabor.py ├── gaborpng.sh ├── test_facerotation.py └── view.py /.gitignore: -------------------------------------------------------------------------------- 1 | # CMake 2 | CMakeCache.txt 3 | CMakeFiles 4 | Makefile 5 | cmake_install.cmake 6 | install_manifest.txt 7 | 8 | # IDE 9 | *.sublime-project 10 | *.sublime-workspace 11 | *.swp 12 | *.swo 13 | *~ 14 | .ycm_* 15 | 16 | # Latex 17 | report/*.aux 18 | report/*.log 19 | report/*.out 20 | report/*.pdf 21 | report/*.bbl 22 | report/*.blg 23 | 24 | # folders 25 | build/** 26 | dataset/** 27 | assets/** 28 | papers/ 29 | 30 | # Compiled 31 | *.o 32 | *.so 33 | *.pyc 34 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | project(emotime) 4 | 5 | find_package(OpenCV REQUIRED) 6 | include_directories(${OpenCV_INCLUDE_DIRS}) 7 | 8 | set(ASSETDIR "${emotime_SOURCE_DIR}/assets" ) 9 | set(HEADERDIR "${emotime_SOURCE_DIR}/include" ) 10 | set(SRCDIR "${emotime_SOURCE_DIR}/src" ) 11 | 12 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" ) 13 | 14 | add_subdirectory(src) 15 | 16 | 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Emotime 2 | ======= 3 | 4 | _Recognizing emotional states in faces_ 5 | 6 | ---------------------------------------------- 7 | 8 | Authors: Luca Mella, Daniele Bellavista 9 | 10 | Contributors: Rohit Krishnan 11 | 12 | Development Status: Experimental 13 | 14 | Copyleft: [CC-BY-NC 2013](http://creativecommons.org/licenses/by-nc/3.0/) 15 | 16 | ---------------------------------------------- 17 | 18 | ## Goal 19 | This project aims to recognize main facial expressions (neutral, anger, disgust, fear, joy, sadness, surprise) in image 20 | sequences using the approaches described in: 21 | 22 | * [Dynamics of Facial Expression Extracted Automatically from Video](http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=1384873) 23 | * [Fully Automatic Facial Action Recognition in Spontaneous Behavior](http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=1613024) 24 | 25 | ## References 26 | 27 | Here is listed some interesting material about machine learning, opencv, gabor transforms and other 28 | stuff that could be useful to get in this topic: 29 | 30 | * [AdaBoost and the Super Bowl of Classifiers](http://www.inf.fu-berlin.de/inst/ag-ki/rojas_home/documents/tutorials/adaboost4.pdf) 31 | * [Tutorial on Gabor Filters](http://mplab.ucsd.edu/tutorials/gabor.pdf) 32 | * [Gabor wavelet transform and its application](http://disp.ee.ntu.edu.tw/~pujols/Gabor%20wavelet%20transform%20and%20its%20application.pdf) 33 | * [Gabor Filter Visualization](http://www.cs.umd.edu/class/spring2005/cmsc838s/assignment-projects/gabor-filter-visualization/report.pdf) 34 | * [Meta-Analyis of the First Facial Expression Recognition Challenge](http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6222016) 35 | 36 | ## Project Structure 37 | 38 | src 39 | \-->dataset Scripts for dataset management 40 | \-->facecrop Utilities and modules for face cropping and registration 41 | \-->gaborbank Utilities and modules for generating gabor filters and image filtering 42 | \-->adaboost Utilities and modules for adaboost train, prediction, and feature selection 43 | \-->svm Utilities and modules for svm training and prediction 44 | \-->detector Multiclass detector and preprocessor 45 | \-->utils String and IO utilities, CSV supports, and so on.. 46 | doc Documentation (doxigen) 47 | report Class project report (latex) 48 | resources Containing third party resources (eg. OpenCV haar classifiers) 49 | assets Binary folder (I know, I know, it is not beautiful) 50 | test Some testing scripts here 51 | 52 | ## Build 53 | 54 | Dependencies: 55 | 56 | * `CMake >= 2.8` 57 | * `Python >= 2.7, < 3.0` 58 | * `OpenCV >= 2.4.5` 59 | 60 | Compiling on linux: 61 | 62 | * `mkdir build` 63 | * `cd build` 64 | * `cmake .. ; make ; make install` - now the `asset` folder should be populated 65 | 66 | Cross-compiling for windows: 67 | 68 | * Using CMake or CMakeGUI, select emotime as source folder and configure. 69 | * If it complains about setting the variable `OpenCV_DIR` set it to the appropriate path so that: 70 | - C:/path/to/opencv/dir/ contains the libraries (`*.lib`) 71 | - C:/path/to/opencv/dir/include contains the include directories (opencv and opencv2) 72 | - **IF the include directory is missing** the project will likely not be able 73 | to compile due to missing reference to `opencv2/opencv` or similar. 74 | * Then generate the project and compile it. 75 | * This was tested with Visual Studio 12 64 bit. 76 | 77 | ## Detection and Prediction 78 | 79 | Proof of concept model trained using faces extracted using the detector `cbcl1` are available for download, mulclass strategy ~~[1 vs all](https://dl.dropboxusercontent.com/u/7618747/dataset_svm_354_cbcl1_1vsall.zip) and [many vs many](https://dl.dropboxusercontent.com/u/7618747/dataset_svm_354_cbcl1_1vsallext.zip)~~. 80 | 81 | __NOTE: Trained models for latest version of the code are available in the ~~[v1.2 release page](https://github.com/luca-m/emotime/releases/tag/v1.2-experimental)~~ (deprecated). Other trained model working better with master branch are available [here](https://copy.com/c4rOYmPgAYrdxdoS)__ 82 | 83 | _NOTE: watch for illumination! At the moment optimal results can be obtained in live webcam sessions using direct illumination directed to the user's face. Don't worry you are not required to blind you with a headlight._ 84 | 85 | _If you'd like to try emotime without any further complication you should take a look to the [x86_64 release](https://github.com/luca-m/emotime/releases/tag/v1.1-experimental). ( __obsolete__ ) _ 86 | 87 | 88 | ### Usage 89 | 90 | Video gui: 91 | 92 | echo "VIDEOPATH" | ./emotimevideo_cli FACEDETECTORXML (EYEDETECTORXML|none) WIDTH HEIGHT NWIDTHS NLAMBDAS NTHETAS (svm|ada) (TRAINEDCLASSIFIERSXML)+ 93 | 94 | Cam gui: 95 | 96 | ./emotimegui_cli FACEDETECTORXML (EYEDETECTORXML|none) WIDTH HEIGHT NWIDTHS NLAMBDAS NTHETAS (svm|ada) (TRAINEDCLASSIFIERSXML)+ 97 | 98 | Or using the python script: 99 | 100 | python gui.py --cfg --mode svm --eye-correction 101 | 102 | #### Binary Release and Trained Models 103 | 104 | If you just want to take a quick look to the project we strongly suggest to go to the [release section](https://github.com/luca-m/emotime/releases) and download compiled binaries for _Linux 64bit_, then: 105 | 106 | * download and unzip the binaries in an empty folder 107 | * run `./download_trained_models.sh` 108 | * Then cd assets and `./emotimegui_cli ../resources/haarcascade_frontalface_cbcl1.xml none 48 48 3 5 4 svm ../dataset_svm_354_cbcl1_1vsallext/classifiers/svm/*` 109 | 110 | 111 | ## Training 112 | 113 | After `mkdir build; cd build; cmake ..; make ; make install` go to the `assets` folder and: 114 | 115 | 1. Initialize a dataset using: 116 | 117 | python datasetInit.py -cfg 118 | 119 | 2. Then fill it with your images or use the Cohn-Kanade importing script: 120 | 121 | python datasetFillCK --cfg 122 | 123 | 3. Now you are ready to train models: 124 | 125 | python train_models.py --cfg --mode svm --prep-train-mode [1vsall|1vsallext] 126 | 127 | 128 | ### Dataset 129 | 130 | The [Cohn-Kanade database](http://www.consortium.ri.cmu.edu/ckagree/) is one of the most used faces database. Its extended version (CK+) contains also [FACS](http://en.wikipedia.org/wiki/Facial_Action_Coding_System) 131 | code labels (aka Action Units) and emotion labels (neutral, anger, contempt, disgust, fear, happy, sadness, surprise). 132 | 133 | ## ~~Validation~~ (old, check v1.2 release page) 134 | 135 | _First, rough evaluation of the performance of the system_ 136 | Validation test involved the whole system `face detector + emotion classifier`, so should not be considered relative to the _emotion classifier_ itself. 137 | 138 | Of course, a more fine validation shuld be tackled in order to evaluate emotion classifier alone. 139 | For the sake of completeness the reader have to know that the _cbcl1_ face model is a good face locator rather than detector, roughly speaking it detects less but is more precise. 140 | 141 | Following results are commented with my personal - totally informal - evaluation after live webcam session. 142 | 143 | ```text 144 | multicalss method: 1vsAllExt 145 | face detector: cbcl1 146 | eye correction: no 147 | width: 48 148 | height: 48 149 | nwidths: 3 150 | nlambdas: 5 151 | nthetas: 4 152 | 153 | Sadness <-- Not good in live webcam sessions too 154 | sadness -> 0.67% 155 | surprise -> 0.17% 156 | anger -> 0.17% 157 | Neutral <-- Good in live webcam sessions 158 | neutral -> 0.90% 159 | contempt -> 0.03% 160 | anger -> 0.03% 161 | fear -> 0.02% 162 | surprise -> 0.01% 163 | Disgust <-- Good in live webcam sessions 164 | disgust -> 1.00% 165 | Anger <-- Good in live webcam sessions 166 | anger -> 0.45% 167 | neutral -> 0.36% 168 | disgust -> 0.09% 169 | contempt -> 0.09% 170 | Surprise <-- Good in live webcam sessions 171 | surprise -> 0.94% 172 | neutral -> 0.06% 173 | Fear <-- Almost Good in live webcam sessions 174 | fear -> 0.67% 175 | surprise -> 0.17% 176 | happy -> 0.17% 177 | Contempt <-- Not good in live webcam sessions 178 | neutral -> 0.50% 179 | contempt -> 0.25% 180 | anger -> 0.25% 181 | Happy <-- Good in live webcam sessions 182 | happy -> 1.00% 183 | ``` 184 | 185 | ```text 186 | multicalss method: 1vsAll 187 | face detector: cbcl1 188 | eye correction: no 189 | width: 48 190 | height: 48 191 | nwidths: 3 192 | nlambdas: 5 193 | nthetas: 4 194 | 195 | Sadness <-- Not good in live webcam sessions too 196 | unknown -> 0.50% 197 | sadness -> 0.33% 198 | fear -> 0.17% 199 | Neutral <-- Good in live webcam sessions 200 | neutral -> 0.73% 201 | unknown -> 0.24% 202 | surprise -> 0.01% 203 | fear -> 0.01% 204 | contempt -> 0.01% 205 | Disgust <-- Good in live webcam sessions 206 | disgust -> 0.82% 207 | unknown -> 0.18% 208 | Anger <-- Almost sufficient in live webcam sessions 209 | anger -> 0.36% 210 | neutral -> 0.27% 211 | unknown -> 0.18% 212 | disgust -> 0.09% 213 | contempt -> 0.09% 214 | Surprise <-- Good in live webcam sessions 215 | surprise -> 0.94% 216 | neutral -> 0.06% 217 | Fear <-- Sufficient in live webcam sessions 218 | fear -> 0.67% 219 | surprise -> 0.17% 220 | happy -> 0.17% 221 | Contempt <-- Not good in live webcam sessions too 222 | unknown -> 1.00% 223 | Happy <-- Good in live webcam sessions 224 | happy -> 1.00% 225 | ``` 226 | 227 | Also main difference between the _1vsAll_ and the _1vsAllExt_ mode experimented in livecam sessions are related to the amount of unknown states registered and the stability of the detected states. 228 | In detail 1vsAll multiclass method provide more less noisy detections during a live web-cam session, 1vsAllExt mode instead is able to always predict a valid state for each frame processed, but sometimes it result to be more unstable during the expression transition. 229 | 230 | 231 | Sorry for the lack of fine tuning and detail, but it is a spare time project at the moment.. 232 | If you have any idea or suggestion feel free to write us! 233 | 234 | 235 | ## Further Development 236 | 237 | * Tuning GaborBank parameters for accuracy enhancement. 238 | * Tuning image sizes for better real-time performance. 239 | * Better handle illumination, detections are good when frontal light is in place (keep it in mind when you use it with your camera). 240 | -------------------------------------------------------------------------------- /assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/assets/.gitkeep -------------------------------------------------------------------------------- /build/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/build/.gitkeep -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | html/ 2 | latex/ 3 | -------------------------------------------------------------------------------- /report/.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | main.tex.latexmain 3 | main.pdf 4 | -------------------------------------------------------------------------------- /report/architecture.tex: -------------------------------------------------------------------------------- 1 | \section{Project Architecture} 2 | 3 | The project is composed of the following entities: 4 | 5 | \begin{description} 6 | \item[FaceDetector:] a utility class that extracts faces from images. 7 | \item[GaborBank:] generator of features using Gabor filters. 8 | \item[Classifier:] generic 2-class classifier, which can be 9 | trained to predict a feature matrix. 10 | \item[EmoDetector:] using a set of classifiers, it realizes multi-class 11 | detection for emotions. 12 | \item[GUI:] generic GUI that uses an EmoDetector to predict the emotion. 13 | \end{description} 14 | 15 | Instead of creating a single executable that does everything, we chose a 16 | \emph{linux-like} approach of having a set of tools, where each of them 17 | performs a single task. 18 | 19 | Thus, we created a set of efficient tools, written in \code{C++}, which performs 20 | training and detection, while high-level tasks such as data and training 21 | organization are done by \code{Python} scripts. 22 | 23 | \subsection{Face detection} 24 | 25 | The face detection task is realized by the class \code{FaceDetector}. It uses 26 | trained \emph{Haar Cascades} classifiers to detect and crop the first face 27 | from an image. 28 | 29 | Since the face can be partially occluded or rotated around the principal axis, 30 | mere detection is not enough. We implemented only the roll correction, by using 31 | another \emph{Haar Cascades} classifier to detect the eyes coordinates. If 32 | the eyes are not on the same axis, the image is rotated. 33 | 34 | The face detection receives an image as an input and outputs a matrix containing 35 | the cropped and adjusted face. 36 | 37 | \subsection{Gabor bank} 38 | 39 | The class \code{GaborBank} is a generator and container of \code{GaborKernel}. 40 | It permits the generation of various Gabor kernels with different orientations, 41 | dimensions and wavelengths. 42 | 43 | Once filled, \code{GaborBank} can filter images, returning a matrix that 44 | contains a stack of images, one for each filter present in the bank. 45 | 46 | \subsection{Classifier} 47 | 48 | \code{Classifier} is a generic 2-class classifier, which can be trained to 49 | predict the label of a feature matrix. The specializations classes we implemented are 50 | \code{SVMClassifier} and \code{AdaBoostClassifier}. 51 | 52 | The training set is composed of \emph{negative} (class 0) and \emph{positive} 53 | (class 1) samples. The output is a trained classifier. We used the 54 | \emph{OpenCV}, \code{CvSVM} and \code{CvBoost}, which permit to save and load 55 | their internal state. 56 | 57 | \subsection{Emotion detector} 58 | 59 | Once the 2-class classifiers are trained, they can be used together to realize 60 | a multi-class classifiers. This goal is achieved by \code{EmoDetector}. It is 61 | composed of multiple \code{Classifier} and combines their prediction in various 62 | ways to detect the emotion. 63 | 64 | The input is an image, which is preprocessed: the face is extracted and then 65 | features are computed. Then result is the detected emotion with its prediction 66 | score. 67 | 68 | \subsection{GUI} 69 | 70 | The GUI is realized using the \emph{OpenCV} API. A generic GUI (\code{AGui}) is 71 | composed of a \code{FacePreProcessor}, which extracts features from images, an 72 | \code{EmoDetector} and a \code{ACapture}. 73 | 74 | An \code{ACapture} is a generic frame capture. The specializations we 75 | implemented permit the capture from videos, images and webcams. 76 | 77 | \subsection{Tools and scripts} 78 | 79 | Tools are \code{C++} programs performing high-performance tasks. In order 80 | to be generic, they require every algorithm parameter as input argument. 81 | 82 | \begin{description} 83 | \item[facecrop\_cli] reads an image and outputs a new image with the cropped 84 | face. It also performs eye-adjusting. 85 | \item[gaborbank\_cli] reads an image and outputs a file containing the 86 | extracted features. 87 | \item[train\_cli] reads a training file, trains a classifier and writes the 88 | trained state in an output file in \emph{xml} format. 89 | \item[emo\_detector\_cli] reads images and detects emotions using trained 90 | classifiers. 91 | \item[emotimegui\_cli] captures the webcam and performs real-time emotion 92 | detection. 93 | \end{description} 94 | 95 | The duty of python scripts is to organize the training process, by using the 96 | \code{C++} tools. The configuration, such as tools name and algorithm 97 | parameters, are read from a configuration file. 98 | 99 | \begin{description} 100 | \item[datasetInit.py] initializes the directory hierarchy needed by the 101 | training process. This directory is called \code{dataset} directory. 102 | \item[datasetFillCK.py] fills the \code{dataset} with images from the CK database. 103 | \item[datasetCropFaces.py] applies the facecrop tool to the images, saving 104 | them in the \code{dataset}. 105 | \item[datasetFeatures.py] uses the gaborbank tool to extract features from 106 | the faces. The features are saved as \emph{yml} files. 107 | \item[datasetPrepTrain.py] prepares the training files by combining the 108 | features files using various strategies. The outputs are \emph{csv} files 109 | containing the positive and negative samples. 110 | \item[datasetTrain.py] iterates the \emph{csv} training files and launches a 111 | multi-threading training using the train tool. Its outputs are trained 112 | classifiers, which are saved in the \code{dataset} as \code{xml} files. 113 | \item[datasetVerifyPrediction.py] uses the emo\_detector tool to verify the 114 | prediction of the trained classifiers. We don't have a \emph{validation 115 | set}, so we just test the prediction on the \emph{training set}. 116 | \end{description} 117 | -------------------------------------------------------------------------------- /report/bib/bib.bib: -------------------------------------------------------------------------------- 1 | @INPROCEEDINGS{Littlewort04dynamicsof, 2 | author = {Gwen Littlewort and Marian Stewart Bartlett and Ian Fasel and Joshua Susskind and Javier Movellan}, 3 | title = {Dynamics of Facial Expression Extracted Automatically from Video}, 4 | booktitle = {J. Image \& Vision Computing}, 5 | year = {2004}, 6 | pages = {615--625} 7 | } 8 | 9 | @INPROCEEDINGS{Bartlett06fullyautomatic, 10 | author = {Marian Stewart Bartlett and Gwen Littlewort and Mark Frank and Claudia Lainscsek and Ian Fasel and Javier Movellan}, 11 | title = {Fully automatic facial action recognition in spontaneous behavior}, 12 | booktitle = {in 7th International Conference on Automatic Face and Gesture Recognition}, 13 | year = {2006}, 14 | pages = {223--230} 15 | } 16 | 17 | @INPROCEEDINGS{Viola01rapidobject, 18 | author = {Paul Viola and Michael Jones}, 19 | title = {Rapid object detection using a boosted cascade of simple features}, 20 | booktitle = {}, 21 | year = {2001}, 22 | pages = {511--518} 23 | } 24 | 25 | @ARTICLE{Lades93distortioninvariant, 26 | author = {Martin Lades and Jan C. Vorbrüggen and Joachim Buhmann and Jörg Lange and Christoph V. D. Malsburg and Rolf P. Würtz and Wolfgang Konen}, 27 | title = {Distortion Invariant Object Recognition in the Dynamic Link Architecture}, 28 | journal = {IEEE Trans. Computers}, 29 | year = {1993}, 30 | volume = {42}, 31 | pages = {300--311} 32 | } 33 | 34 | @ARTICLE{Friedman98additivelogistic, 35 | author = {Jerome Friedman and Trevor Hastie and Robert Tibshirani}, 36 | title = {Additive Logistic Regression: a Statistical View of Boosting}, 37 | journal = {Annals of Statistics}, 38 | year = {1998}, 39 | volume = {28}, 40 | pages = {2000} 41 | } 42 | 43 | @ARTICLE{rojas2009adaboost, 44 | added-at = {2013-04-07T20:20:44.000+0200}, 45 | author = {Rojas, Raúl}, 46 | biburl = {http://www.bibsonomy.org/bibtex/27a6b67dc2a8758139dd270692c3929f1/bsc}, 47 | interhash = {12dab9863857f6a0079c677331f3ad30}, 48 | intrahash = {7a6b67dc2a8758139dd270692c3929f1}, 49 | keywords = {AdaBoost ba2013}, 50 | timestamp = {2013-04-07T20:20:44.000+0200}, 51 | title = {AdaBoost and the Super Bowl of Classifiers A Tutorial Introduction to Adaptive Boosting}, 52 | year = {2009} 53 | } 54 | 55 | @MISC{gaborTutorial, 56 | author={Javier R., Movellan}, 57 | title={AdaBoost and the Super Bowl of Classifiers A Tutorial Introduction to Adaptive Boosting}, 58 | howpublished={\url{mplab.ucsd.edu/tutorials/gabor.pdf}} 59 | } 60 | 61 | @MISC{gaborParams, 62 | author={ N. , Petkov and M.B. , Wieling }, 63 | title={Gabor filter for image processing and computer vision}, 64 | howpublished={\url{http://matlabserver.cs.rug.nl/edgedetectionweb/web/edgedetection_params.html}} 65 | } 66 | 67 | @ARTICLE{gaborBandwidth, 68 | author = {C. Grigorescu and N. Petkov and M. A. Westenberg}, 69 | title = {Contour detection based on nonclassical receptive field inhibition}, 70 | journal = {IEEE Transactions on Image Processing}, 71 | year = {2003}, 72 | volume = {12}, 73 | number = {7}, 74 | pages = {729-739} 75 | } 76 | 77 | @MISC{gaborApplication, 78 | author={Wei-lun, Chao}, 79 | title={Gabor wavelet transform and its application}, 80 | howpublished={\url{http://www.researchgate.net/go.Deref.html?url=http%3A%2F%2Fdisp.ee.ntu.edu.tw%2F~pujols%2FGabor%2520wavelet%2520transform%2520and%2520its%2520application.pdf}} 81 | } 82 | 83 | @BOOK{learningOpenCV2008, 84 | author = {Bradski, Dr. Gary Rost and Kaehler, Adrian}, 85 | title = {Learning Opencv, 1st Edition}, 86 | year = {2008}, 87 | isbn = {9780596516130}, 88 | edition = {First}, 89 | publisher = {O'Reilly Media, Inc.} 90 | } 91 | 92 | @INPROCEEDINGS{Kanade2000, 93 | author = {Kanade, Takeo and Tian, Yingli and Cohn, Jeffrey F.}, 94 | title = {Comprehensive Database for Facial Expression Analysis}, 95 | booktitle = {Proceedings of the Fourth IEEE International Conference on Automatic Face and Gesture Recognition 2000}, 96 | series = {FG '00}, 97 | year = {2000}, 98 | isbn = {0-7695-0580-5}, 99 | pages = {46--}, 100 | url = {http://dl.acm.org/citation.cfm?id=795661.796155}, 101 | acmid = {796155}, 102 | publisher = {IEEE Computer Society}, 103 | address = {Washington, DC, USA} 104 | } 105 | 106 | %TODO: svm references 107 | 108 | @MISC{libsvm, 109 | author = {Chih-Chung Chang and Chih-Jen Lin}, 110 | title = {LIBSVM: A Library for Support Vector Machines}, 111 | year = {2001}, 112 | month = {march}, 113 | howpublished = {\url{http://www.csie.ntu.edu.tw/~cjlin/papers/libsvm.pdf}} 114 | } 115 | 116 | @MISC{opencvSVM, 117 | author = {SVM team}, 118 | title = {Support Vector Machines}, 119 | howpublished = {\url{http://docs.opencv.org/modules/ml/doc/support_vector_machines.html}}, 120 | } 121 | 122 | 123 | -------------------------------------------------------------------------------- /report/goal.tex: -------------------------------------------------------------------------------- 1 | \section{Goal} 2 | 3 | This class project boldly aims to face the challenge of the emotional state recognition in human faces through experimentation of some of the latest techniques available to develop a working prototype of an \emph{emotion detector}, aka \code{Emotime}. 4 | 5 | \subsubsection*{Code Base and Other Details} 6 | 7 | Project source code and further details are available at \url{https://github.com/luca-m/emotime/}. 8 | -------------------------------------------------------------------------------- /report/images/db_anger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/db_anger.png -------------------------------------------------------------------------------- /report/images/db_contempt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/db_contempt.png -------------------------------------------------------------------------------- /report/images/db_fear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/db_fear.png -------------------------------------------------------------------------------- /report/images/db_surprise.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/db_surprise.png -------------------------------------------------------------------------------- /report/images/exampl_happy2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/exampl_happy2.png -------------------------------------------------------------------------------- /report/images/example_anger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/example_anger.png -------------------------------------------------------------------------------- /report/images/example_happy1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/example_happy1.png -------------------------------------------------------------------------------- /report/images/example_happy3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/example_happy3.png -------------------------------------------------------------------------------- /report/images/example_sad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/example_sad.png -------------------------------------------------------------------------------- /report/images/example_surprise.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/example_surprise.png -------------------------------------------------------------------------------- /report/images/gabor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/gabor.png -------------------------------------------------------------------------------- /report/images/haarfeatures.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luca-m/emotime/643a5c09144b515a102942a178a3b7ce0e2cdc92/report/images/haarfeatures.png -------------------------------------------------------------------------------- /report/main.tex: -------------------------------------------------------------------------------- 1 | \documentclass[11pt]{article} 2 | 3 | \newcommand{\xautha}{Luca Mella} 4 | \newcommand{\xauthb}{Daniele Bellavista} 5 | \newcommand{\xtitle}{Emotime: Recognizing emotional states in faces\\{\small Elaborazione Delle Immagini LM}\\{University of Bologna}} 6 | 7 | \usepackage{amsmath} 8 | \usepackage{placeins} 9 | \usepackage{float} 10 | \usepackage{ifpdf,ifluatex} 11 | \ifluatex 12 | \usepackage{fontspec} 13 | \else 14 | \usepackage[utf8x]{inputenc} 15 | \fi 16 | 17 | \usepackage{url} 18 | \ifpdf 19 | \usepackage{hyperref} 20 | \hypersetup{unicode=true, 21 | pdftitle={\xtitle}, 22 | pdfauthor={\xautha, \xauthb}, 23 | } 24 | \fi 25 | 26 | \usepackage{chngcntr} 27 | \counterwithin{figure}{subsection} 28 | \counterwithin{table}{subsection} 29 | 30 | \usepackage{xspace} 31 | \usepackage[usenames,dvipsnames]{xcolor} 32 | \usepackage{graphicx} 33 | 34 | \usepackage{listings} 35 | \lstset{% 36 | breakatwhitespace=false, 37 | breaklines=true, 38 | captionpos=t, 39 | extendedchars=true, 40 | literate= {à}{{\`a}}1 41 | {η}{{$\eta$}}1 42 | {μ}{{$\mu$}}1 43 | {ρ}{{$\rho$}}1, 44 | frame=single, 45 | keepspaces=true, 46 | basicstyle=\ttfamily\footnotesize, 47 | keywordstyle=\color{blue}, 48 | commentstyle=\color{OliveGreen}, 49 | stringstyle=\color{RedOrange}, 50 | numbers=left, 51 | numberstyle=\tiny\color{CadetBlue}, 52 | rulecolor=\color{black}, 53 | showspaces=false, 54 | showstringspaces=false, 55 | showtabs=false, 56 | tabsize=2, 57 | title=\lstname 58 | } 59 | 60 | \usepackage{algorithm} 61 | \usepackage{algorithmic} 62 | \usepackage{amsfonts} 63 | 64 | \newcommand{\action}[1]{\texttt{#1}\xspace} 65 | \newcommand{\code}[1]{{\small{\texttt{#1}}}\xspace} 66 | \newcommand{\codescript}[1]{{\scriptsize{\texttt{#1}}}\xspace} 67 | 68 | % Cross-referencing 69 | \newcommand{\labelsec}[1]{\label{sec:#1}} 70 | \newcommand{\xs}[1]{\sectionname~\ref{sec:#1}} 71 | \newcommand{\xsp}[1]{\sectionname~\ref{sec:#1} \onpagename~\pageref{sec:#1}} 72 | \newcommand{\labelssec}[1]{\label{ssec:#1}} 73 | \newcommand{\xss}[1]{\subsectionname~\ref{ssec:#1}} 74 | \newcommand{\xssp}[1]{\subsectionname~\ref{ssec:#1} \onpagename~\pageref{ssec:#1}} 75 | \newcommand{\labelsssec}[1]{\label{sssec:#1}} 76 | \newcommand{\xsss}[1]{\subsectionname~\ref{sssec:#1}} 77 | \newcommand{\xsssp}[1]{\subsectionname~\ref{sssec:#1} \onpagename~\pageref{sssec:#1}} 78 | \newcommand{\labelfig}[1]{\label{fig:#1}} 79 | \newcommand{\xf}[1]{\figurename~\ref{fig:#1}} 80 | \newcommand{\xfp}[1]{\figurename~\ref{fig:#1} \onpagename~\pageref{fig:#1}} 81 | \newcommand{\labeltab}[1]{\label{tab:#1}} 82 | \newcommand{\xt}[1]{\tablename~\ref{tab:#1}} 83 | \newcommand{\xtp}[1]{\tablename~\ref{tab:#1} \onpagename~\pageref{tab:#1}} 84 | 85 | \begin{document} 86 | \title{\xtitle} 87 | \author{\xautha\\\xauthb} 88 | 89 | \maketitle 90 | \newpage 91 | %\makeindex 92 | \newpage 93 | 94 | \input{goal.tex} 95 | 96 | \input{approach.tex} 97 | 98 | \input{workflow.tex} 99 | 100 | \input{parameters.tex} 101 | 102 | \input{architecture.tex} 103 | 104 | \input{organization.tex} 105 | 106 | \input{results.tex} 107 | 108 | 109 | \nocite{*} 110 | \bibliographystyle{plain} 111 | \bibliography{bib/bib}{} 112 | 113 | \end{document} 114 | -------------------------------------------------------------------------------- /report/organization.tex: -------------------------------------------------------------------------------- 1 | \section{Project organization and usage} 2 | 3 | The project is managed by \emph{CMake} and it's organized as follows: 4 | 5 | \begin{verbatim} 6 | 7 | src 8 | \-->dataset Scripts for dataset management 9 | \-->detector Multi-class detector and preprocessor 10 | \-->facedetector Face cropping, adjusting and registration 11 | \-->gaborbank Generation of Gabor filters and image filtering 12 | \-->gui Graphical interfaces and interaction with videos and webcam 13 | \-->training 2-class training and prediction 14 | \-->utils String and IO utilities, CSV supports, and so on.. 15 | doc Documentation (doxigen) 16 | report Report sources 17 | resources Containing third party resources (e.g. OpenCV haar classifiers) 18 | assets Binary folder 19 | 20 | \end{verbatim} 21 | 22 | In order to configure, compile and run this project one must have: 23 | 24 | \begin{itemize} 25 | \item CMake $\ge$ 2.8 26 | \item Opencv $\ge$ 2.4.5 27 | \item Python $\ge$ 2.7 28 | \end{itemize} 29 | 30 | Although python is not strictly required, the usage of the individual tools 31 | without python is not recommended. 32 | 33 | \subsection{Compilation on Linux} 34 | 35 | Just like any other \emph{CMake} project, the first step is to initialize the 36 | compilation by doing: 37 | 38 | \begin{verbatim} 39 | mkdir build 40 | cd build 41 | cmake ../ 42 | \end{verbatim} 43 | 44 | The \code{make install} command, or equivalent, compiles and moves binaries and 45 | scripts in the \code{assets} directory. 46 | 47 | \subsection{Compilation on Windows} 48 | 49 | We have tested the compilation and usage on Windows 8.1 using Visual Studio 12 50 | (64-bit). However, it should work with any version: 51 | 52 | \begin{enumerate} 53 | \item Using CMake or CMakeGUI, select emotime as source folder and configure. 54 | \item If it complains about setting the variable \code{OpenCV\_DIR}, set it to the appropriate path, so that: 55 | \begin{enumerate} 56 | \item \code{C:/path/to/opencv/dir/} contains the libraries (\code{*.lib}) 57 | \item \code{C:/path/to/opencv/dir/include} contains the include 58 | directories (opencv and opencv2). \textbf{IF the include directory is 59 | not in the right position}, the project will likely not be able to 60 | compile due to missing reference to \code{opencv2/opencv} or similar. 61 | \end{enumerate} 62 | \item Then generate the project and compile the project \code{ALL\_BUILD} 63 | \item If the compilation succeeds, compile the project \code{INSTALL} 64 | \item Now all scripts, configuration and tools should be in the directory 65 | \code{assets} in the project source directory. 66 | \end{enumerate} 67 | 68 | \textbf{Note:} in order to execute the CLI programs, the opencv dlls must be in 69 | the \code{PATH}. 70 | 71 | \subsection{Usage} 72 | 73 | The general usage is to initialize the \code{dataset} directory, fill it with 74 | images from the \code{CK} database, train the classifiers with those images and 75 | launch the GUI\@. Algorithm parameters are set using a configuration file. An 76 | example of the configuration file is \code{assets/example\_dataset.cfg}. This file 77 | will be overwritten at every \code{make install}. 78 | 79 | By default python scripts search the configuration file named 80 | \code{./dataset.cfg}, but with the parameter \code{--cfg /path/to/cfg} any 81 | configuration file can be used. 82 | 83 | \subsubsection*{Initialize and fill a dataset} 84 | 85 | \begin{verbatim} 86 | python2 datasetInit.py --cfg dataset.cfg /path/to/dataset 87 | 88 | python2 datasetFillCK.py /path/to/dataset \ 89 | /path/to/cohn/kanade \ 90 | /path/to/cohn/kanade/emotions 91 | \end{verbatim} 92 | 93 | \subsubsection*{Classifier training} 94 | 95 | Your console's working directory must be the \code{assets} directory, 96 | with all the scripts and clis present. Assuming your configuration is in the 97 | assets directory, named \code{dataset.cfg}, the script \code{train\_models.py} 98 | automatize the training procedure. \textbf{Remember} this script must be called 99 | inside the \code{assets} directory. 100 | 101 | \begin{verbatim} 102 | python ./train_models.py --cfg dataset.cfg --mode svm\ 103 | --prep-train-mode {1vsAll, 1vsAllExt, 1vs1}\ 104 | [--eye-correction]\ 105 | /path/to/dataset 106 | \end{verbatim} 107 | 108 | Or also (in order): 109 | 110 | \begin{verbatim} 111 | python datasetCropFaces.py [--eye-correction] /path/to/dataset 112 | 113 | python datasetFeatures.py /path/to/dataset 114 | 115 | python datasetPrepTrain.py --mode {1vs1,1vsAll,1vsAllExt}\ 116 | /path/to/dataset 117 | 118 | python datasetTrain.py --mode svm /path/to/dataset 119 | 120 | python datasetVerifyPrediction.py --mode svm\ 121 | [--eye-correction] /path/to/dataset 122 | \end{verbatim} 123 | 124 | \subsubsection*{GUI webcam} 125 | 126 | After the classifiers have been trained, you can use them to launch the emotime 127 | GUI\@. The GUI uses the webcam to detect emotions. 128 | 129 | \begin{verbatim} 130 | 131 | python gui.py --cfg dataset.cfg --mode svm [--eye-correction]\ 132 | /path/to/dataset 133 | \end{verbatim} -------------------------------------------------------------------------------- /report/parameters.tex: -------------------------------------------------------------------------------- 1 | \section{Parameters} 2 | 3 | In this section, we will discuss the parameter used for the algorithm adopted in the class project. 4 | 5 | \subsection{Face Detection Parameters} 6 | 7 | OpenCV supports several parameters for multi-scale detection, from the classifier to use, to scale factor and minimum object size. We adopted an empirical approach and used quick-and-dirty python script to test face detection parameters directly using OpenCV implementation: we figured out that considering a 640$\times$490 figure as the minimum face size, size of 120$\times$200 produces good detection on available samples (face/image area ratio around 7-10\%). 8 | But in order to ensure detection for all treatable faces, we set up a minimum size of 30$\times$60. This value is related to the resizing performed before feature extraction, in our experiments we chose 48$\times$48. 9 | 10 | Regarding the eye detection step, we decided to set the minimum object threshold to a dynamical value calculated using knowledge of face area size and a qualitative estimation of the human face proportions. In detail, we considered that the width of an eye cannot be smaller than $\frac{1}{5}$ of the face width. 11 | 12 | Also we tried several face detectors in order to see which one fits better for our purposes, for example we considered the \code{haarcascade\_frontalface\_ default.xml} and the \code{haarcascade\_frontalface\_cbcl1.xml}. The latter is interesting because it detects the inner part of the face, discarding the surroundings. This choice has side effects on results~\ref{res:issues}. 13 | 14 | \subsection{Gabor Kernels Parameters} 15 | 16 | Gabor parameter selection is one of the most delicate parts of the tuning process, it directly affects the features evaluated for the emotional state classification. 17 | %Hopefully \cite{Littlewort04dynamicsof, Bartlett06fullyautomatic, Lades93distortioninvariant} provide some hints on the Gabor bank parameters that can be used to achieve good results, in detail the number of $\lambda$ (wavelength) to use, number of $\theta$ (orientation) and some clues about interesting wavelength ranges are provided: 18 | In our experimentation, we decided to generate Gabor filter banks using relationship reported in previous section, and we have empirically fixed following parameters domain: 19 | 20 | \begin{itemize} 21 | \item bandwidth $b \in 1.3,1.6$, %around $\sim \pi \frac{2}{5}, \frac{\pi}{2} $, 22 | ranging in common values of bandwidth\cite{gaborBandwidth}\footnote{\url{http://www.cs.rug.nl/~imaging/simplecell.html}}. 23 | \item $\sigma \in 1,6$, and consequentially $\lambda = \frac{\lambda}{\sigma} \sigma $ 24 | \item $\theta \in ( 0 \ldots \frac{\pi}{2} )$ 25 | \item kernel size determined using aspect ratio and scale of the Gaussian support ($\frac{\sigma}{\gamma}$), using the parameters above results from $5$ to $25$ pixels. 26 | %\item $\lambda \in ( \frac{\pi}{32} \ldots \frac{\pi}{2} ) $ as suggested in \cite{Lades93distortioninvariant} 27 | %\item suggested 5 $\lambda$ and 8 $\theta$ subdivisions 28 | %\item same aspect ration $\gamma$ for each kernel in the bank 29 | \end{itemize} 30 | 31 | We also adopted a flexible approach by keeping subdivision number configurable in order to be able to refine parameters in further development stages. 32 | %Least, no clear indication about the size of the kernel, so we decided to support multiple kernel size ranging from $7$ to $17$, this values have been fixed in the first testing phase, using the developed Gabor utilities for visualizing the resulting bank. 33 | 34 | \subsection{Boosting Parameters} 35 | 36 | Boosting parameters were considered in the early development stages and were ignored in the later phases due to the huge amount of time needed for \code{AdaBoost} training. However parameter choice involves: 37 | 38 | \begin{itemize} 39 | \item Variant of the algorithm to use, in our case \code{Real AdaBoost}, \code{Discrete AdaBoost} and \code{Gentle AdaBoost}. 40 | \item Classifier trim weight, as the lower accepted weight for a weak classifier. 41 | \item Depth of the trained decision tree. 42 | \end{itemize} 43 | 44 | We have no clue about these parameters so we tried several configurations and better results have been obtained via \emph{Gentle AdaBoost}, trimming weak classifiers provides a speed-up but it is a sensible operation due to distribution of classifier weights. However, as reported in~\ref{res:issues}, we have temporarily dismissed the boosting approach due to lack of time (and/or servers). 45 | 46 | \subsection{SVM Parameters} 47 | 48 | Linear SVM with soft max has only one parameter, the value \emph{C}, which 49 | defines how much to consider the misclassification. Since we hadn't any 50 | validation set, we couldn't verify how much this value could affect the train. 51 | We decided to leave $C=0.5$, for no particular reason. 52 | 53 | \subsection{Multi-class Strategies} 54 | 55 | Each classification algorithm adopted works only for binary classification tasks, and our problem involves 7 distinct classes. For this reason, we follow the indications in \cite{Littlewort04dynamicsof, Bartlett06fullyautomatic} and implement a quite general support, building a voting scheme based multi-class classifier which relies on binary classifiers. 56 | In detail, we support multi-class training and detection in the following configurations: 57 | 58 | \begin{itemize} 59 | \item \emph{1 vs 1}, binary classifiers separate single classes from each other. 60 | \item \emph{1 vs all}, separates a class from the others. 61 | \item \emph{many vs many}, separates groups of classes. 62 | \end{itemize} 63 | 64 | Indications in papers suggest that \emph{1 vs all} and \emph{many vs many} should be better approaches. -------------------------------------------------------------------------------- /report/results.tex: -------------------------------------------------------------------------------- 1 | \section{Results} 2 | 3 | We performed training with several dataset configurations, but here we'll report only the most significant, however the reader may find implementation details, source code, trained models and log at \url{https://github.com/luca-m/emotime/}. 4 | 5 | \subsection{Training Results} 6 | 7 | Generalization performance is not formerly available due to the lack of a meaningful validation set\ref{res:issues}, so any generalization performance consideration should be taken as qualitative indications. Having said that, we reported good performance in the detection of ``happiness'', ``anger'', ``disgust'', ``neutrality'' and ``surprise'', less effective is the detection of ``contempt'', ``sadness'' and ``fear''. We observed emotion detection happens on expression peeks which can be observed especially during face expression transitions, this behaviour could be caused by our current implementation of the binary classifiers: the SVM classifiers adopted provide a \emph{boolean information} about its prediction, an estimation of distance from the sample and the boundary hyperplane should enable a more gradual detection. 8 | 9 | We also observed that the \code{1vsAll} multi-class method provide more less noisy detections during a live web-cam session, \code{1vsAllExt} mode instead is able to always predict a valid state for each frame processed, but sometimes it result to be more unstable during the expression transition. 10 | 11 | \subsection{Issues} 12 | \label{res:issues} 13 | 14 | Here is a brief discussion about problems and issues in general which remain unsolved due to the limited amount of time available for this class project. 15 | 16 | \subsubsection*{Training Samples Number} 17 | 18 | One of the main problems noticed during testing on arbitrary video or webcam is that detection of some emotions is not as good as some other ones, a possible explanation relies on the limited amount of samples of the adopted dataset: some emotions have only 10/20 samples, which is certainly not a huge number.\\ 19 | Also the Cohn-Kanade Expression Database represents only the first step in several articles we have read, typically other dataset flanked to it (MMI, FERET, JAFFE, \ldots). 20 | 21 | \subsubsection*{Validation Dataset} 22 | 23 | The availability of a single, basic dataset did not give us the opportunity to calculate the confusion matrix on validation set for assessing generalization performance. 24 | %Also gaining access to datasets is not properly immediate: not automated registration, licence agreement and university affiliation are necessary. 25 | 26 | \subsubsection*{Boosting Training Time} 27 | 28 | The training time of multiple AdaBoost classifiers with $O(10^4)$ features is not negligible, it took more than a day to complete the whole training process. For this reason, we chose to temporarily skip the feature selection approach based on the boosting results and focus on SVM with linear kernel. 29 | 30 | This means that our performance is not optimal because we may calculate some features which are actually not crucial in the decision process, nevertheless real-time performance is achieved. 31 | 32 | \subsubsection*{Face Detector Side Effect} 33 | 34 | We empirically noted that the face detector adopted for training and detection has side effect on generalization performance, a more formal discussion is not possible due to the lack of validation dataset availability. However we noted that using \code{haarcascade\_frontalface\_default}, the face detection rate increases, but part of the ROI of the face is not useful for emotion detection due to the presence of part of background, hair and ears, giving worst generalization performance. 35 | 36 | Using \code{haarcascade\_frontalface\_cbcl1}, only the inner part of the face is detected, this leads to a lower face detection rate but the ROI results more dense of meaningful features. Since we resize ROI to a 48$\times$48 rectangle, which are relatively small images, the effect of the trade-off described above is not negligible. 37 | 38 | \subsection{Examples} 39 | 40 | Following pictures represent screenshots captured during live test session with webcam and sample videos. 41 | 42 | \begin{figure} 43 | \centering 44 | \includegraphics[width=7cm]{images/example_happy3.png} 45 | \label{fig:example_happy3} 46 | \caption{Happiness detection on a video from MMI dataset} 47 | \end{figure} 48 | 49 | \begin{figure} 50 | \centering 51 | \includegraphics[width=7cm]{images/example_surprise.png} 52 | \label{fig:example_surprise} 53 | \caption{Surprise detection on a video from MMI dataset} 54 | \end{figure} 55 | 56 | \begin{figure} 57 | \centering 58 | \includegraphics[width=7cm]{images/example_happy1.png} 59 | \label{fig:example_happy1} 60 | \caption{Happiness detection on live webcam capture} 61 | \end{figure} 62 | 63 | \begin{figure} 64 | \centering 65 | \includegraphics[width=7cm]{images/example_anger.png} 66 | \label{fig:example_anger} 67 | \caption{Anger detection on a video from MMI dataset} 68 | \end{figure} 69 | 70 | \begin{figure} 71 | \centering 72 | \includegraphics[width=7cm]{images/example_sad.png} 73 | \label{fig:example_sad} 74 | \caption{Sadness detection on live webcam capture} 75 | \end{figure} 76 | 77 | \begin{figure} 78 | \centering 79 | \includegraphics[width=7cm]{images/exampl_happy2.png} 80 | \label{fig:exampl_happy2} 81 | \caption{Happiness for happiness detection on live webcam capture} 82 | \end{figure} 83 | 84 | \begin{figure} 85 | \centering 86 | \includegraphics[width=7cm]{./images/db_fear.png} 87 | \caption{Fear detection on live webcam capture} 88 | \label{fig:exampl_fear} 89 | \end{figure} 90 | 91 | \begin{figure} 92 | \centering 93 | \includegraphics[width=7cm]{./images/db_anger.png} 94 | \caption{Anger detection on live webcam capture} 95 | \label{fig:exampl_anger} 96 | \end{figure} 97 | 98 | \begin{figure} 99 | \centering 100 | \includegraphics[width=7cm]{./images/db_contempt.png} 101 | \caption{Contempt detection on live webcam capture} 102 | \label{fig:exampl_contempt} 103 | \end{figure} 104 | 105 | \begin{figure} 106 | \centering 107 | \includegraphics[width=7cm]{./images/db_surprise.png} 108 | \caption{Surprise detection on live webcam capture} 109 | \label{fig:exampl_surprise} 110 | \end{figure} 111 | \newpage 112 | -------------------------------------------------------------------------------- /report/workflow.tex: -------------------------------------------------------------------------------- 1 | \section{Workflow} 2 | 3 | The original workflow was: 4 | 5 | \begin{enumerate} 6 | \item Dataset preparation and AdaBoost training. 7 | \begin{itemize} 8 | \item \textbf{LM} Python tool development for automatic training. 9 | \item \textbf{LM} Tools and functions for AdaBoost training using OpenCV. 10 | \item \textbf{DB} Tools and functions for feature extraction by means of Gabor Filter. 11 | \end{itemize} 12 | \item AdaBoost feature selection and SVM training. 13 | \begin{itemize} 14 | \item \textbf{LM} AdaBoost feature selection and specialized Gabor filter. 15 | \item \textbf{DB} Tools and functions for SVM training using OpenCV. 16 | \end{itemize} 17 | \item Testing 18 | \begin{itemize} 19 | \item \textbf{LM} Prediction tools and functions for AdaBoost prediction 20 | \item \textbf{DB} Prediction tools and functions for SVM prediction 21 | \end{itemize} 22 | \item End-User application 23 | \begin{itemize} 24 | \item \textbf{LM,DB} GUI for video, webcam and images. 25 | \item \textbf{DB} Release of dynamic libraries and trained classifiers. 26 | \end{itemize} 27 | \end{enumerate} 28 | 29 | However during the completion of point 1, we faced various problems using 30 | AdaBoost (more details in~\ref{res:issues}) and we discarded it. Thus, the 31 | workflow adapted: 32 | 33 | \begin{enumerate} 34 | \item Dataset preparation and AdaBoost training. 35 | \begin{itemize} 36 | \item \textbf{LM} Python tool development for automatic training. 37 | \item \textbf{LM} Tools and functions for AdaBoost training using OpenCV. 38 | \item \textbf{DB} Tools and functions for feature extraction by means of Gabor Filter. 39 | \end{itemize} 40 | \item Improved feature extraction and SVM training. 41 | \begin{itemize} 42 | \item \textbf{LM} Implementation of face rotation using eye position. 43 | \item \textbf{DB} Tools and functions for SVM training using OpenCV. 44 | \end{itemize} 45 | \item Testing and Refractoring 46 | \begin{itemize} 47 | \item \textbf{LM} Tools and GUIs for determination of Gabor parameters. 48 | \item \textbf{DB} Prediction tools and functions for prediction. 49 | \item \textbf{LM,DB} Code refractoring from functions to classes. 50 | \end{itemize} 51 | \item Parameters optimization 52 | \begin{itemize} 53 | \item \textbf{LM,DB} Study of papers and formulae for determination of 54 | SVM parameters. 55 | \item \textbf{LM,DB} Study of papers and formulae for determination of 56 | Gabor parameters. 57 | \end{itemize} 58 | \item End-User application 59 | \begin{itemize} 60 | \item \textbf{LM,DB} GUI for video, webcam and images. 61 | \item \textbf{DB} Release of dynamic libraries and trained classifiers. 62 | \end{itemize} 63 | \end{enumerate} 64 | 65 | The study of SVM parameters was discarded for the absence of a validation set. 66 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | IF (MSVC) 3 | # Setting the xecutables as console applications 4 | SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:CONSOLE") 5 | ENDIF (MSVC) 6 | 7 | include_directories(${SRCDIR}/training) 8 | include_directories(${SRCDIR}/facedetector) 9 | include_directories(${SRCDIR}/gaborbank) 10 | include_directories(${SRCDIR}/utils) 11 | include_directories(${SRCDIR}/detector) 12 | include_directories(${SRCDIR}/gui) 13 | 14 | SET(EMOTIME_LIB_FILES "") 15 | SET(EMOTIME_H_FILES "") 16 | 17 | add_subdirectory(training) 18 | add_subdirectory(facedetector) 19 | add_subdirectory(gaborbank) 20 | add_subdirectory(dataset) 21 | add_subdirectory(utils) 22 | add_subdirectory(detector) 23 | add_subdirectory(gui) 24 | 25 | find_package(OpenCV COMPONENTS opencv_core opencv_ml opencv_imgproc 26 | opencv_highgui opencv_objdetect REQUIRED) 27 | 28 | # Emotime library 29 | add_library(emotime SHARED facedetector/FaceDetector.cpp 30 | detector/EmoDetector.cpp 31 | detector/BoostEmoDetector.cpp 32 | detector/SVMEmoDetector.cpp 33 | training/Classifier.cpp 34 | training/AdaBoostClassifier.cpp 35 | training/SVMClassifier.cpp 36 | detector/FacePreProcessor.cpp 37 | gaborbank/GaborBank.cpp 38 | gaborbank/GaborKernel.h 39 | utils/matrix_io.cpp 40 | utils/string_utils.cpp 41 | ) 42 | 43 | target_link_libraries(emotime ${OpenCV_LIBS}) 44 | 45 | INSTALL(TARGETS emotime DESTINATION ${ASSETDIR}) 46 | 47 | INSTALL(FILES facedetector/FaceDetector.h 48 | utils/matrix_io.h utils/string_utils.h 49 | gaborbank/GaborBank.h gaborbank/GaborKernel.h 50 | detector/EmoDetector.h detector/BoostEmoDetector.h detector/SVMEmoDetector.h detector/FacePreProcessor.h 51 | DESTINATION "${HEADERDIR}") 52 | 53 | -------------------------------------------------------------------------------- /src/dataset/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Configuration 3 | CONFIGURE_FILE(example_dataset.cfg ${ASSETDIR} COPYONLY) 4 | CONFIGURE_FILE(datasetConfigParser.py ${ASSETDIR} COPYONLY) 5 | # Dataset tools 6 | CONFIGURE_FILE(datasetInit.py ${ASSETDIR} COPYONLY) 7 | CONFIGURE_FILE(datasetFillCK.py ${ASSETDIR} COPYONLY) 8 | CONFIGURE_FILE(datasetCropFaces.py ${ASSETDIR} COPYONLY) 9 | CONFIGURE_FILE(datasetFeatures.py ${ASSETDIR} COPYONLY) 10 | CONFIGURE_FILE(datasetPrepTrain.py ${ASSETDIR} COPYONLY) 11 | CONFIGURE_FILE(datasetTrain.py ${ASSETDIR} COPYONLY) 12 | CONFIGURE_FILE(datasetVerifyPrediction.py ${ASSETDIR} COPYONLY) 13 | # GUI tool 14 | CONFIGURE_FILE(gui.py ${ASSETDIR} COPYONLY) 15 | # Scripts 16 | CONFIGURE_FILE(train_models.sh ${ASSETDIR} COPYONLY) 17 | CONFIGURE_FILE(train_models.py ${ASSETDIR} COPYONLY) 18 | CONFIGURE_FILE(cleandataset.sh ${ASSETDIR} COPYONLY) 19 | -------------------------------------------------------------------------------- /src/dataset/cleandataset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## cleandataset.sh 3 | ## 4 | ## Copyright (C) 2014 luca.mella@studio.unibo.it 5 | ## 6 | ## Description: 7 | ## 8 | ## NA 9 | ## 10 | ## Usage: 11 | ## 12 | ## cleandataset OPTIONS DSETPATH 13 | ## 14 | ## Parameters: 15 | ## 16 | ## DSETPATH Dataset path 17 | ## 18 | ## Options: 19 | ## 20 | ## -h, --help This help message 21 | ## --faces Clear faces 22 | ## --features Clear features 23 | ## --classifiers Clear classifiers 24 | ## 25 | ## Example: 26 | ## 27 | ## cleandataset -h 28 | ## 29 | 30 | # ---------------- 31 | # UTILITY FUNCTIONS 32 | # ---------------- 33 | 34 | echoerr() { 35 | echo "$@" 1>&2; 36 | } 37 | usage() { 38 | [ "$*" ] && echoerr "$0: $*" 39 | sed -n '/^##/,/^$/s/^## \{0,1\}//p' "$0" 1>&2 40 | exit 2 41 | } 42 | 43 | # ---------------- 44 | # FUNCTIONS 45 | # ---------------- 46 | 47 | 48 | # ---------------- 49 | # MAIN 50 | # ---------------- 51 | CLEAR_FACES=0 52 | CLEAR_FEATURES=0 53 | CLEAR_CLASSIFIERS=0 54 | # Options parsing 55 | while [ $# -gt 0 ]; do 56 | case $1 in 57 | (-h|--help) usage 2>&1;; 58 | (--faces) CLEAR_FACES=1;shift;; 59 | (--features) CLEAR_FEATURES=1;shift;; 60 | (--classifiers) CLEAR_CLASSIFIERS=1;shift;; 61 | (--) shift; break;; 62 | (-*) usage "$1: unknown option";; 63 | (*) break;; 64 | esac 65 | done 66 | 67 | DSET=$1 68 | if [ "$DSET" == "" ]; then 69 | usage 2>&1 70 | fi 71 | 72 | if [ $CLEAR_FACES -eq 1 ]; then 73 | for f in `ls -1 $DSET/training/faces | xargs`; 74 | do 75 | rm -fr $DSET/training/faces/$f/* 76 | done 77 | fi 78 | 79 | if [ $CLEAR_FEATURES -eq 1 ]; then 80 | for f in `ls -1 $DSET/training/features | xargs`; 81 | do 82 | rm -fr $DSET/training/features/$f/* 83 | done 84 | fi 85 | 86 | rm $DSET/trainfiles/* 87 | 88 | 89 | if [ $CLEAR_CLASSIFIERS -eq 1 ]; then 90 | clbackup="$DSET/classifiers.$(date +"%Y%m%d-%H%M")" 91 | mkdir -p "$clbackup" 92 | mv $DSET/classifiers/* $clbackup 93 | mkdir $DSET/classifiers/svm 94 | mkdir $DSET/classifiers/ada 95 | fi 96 | 97 | -------------------------------------------------------------------------------- /src/dataset/datasetConfigParser.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | 4 | """ 5 | import ConfigParser as cp 6 | from string import upper 7 | 8 | def fill_it(parser, name, left_side): 9 | 10 | if left_side: 11 | fstr = '{0}_{1}' 12 | else: 13 | fstr = '{1}_{0}' 14 | 15 | config = {} 16 | for opt in parser.options(name): 17 | val = parser.get(name,opt) 18 | if upper(opt) == 'TOOL': 19 | try: 20 | with open(val): 21 | pass 22 | except: 23 | val += '.exe' 24 | with open(val): 25 | pass 26 | config[fstr.format(upper(opt), name)] = val 27 | 28 | return config 29 | 30 | def parse_python_config(configFile): 31 | config={} 32 | execfile(configFile, config) 33 | return config 34 | 35 | def parse_ini_config(configFile): 36 | config={} 37 | parser=cp.ConfigParser() 38 | parser.read(configFile) 39 | 40 | classes=[] 41 | for opt in parser.options('CLASSES'): 42 | classes.append(opt) 43 | config['CLASSES']=classes 44 | 45 | size={} 46 | for opt in parser.options('SIZE'): 47 | size[opt]=parser.get('SIZE',opt) 48 | config['SIZE']=size 49 | 50 | 51 | config.update(fill_it(parser, 'TRAINING', False)) 52 | config.update(fill_it(parser, 'VALIDATION', False)) 53 | config.update(fill_it(parser, 'FOLDER', True)) 54 | config.update(fill_it(parser, 'SUFFIX', True)) 55 | 56 | config.update(fill_it(parser, 'FACECROP', False)) 57 | config.update(fill_it(parser, 'GABOR', False)) 58 | config.update(fill_it(parser, 'TRAIN', False)) 59 | config.update(fill_it(parser, 'DETECTION', False)) 60 | config.update(fill_it(parser, 'GUI', False)) 61 | 62 | return config 63 | 64 | -------------------------------------------------------------------------------- /src/dataset/datasetCropFaces.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | import os 3 | import subprocess 4 | import argparse 5 | import datasetConfigParser as dcp 6 | 7 | from os.path import join 8 | from os.path import isfile 9 | 10 | import sys 11 | 12 | def dataset_cropFaces(dsFolder, config, eye_correction, validation=False): 13 | """ 14 | Crop faces in dataset 15 | """ 16 | print("INFO: start cropping faces with 'facecrop'") 17 | 18 | if eye_correction: 19 | info_str = "with eye correction" 20 | else: 21 | info_str = "without eye correction" 22 | 23 | IMAGES_FOLDER = config['TRAINING_IMAGES'] 24 | FACES_FOLDER = config['TRAINING_FACES'] 25 | if validation: 26 | IMAGES_FOLDER = config['VALIDATION_IMAGES'] 27 | FACES_FOLDER = config['VALIDATION_FACES'] 28 | for c in config['CLASSES']: 29 | failed=0 30 | ipath=join(dsFolder, join(IMAGES_FOLDER, c)) 31 | opath=join(dsFolder, join(FACES_FOLDER, c)) 32 | imgs=[ f for f in os.listdir(ipath) if isfile(join(ipath, f))] 33 | for i in xrange(0, len(imgs)): 34 | im = imgs[i] 35 | iimPath = join(ipath, im) 36 | oimPath = join(opath, im) 37 | 38 | sys.stdout.write("INFO: running facecropping %s on %s (%d of %d, %d failed)\r"%(info_str, c, (i + 1), len(imgs),failed)) 39 | sys.stdout.flush() 40 | 41 | args = [config['FACECROP_TOOL'], config['FACECROP_FACE_DETECTOR_CFG']] 42 | if eye_correction: 43 | args.append(config['FACECROP_EYE_DETECTOR_CFG']) 44 | args.extend([str(iimPath), str(oimPath)]) 45 | 46 | retcode = subprocess.call(args) 47 | if retcode is not 0: 48 | failed+=1 49 | #print("\nWARN: execution has returned error %d" % retcode) 50 | print "" 51 | 52 | if __name__ == "__main__": 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 55 | parser.add_argument("--eye-correction", action="store_true", help="Apply eye correction to faces") 56 | parser.add_argument("--validation", action="store_true", help="true if it is validation") 57 | parser.add_argument("dsFolder", help="Dataset base folder") 58 | args = parser.parse_args() 59 | try: 60 | config={} 61 | config=dcp.parse_ini_config(args.cfg) 62 | dataset_cropFaces(args.dsFolder, config, args.eye_correction, validation=args.validation) 63 | except Exception as e: 64 | print("ERR: something wrong (%s)" % str(e)) 65 | sys.exit(1) 66 | 67 | -------------------------------------------------------------------------------- /src/dataset/datasetFeatures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | Calculate Gabor features features 4 | """ 5 | import os 6 | import argparse 7 | import subprocess 8 | import datasetConfigParser as dcp 9 | import sys 10 | import multiprocessing 11 | 12 | from os.path import join 13 | from os.path import isfile 14 | 15 | def _subproc_call(args): 16 | """ Wrap a subprocess.call """ 17 | param, comstr = args 18 | retcode=subprocess.call(param, shell=False ) 19 | sys.stdout.write('.') 20 | sys.stdout.flush() 21 | if retcode==0: 22 | return (comstr,True) 23 | else: 24 | print("ERR: '%s' has encountered problems" % comstr) 25 | return (comstr,False) 26 | 27 | def dataset_calcGaborBank(dsFolder, config, validation=False): 28 | """ 29 | Calculate features using a gabor filters bank 30 | """ 31 | bagoftask = [] 32 | 33 | FACES_FOLDER = config['TRAINING_FACES'] 34 | FEATURES_FOLDER = config['TRAINING_FEATURES'] 35 | if validation: 36 | FACES_FOLDER = config['VALIDATION_FACES'] 37 | FEATURES_FOLDER = config['VALIDATION_FEATURES'] 38 | 39 | for c in config['CLASSES']: 40 | facesFolder=join(dsFolder, join(FACES_FOLDER, c)) 41 | featsFolder=join(dsFolder, join(FEATURES_FOLDER, c)) 42 | faces=[ f for f in os.listdir(facesFolder) if isfile(join(facesFolder, f))] 43 | _NJOBS = len(faces) 44 | for i in xrange(0, len(faces)): 45 | face = faces[i] 46 | faceFile=join(facesFolder, face) 47 | featFolder=join(featsFolder, os.path.splitext(face)[0]) + config['FILTERED_FOLDER_SUFFIX'] 48 | try: 49 | os.mkdir(featFolder) 50 | except Exception: 51 | pass 52 | 53 | featFile=join(featFolder, config['GABOR_FEAT_FNAME']) 54 | cmd=[config['GABOR_TOOL'], str(config['SIZE']['width']), str(config['SIZE']['height']), 55 | config['GABOR_NWIDTHS'], config['GABOR_NLAMBDAS'], config['GABOR_NTHETAS'], 56 | str(faceFile), str(featFile)] 57 | if 'GABOR_FILTER_FILE' in config.keys(): 58 | if config['GABOR_FILTER_FILE'] != 'NA': 59 | cmd.append(config['GABOR_FILTER_FILE']) 60 | 61 | bagoftask.append((cmd,'GaborFilter {0}'.format(faceFile))) 62 | 63 | # Spawining parallel task 64 | nprocs = max(1, int(multiprocessing.cpu_count() * abs(float(config['TRAIN_SVM_CPU_USAGE'])))) 65 | results = [] 66 | pool = multiprocessing.Pool(processes= nprocs) 67 | pool.map_async(_subproc_call, bagoftask, callback = results.append).get(2**32) # workaround for properly handling SIGINT 68 | pool.close() 69 | pool.join() 70 | 71 | def dataset_calcFeatures(dsFolder, config, validation=False): 72 | """ Calculate features on dataset""" 73 | print("INFO: calculating gabor features") 74 | dataset_calcGaborBank(dsFolder,config, validation=validation) 75 | 76 | if __name__ == "__main__": 77 | parser = argparse.ArgumentParser() 78 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 79 | parser.add_argument("--validation", action="store_true", help="true if it is validation") 80 | parser.add_argument("dsFolder", help="Dataset base folder") 81 | args = parser.parse_args() 82 | try: 83 | config={} 84 | config=dcp.parse_ini_config(args.cfg) 85 | dataset_calcFeatures(args.dsFolder, config, validation=args.validation) 86 | except Exception as e: 87 | print("ERR: something wrong (%s)" % str(e)) 88 | sys.exit(1) 89 | 90 | 91 | -------------------------------------------------------------------------------- /src/dataset/datasetFillCK.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | Import the Chon-Kanade plus database to a dataset. 4 | """ 5 | import os 6 | import sys 7 | import shutil 8 | import argparse 9 | import datasetConfigParser as dcp 10 | import random 11 | 12 | from string import strip 13 | from os.path import join 14 | from os.path import isfile 15 | from os.path import isdir 16 | 17 | 18 | def dataset_fillCohnKanade( dsFolder, ckFolder, ckEmoFolder, config, vperc=0.3, vseed=0): 19 | """ 20 | Fill dataset with Cohn Kanade + data. 21 | """ 22 | 23 | subjects=[ x for x in os.listdir(ckFolder) if isdir(join(ckFolder, x)) ] 24 | print "INFO: %d subjects found in CK+ database" % len(subjects) 25 | 26 | for subj in subjects: 27 | print "INFO: Processing subject %s " % subj 28 | 29 | labelFolders=[x for x in os.listdir(join(ckEmoFolder, subj)) if isdir(join(ckEmoFolder, join(subj, x)))] 30 | imageFolders=[x for x in os.listdir(join(ckFolder, subj)) if isdir(join(ckEmoFolder, join(subj, x)))] 31 | 32 | shots=[x for x in imageFolders if x in labelFolders] 33 | for s in shots: 34 | print "INFO: Processing shot %s " % s 35 | 36 | pics=[x for x in os.listdir( join(ckFolder, join(subj,s)) ) if isfile(join(ckFolder, join(subj, join(s, x))))] 37 | pics.sort() 38 | labels=[x for x in os.listdir( join(ckEmoFolder, join(subj, s)) ) if isfile( join(ckEmoFolder, join(subj, join(s, x)) )) ] 39 | if len(labels)<1 or len(pics)<1: 40 | # label forlder could contain no file at all, in this case skip the current shot or mark it as neutral? 41 | print "WARN: subject %s shot %s has #%d emo labels and #%d pictures, (skip:incomplete)" %( subj, s, len(labels), len(pics)) 42 | continue 43 | emo=None 44 | with open(join(ckEmoFolder, join(subj, join(s, labels[0]))), "r") as f: 45 | buf=f.read() 46 | if len(buf)==0: 47 | print "WARN: subject %s shot %s has void emo label '%s', (skip:noemo)" % (subj, s, join(ckEmoFolder, join(subj, join(s, labels[0])))) 48 | # A label file could be void, in this case skip the current shot 49 | continue 50 | try: 51 | emo=config['CLASSES'][int(float(strip(buf)))] 52 | except: 53 | print "ERR: cannot parse emotional label for subject %s shot %s (skip:unknown_emo)" % (subj, s) 54 | continue 55 | 56 | # Last picture is the final emotion (most intense), first picture is neutral 57 | to_copy = [(pics[-1], emo), (pics[0], config['CLASSES'][0])] 58 | 59 | for pic, emo in to_copy: 60 | print "INFO: Picture '%s' has been marked as %s" % (pic, emo) 61 | orig = join(ckFolder, join(subj, join(s, pic))) 62 | IMAGES_FOLDER = config['TRAINING_IMAGES'] 63 | if random.random() <= vperc: 64 | IMAGES_FOLDER = config['VALIDATION_IMAGES'] 65 | dest = join(dsFolder, join(IMAGES_FOLDER, join(emo, pic))) 66 | try: 67 | shutil.copy(orig, dest) 68 | except: 69 | print "ERR: cannot copy image '%s' to dataset '%s' "%(orig, dest) 70 | continue 71 | 72 | if __name__ == "__main__": 73 | parser=argparse.ArgumentParser() 74 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 75 | parser.add_argument("--validation-perc",dest='vperc',type=float, default=0.3, help="Validation set percentage (0-1.0)") 76 | parser.add_argument("--validation-seed",dest='vseed',type=int, default=0, help="Seed used to decide when a image belong to training or validation set") 77 | parser.add_argument("dsFolder", help="Dataset Base folder (will be filled with images)") 78 | parser.add_argument("ckFolder", help="Cohn-Kanade image database folder") 79 | parser.add_argument("ckEmoFolder", help="Cohn-Kanade (CK+) emotion label folder") 80 | args=parser.parse_args() 81 | try: 82 | if args.vperc < 0.0 or args.vperc > 1.0: 83 | raise Exception("validation percentage must be in range 0-1 (%f)" % args.vperc) 84 | random.seed(args.vseed) 85 | config={} 86 | config=dcp.parse_ini_config(args.cfg) 87 | dataset_fillCohnKanade(args.dsFolder, args.ckFolder, args.ckEmoFolder, config, vperc=args.vperc, vseed=args.vseed) 88 | except Exception as e: 89 | print "ERR: something wrong (%s)" % str(e) 90 | sys.exit(1) 91 | 92 | -------------------------------------------------------------------------------- /src/dataset/datasetInit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | Initialize a dataset 4 | """ 5 | import os 6 | import sys 7 | import argparse 8 | import datasetConfigParser as dcp 9 | 10 | from os.path import join 11 | 12 | def dataset_init(dsPath, config): 13 | """ 14 | Initialize dataset 15 | """ 16 | for clazz in config['CLASSES']: 17 | pth = join(dsPath, join(config['TRAINING_IMAGES'], clazz)) 18 | if not os.path.exists(pth): 19 | os.makedirs(pth) 20 | pth = join(dsPath, join(config['TRAINING_FACES'], clazz)) 21 | if not os.path.exists(pth): 22 | os.makedirs(pth) 23 | pth=join(dsPath, join(config['TRAINING_FEATURES'], clazz)) 24 | if not os.path.exists(pth): 25 | os.makedirs(pth) 26 | pth = join(dsPath, join(config['VALIDATION_IMAGES'], clazz)) 27 | if not os.path.exists(pth): 28 | os.makedirs(pth) 29 | pth = join(dsPath, join(config['VALIDATION_FACES'], clazz)) 30 | if not os.path.exists(pth): 31 | os.makedirs(pth) 32 | pth=join(dsPath, join(config['VALIDATION_FEATURES'], clazz)) 33 | if not os.path.exists(pth): 34 | os.makedirs(pth) 35 | 36 | pth=join(dsPath, config['TRAIN_FOLDER']) 37 | if not os.path.exists(pth): 38 | os.makedirs(pth) 39 | pth=join(dsPath, config['CLASSIFIER_FOLDER']) 40 | if not os.path.exists(pth): 41 | os.makedirs(pth) 42 | pth=join(dsPath, config['CLASSIFIER_SVM_FOLDER']) 43 | if not os.path.exists(pth): 44 | os.makedirs(pth) 45 | pth=join(dsPath, config['CLASSIFIER_ADA_FOLDER']) 46 | if not os.path.exists(pth): 47 | os.makedirs(pth) 48 | # Copy configuration 49 | #with open(cfgFile, "r") as conf: 50 | #configuration = conf.read() 51 | #with open(join(dsPath, dsCfgName), "w") as nconf: 52 | #nconf.write(configuration) 53 | 54 | if __name__ == "__main__": 55 | parser = argparse.ArgumentParser() 56 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 57 | parser.add_argument("dsFolder", help="Dataset folder path") 58 | args = parser.parse_args() 59 | 60 | try: 61 | config={} 62 | config=dcp.parse_ini_config(args.cfg) 63 | dataset_init(args.dsFolder, config) 64 | except Exception as e: 65 | print("ERR: something wrong (%s)" % str(e)) 66 | sys.exit(1) 67 | 68 | -------------------------------------------------------------------------------- /src/dataset/datasetPrepTrain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | 4 | """ 5 | import cv2 6 | import sys 7 | import os 8 | import itertools 9 | import argparse 10 | import multiprocessing 11 | import datasetConfigParser as dcp 12 | import numpy as np 13 | 14 | from string import lower 15 | from os.path import join 16 | from os.path import isfile 17 | from os.path import isdir 18 | from os.path import splitext 19 | from os.path import basename 20 | from os.path import abspath 21 | 22 | def _dataset_load_matrix(filepath): 23 | """ """ 24 | name,ext=splitext(basename(filepath)) 25 | if ext in ['.yml','.xml']: 26 | cvmat=cv2.cv.Load(filepath, name=name) 27 | return np.asarray(cvmat) 28 | else: 29 | return cv2.imread(filepath, cv2.CV_LOAD_IMAGE_GRAYSCALE) 30 | 31 | def _dataset_multiclass1toAllExt(config, ngroups=3): 32 | """ """ 33 | for z in _dataset_multiclass1toAll(config): 34 | yield z 35 | for i in xrange(1, ngroups + 1): 36 | for z in [ (x, [y for y in config['CLASSES'] if y not in x]) for x in 37 | itertools.combinations(config['CLASSES'], i)]: 38 | yield z 39 | 40 | def _dataset_multiclass1toAll(config): 41 | """ """ 42 | for z in [ ([x],[y for y in config['CLASSES'] if y != x]) for x in config['CLASSES']]: 43 | yield z 44 | 45 | def _dataset_multiclass1to1(config): 46 | """ """ 47 | for x,y in itertools.combinations( config['CLASSES'] , 2 ): 48 | yield ([x],[y]) 49 | 50 | def _dataset_prepare(param): 51 | """ dataset_prepare wrapper """ 52 | return dataset_prepare( param[0], param[1], param[2]) 53 | 54 | def dataset_prepare( (goodClass,badClass), dsFolder, config): 55 | """ 56 | Prepare train file with positive and negative samples. 57 | """ 58 | badClass = sorted(badClass) 59 | goodClass = sorted(goodClass) 60 | goodPath=[join(dsFolder, join(config['TRAINING_FEATURES'], x)) for x in goodClass] 61 | badPath= [join(dsFolder, join(config['TRAINING_FEATURES'], x)) for x in badClass] 62 | # 63 | # Note: a goodFolder should contain the filtered images (various orientation and frequency) 64 | # of a single sample image. So each line of the training file will be composed by its 65 | # MARKER (Good or Bad) plus all the pixel values of the filtered images 66 | # 67 | goodFolders=[] 68 | for x in goodPath: 69 | goodFolders.extend( [join(x, f) for f in os.listdir(x) if isdir(join(x, f)) and f.endswith(config['FILTERED_FOLDER_SUFFIX']) ] ) 70 | goodFolders.sort() 71 | 72 | badFolders=[] 73 | for x in badPath: 74 | badFolders.extend( [join(x, f) for f in os.listdir(x) if isdir(join(x, f)) and f.endswith(config['FILTERED_FOLDER_SUFFIX']) ] ) 75 | badFolders.sort() 76 | 77 | outfpath=join( join(dsFolder, config['TRAIN_FOLDER']), "%s_vs_%s%s" % ( '_'.join(goodClass), '_'.join(badClass), config['FEATURE_FILE_SUFFIX']) ) 78 | 79 | with open(outfpath, "w") as tf: 80 | # 81 | # POSITIVE SAMPLES 82 | # 83 | for fold in goodFolders: 84 | goodImgs=[ f for f in os.listdir(fold) if isfile(join(fold, f))] 85 | goodImgs.sort() 86 | for f in goodImgs: 87 | #print "INFO: Processing (P) '%s'" % join(fold, f) 88 | tf.write("P,%s" % abspath(join(fold, f))) # POSITIVE 89 | tf.write("\n") 90 | # 91 | # NEGATIVE SAMPLES 92 | # 93 | for fold in badFolders: 94 | badImgs=[f for f in os.listdir(fold) if isfile(join(fold, f))] 95 | badImgs.sort() 96 | for f in badImgs: 97 | #print "INFO: Processing (N) '%s'" % join(fold, f) 98 | tf.write("N,%s" % abspath(join(fold, f))) # POSITIVE 99 | tf.write("\n") 100 | 101 | #print "INFO: Done" 102 | return 103 | 104 | def dataset_prepTrainFiles(dsFolder, multiclassMode, config): 105 | """ 106 | Prepare training files 107 | """ 108 | bagoftask=[] 109 | 110 | if multiclassMode=='1vs1': 111 | print "INFO: preparing training files for 1 to 1 multiclass" 112 | for x in _dataset_multiclass1to1(config): 113 | #dataset_prepare(x, dsFolder, config) 114 | bagoftask.append((x, dsFolder, config)) 115 | 116 | if multiclassMode=='1vsall': 117 | print "INFO: preparing training files for 1 to All multiclass" 118 | for x in _dataset_multiclass1toAll(config): 119 | #dataset_prepare(x, dsFolder, config) 120 | bagoftask.append((x, dsFolder, config)) 121 | 122 | if multiclassMode == '1vsallext': 123 | print "INFO: preparing training files for 1 to All Extended multiclass" 124 | for x in _dataset_multiclass1toAllExt(config): 125 | #dataset_prepare(x, dsFolder, config) 126 | bagoftask.append((x, dsFolder, config)) 127 | 128 | nprocs = max(1, int(multiprocessing.cpu_count()*abs(float(config['TRAIN_ADA_CPU_USAGE'])))) 129 | pool = multiprocessing.Pool(processes=nprocs) 130 | pool.map_async(_dataset_prepare, bagoftask).get(2**32) # Workaround for handling SIGINT properly 131 | pool.close() 132 | pool.join() 133 | 134 | if __name__ == "__main__": 135 | parser = argparse.ArgumentParser() 136 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 137 | parser.add_argument("--mode", default="1vsall", choices=['1vs1', '1vsall', '1vsallext'], help="Training mode for multiclass classification") 138 | parser.add_argument("dsFolder",help="Dataset base folder") 139 | args = parser.parse_args() 140 | try: 141 | config={} 142 | config=dcp.parse_ini_config(args.cfg) 143 | dataset_prepTrainFiles(args.dsFolder, lower(args.mode), config) 144 | except Exception as e: 145 | print "ERR: something wrong (%s)" % str(e) 146 | sys.exit(1) 147 | 148 | -------------------------------------------------------------------------------- /src/dataset/datasetTrain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | Train with Adaboost and select relevant features 4 | """ 5 | import argparse 6 | import sys 7 | import os 8 | import subprocess 9 | import multiprocessing 10 | import datasetConfigParser as dcp 11 | 12 | from string import lower 13 | from os.path import join 14 | 15 | def _subproc_call(args): 16 | """ Wrap a subprocess.call """ 17 | param, comstr = args 18 | retcode=subprocess.call( param, shell=False ) 19 | #comstr=' '.join(param) 20 | if retcode==0: 21 | print("INFO: done %s"%comstr) 22 | return (comstr,True) 23 | else: 24 | print("ERR: '%s' has encountered problems" % comstr) 25 | return (comstr,False) 26 | 27 | def dataset_train(smode, trainFolder, outFolder, config): 28 | """ 29 | Train svm classifiers 30 | """ 31 | bagoftask = [] 32 | print("INFO: starting svm training") 33 | for f in os.listdir(trainFolder): 34 | of = os.path.splitext(f)[0] + '.xml' 35 | bagoftask.append(([config['TRAIN_TOOL'], smode, '{0}'.format(join(trainFolder, 36 | f)), '{0}'.format(join(outFolder, of))], os.path.splitext(f)[0])) 37 | nprocs = max(1, int(multiprocessing.cpu_count() * abs(float(config['TRAIN_SVM_CPU_USAGE'])))) 38 | results = [] 39 | pool = multiprocessing.Pool(processes= nprocs) 40 | pool.map_async(_subproc_call, bagoftask, callback = 41 | results.append).get(2**32) # workaround for properly handling SIGINT 42 | pool.close() 43 | pool.join() 44 | 45 | print("INFO: %s training finished."%smode) 46 | return results 47 | 48 | def dataset_run_training(dsFolder, config, mode): 49 | """ 50 | Start training 51 | """ 52 | trainFldr = join(dsFolder, config['TRAIN_FOLDER']) 53 | if mode == "ada": 54 | smode = "ada" 55 | classifFldr = join(dsFolder, config['CLASSIFIER_ADA_FOLDER']) 56 | else: 57 | smode = "svm" 58 | classifFldr = join(dsFolder, config['CLASSIFIER_SVM_FOLDER']) 59 | dataset_train(smode, trainFldr, classifFldr, config) 60 | 61 | if __name__ == "__main__": 62 | parser = argparse.ArgumentParser() 63 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 64 | parser.add_argument("dsFolder", help="Dataset base folder") 65 | parser.add_argument("--mode", default="svm", choices=['ada', 'svm'], help="training mode: ada (AdaBoost) or svm") 66 | args = parser.parse_args() 67 | 68 | try: 69 | config={} 70 | config=dcp.parse_ini_config(args.cfg) 71 | dataset_run_training(args.dsFolder, config, lower(args.mode)) 72 | except Exception as e: 73 | print("ERR: something wrong (%s)" % str(e)) 74 | sys.exit(1) 75 | 76 | -------------------------------------------------------------------------------- /src/dataset/datasetVerifyPrediction.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import argparse 4 | import sys 5 | import datasetConfigParser as dcp 6 | import os 7 | import subprocess 8 | import re 9 | 10 | from subprocess import PIPE 11 | 12 | def dataset_verify_prediction(dsfolder, config, mode, eye_detection): 13 | results = dataset_do_prediction(dsfolder, config, mode, eye_detection) 14 | dataset_process_results(results) 15 | 16 | def dataset_process_results(results): 17 | for expected, l in results.items(): 18 | percs = {} 19 | for (res, val) in l: 20 | if not percs.has_key(res): 21 | percs[res] = 1.0 / len(l) 22 | else: 23 | percs[res] += 1.0 / len(l) 24 | percs = sorted([(v, r) for r,v in percs.items()], reverse=True) 25 | 26 | print expected.capitalize() 27 | for (v, k) in percs: 28 | print "\t", k, '->', "%.2f%%"%v 29 | 30 | 31 | def dataset_do_prediction(dsfolder, config, mode, eye_detection, do_prints=True): 32 | faces_dir = os.path.join(dsfolder, config['VALIDATION_IMAGES']) 33 | 34 | if mode == 'svm': 35 | class_dir = os.path.join(dsfolder, config['CLASSIFIER_SVM_FOLDER']) 36 | mode_s = 'svm' 37 | else: 38 | class_dir = os.path.join(dsfolder, config['CLASSIFIER_ADA_FOLDER']) 39 | mode_s = 'ada' 40 | 41 | execut = config['DETECTION_TOOL'] 42 | print "INFO: detector tool '%s %s', eye detection: %r"%(execut, mode_s, eye_detection) 43 | 44 | classificators = [] 45 | for f in os.listdir(class_dir): 46 | abs_f = os.path.join(class_dir, f) 47 | if os.path.isfile(abs_f): 48 | classificators.append(abs_f) 49 | 50 | #print "INFO: classifiers %s"%str(classificators) 51 | 52 | results = {} 53 | args = [execut, mode_s, config['FACECROP_FACE_DETECTOR_CFG']] 54 | if eye_detection: 55 | args.append(config['FACECROP_EYE_DETECTOR_CFG']) 56 | else: 57 | args.append('none') 58 | args += [config['SIZE']['width'], config['SIZE']['height'], 59 | config['GABOR_NWIDTHS'], config['GABOR_NLAMBDAS'], 60 | config['GABOR_NTHETAS']] + classificators 61 | 62 | res_reg = re.compile("predicted: (\w*) with score (.*)") 63 | for emo in os.listdir(faces_dir): 64 | emo_dir = os.path.join(faces_dir, emo) 65 | 66 | if not os.path.isdir(emo_dir): 67 | continue 68 | 69 | faces_list = [os.path.join(emo_dir, f) for f in os.listdir(emo_dir)] 70 | faces = '\n'.join(faces_list) 71 | if do_prints: 72 | print "Predicting:", emo, "(%d faces)"%len(faces_list) 73 | 74 | p = subprocess.Popen(args, stdout=PIPE, stdin=PIPE, stderr=PIPE) 75 | out = p.communicate(input=faces) 76 | results[emo] = re.findall(res_reg, out[0]) 77 | 78 | if do_prints: 79 | print "" 80 | return results 81 | 82 | if __name__ == "__main__": 83 | parser = argparse.ArgumentParser() 84 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 85 | parser.add_argument("dsFolder", help="Dataset base folder") 86 | parser.add_argument("-v", "--verbose", action='store_true', help="verbosity") 87 | parser.add_argument("--mode", default="adaboost", choices=['adaboost', 'svm'], help="training mode: adaboost or svm") 88 | parser.add_argument("--eye-correction", action="store_true", help="Perform eye correction on images") 89 | args = parser.parse_args() 90 | 91 | try: 92 | config = {} 93 | config = dcp.parse_ini_config(args.cfg) 94 | dataset_verify_prediction(args.dsFolder, config, args.mode, args.eye_correction) 95 | except Exception as e: 96 | print "ERR: something wrong (%s)" % str(e) 97 | sys.exit(1) 98 | -------------------------------------------------------------------------------- /src/dataset/example_dataset.cfg: -------------------------------------------------------------------------------- 1 | ## Example configuration file. This file will be overwritten 2 | ## everytime you perfomrs an install 3 | 4 | # Classes to recognize 5 | [CLASSES] 6 | neutral: 0 7 | anger: 1 8 | contempt: 2 9 | disgust: 3 10 | fear: 4 11 | happy: 5 12 | sadness: 6 13 | surprise: 7 14 | 15 | [TRAINING] 16 | # Folder containing the raw images (should contain a subfolder for each classes to recognize) 17 | IMAGES: training/images 18 | # Folder containing the cropped faces (should contain a subfolder for each classes to recognize) 19 | FACES: training/faces 20 | # Folder containing the features calculated for each face image. 21 | # Should have a first layer of subfolder named as classes to recognize, and inside each 22 | # class subfolder should contains a folder for each face image that belongs to that class 23 | FEATURES: training/features 24 | 25 | [VALIDATION] 26 | # Folder containing the raw images (should contain a subfolder for each classes to recognize) 27 | IMAGES: validation/images 28 | # Folder containing the cropped faces (should contain a subfolder for each classes to recognize) 29 | FACES: validation/faces 30 | # Folder containing the features calculated for each face image. 31 | # Should have a first layer of subfolder named as classes to recognize, and inside each 32 | # class subfolder should contains a folder for each face image that belongs to that class 33 | FEATURES: validation/features 34 | 35 | [FOLDER] 36 | # Folder containing the csv files of the features for training set and validation set (one file per classifier) 37 | TRAIN: trainfiles 38 | # Folder containing the trained classifiers (auspiciously should be opencv .xml files) 39 | CLASSIFIER: classifiers 40 | CLASSIFIER_ADA: classifiers/ada 41 | CLASSIFIER_SVM: classifiers/svm 42 | 43 | [SUFFIX] 44 | # Suffix of the folder which contains Gabor filtered images 45 | FILTERED_FOLDER: _feats 46 | # Suffix of the file containing features in csv format 47 | FEATURE_FILE: _feats.csv 48 | 49 | [FACECROP] 50 | TOOL: ./facecrop_cli 51 | FACE_DETECTOR_CFG: ../resources/haarcascade_frontalface_default.xml 52 | EYE_DETECTOR_CFG: ../resources/haarcascade_eye.xml 53 | 54 | [SIZE] 55 | # Size of the images before features extraction 56 | width: 32 57 | height: 32 58 | 59 | [GABOR] 60 | TOOL: ./gaborbank_cli 61 | FILTER_FILE: NA 62 | FEAT_FNAME: gabor.yml 63 | NWIDTHS: 1 64 | NLAMBDAS: 5 65 | NTHETAS: 6 66 | 67 | [TRAIN] 68 | # Write samples features directly in the csv file prepared for training, this way is possible to use CvMLData.read_csv for 69 | # loading sample data. If "False" do not embed any feature in the csv file, instead specify the file path where to find 70 | # the feature matrix of the sample. 71 | TOOL: ./train_cli 72 | ADA_CPU_USAGE: 0.5 73 | ADA_FILTER_FNAME: adaboost_featselection.dat 74 | SVM_CPU_USAGE: 0.5 75 | SVM_FILTER_FNAME: svm_featselection.dat 76 | 77 | [DETECTION] 78 | TOOL: ./emo_detector_cli 79 | 80 | [GUI] 81 | TOOL: ./emotimegui_cli 82 | -------------------------------------------------------------------------------- /src/dataset/gui.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import argparse 4 | import datasetConfigParser as dcp 5 | import os 6 | import sys 7 | 8 | def launch_gui(dsfolder, config, mode, eye_detection, do_prints=True): 9 | 10 | if mode == 'svm': 11 | class_dir = os.path.join(dsfolder, config['CLASSIFIER_SVM_FOLDER']) 12 | mode_s = 'svm' 13 | else: 14 | class_dir = os.path.join(dsfolder, config['CLASSIFIER_ADA_FOLDER']) 15 | mode_s = 'ada' 16 | 17 | execut = config['GUI_TOOL'] 18 | 19 | classificators = [] 20 | for f in os.listdir(class_dir): 21 | abs_f = os.path.join(class_dir, f) 22 | if os.path.isfile(abs_f): 23 | classificators.append(abs_f) 24 | 25 | args = [execut, config['FACECROP_FACE_DETECTOR_CFG']] 26 | if eye_detection: 27 | args.append(config['FACECROP_EYE_DETECTOR_CFG']) 28 | else: 29 | args.append('none') 30 | args += [config['SIZE']['width'], config['SIZE']['height'], 31 | config['GABOR_NWIDTHS'], config['GABOR_NLAMBDAS'], 32 | config['GABOR_NTHETAS'], mode_s] + classificators 33 | 34 | print "Launching emotime gui! Smile :)" 35 | os.execv(execut, args); 36 | 37 | 38 | if __name__ == "__main__": 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--cfg", default="dataset.cfg", help="Dataset config file name") 41 | parser.add_argument("dsFolder", help="Dataset base folder") 42 | parser.add_argument("-v", "--verbose", action='store_true', help="verbosity") 43 | parser.add_argument("--mode", default="svm", choices=['ada', 'svm'], help="training mode: ada or svm") 44 | parser.add_argument("--eye-correction", action="store_true", help="Perform eye correction on images") 45 | args = parser.parse_args() 46 | 47 | try: 48 | config = {} 49 | config = dcp.parse_ini_config(args.cfg) 50 | launch_gui(args.dsFolder, config, args.mode, args.eye_correction) 51 | except Exception as e: 52 | print "ERR: something wrong (%s)" % str(e) 53 | sys.exit(1) 54 | -------------------------------------------------------------------------------- /src/dataset/train_models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import argparse 4 | import subprocess 5 | import sys 6 | import os 7 | 8 | if __name__ == "__main__": 9 | 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument("--cfg", help="Dataset config file name", default='dataset.cfg') 12 | parser.add_argument("dsFolder", help="Dataset base folder") 13 | parser.add_argument("--mode", choices=['adaboost', 'svm'], help="training mode: adaboost or svm", required=True) 14 | parser.add_argument("--prep-train-mode", choices=['1vsallext', '1vsall', '1vs1'], help="Training set preparation mode: 1vsall, 1vsall extended, 1vs1", required=True) 15 | parser.add_argument("--eye-correction", action="store_true", help="Apply eye correction to faces") 16 | parser.add_argument("--skip-facecrop", action="store_true", help="WARNING: To be set only if the facecropping was already performed with the same configuration file!") 17 | parser.add_argument("--skip-feature", action="store_true", help="WARNING: To be set only if the (gabor) feature calculation was already performed with the same configuration file!") 18 | parser.add_argument("--clean", action="store_true", help="Clean training dataset before start") 19 | 20 | args = parser.parse_args() 21 | 22 | print(" ** Emotime ** ") 23 | print(" > Luca Mella") 24 | print(" > Daniele Bellavista") 25 | print("") 26 | print(" [*] Remember! Before using the training, the dataset folder must be") 27 | print(" initialized with datasetInit.py and datasetFillCK.py") 28 | print("") 29 | 30 | if not os.path.isfile(args.cfg): 31 | print(" [#] ERROR: configuration file", args.cfg, "doesn't exists at path", os.path.abspath(args.cfg)) 32 | print("") 33 | sys.exit(1) 34 | 35 | if not os.path.isdir(args.dsFolder): 36 | print(" [#] ERROR: dataset folder", args.dsFolder, "doesn't exists at path", os.path.abspath(args.dsFolder)) 37 | print("") 38 | sys.exit(1) 39 | 40 | print(" [*] Parameters: ") 41 | print(" [>] Dataset folder: " + args.dsFolder) 42 | print(" [>] Configuration file: " + args.cfg) 43 | print(" [>] Trainig mode: " + args.mode) 44 | print(" [>] Trainig preparation mode: " + args.prep_train_mode) 45 | print(" [>] Eye correction: " + repr(args.eye_correction)) 46 | print(" [>] Skip facecrop (WARNING): " + repr(args.skip_facecrop)) 47 | 48 | base_args = ['--cfg', args.cfg, args.dsFolder] 49 | prep_mode_args = ['--mode', args.prep_train_mode] 50 | mode_args = ['--mode', args.mode] 51 | eye_args = [] 52 | if args.eye_correction: 53 | eye_args.append('--eye-correction') 54 | 55 | 56 | ########################### Face Cropping 57 | if not args.skip_facecrop: 58 | print(" [1] Cropping faces...") 59 | 60 | if subprocess.call(['python', './datasetCropFaces.py'] + eye_args + base_args) is not 0: 61 | print(" [#] An Error occured! Exiting...") 62 | sys.exit(1) 63 | else: 64 | print(" [1] Skipping face crop!") 65 | ########################################## 66 | 67 | ########################### Features computation 68 | if not args.skip_feature: 69 | print(" [2] Computing features using bank of gabor magniture filters...") 70 | 71 | if subprocess.call(['python', './datasetFeatures.py'] + base_args) is not 0: 72 | print(" [#] An Error occured! Exiting...") 73 | sys.exit(1) 74 | else: 75 | print(" [2] Skipping feature calculation!") 76 | ########################################## 77 | 78 | ########################### Training files preparation 79 | print(" [3] Preparing CSV files with training data using %s..."%args.prep_train_mode) 80 | 81 | # Removing old csv files 82 | train_fold = os.path.join(args.dsFolder, 'training') 83 | [os.remove(os.path.join(train_fold, f)) for f in os.listdir(train_fold) if f.endswith(".csv")] 84 | 85 | if subprocess.call(['python', './datasetPrepTrain.py'] + prep_mode_args + base_args) is not 0: 86 | print(" [#] An Error occured! Exiting...") 87 | sys.exit(1) 88 | ########################################## 89 | 90 | ########################### Training 91 | print(" [4] Training with %s and selecting relevant features..."%args.mode) 92 | 93 | if subprocess.call(['python', './datasetTrain.py'] + mode_args + base_args) is not 0: 94 | print(" [#] An Error occured! Exiting...") 95 | sys.exit(1) 96 | ########################################## 97 | 98 | ########################### Verification 99 | print(" [5] Verifying the prediction...") 100 | 101 | if subprocess.call(['python', './datasetVerifyPrediction.py'] + mode_args + eye_args + base_args) is not 0: 102 | print(" [#] An Error occured! Exiting...") 103 | sys.exit(1) 104 | ########################################## 105 | -------------------------------------------------------------------------------- /src/dataset/train_models.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## Description: 4 | ## 5 | ## Train script 6 | ## 7 | ## Usage: 8 | ## 9 | ## train_models.sh OPTIONS MODE PREPTRAINMODE DATASET 10 | ## 11 | ## Parameters: 12 | ## 13 | ## MODE ada, svm 14 | ## PREPTRAINMODE 1vs1, 1vsall, 1vsallext 15 | ## DATASET Dataset folder 16 | ## 17 | ## Options: 18 | ## 19 | ## -h, --help This help message 20 | ## --no-eye Do not perform eye-based face rotation correction 21 | ## --skip-facecrop Do not perform face cropping (the operation must 22 | ## have been completed previously) 23 | ## 24 | ## Example: 25 | ## 26 | ## ./train_models.sh -h 27 | ## 28 | 29 | # ---------------- 30 | # UTILITY FUNCTIONS 31 | # ---------------- 32 | 33 | echoerr() { 34 | echo "$@" 1>&2; 35 | } 36 | usage() { 37 | [ "$*" ] && echoerr "$0: $*" 38 | sed -n '/^##/,/^$/s/^## \{0,1\}//p' "$0" 1>&2 39 | exit 2 40 | } 41 | 42 | # default params 43 | DS_FOLDER=../dataset 44 | DS_CONFIG=default.cfg 45 | MODE=adaboost 46 | PREPTRAINMODE=1vsAll 47 | EYE="--eye-correction" 48 | SKIP_FACECROP=0 49 | 50 | # ---------------- 51 | # MAIN 52 | # ---------------- 53 | 54 | # Options parsing 55 | while [ $# -gt 0 ]; do 56 | case $1 in 57 | (-h|--help) usage 2>&1 ;; 58 | (--no-eye) EYE=''; shift ;; 59 | (--skip-facecrop) SKIP_FACECROP=1; shift ;; 60 | (--) shift; break ;; 61 | (-*) usage "$1: unknown option" ;; 62 | (*) break;; 63 | esac 64 | done 65 | 66 | if [[ $# -lt 3 ]] ; then 67 | echoerr "ERR: Missing mandatory parameter" 68 | exit -1 69 | fi 70 | 71 | # Parameter parsing 72 | if [[ $1 == "svm" ]]; then 73 | MODE='svm' 74 | elif [[ $1 == "adaboost" ]]; then 75 | MODE='adaboost' 76 | fi 77 | 78 | case ${2,,} in 79 | "1vsall") 80 | PREPTRAINMODE='1vsAll' 81 | ;; 82 | "1vsallext") 83 | PREPTRAINMODE='1vsAllExt' 84 | ;; 85 | "1vs1") 86 | PREPTRAINMODE='1vs1' 87 | ;; 88 | *) 89 | ;; 90 | esac 91 | 92 | DS_FOLDER="$3" 93 | 94 | if [[ $SKIP_FACECROP -eq 0 ]] ; then 95 | echo "1) Cropping faces:" 96 | python2 ./datasetCropFaces.py $DS_FOLDER $EYE 97 | else 98 | echo "1) Cropping faces:" 99 | echo " skipping..." 100 | fi 101 | echo "------------------------" 102 | echo "2) Calculating features using bank of Gabor magnitude filters:" 103 | python2 ./datasetFeatures.py $DS_FOLDER 104 | echo "------------------------" 105 | # Non dataset dependent standard cleanup 106 | rm $DS_FOLDER/training/*.csv 107 | echo "3) Preparing CSV file containing training data:" 108 | python2 ./datasetPrepTrain.py $DS_FOLDER --mode $PREPTRAINMODE 109 | echo "------------------------" 110 | echo "4) Training with $MODE and selecting relevant features:" 111 | python2 ./datasetTrain.py $DS_FOLDER --mode $MODE 112 | echo "------------------------" 113 | echo "5) Verifying the prediction of $MODE:" 114 | python2 ./datasetVerifyPrediction.py --mode $MODE $DS_FOLDER $EYE 115 | echo "------------------------" 116 | 117 | -------------------------------------------------------------------------------- /src/detector/BoostEmoDetector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file BoostEmoDetector.cpp 4 | * @author Daniele Bellavista (daniele.bellavista@studio.unibo.it) 5 | * @date 12/27/2013 12:12:58 PM 6 | * @brief 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include "BoostEmoDetector.h" 13 | #include "AdaBoostClassifier.h" 14 | 15 | namespace emotime { 16 | 17 | BoostEmoDetector::BoostEmoDetector(int boost_type, double trim_weight, int max_depth) : EmoDetector() { 18 | this->boost_type = boost_type; 19 | this->trim_weight = trim_weight; 20 | this->max_depth = max_depth; 21 | } 22 | 23 | BoostEmoDetector::BoostEmoDetector(int boost_type, double trim_weight, int max_depth, 24 | std::map, Classifier*> > 25 | detmap_ext) : EmoDetector(detmap_ext) { 26 | this->boost_type = boost_type; 27 | this->trim_weight = trim_weight; 28 | this->max_depth = max_depth; 29 | } 30 | 31 | 32 | Classifier* BoostEmoDetector::createClassifier() { 33 | return new AdaBoostClassifier(this->boost_type, this->trim_weight, this->max_depth); 34 | } 35 | 36 | std::pair BoostEmoDetector::predict(cv::Mat& frame) { 37 | return EmoDetector::predictVotingOneVsAllExt(frame); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/detector/BoostEmoDetector.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file BoostEmoDetector.h 4 | * @date 02/08/2014 11:47:13 AM 5 | * @brief Definition of BoostEmoDetector 6 | * 7 | */ 8 | 9 | #ifndef _H_BOST_EMO_DETECTOR 10 | #define _H_BOST_EMO_DETECTOR 11 | 12 | #include "EmoDetector.h" 13 | 14 | namespace emotime{ 15 | 16 | /** 17 | * @class BoostEmoDetector 18 | * 19 | * @date 12/27/2013 11:59:26 AM 20 | * 21 | * @brief Emotion detector specialization using AdaBoost 22 | * 23 | */ 24 | class BoostEmoDetector : public EmoDetector { 25 | public: 26 | 27 | 28 | /** 29 | * @brief Initialize the emodetector with boost parameters and 30 | * empty classifiers. 31 | * 32 | * @param[in] boost_type Type of the opencv boosting algorithm 33 | * @param[in] trim_weight The opencv trim weight value 34 | * @param[in] max_depth Algorithm max depth 35 | * 36 | * @see AdaBoostClassifier 37 | * 38 | */ 39 | BoostEmoDetector(int boost_type, double trim_weight, int max_depth); 40 | 41 | /** 42 | * @brief Initialize the emodetector with boost parameters and 43 | * classifiers. 44 | * 45 | * @param[in] boost_type Type of the opencv boosting algorithm 46 | * @param[in] trim_weight The opencv trim weight value 47 | * @param[in] max_depth Algorithm max depth 48 | * @param[in] detmap_ext Mapping between emotions and classifier. 49 | * 50 | * @see AdaBoostClassifier 51 | * 52 | */ 53 | BoostEmoDetector(int boost_type, double trim_weight, int max_depth, 54 | std::map, Classifier*> > 55 | detmap_ext); 56 | 57 | std::pair predict(cv::Mat& frame); 58 | 59 | protected: 60 | 61 | Classifier* createClassifier(); 62 | 63 | private: 64 | 65 | /// Type of the opencv boosting algorithm 66 | int boost_type; 67 | /// The opencv trim weight value 68 | double trim_weight; 69 | /// Algorithm max depth 70 | int max_depth; 71 | }; 72 | 73 | } 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /src/detector/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | find_package(OpenCV COMPONENTS opencv_core opencv_ml opencv_imgproc opencv_highgui opencv_objdetect REQUIRED) 3 | 4 | add_executable(emo_detector_cli emo_detector_cli.cpp 5 | EmoDetector.cpp 6 | BoostEmoDetector.cpp 7 | SVMEmoDetector.cpp 8 | FacePreProcessor.cpp 9 | ../training/Classifier.cpp 10 | ../training/SVMClassifier.cpp 11 | ../training/AdaBoostClassifier.cpp 12 | ../utils/matrix_io.cpp 13 | ../gaborbank/GaborBank.cpp 14 | ../utils/string_utils.cpp 15 | ../facedetector/FaceDetector.cpp) 16 | 17 | target_link_libraries(emo_detector_cli ${OpenCV_LIBS} ) 18 | 19 | INSTALL(TARGETS emo_detector_cli DESTINATION ${ASSETDIR} ) 20 | -------------------------------------------------------------------------------- /src/detector/EmoDetector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file EmoDetector.cpp 3 | * @date 01/10/2014 01:12:27 AM 4 | * @brief 5 | * 6 | * @details 7 | * 8 | */ 9 | 10 | #include "matrix_io.h" 11 | #include "string_utils.h" 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include "EmoDetector.h" 23 | 24 | using std::pair; 25 | using std::map; 26 | using std::string; 27 | using std::vector; 28 | using std::make_pair; 29 | 30 | using std::numeric_limits; 31 | 32 | using std::cerr; 33 | using std::endl; 34 | 35 | using cv::Mat; 36 | 37 | namespace emotime { 38 | 39 | string emotionStrings(Emotion emo){ 40 | switch (emo) { 41 | case NEUTRAL: return string("neutral"); 42 | case ANGER: return string("anger"); 43 | case CONTEMPT: return string("contempt"); 44 | case DISGUST: return string("disgust"); 45 | case FEAR: return string("fear"); 46 | case HAPPY: return string("happy"); 47 | case SADNESS: return string("sadness"); 48 | case SURPRISE: return string("surprise"); 49 | case OTHERS: return string("others"); 50 | default: return string("unknown"); 51 | } 52 | }; 53 | 54 | void EmoDetector::init(const std::map, Classifier*> >& detmap_ext) { 55 | this->detectors_ext = detmap_ext; 56 | 57 | for(map, Classifier*> >::const_iterator ii = detmap_ext.begin(); 58 | ii != detmap_ext.end(); ++ii) { 59 | vector emv = ii->second.first; 60 | this->detectors_ext.insert(make_pair(ii->first, make_pair(emv, ii->second.second))); 61 | } 62 | } 63 | 64 | EmoDetector::EmoDetector() { 65 | 66 | } 67 | 68 | EmoDetector::EmoDetector(std::map, Classifier*> >& detmap_ext) { 69 | init(detmap_ext); 70 | } 71 | 72 | EmoDetector::~EmoDetector() { 73 | for(map, Classifier*> >::const_iterator ii = this->detectors_ext.begin(); 74 | ii != this->detectors_ext.end(); ++ii) { 75 | delete ii->second.second; 76 | } 77 | detectors_ext.clear(); 78 | } 79 | 80 | void EmoDetector::init(std::vector& classifier_paths) { 81 | map, Classifier*> > classifiers; 82 | 83 | for(size_t i = 0; i < classifier_paths.size(); i++) { 84 | 85 | string clpath = classifier_paths[i]; 86 | Classifier* cvD = this->createClassifier(); 87 | cvD->load(clpath); 88 | 89 | string fname = matrix_io_fileBaseName(clpath); 90 | Emotion emo = UNKNOWN; 91 | 92 | vector emotions_list = split_string(fname, "_"); 93 | vector fin_emo_list; 94 | fin_emo_list.reserve(emotions_list.size()); 95 | string label = ""; 96 | 97 | for(vector::iterator it = emotions_list.begin(); it != 98 | emotions_list.end(); ++it) { 99 | emo = UNKNOWN; 100 | if (*it == "vs") { 101 | break; 102 | } else if (*it == emotionStrings(NEUTRAL)) { 103 | emo = NEUTRAL; 104 | } else if (*it == emotionStrings(ANGER)) { 105 | emo = ANGER; 106 | } else if (*it == emotionStrings(CONTEMPT)) { 107 | emo = CONTEMPT; 108 | } else if (*it == emotionStrings(DISGUST)) { 109 | emo = DISGUST; 110 | } else if (*it == emotionStrings(FEAR)) { 111 | emo = FEAR; 112 | } else if (*it == emotionStrings(HAPPY)) { 113 | emo = HAPPY; 114 | } else if (*it == emotionStrings(SADNESS)) { 115 | emo = SADNESS; 116 | } else if (*it == emotionStrings(SURPRISE)) { 117 | emo = SURPRISE; 118 | } 119 | if(emo != UNKNOWN) { 120 | if(label.size() > 0) { 121 | label.append("_"); 122 | } 123 | label.append(emotionStrings(emo)); 124 | fin_emo_list.push_back(emo); 125 | } 126 | } 127 | pair, Classifier*> value(fin_emo_list, cvD); 128 | pair, Classifier*> > entry(label, value); 129 | classifiers.insert(entry); 130 | } 131 | 132 | init(classifiers); 133 | } 134 | 135 | pair EmoDetector::predictVotingOneVsAllExt(cv::Mat& frame) { 136 | 137 | map votes; 138 | 139 | if (detectors_ext.size() == 0) { 140 | return make_pair(UNKNOWN, 0.0f); 141 | } 142 | 143 | votes.insert(make_pair(NEUTRAL, 0.f)); 144 | votes.insert(make_pair(CONTEMPT , 0.f)); 145 | votes.insert(make_pair(DISGUST, 0.f)); 146 | votes.insert(make_pair(SADNESS, 0.f)); 147 | votes.insert(make_pair(ANGER, 0.f)); 148 | votes.insert(make_pair(HAPPY, 0.f)); 149 | votes.insert(make_pair(FEAR, 0.f)); 150 | votes.insert(make_pair(UNKNOWN, 0.f)); 151 | 152 | for(map, Classifier*> >::iterator ii = 153 | this->detectors_ext.begin(); ii != this->detectors_ext.end(); ++ii) { 154 | 155 | vector emo = ii->second.first; // detected emotions 156 | Classifier* cl = ii->second.second; 157 | 158 | float prediction = cl->predict(frame); 159 | 160 | for(vector::iterator emo_it = emo.begin(); emo_it != emo.end(); ++emo_it) { 161 | map::iterator it = votes.find(*emo_it); 162 | if (it == votes.end()) { 163 | votes.insert(make_pair(*emo_it, prediction)); 164 | } else{ 165 | if (prediction > 0.5) { 166 | it->second += 0.5; //1.0; 167 | } else { 168 | //for(map::iterator votes_it = votes.begin(); votes_it != votes.end(); ++votes_it) { 169 | // vector::iterator e_it = find(emo.begin(), emo.end(), votes_it->first); 170 | // if (e_it == emo.end()) { 171 | // // if I dont find emotion in detected emotion 172 | // votes_it->second+=1.0; 173 | // } 174 | //} 175 | it->second -= 0.0; 176 | } 177 | } 178 | } 179 | } 180 | 181 | pair max_pair = make_pair(UNKNOWN, numeric_limits::min()); 182 | 183 | for( map::iterator ii = votes.begin(); ii != votes.end(); 184 | ++ii) { 185 | if (ii->second > max_pair.second) { 186 | max_pair.first = ii->first; 187 | max_pair.second = ii->second; 188 | } 189 | } 190 | 191 | return max_pair; 192 | } 193 | 194 | pair EmoDetector::predictBestWinsOneVsAll(cv::Mat& frame) { 195 | 196 | pair best(UNKNOWN, numeric_limits::min()); 197 | 198 | if (detectors_ext.size() == 0) { 199 | return make_pair(UNKNOWN, 0.0f); 200 | } 201 | 202 | for (map, Classifier*> >::iterator ii = 203 | this->detectors_ext.begin(); ii != this->detectors_ext.end(); ++ii) { 204 | 205 | if (ii->second.first.size() != 1) { 206 | continue; 207 | } 208 | Emotion emo = ii->second.first[0]; 209 | Classifier* cl = ii->second.second; 210 | 211 | float prediction = cl->predict(frame); 212 | 213 | if (best.second < prediction) { 214 | best.first = emo; 215 | best.second = prediction; 216 | } 217 | } 218 | 219 | return best; 220 | } 221 | 222 | pair EmoDetector::predictMajorityOneVsAll(cv::Mat& frame){ 223 | map votes; 224 | 225 | if (detectors_ext.size() == 0) { 226 | return make_pair(UNKNOWN, 0.0f); 227 | } 228 | 229 | for (map, Classifier*> >::iterator ii = 230 | this->detectors_ext.begin(); ii != this->detectors_ext.end(); ++ii) { 231 | 232 | if (ii->second.first.size() != 1) { 233 | continue; 234 | } 235 | Emotion emo = ii->second.first[0]; 236 | Classifier* cl = ii->second.second; 237 | 238 | float prediction = cl->predict(frame); 239 | 240 | map::iterator it = votes.find(emo); 241 | if (it == votes.end()) { 242 | votes.insert(make_pair(emo, prediction)); 243 | } else { 244 | it->second+=prediction; 245 | } 246 | } 247 | 248 | pair max_pair = make_pair(UNKNOWN, numeric_limits::min()); 249 | for( map::iterator ii=votes.begin(); ii!=votes.end(); ++ii){ 250 | if (ii->second > max_pair.second){ 251 | max_pair.first=ii->first; 252 | max_pair.second=ii->second; 253 | } 254 | } 255 | 256 | return max_pair; 257 | } 258 | 259 | pair EmoDetector::predict(cv::Mat& frame) { 260 | return predictMajorityOneVsAll(frame); 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /src/detector/EmoDetector.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file EmoDetector.h 4 | * @brief Contains definition of EmoDetector and Emotion 5 | * 6 | */ 7 | 8 | #ifndef _H_EMO_DETECTOR 9 | #define _H_EMO_DETECTOR 10 | 11 | #include "matrix_io.h" 12 | #include "string_utils.h" 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include "Classifier.h" 23 | 24 | using namespace cv; 25 | 26 | namespace emotime { 27 | 28 | /// Emotion identification 29 | enum Emotion { 30 | /// NEUTRAL emotion 31 | NEUTRAL = 0, 32 | /// ANGER emotion 33 | ANGER = 1, 34 | /// CONTEMPT emotion 35 | CONTEMPT = 2, 36 | /// DISGUST emotion 37 | DISGUST = 3, 38 | /// FEAR emotion 39 | FEAR = 4, 40 | /// HAPPY emotion 41 | HAPPY = 5, 42 | /// SADNESS emotion 43 | SADNESS = 6, 44 | /// SURPRISE emotion 45 | SURPRISE = 7, 46 | /// OTHERS emotion 47 | OTHERS = 8, 48 | /// UNKNOWN emotion 49 | UNKNOWN = 9 50 | }; 51 | 52 | /** 53 | * @brief Retrieve the string associated with an emotion 54 | * 55 | * @param[in] emo The enum value 56 | * 57 | * @return A string representation 58 | */ 59 | std::string emotionStrings(Emotion emo); 60 | 61 | /** 62 | * @class EmoDetector 63 | * 64 | * @brief Generic class for performing multi-class classification using 65 | * binary classifiers. 66 | * 67 | * @details 68 | * 69 | */ 70 | class EmoDetector { 71 | 72 | public: 73 | 74 | EmoDetector(); 75 | 76 | /** 77 | * @brief Initialize the detectors in extended mode (one detector multiple emotion) 78 | * 79 | * @param[in] detmap_ext A map to name -> (vector, detector) 80 | * 81 | */ 82 | EmoDetector(std::map, Classifier*> >& detmap_ext); 83 | 84 | /** 85 | * @brief Initialize the detector from classifier paths 86 | * 87 | * @param[in] classifiers_path Path of various classifier files. 88 | * 89 | * @details Path must be in the format: emop1_emop2_emopn_vs_emon1_emon2_emonm.xml 90 | * Where emop* is the emotion recognized and emon* is the emotion not recognized. 91 | * 92 | */ 93 | void init(std::vector& classifiers_path); 94 | 95 | /** 96 | * Release an EmoDetector 97 | */ 98 | virtual ~EmoDetector(); 99 | 100 | /** 101 | * Return true if the given emotion is present 102 | * 103 | * @param[in] name The name to search 104 | * 105 | * @return true if the given emotion is present 106 | * 107 | */ 108 | bool contains(std::string& name); 109 | 110 | /** 111 | * @brief Predict the emotion using the extended classification. 112 | * 113 | * @param[in] frame The image to predict 114 | * 115 | * @return The prediction and its result 116 | * 117 | * @details 118 | */ 119 | std::pair predictVotingOneVsAllExt(cv::Mat& frame); 120 | 121 | /** 122 | * @brief Predict the emotion using the best-wins approach 123 | * 124 | * @param[in] frame The image to predict 125 | * 126 | * @return The prediction and its result 127 | * 128 | * @details 129 | */ 130 | std::pair predictBestWinsOneVsAll(cv::Mat& frame); 131 | 132 | /** 133 | * @brief Predict the class of the sample frame using a majority voting strategy. 134 | * 135 | * @param[in] frame The image to predict 136 | * 137 | * @return The prediction and its result 138 | * 139 | * @details 140 | * 141 | */ 142 | std::pair predictMajorityOneVsAll(cv::Mat& frame); 143 | 144 | /** 145 | * @brief Apply the default prediction method 146 | * 147 | * @param[in] frame The image to predict 148 | * 149 | * @return The predicted emotion 150 | */ 151 | virtual std::pair predict(cv::Mat& frame); 152 | 153 | protected: 154 | 155 | /** 156 | * @brief Instantiate a classifier 157 | * 158 | * @return A new classifier 159 | * 160 | */ 161 | virtual Classifier* createClassifier() = 0; 162 | 163 | private: 164 | 165 | /// Detectors for generic approaches (each detector matches one or more emotion) 166 | std::map, Classifier*> > detectors_ext; 167 | 168 | /** 169 | * @brief Initialize the detectors in extended mode (one detector multiple emotion) 170 | * 171 | * @param[in] detmap_ext A map to name -> (vector, detector) 172 | * 173 | */ 174 | void init(const std::map, Classifier*> >& detmap_ext); 175 | 176 | }; // end of EmoDetector 177 | } 178 | 179 | #endif 180 | -------------------------------------------------------------------------------- /src/detector/FacePreProcessor.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file FacePreProcessor.cpp 3 | * 4 | * @date 01/10/2014 12:48:37 AM 5 | * @brief 6 | * 7 | * @details 8 | * 9 | */ 10 | 11 | #include "FacePreProcessor.h" 12 | 13 | using std::string; 14 | using cv::Mat; 15 | 16 | namespace emotime { 17 | 18 | void FacePreProcessor::init(std::string faceDetectorConfig, std::string eyesDetectorConfig, int 19 | width, int height, double nwidths, double nlambdas, double nthetas) { 20 | 21 | this->imgsize.width = width; 22 | this->imgsize.height = height; 23 | this->nwidths = nwidths; 24 | this->nlambdas = nlambdas; 25 | this->nthetas = nthetas; 26 | if (eyesDetectorConfig.size() > 0) { 27 | this->facedet = FaceDetector(faceDetectorConfig, eyesDetectorConfig); 28 | } else { 29 | this->facedet = FaceDetector(faceDetectorConfig); 30 | } 31 | this->gaborbank.fillGaborBank(this->nwidths, this->nlambdas, this->nthetas); 32 | } 33 | 34 | FacePreProcessor::FacePreProcessor(std::string faceDetectorConfig, std::string 35 | eyesDetectorConfig, int width, int height, double nwidths, double 36 | nlambdas, double nthetas) { 37 | init(faceDetectorConfig, eyesDetectorConfig, width, height, nwidths, 38 | nlambdas, nthetas); 39 | } 40 | 41 | FacePreProcessor::FacePreProcessor(std::string faceDetectorConfig, int width, int 42 | height, double nwidths, double nlambdas, double nthetas) { 43 | init(faceDetectorConfig, "", width, height, nwidths, nlambdas, nthetas); 44 | } 45 | 46 | FacePreProcessor::~FacePreProcessor() { 47 | 48 | } 49 | 50 | bool FacePreProcessor::preprocess(Mat& img, Mat& featvec) { 51 | Mat face, filtered, fvec; 52 | bool res; 53 | res = this->extractFace(img, face); 54 | if (res) { 55 | res = this->filterImage(face, filtered); 56 | if (res) { 57 | res = this->toFeaturesVector(filtered, featvec); 58 | } 59 | } 60 | return res; 61 | } 62 | 63 | bool FacePreProcessor::extractFace(Mat& src, Mat& out) { 64 | Mat face; 65 | bool faceFound=this->facedet.detect(src, face); 66 | if (!faceFound){ 67 | return false; 68 | } 69 | face.copyTo(out); 70 | //face.release(); 71 | return true; 72 | } 73 | 74 | bool FacePreProcessor::filterImage(Mat& src, Mat& out) { 75 | Mat filtered; 76 | filtered = this->gaborbank.filterImage(src, this->imgsize); 77 | filtered.copyTo(out); 78 | //filtered.release(); 79 | return true; 80 | } 81 | 82 | bool FacePreProcessor::toFeaturesVector(Mat& src, Mat& out) { 83 | Mat feat; 84 | feat = src.reshape(1/*chan*/, 1/*rows*/); 85 | feat.copyTo(out); 86 | return true; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/detector/FacePreProcessor.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file FacePreProcessor.h 4 | * @brief Definition of FacePreProcessor 5 | * 6 | */ 7 | 8 | #ifndef FACEPREPROCESSOR_H 9 | #define FACEPREPROCESSOR_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "FaceDetector.h" 21 | #include "GaborBank.h" 22 | 23 | namespace emotime{ 24 | 25 | /** 26 | * @class FacePreProcessor 27 | * 28 | * @brief Class for preprocessing a face before training. 29 | * 30 | * @details 31 | * 32 | */ 33 | class FacePreProcessor{ 34 | 35 | public: 36 | 37 | 38 | /** 39 | * @brief Creates a FacePreProcessor specifying FaceDetector and GaborBank parameters. 40 | * 41 | * @param[in] faceDetectorConfig The configuration file for face detection 42 | * @param[in] eyesDetectorConfig The configuration file for eye 43 | * detection. If the value is "none", the eye 44 | * configuration is ignored. 45 | * @param[in] width Resized with of the face 46 | * @param[in] height Resized height of the face 47 | * @param[in] nwidths nwidhts parameter of GaborBank::fillGaborBank 48 | * @param[in] nlambdas nlambdas parameter of GaborBank::fillGaborBank 49 | * @param[in] nthetas nthetas parameter of GaborBank::fillGaborBank 50 | * 51 | * @see FaceDetector 52 | * @see GaborBank 53 | * 54 | */ 55 | FacePreProcessor(std::string faceDetectorConfig, std::string eyesDetectorConfig, 56 | int width, int height, double nwidths, double nlambdas, double 57 | nthetas); 58 | 59 | /** 60 | * @brief Creates a FacePreProcessor specifying FaceDetector 61 | * and GaborBank parameters. This constructor disables the eye detection. 62 | * 63 | * @param[in] faceDetectorConfig The configuration file for face detection 64 | * @param[in] width Resized with of the face 65 | * @param[in] height Resized height of the face 66 | * @param[in] nwidths nwidhts parameter of GaborBank::fillGaborBank 67 | * @param[in] nlambdas nlambdas parameter of GaborBank::fillGaborBank 68 | * @param[in] nthetas nthetas parameter of GaborBank::fillGaborBank 69 | * 70 | * @see FaceDetector 71 | * @see GaborBank 72 | * 73 | */ 74 | FacePreProcessor(std::string faceDetectorConfig, int width, int height, double 75 | nwidths, double nlambdas, double nthetas); 76 | 77 | ~FacePreProcessor(); 78 | 79 | /** 80 | * @brief Extract the features from the given image 81 | * 82 | * @param[in] src The image to analyze 83 | * @param[in,out] out The feature vector 84 | * 85 | * @return True if the operation succedede 86 | * 87 | */ 88 | bool preprocess(cv::Mat& src, cv::Mat& out); 89 | 90 | /** 91 | * @brief Extract the face from the given image 92 | * 93 | * @param[in] src The image to analyze 94 | * @param[in,out] out The image to set 95 | * 96 | * @return True if the face was extracted 97 | * 98 | */ 99 | bool extractFace(cv::Mat& src, cv::Mat& out); 100 | 101 | /** 102 | * @brief Filter the given image 103 | * 104 | * @param[in] src The image to filter 105 | * @param[in,out] out The image to set 106 | * 107 | * @return True if the image was filtered 108 | * 109 | */ 110 | bool filterImage(cv::Mat& src, cv::Mat& out); 111 | 112 | private: 113 | 114 | /// Face detector utility 115 | FaceDetector facedet; 116 | /// Gabor bank to use 117 | GaborBank gaborbank; 118 | /// The resize measure 119 | cv::Size imgsize; 120 | /// Gabor param: nwidths 121 | double nwidths; 122 | /// Gabor param: nlambdas 123 | double nlambdas; 124 | /// Gabor param: nthetas 125 | double nthetas; 126 | 127 | /** 128 | * @brief Convert the given filtered image to a feature vector, 129 | * suitable for training. 130 | * 131 | * @param[in] src The filtered image 132 | * @param[in,out] out The vector to set 133 | * 134 | * @return True if the operation succeded 135 | * 136 | */ 137 | bool toFeaturesVector(cv::Mat& src, cv::Mat& out); 138 | 139 | /** 140 | * @brief Initialize the FacePreProcessor 141 | * 142 | * @param[in] faceDetectorConfig Config file for face detection 143 | * @param[in] eyeDetectorConfig Config file for eye detection. If 144 | * empty, no eye detection will be performed. 145 | * @param[in] width The width resize 146 | * @param[in] height The height resize 147 | * @param[in] nwidths N-Width gabor param 148 | * @param[in] nlabmdas N-Width gabor param 149 | * @param[in] nthetas N-Width gabor param 150 | * 151 | */ 152 | void init(std::string faceDetectorConfig, std::string eyesDetectorConfig, int 153 | width, int height, double nwidths, double nlambdas, double nthetas); 154 | }; 155 | 156 | } 157 | #endif /* !FACEPREPROCESSOR_H */ 158 | 159 | -------------------------------------------------------------------------------- /src/detector/SVMEmoDetector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file SVMEmoDetector.cpp 4 | * @date 12/27/2013 12:12:58 PM 5 | * @brief Implementation of SVMEmoDetector 6 | * 7 | * @details 8 | * 9 | */ 10 | 11 | #include "SVMEmoDetector.h" 12 | #include "SVMClassifier.h" 13 | 14 | namespace emotime { 15 | 16 | SVMEmoDetector::SVMEmoDetector(double C_factor, int max_iteration, double error_margin) : EmoDetector() { 17 | this->C_factor = C_factor; 18 | this->max_iteration = max_iteration; 19 | this->error_margin = error_margin; 20 | } 21 | 22 | SVMEmoDetector::SVMEmoDetector(double C_factor, int max_iteration, double error_margin, 23 | std::map, Classifier*> > 24 | detmap_ext) : EmoDetector(detmap_ext) { 25 | this->C_factor = C_factor; 26 | this->max_iteration = max_iteration; 27 | this->error_margin = error_margin; 28 | } 29 | 30 | 31 | Classifier* SVMEmoDetector::createClassifier() { 32 | return new SVMClassifier(this->C_factor, this->max_iteration, this->error_margin); 33 | } 34 | 35 | std::pair SVMEmoDetector::predict(cv::Mat& frame) { 36 | return EmoDetector::predictVotingOneVsAllExt(frame); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/detector/SVMEmoDetector.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file SVMEmoDetector.h 4 | * @brief Defines SVMEmoDetector 5 | * 6 | */ 7 | 8 | #ifndef __SVMEMODETECTOR_INC__ 9 | #define __SVMEMODETECTOR_INC__ 10 | 11 | #include "EmoDetector.h" 12 | 13 | namespace emotime { 14 | 15 | /** 16 | * @class SVMEmoDetector 17 | * @date 12/27/2013 11:59:26 AM 18 | * 19 | * @brief EmoDetector specialization using SVMClassifier 20 | * 21 | */ 22 | class SVMEmoDetector : public EmoDetector { 23 | public: 24 | 25 | /** 26 | * @brief Creates an SVMEmoDetector with svm parameters and 27 | * empty classifiers. 28 | * 29 | * @param[in] C_factor The algorithm C factor 30 | * @param[in] max_iteration Maximum number of iteration termination criteria 31 | * @param[in] error_margin Minimum error termination criteria 32 | * 33 | * @see SVMClassifier 34 | */ 35 | SVMEmoDetector(double C_factor, int max_iteration, double error_margin); 36 | 37 | /** 38 | * @brief Creates an SVMEmoDetector with svm parameters and 39 | * classifiers. 40 | * 41 | * @param[in] C_factor The algorithm C factor 42 | * @param[in] max_iteration Maximum number of iteration termination criteria 43 | * @param[in] error_margin Minimum error termination criteria 44 | * @param[in] detmap_ext Mapping between emotions and classifier. 45 | * 46 | * @see SVMClassifier 47 | */ 48 | SVMEmoDetector(double C_factor, int max_iteration, double error_margin, 49 | std::map, Classifier*> > 50 | detmap_ext); 51 | 52 | std::pair predict(cv::Mat& frame); 53 | 54 | protected: 55 | 56 | Classifier* createClassifier(); 57 | 58 | private: 59 | 60 | /// The algorithm C factor 61 | double C_factor; 62 | /// Maximum number of iteration termination criteria 63 | int max_iteration; 64 | /// Minimum error termination criteria 65 | double error_margin; 66 | 67 | 68 | }; /* ----- end of class SVMEmoDetector ----- */ 69 | 70 | } 71 | 72 | 73 | #endif /* ----- #ifndef __SVMEMODETECTOR_INC__ ----- */ 74 | -------------------------------------------------------------------------------- /src/detector/emo_detector_cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file emo_detector_cli.cpp 4 | * @author Daniele Bellavista (daniele.bellavista@studio.unibo.it) 5 | * @date 12/27/2013 12:18:32 PM 6 | * @brief 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include "SVMEmoDetector.h" 13 | #include "BoostEmoDetector.h" 14 | #include "matrix_io.h" 15 | #include "FacePreProcessor.h" 16 | 17 | #include "TrainingParameters.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace cv; 28 | using namespace emotime; 29 | 30 | using std::cout; 31 | using std::endl; 32 | using std::cerr; 33 | using std::string; 34 | using std::pair; 35 | 36 | /** 37 | * @brief Prints the CLI banner 38 | * 39 | */ 40 | void banner(); 41 | 42 | /** 43 | * @brief Prints the CLI help 44 | * 45 | */ 46 | void help(); 47 | 48 | void help() { 49 | cout << "Usage:" << endl; 50 | cout << " emo_detector_cli (svm|ada) (|none) {}" << endl; 51 | cout << "Parameters:" << endl; 52 | cout << " (svm|ada) - Use ada or svm" << endl; 53 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for face detection" << endl; 54 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for eye detection. If the file is 'none', no eye correction is performed." << endl; 55 | cout << " - Width of the image, the input image will be scaled" << endl; 56 | cout << " - Height of the image, the input image will be scaled" << endl; 57 | cout << " - " << endl; 58 | cout << " - " << endl; 59 | cout << " - " << endl; 60 | cout << " - The trained svm for detecting an expression " << endl; 61 | cout << " Name format: emotion1_emotion2_..._vs_emotion... where emotion* is one of (neutral, contempt, disgust, fear, sadness, surprise)" << endl; 62 | cout << endl; 63 | } 64 | 65 | void banner() { 66 | cout << "EmoDetector Utility:" << endl; 67 | cout << " Detect emotions using trained classificator" << endl; 68 | } 69 | 70 | /** 71 | * @brief Main 72 | * 73 | * @param[in] argc 74 | * @param[in] argv 75 | * 76 | * @returns 1 77 | * 78 | */ 79 | int main(int argc, const char *argv[]) { 80 | if (argc < 9) { 81 | banner(); 82 | help(); 83 | cerr << "ERR: missing parameters" << endl; 84 | return -3; 85 | } 86 | string infile; // = string(argv[1]); 87 | string method(argv[1]); 88 | string config(argv[2]); 89 | string config_e(argv[3]); 90 | 91 | cv::Size size(0,0); 92 | int nwidths, nlambdas, nthetas; 93 | size.width = abs(atoi(argv[4])); 94 | size.height = abs(atoi(argv[5])); 95 | nwidths = abs(atoi(argv[6])); 96 | nlambdas= abs(atoi(argv[7])); 97 | nthetas = abs(atoi(argv[8])); 98 | vector classifier_paths; 99 | 100 | if (argc >= 10) { 101 | // Read boost XML paths 102 | for (int i = 9; i < argc; i++) { 103 | string clpath(argv[i]); 104 | classifier_paths.push_back(string(argv[i])); 105 | } 106 | } else { 107 | cerr << "ERR: you must specify some classifiers" << endl; 108 | return -2; 109 | } 110 | 111 | FacePreProcessor* preprocessor; 112 | EmoDetector* emodetector; 113 | 114 | try { 115 | 116 | if (config_e == "none") { 117 | preprocessor = new FacePreProcessor(config, size.width, size.height, 118 | nwidths, nlambdas, nthetas); 119 | } else { 120 | preprocessor = new FacePreProcessor(config, config_e, size.width, 121 | size.height, nwidths, nlambdas, nthetas); 122 | } 123 | 124 | if (method == "svm") { 125 | emodetector = new SVMEmoDetector(kCfactor, kMaxIteration, kErrorMargin); 126 | } else { 127 | emodetector = new BoostEmoDetector(kBoostType, kTrimWeight, kMaxDepth); 128 | } 129 | 130 | emodetector->init(classifier_paths); 131 | 132 | cout << "Insert the image file path: " << endl; 133 | while(std::getline(std::cin, infile)) { 134 | try { 135 | cout << "Processing '" << infile << "'" << endl; 136 | Mat img = matrix_io_load(infile); 137 | Mat features; 138 | bool canPreprocess = preprocessor->preprocess(img, features); 139 | if (!canPreprocess) { 140 | cerr << "ERR: Cannot preprocess this image '" << infile << "'" << endl; 141 | continue; 142 | } 143 | pair prediction = emodetector->predict(features); 144 | cout << "Emotion predicted: " << emotionStrings(prediction.first) << " with score " << prediction.second << endl; 145 | cout << "Insert the image file path: " << endl; 146 | } catch (int ee) { 147 | cerr << "ERR: Something wrong with '" << infile << "' (" << ee << ")" << endl; 148 | } 149 | } 150 | 151 | delete emodetector; 152 | delete preprocessor; 153 | } catch (int e) { 154 | cerr << "ERR: Exception #" << e << endl; 155 | return -e; 156 | } 157 | 158 | return 0; 159 | } 160 | -------------------------------------------------------------------------------- /src/facedetector/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | find_package(OpenCV COMPONENTS opencv_core opencv_imgproc opencv_highgui opencv_objdetect REQUIRED) 3 | 4 | add_executable( facecrop_cli facecrop_cli.cpp FaceDetector.cpp ../utils/matrix_io.cpp ) 5 | target_link_libraries( facecrop_cli ${OpenCV_LIBS} ) 6 | 7 | INSTALL(TARGETS facecrop_cli DESTINATION ${ASSETDIR} ) 8 | -------------------------------------------------------------------------------- /src/facedetector/FaceDetector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file FaceDetector.cpp 4 | * 5 | * @date 01/10/2014 12:26:04 AM 6 | * @brief Implementation for FaceDetector 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include "FaceDetector.h" 13 | #include 14 | 15 | using cv::Mat; 16 | using cv::Point; 17 | using cv::Size; 18 | using cv::Rect; 19 | using cv::CascadeClassifier; 20 | 21 | using std::string; 22 | using std::vector; 23 | 24 | namespace emotime { 25 | 26 | FaceDetector::FaceDetector(std::string face_config_file, std::string eye_config_file){ 27 | 28 | if (face_config_file.find(std::string("cbcl1"))!=std::string::npos){ 29 | this->faceMinSize=Size(30,30); 30 | } else { 31 | this->faceMinSize=Size(60,60); 32 | } 33 | 34 | cascade_f.load(face_config_file); 35 | if (eye_config_file != string("none") && eye_config_file != string("")) { 36 | cascade_e.load(eye_config_file); 37 | assert(!cascade_e.empty()); 38 | this->doEyesRot = true; 39 | } else { 40 | this->doEyesRot = false; 41 | } 42 | assert(!cascade_f.empty()); 43 | this->clahe = cv::createCLAHE(kCLAHEClipLimit, kCLAHEGridSize); 44 | } 45 | 46 | FaceDetector::FaceDetector(std::string face_config_file) { 47 | cascade_f.load(face_config_file); 48 | this->doEyesRot = false; 49 | assert(!cascade_f.empty()); 50 | this->clahe = cv::createCLAHE(kCLAHEClipLimit, kCLAHEGridSize); 51 | } 52 | 53 | FaceDetector::FaceDetector() { 54 | 55 | } 56 | 57 | FaceDetector::~FaceDetector() { 58 | 59 | } 60 | 61 | bool FaceDetector::detectFace(cv::Mat& img, cv::Rect& face) { 62 | vector faces; 63 | // detect faces 64 | assert(!cascade_f.empty()); 65 | this->faceMinSize.height = img.rows / 3; 66 | this->faceMinSize.width = img.cols / 4; 67 | cascade_f.detectMultiScale(img, faces, 1.1, 2, 0|CV_HAAR_SCALE_IMAGE, this->faceMinSize ); 68 | 69 | if (faces.size() == 0){ 70 | return false; 71 | } 72 | // Pick the face with maximum area 73 | unsigned int maxI=-1; 74 | int maxArea=-1; 75 | int area=-1; 76 | for (unsigned int i=0;imaxArea){ 79 | maxI=i; 80 | maxArea=area; 81 | } 82 | } 83 | face.x = faces.at(maxI).x; 84 | face.y = faces.at(maxI).y; 85 | face.width = faces.at(maxI).width; 86 | face.height = faces.at(maxI).height; 87 | faces.clear(); 88 | return true; 89 | } 90 | 91 | bool FaceDetector::detectEyes(cv::Mat& img, cv::Point& eye1, cv::Point& eye2){ 92 | vector eyes; 93 | // detect faces 94 | assert(!cascade_e.empty()); 95 | // Min widths and max width are taken from eyes proportions 96 | cascade_e.detectMultiScale(img, eyes, 1.1, 2, 0|CV_HAAR_SCALE_IMAGE, 97 | Size(img.size().width/5, img.size().width/(5*2))); 98 | 99 | if (eyes.size() < 2) { 100 | eyes.clear(); 101 | return false; 102 | } 103 | 104 | // Pick eyes with maximum area 105 | int val1=-1; 106 | int tmp=-1; 107 | int x,y,w,h; 108 | Point tmpe; 109 | int val2=-1; 110 | int area=-1; 111 | for (unsigned int i=0;ival1 && val1>val2){ 118 | tmp=val1; 119 | tmpe.x=eye1.x; 120 | tmpe.y=eye1.y; 121 | val1=area; 122 | eye1.x=x+w/2; 123 | eye1.y=y+h/2; 124 | val2=tmp; 125 | eye2.x=tmpe.x; 126 | eye2.y=tmpe.y; 127 | }else if (area>val2 && val2>val1){ 128 | tmp=val2; 129 | tmpe.x=eye2.x; 130 | tmpe.y=eye2.y; 131 | val2=area; 132 | eye2.x=x+w/2; 133 | eye2.y=y+h/2; 134 | val1=tmp; 135 | eye1.x=tmpe.x; 136 | eye1.y=tmpe.y; 137 | } else if (area>val1){ 138 | // second 139 | val1=area; 140 | eye1.x=x+w/2; 141 | eye1.y=y+h/2; 142 | } else if (area>val2){ 143 | // second 144 | val2=area; 145 | eye2.x=x+w/2; 146 | eye2.y=y+h/2; 147 | } 148 | } 149 | eyes.clear(); 150 | return true; 151 | } 152 | 153 | bool FaceDetector::detect(cv::Mat& img, cv::Mat& face) { 154 | bool hasFace; 155 | bool hasEyes; 156 | Rect faceRegion; 157 | Mat plainFace; 158 | Point eye1,eye2; 159 | 160 | if (img.rows == 0 || img.rows == 0){ 161 | return false; 162 | } 163 | 164 | Mat imgGray(img.size(),CV_8UC1); 165 | if (img.channels()>2){ 166 | cvtColor(img, imgGray, CV_BGR2GRAY); 167 | }else{ 168 | img.copyTo(imgGray); 169 | } 170 | // Scale image for better performance 171 | Size max_s, curr_s, tgt_s; 172 | float ratio = 0; 173 | max_s.width = 500; 174 | max_s.height = 500; 175 | curr_s.width = imgGray.cols; 176 | curr_s.height= imgGray.rows; 177 | if(curr_s.width > max_s.width){ 178 | ratio = max_s.width / (float) curr_s.width; // get ratio for scaling image 179 | tgt_s.width = max_s.width; // Set new width 180 | tgt_s.height = curr_s.height * ratio; // Scale height based on ratio 181 | } 182 | if(curr_s.height > max_s.height){ 183 | ratio = max_s.height / (float) curr_s.height; // get ratio for scaling image 184 | tgt_s.height = max_s.height; // Set new height 185 | tgt_s.width = curr_s.width * ratio; // Scale width based on ratio 186 | } 187 | resize(imgGray, imgGray, tgt_s , CV_INTER_AREA); 188 | 189 | //equalizeHist(imgGray, imgGray); 190 | this->clahe->apply(imgGray,imgGray); 191 | hasFace=detectFace(imgGray, faceRegion); 192 | 193 | if (!hasFace){ 194 | return false; 195 | } 196 | // detect eyes and locate points 197 | plainFace=imgGray(faceRegion); 198 | if (doEyesRot){ 199 | hasEyes=detectEyes(plainFace, eye1, eye2); 200 | if (hasEyes){ 201 | // eyes are initially relative to face patch 202 | eye1.x+=faceRegion.x; 203 | eye2.x+=faceRegion.x; 204 | eye1.y+=faceRegion.y; 205 | eye2.y+=faceRegion.y; 206 | Point left,right,upper,lower,tribase,eyecenter; 207 | if (eye1.xclahe->apply(face,face); 242 | //equalizeHist(face, face); 243 | imgGray.release(); 244 | return true; 245 | } 246 | 247 | }/* namespace facecrop */ 248 | -------------------------------------------------------------------------------- /src/facedetector/FaceDetector.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file FaceDetector.h 4 | * 5 | * @date 01/10/2014 12:26:04 AM 6 | * @brief Defintion of FaceDetector 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #ifndef FACEDETECTOR_H_ 13 | #define FACEDETECTOR_H_ 14 | 15 | //#define TRAINING_BUILD 16 | 17 | #include 18 | 19 | namespace emotime { 20 | 21 | /// Performance saving constant. Limit the dimension of the face retrieved. (NOT GOOD FOR TRAINING) 22 | const cv::Size kFaceSizeLimit = cv::Size(128,128); 23 | /// Scale factor determines how much the image size is reduced at each scaling operation 24 | const float kScaleFactor=1.3; 25 | const double kCLAHEClipLimit = 2.0; 26 | const cv::Size kCLAHEGridSize = cv::Size(8,8); 27 | #ifdef TRAINING_BUILD 28 | /// Maximum rotation allowed for eye correction 29 | const float kMaxRotationAngle = 10.0f; 30 | #else 31 | /// Maximum rotation allowed for eye correction 32 | const float kMaxRotationAngle = 20.0f; 33 | #endif 34 | 35 | /** 36 | * @class FaceDetector 37 | * 38 | * @brief Face detector using Cascade classifier for face and eye. 39 | * 40 | * @details 41 | * 42 | */ 43 | class FaceDetector { 44 | 45 | public: 46 | 47 | /** 48 | * @brief Creates a face detector with both face and eye 49 | * detection. 50 | * 51 | * @param[in] face_config_file The configuration file for face detection 52 | * @param[in] eye_config_file The configuration file for eye 53 | * detection. If the value is "none", the eye 54 | * configuration is ignored. 55 | * 56 | */ 57 | FaceDetector(std::string face_config_file, std::string eye_config_file); 58 | 59 | /** 60 | * @brief Creates a face detector 61 | * @param[in] face_config_file The configuration file for face detection 62 | * 63 | */ 64 | FaceDetector(std::string face_config_file); 65 | 66 | /** 67 | * @brief Create an empty face detector. Should NOT be used! 68 | * 69 | */ 70 | FaceDetector(); 71 | 72 | virtual ~FaceDetector(); 73 | 74 | /** 75 | * @brief Detect the face and adjust roll basing on eye position. 76 | * 77 | * @param[in] img The image to analyze 78 | * @param[in,out] face The resulting face 79 | * 80 | * @return True if a face was identified 81 | * 82 | * @details If the eye detector was loaded and doEyesRot 83 | * is true, then the face is rotated basing on eye position to 84 | * compensate roll. 85 | * 86 | * @see detectFace 87 | * @see detectEyes 88 | * 89 | */ 90 | virtual bool detect(cv::Mat& img, cv::Mat& face); 91 | 92 | protected: 93 | 94 | /// Cascade classifier for face 95 | cv::CascadeClassifier cascade_f; 96 | /// Cascade classifier for eyes 97 | cv::CascadeClassifier cascade_e; 98 | /// Minimum size of the face 99 | cv::Size faceMinSize; 100 | /// Perform the eye rotation? 101 | bool doEyesRot; 102 | /// Contrast Limited Adaptive Histogram Equalization 103 | cv::Ptr clahe; 104 | /** 105 | * @brief Detect a face, setting a region with the coordinate of the first face found. 106 | * 107 | * @param[in] img The image to analyze 108 | * @param[in,out] face The rectangle surrounding the face 109 | * 110 | * @return True if a face was identified 111 | */ 112 | virtual bool detectFace(cv::Mat& img, cv::Rect& face); 113 | 114 | /** 115 | * @brief Detects two eyes, returning their points 116 | * 117 | * @param[in] img The image to analyze 118 | * @param[in,out] eye1 First eye coordinates 119 | * @param[in,out] eye2 Second eye coordinates 120 | * 121 | * @return True if the eyes were identified 122 | */ 123 | virtual bool detectEyes(cv::Mat& img, cv::Point& eye1, cv::Point& eye2); 124 | 125 | }; 126 | 127 | } /* namespace facecrop */ 128 | 129 | #endif /* FACEDETECTOR_H_ */ 130 | -------------------------------------------------------------------------------- /src/facedetector/facecrop_cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file facecrop_cli.cpp 4 | * 5 | * @date 01/10/2014 12:33:58 AM 6 | * @brief CLI utils for face cropping 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include 13 | 14 | #include "FaceDetector.h" 15 | #include "matrix_io.h" 16 | 17 | using namespace emotime; 18 | 19 | using std::cerr; 20 | using std::cout; 21 | using std::endl; 22 | using std::string; 23 | 24 | using cv::Mat; 25 | 26 | /** 27 | * @brief Prints the CLI banner 28 | * 29 | */ 30 | void banner(); 31 | 32 | /** 33 | * @brief Prints the CLI help 34 | * 35 | */ 36 | void help(); 37 | 38 | void help() { 39 | cout << "Usage:" << endl; 40 | cout << " facecrop_cli [] " << endl; 41 | cout << "Parameters:" << endl; 42 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP)" << endl; 43 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP)" << endl; 44 | cout << " - Input image" << endl; 45 | cout << " - Output image" << endl; 46 | cout << endl; 47 | } 48 | void banner(){ 49 | cout << "FaceCrop Utility:" << endl; 50 | cout << " Crop faces from images" << endl; 51 | } 52 | 53 | /** 54 | * @brief Main 55 | * 56 | * @param[in] argc 57 | * @param[in] argv 58 | * 59 | * @returns 1 60 | * 61 | */ 62 | int main( int argc, const char* argv[] ){ 63 | if (argc < 4) { 64 | banner(); 65 | help(); 66 | cerr << "ERR: missing parameters" << endl; 67 | return -3; 68 | } 69 | string config_f(argv[1]); 70 | string* config_e = NULL; 71 | int i = 2; 72 | 73 | if (argc == 5) { 74 | config_e = new string(argv[i++]); 75 | } 76 | string infile(argv[i++]); 77 | string outfile(argv[i++]); 78 | 79 | FaceDetector* detector; 80 | if (config_e != NULL) { 81 | detector = new FaceDetector(config_f, *config_e); 82 | } else { 83 | detector = new FaceDetector(config_f); 84 | } 85 | delete config_e; 86 | try { 87 | Mat img = matrix_io_load(infile); 88 | Mat cropped; 89 | if (detector->detect(img, cropped)){ 90 | matrix_io_save(cropped, outfile); 91 | } else { 92 | //cerr << "ERR: no face found.." << endl; 93 | return 1; 94 | } 95 | delete detector; 96 | } 97 | catch (int e) { 98 | cerr << "ERR: Exception #" << e << endl; 99 | return -e; 100 | } 101 | return 0; 102 | } 103 | -------------------------------------------------------------------------------- /src/gaborbank/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | find_package(OpenCV COMPONENTS opencv_core opencv_imgproc opencv_highgui REQUIRED) 3 | 4 | add_executable(gaborbank_cli gaborbank_cli.cpp GaborBank.cpp ../utils/matrix_io.cpp ) 5 | target_link_libraries(gaborbank_cli ${OpenCV_LIBS}) 6 | 7 | INSTALL(TARGETS gaborbank_cli DESTINATION ${ASSETDIR}) 8 | 9 | -------------------------------------------------------------------------------- /src/gaborbank/GaborKernel.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file GaborKernel.h 4 | * @brief Contains the declaration of GaborKernel 5 | * 6 | */ 7 | 8 | #ifndef __GABORKERNEL_INC__ 9 | #define __GABORKERNEL_INC__ 10 | 11 | #include 12 | 13 | namespace emotime { 14 | 15 | /** 16 | * @class GaborKernel 17 | * @author 18 | * @date 19 | * 20 | * @brief Representation of a gabor kernel 21 | * 22 | * @details 23 | * 24 | */ 25 | class GaborKernel { 26 | 27 | public: 28 | 29 | /** 30 | * @brief Creates a gabor kernel specifying real and imaginary parts 31 | * 32 | * @param[in] r The real part 33 | * @param[in] i The imaginary part 34 | * 35 | */ 36 | GaborKernel(cv::Mat r, cv::Mat i) { 37 | r.copyTo(real); 38 | i.copyTo(imag); 39 | } 40 | 41 | ~GaborKernel() { 42 | real.release(); 43 | imag.release(); 44 | } 45 | 46 | /** 47 | * @brief Return the real part 48 | * 49 | * @return The real part of the filter 50 | */ 51 | cv::Mat getReal() { 52 | return real; 53 | } 54 | 55 | /** 56 | * @brief Return the imaginary part 57 | * 58 | * @return The imaginary part of the filter 59 | */ 60 | cv::Mat getImag() { 61 | return imag; 62 | } 63 | 64 | private: 65 | /// Real part 66 | cv::Mat real; 67 | /// Immaginary part 68 | cv::Mat imag; 69 | }; 70 | 71 | } 72 | 73 | #endif /* ----- #ifndef __GABORKERNEL_INC__ ----- */ 74 | -------------------------------------------------------------------------------- /src/gaborbank/gaborbank_cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file gaborbank_cli.cpp 4 | * @author Daniele Bellavista (daniele.bellavista@studio.unibo.it) 5 | * @date 12/04/2013 10:33:42 AM 6 | * @brief CLI interface for filtering an image using Gabor's Filters. 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "GaborBank.h" 17 | #include "matrix_io.h" 18 | 19 | using namespace emotime; 20 | 21 | using std::cout; 22 | using std::cerr; 23 | using std::endl; 24 | using std::string; 25 | 26 | using cv::Mat; 27 | 28 | /** 29 | * @brief Prints the CLI banner 30 | * 31 | */ 32 | void banner(); 33 | 34 | /** 35 | * @brief Prints the CLI help 36 | * 37 | */ 38 | void help(); 39 | 40 | void help() { 41 | cout << "Usage:" << endl; 42 | cout << " gaborbank_cli " << endl; 43 | cout << "Parameters:" << endl; 44 | cout << " - Width of the image, the input image will be scaled" << endl; 45 | cout << " - Height of the image, the input image will be scaled" << endl; 46 | cout << " - " << endl; 47 | cout << " - " << endl; 48 | cout << " - " << endl; 49 | cout << " - Input image" << endl; 50 | cout << " - Output file where to store filtered images" << endl; 51 | cout << endl; 52 | } 53 | 54 | void banner() { 55 | cout << "GaborBank Utility:" << endl; 56 | cout << " Filter with a bank of Gabor filters with different " << endl; 57 | cout << " orientations and frequencies. (Gabor magnitude)" << endl; 58 | } 59 | 60 | /** 61 | * @brief Main 62 | * 63 | * @param[in] argc 64 | * @param[in] argv 65 | * 66 | * @returns 1 67 | * 68 | */ 69 | int main(int argc, const char* argv[]){ 70 | if (argc < 5) { 71 | banner(); 72 | help(); 73 | cerr << "ERR: missing parameters" << endl; 74 | return -3; 75 | } 76 | cv::Size size; 77 | size.width = abs(atoi(argv[1])); 78 | size.height = abs(atoi(argv[2])); 79 | double nwidths = (atof(argv[3])); 80 | double nlambdas = (atof(argv[4])); 81 | double nthetas = (atof(argv[5])); 82 | string infile(argv[6]); 83 | string outfile(argv[7]); 84 | 85 | Mat img = matrix_io_load(infile); 86 | GaborBank bank; 87 | bank.fillGaborBank((double) nwidths, (double) nlambdas, (double) nthetas); 88 | Mat dest = bank.filterImage(img, size); 89 | matrix_io_save(dest, outfile); 90 | dest.release(); 91 | } 92 | -------------------------------------------------------------------------------- /src/gui/ACapture.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ACapture.cpp 3 | * @brief Implementation of ACapture 4 | * 5 | */ 6 | 7 | #include "ACapture.h" 8 | 9 | namespace emotime{ 10 | 11 | ACapture::ACapture(bool toGray){ 12 | toGrayscale=toGray; 13 | } 14 | 15 | ACapture::~ACapture(){} 16 | 17 | bool ACapture::nextFrame(Mat & frame){ 18 | Mat frm; 19 | bool hasNext; 20 | hasNext=extractFrame(frm); 21 | if (hasNext){ 22 | if (toGrayscale && frm.channels()>2){ 23 | Mat gray(frm.size(), CV_8UC1); 24 | cvtColor(frm, gray, CV_BGR2GRAY); 25 | gray.copyTo(frame); 26 | } else { 27 | frm.copyTo(frame); 28 | } 29 | return true; 30 | } 31 | return false; 32 | } 33 | 34 | } 35 | 36 | -------------------------------------------------------------------------------- /src/gui/ACapture.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ACapture.h 3 | * @brief Definition of ACapture 4 | * 5 | */ 6 | 7 | 8 | #ifndef ACAPTURE_H 9 | #define ACAPTURE_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | using namespace std; 17 | using namespace cv; 18 | 19 | namespace emotime{ 20 | 21 | /** 22 | * @class ACapture 23 | * @date 12/31/2013 10:53:55 AM 24 | * 25 | * @brief Generic frame capture interface 26 | * 27 | * @details 28 | * 29 | */ 30 | class ACapture{ 31 | private: 32 | /// Retrieve grayscale 33 | bool toGrayscale; 34 | protected: 35 | /** 36 | * @brief Extracts the frame 37 | * 38 | * @param[in,out] frm frame to fill 39 | * 40 | * @return true if frame has been retrieved 41 | * */ 42 | virtual bool extractFrame(Mat & frm)=0; 43 | public: 44 | /** 45 | * @brief Creates an ACaputre specifying if the capture must be performed 46 | * in grayscale. 47 | * 48 | * @param[in] toGray Retrieve grayscal frames 49 | * 50 | * */ 51 | ACapture(bool toGray); 52 | 53 | virtual ~ACapture(); 54 | /** 55 | * @brief Returns true if the device is ready to provide new frames 56 | * 57 | * @return true if capture device is ready to provide new frames 58 | * */ 59 | virtual bool isReady()=0; 60 | /** 61 | * @brief Fills the given image with a new frame. 62 | * 63 | * @param[in,out] frame Frame to fill 64 | * 65 | * @return true if new frame is available 66 | * */ 67 | bool nextFrame(Mat & frame); 68 | }; 69 | 70 | 71 | 72 | } 73 | 74 | 75 | #endif /* !ACAPTURE_H */ 76 | 77 | -------------------------------------------------------------------------------- /src/gui/AGui.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file AGui.cpp 4 | * @brief Contains the implementation for AGui 5 | * 6 | */ 7 | 8 | #include "AGui.h" 9 | 10 | 11 | volatile int quit_signal=0; 12 | #ifdef __unix__ 13 | #include 14 | extern "C" void quit_signal_handler(int signum) { 15 | if (quit_signal!=0) exit(0); // just exit already 16 | quit_signal=1; 17 | cout << "Will quit at next camera frame (repeat to kill now)\n"; 18 | } 19 | #endif 20 | 21 | namespace emotime{ 22 | 23 | AGui::AGui(ACapture* capt, FacePreProcessor* fp, EmoDetector* detect, int fps, string title) { 24 | capture = capt; 25 | preprocessor = fp; 26 | detector = detect; 27 | mainWinTitle = title; 28 | this->fps = fps; 29 | } 30 | 31 | bool AGui::run() { 32 | if(!init()) { 33 | return false; 34 | } 35 | #ifdef __unix__ 36 | signal(SIGINT,quit_signal_handler); // listen for ctrl-C 37 | #endif 38 | while (nextFrame()) { 39 | if (quit_signal) exit(0); 40 | int key; 41 | if (fps <= 0) { 42 | key = waitKey(0); 43 | } else { 44 | key = waitKey((int) 1000.0 / fps); 45 | } 46 | if((key & 0xFF) == 27) { 47 | break; 48 | } 49 | } 50 | return true; 51 | } 52 | 53 | bool AGui::init() { 54 | namedWindow(mainWinTitle.c_str(), WINDOW_NORMAL); 55 | return true; 56 | } 57 | 58 | bool AGui::nextFrame() { 59 | Mat frame; 60 | Mat featvector; 61 | if (capture->nextFrame(frame)) { 62 | if (preprocessor->preprocess(frame, featvector)) { 63 | pair prediction = detector->predict(featvector); 64 | if (!newFrame(frame, prediction)) { 65 | return false; 66 | } 67 | imshow(mainWinTitle.c_str(), frame); 68 | } 69 | return true; 70 | } else { 71 | return false; 72 | } 73 | } 74 | 75 | } 76 | 77 | -------------------------------------------------------------------------------- /src/gui/AGui.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file AGui.h 4 | * @brief Defines the class AGui 5 | * 6 | */ 7 | 8 | #ifndef AGUI_H 9 | #define AGUI_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "ACapture.h" 17 | #include "EmoDetector.h" 18 | #include "FacePreProcessor.h" 19 | 20 | using std::stringstream; 21 | using std::string; 22 | using std::pair; 23 | 24 | using namespace emotime; 25 | 26 | namespace emotime { 27 | 28 | /** 29 | * @class AGui 30 | * @date 12/31/2013 10:53:55 AM 31 | * 32 | * @brief Generic GUI 33 | * 34 | * @details 35 | * 36 | */ 37 | class AGui { 38 | 39 | public: 40 | 41 | /** 42 | * @brief Creates an AGui 43 | * 44 | * @param[in] capt A capture instance 45 | * @param[in] fp The face preprocessor to use 46 | * @param[in] detect An EmoDetector instance 47 | * @param[in] fps Desired frame per second 48 | * @param[in] title The title of the GUI 49 | * 50 | */ 51 | AGui(ACapture* capt, FacePreProcessor* fp, EmoDetector* detect, int fps, string title="AGui: Main Emotime GUI"); 52 | 53 | /** 54 | * @brief Starts the gui 55 | * 56 | * @return False if something wrong. 57 | * 58 | */ 59 | bool run(); 60 | 61 | protected: 62 | 63 | /// Face preprocessor 64 | FacePreProcessor* preprocessor; 65 | /// Emotion detector 66 | EmoDetector* detector; 67 | /// Capture instance 68 | ACapture* capture; 69 | /// Title for the main window 70 | string mainWinTitle; 71 | /// Desired frames per second 72 | int fps; 73 | 74 | /** 75 | * @brief Initialize the windows 76 | * 77 | * @return Returns true if the initialization succeded 78 | * 79 | */ 80 | virtual bool init(); 81 | 82 | /** 83 | * @brief Produce the next frame 84 | * 85 | * @return False if there is no next frame 86 | * 87 | */ 88 | virtual bool nextFrame(); 89 | 90 | /** 91 | * @brief A new frame is available 92 | * 93 | * @param[in,out] frame The new frame that will be drawn after returing 94 | * @param[in] prediction The predicted emotion 95 | * 96 | * @return False if something wrong 97 | * 98 | */ 99 | virtual bool newFrame(Mat& frame, pair prediction) = 0; 100 | 101 | }; 102 | } 103 | 104 | #endif 105 | 106 | -------------------------------------------------------------------------------- /src/gui/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Emotime Video Debug Gui Tool 2 | 3 | find_package(OpenCV COMPONENTS opencv_core opencv_ml opencv_imgproc 4 | opencv_highgui opencv_objdetect REQUIRED) 5 | 6 | add_executable(emotimevideodebug_cli emotimevideodebug_cli.cpp 7 | ACapture.cpp VideoCapture.cpp AGui.cpp DebugGui.cpp 8 | ../detector/EmoDetector.cpp ../detector/BoostEmoDetector.cpp ../detector/FacePreProcessor.cpp ../detector/SVMEmoDetector.cpp 9 | ../training/Classifier.cpp ../training/AdaBoostClassifier.cpp ../training/SVMClassifier.cpp 10 | ../facedetector/FaceDetector.cpp 11 | ../gaborbank/GaborBank.cpp 12 | ../utils/string_utils.cpp ../utils/matrix_io.cpp ) 13 | target_link_libraries(emotimevideodebug_cli ${OpenCV_LIBS} ) 14 | INSTALL(TARGETS emotimevideodebug_cli DESTINATION ${ASSETDIR} ) 15 | 16 | # Emotime Video Gui Tool 17 | add_executable(emotimevideo_cli emotimevideo_cli.cpp 18 | ACapture.cpp VideoCapture.cpp AGui.cpp EmotimeGui.cpp 19 | ../detector/EmoDetector.cpp ../detector/BoostEmoDetector.cpp ../detector/FacePreProcessor.cpp ../detector/SVMEmoDetector.cpp 20 | ../training/Classifier.cpp ../training/AdaBoostClassifier.cpp ../training/SVMClassifier.cpp 21 | ../facedetector/FaceDetector.cpp 22 | ../gaborbank/GaborBank.cpp 23 | ../utils/string_utils.cpp ../utils/matrix_io.cpp ) 24 | target_link_libraries(emotimevideo_cli ${OpenCV_LIBS} ) 25 | INSTALL(TARGETS emotimevideo_cli DESTINATION ${ASSETDIR} ) 26 | 27 | # Emotime Webcam Gui Tool 28 | add_executable(emotimegui_cli emotimegui_cli.cpp 29 | ACapture.cpp VideoCapture.cpp AGui.cpp EmotimeGui.cpp 30 | ../detector/EmoDetector.cpp ../detector/BoostEmoDetector.cpp ../detector/FacePreProcessor.cpp ../detector/SVMEmoDetector.cpp 31 | ../training/Classifier.cpp ../training/AdaBoostClassifier.cpp ../training/SVMClassifier.cpp 32 | ../facedetector/FaceDetector.cpp 33 | ../gaborbank/GaborBank.cpp 34 | ../utils/string_utils.cpp ../utils/matrix_io.cpp ) 35 | target_link_libraries(emotimegui_cli ${OpenCV_LIBS} ) 36 | INSTALL(TARGETS emotimegui_cli DESTINATION ${ASSETDIR} ) 37 | 38 | # Gabor Parameter Tuning Gui Tool 39 | add_executable(gaborgui_cli gaborgui_cli.cpp 40 | ImageCapture.cpp ACapture.cpp GaborGui.hpp 41 | ../gaborbank/GaborBank.cpp 42 | ../utils/string_utils.cpp ../utils/matrix_io.cpp ) 43 | target_link_libraries(gaborgui_cli ${OpenCV_LIBS} ) 44 | INSTALL(TARGETS gaborgui_cli DESTINATION ${ASSETDIR} ) 45 | 46 | -------------------------------------------------------------------------------- /src/gui/DebugGui.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file DebugGui.cpp 4 | * @brief Contains the implementation of DebugGui 5 | * 6 | */ 7 | 8 | #include "DebugGui.h" 9 | 10 | namespace emotime{ 11 | 12 | DebugGui::DebugGui(ACapture* capt, FacePreProcessor* fp, EmoDetector* detect, int fps) : AGui(capt, fp, detect, fps, "Emotime Video Debug Gui") { 13 | faceWinTitle = string("AGui: Face"); 14 | featsWinTitle = string("AGui: Features"); 15 | } 16 | 17 | bool DebugGui::init() { 18 | if (!AGui::init()) { 19 | return false; 20 | } 21 | //namedWindow(faceWinTitle.c_str(), CV_WINDOW_AUTOSIZE); 22 | namedWindow(featsWinTitle.c_str(), CV_WINDOW_AUTOSIZE); 23 | return true; 24 | } 25 | 26 | bool DebugGui::newFrame(Mat& frame, pair prediction) { 27 | Mat copy; 28 | frame.copyTo(copy); 29 | stringstream ss; 30 | ss << "Emotion=" << emotionStrings(prediction.first) << ", Score=" << prediction.second; 31 | string osd = ss.str(); 32 | cv::putText(frame, osd.c_str(), Point(80,60), FONT_HERSHEY_SIMPLEX, 0.7, Scalar::all(255)); 33 | // QT only 34 | //displayOverlay(mainWinTitle.c_str(), osd.c_str(), 2000); 35 | Mat face; 36 | if (AGui::preprocessor->extractFace(copy, face)) { 37 | Mat gabor; 38 | if (AGui::preprocessor->filterImage(face,gabor)){ 39 | double min; 40 | double max; 41 | cv::minMaxIdx(gabor, &min, &max); 42 | cv::Mat adjMap; 43 | cv::convertScaleAbs(gabor, adjMap, 255 / max); 44 | Mat bigger; 45 | resize(adjMap,bigger,Size(adjMap.size().width*3,adjMap.size().height*3), 0, 0, CV_INTER_LINEAR); 46 | equalizeHist(bigger,bigger); 47 | imshow(featsWinTitle.c_str(), bigger); 48 | } 49 | } 50 | return true; 51 | } 52 | 53 | } 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/gui/DebugGui.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file DebugGui.h 4 | * @brief Defines the class DebugGui 5 | * 6 | */ 7 | 8 | #ifndef DEBUGGUI_H 9 | #define DEBUGGUI_H 10 | 11 | #include "AGui.h" 12 | 13 | namespace emotime{ 14 | 15 | /** 16 | * @author Luca Mella 17 | * @date 18 | * 19 | * @brief GUI for debugging purpose 20 | * 21 | */ 22 | class DebugGui: public AGui { 23 | 24 | public: 25 | 26 | /** 27 | * @brief Creates an ADebugGUI 28 | * 29 | * @param[in] capt A capture instance 30 | * @param[in] fp The face preprocessor to use 31 | * @param[in] detect An emodetector instance 32 | * @param[in] fps Desired frame per second 33 | * 34 | */ 35 | DebugGui(ACapture* capt, FacePreProcessor* fp, EmoDetector* detect, int fps); 36 | 37 | protected: 38 | /// Title for face window 39 | string faceWinTitle; 40 | /// Title for features window 41 | string featsWinTitle; 42 | 43 | /** 44 | * @brief Initialize the GUI 45 | * 46 | * @return False if something wrong 47 | */ 48 | bool init(); 49 | 50 | /** 51 | * @brief Produce a new frame, showing the given prediction 52 | * 53 | * @param[in,out] frame The image to be filled 54 | * @param[in] prediction The prediction to show 55 | * 56 | * @return False if something wrong. 57 | * 58 | */ 59 | bool newFrame(Mat& frame, pair prediction); 60 | 61 | }; 62 | 63 | } 64 | 65 | #endif /* !DEBUGGUI_H */ 66 | 67 | -------------------------------------------------------------------------------- /src/gui/EmotimeGui.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file EmotimeGui.cpp 4 | * @brief Implementation of EmotimeGUI 5 | * 6 | */ 7 | 8 | #include "EmotimeGui.h" 9 | 10 | namespace emotime{ 11 | 12 | EmotimeGui::EmotimeGui(FacePreProcessor* fp, EmoDetector* detect, int fps) : 13 | EmotimeGui::AGui(new WebcamCapture(true), fp, detect, fps, "Emotime!") { 14 | 15 | } 16 | 17 | EmotimeGui::EmotimeGui(ACapture* capture, FacePreProcessor* fp, EmoDetector* 18 | detect, int fps) : EmotimeGui::AGui(capture, fp, detect, fps, 19 | "Emotime!") { 20 | } 21 | 22 | EmotimeGui::~EmotimeGui() { 23 | //delete this->capture; 24 | } 25 | 26 | bool EmotimeGui::newFrame(Mat& frame, pair prediction) { 27 | Mat copy; 28 | frame.copyTo(copy); 29 | stringstream ss, ss2; 30 | ss << "Emotion: " << emotionStrings(prediction.first); 31 | ss2 << "Score: " << prediction.second; 32 | string osd = ss.str(); 33 | string osd2 = ss2.str(); 34 | 35 | cv::putText(frame, osd.c_str(), Point(20,60), FONT_HERSHEY_SIMPLEX, 0.7, Scalar::all(255)); 36 | cv::putText(frame, osd2.c_str(), Point(20,100), FONT_HERSHEY_SIMPLEX, 0.7, Scalar::all(255)); 37 | // QT only 38 | //displayOverlay(mainWinTitle.c_str(), osd.c_str(), 2000); 39 | return true; 40 | } 41 | 42 | 43 | } 44 | 45 | 46 | -------------------------------------------------------------------------------- /src/gui/EmotimeGui.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file EmotimeGui.h 4 | * @brief Definition of EmotimeGUI 5 | * 6 | */ 7 | 8 | #ifndef EMOTIMEGUI_H 9 | #define EMOTIMEGUI_H 10 | 11 | 12 | #include "AGui.h" 13 | #include "WebcamCapture.h" 14 | 15 | namespace emotime{ 16 | 17 | /** 18 | * @class EmotimeGui 19 | * 20 | * @brief Gui that detects emotion. 21 | * 22 | */ 23 | class EmotimeGui : public AGui { 24 | 25 | public: 26 | 27 | /** 28 | * @brief Creates an EmotimeGUI with a webcam capture 29 | * 30 | * @param[in] fp The face preprocessor to use 31 | * @param[in] detect An EmoDetector instance 32 | * @param[in] fps Desired frame per second 33 | * 34 | */ 35 | EmotimeGui(FacePreProcessor* fp, EmoDetector* detect, int fps); 36 | 37 | /** 38 | * @brief Creates an EmotimeGUI with a custom capture 39 | * 40 | * @param[in] capture The ACapture to use 41 | * @param[in] fp The face preprocessor to use 42 | * @param[in] detect An EmoDetector instance 43 | * @param[in] fps Desired frame per second 44 | * 45 | */ 46 | EmotimeGui(ACapture * capture, FacePreProcessor* fp, EmoDetector* detect, int fps); 47 | 48 | ~EmotimeGui(); 49 | 50 | protected: 51 | 52 | 53 | /** 54 | * @brief Prints the prediction over the frame 55 | * 56 | * @param[in,out] frame The frame filled with the prediction 57 | * @param[in] prediction The prediction to display 58 | * 59 | * @returns Always returns true 60 | */ 61 | bool newFrame(Mat& frame, pair prediction); 62 | 63 | private: 64 | 65 | }; 66 | 67 | } 68 | 69 | #endif /* !EMOTIMEGUI_H */ 70 | 71 | -------------------------------------------------------------------------------- /src/gui/GaborGui.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file GaborGui.hpp 4 | * @brief Contains the implementation and definition of GaborGui 5 | * 6 | */ 7 | 8 | 9 | #ifndef GABORGUI_H 10 | #define GABORGUI_H 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "ACapture.h" 18 | #include "GaborBank.h" 19 | 20 | using namespace std; 21 | 22 | namespace emotime{ 23 | 24 | /** 25 | * @brief Gui for visualizing gabor bank effects 26 | * 27 | */ 28 | class GaborGui { 29 | 30 | protected: 31 | 32 | /// The frame capture device 33 | ACapture* capture; 34 | /// The title for the main window 35 | string mainWinTitle; 36 | /// The title for the gabor window 37 | string gaborWinTitle; 38 | /// The size to be used for image resizing 39 | Size size; 40 | /// 41 | int isize; 42 | /// nwidths param of GaborBank::fillGaborBank 43 | int nwidths; 44 | /// nlambdas param of GaborBank::fillGaborBank 45 | int nlambdas; 46 | /// nthetas param of GaborBank::fillGaborBank 47 | int nthetas; 48 | /// The current gui frame 49 | Mat currframe; 50 | 51 | protected: 52 | 53 | /** 54 | * @brief OpenCV trackbar callback function 55 | * 56 | * @param[in] newVal new value 57 | * @param[in] obj The GaburGui object reference 58 | * 59 | */ 60 | static void on_trackbar(int newVal, void *obj) { 61 | double min,max; 62 | GaborBank gaborbank; 63 | Mat frame; 64 | Mat gaborframe; 65 | Mat resized; 66 | Mat scaled; 67 | Mat magnified; 68 | #ifdef DEBUG 69 | cout<<"DEBUG: Parameters changed, reconfiguring.. (this@"<size.width=ths->isize; 77 | ths->size.height=ths->isize; 78 | ths->currframe.copyTo(frame); 79 | //resize(ths->currframe, resized, ths->size, 0, 0, CV_INTER_AREA); 80 | cout<<"DEBUG: recreating gabor filter bank "<nwidths<<","<nlambdas<<","<nthetas <nwidths, ths->nlambdas, ths->nthetas); 83 | gaborframe=gaborbank.filterImage(frame, ths->size); 84 | cout<<"DEBUG: preparing for visualization"<isize)*72, 95 | (scaled.size().height/ths->isize)*72), 96 | 0, 0, CV_INTER_LINEAR); 97 | imshow(ths->mainWinTitle.c_str(), ths->currframe); 98 | imshow(ths->gaborWinTitle.c_str(), magnified); 99 | } 100 | 101 | public: 102 | 103 | 104 | /** 105 | * @brief Creates a GaborGui with the given ACapture 106 | * 107 | * @param[in] cap The ACapture to use 108 | * 109 | */ 110 | GaborGui(ACapture* cap) { 111 | capture=cap; 112 | size=Size(56,56); 113 | isize=56; 114 | nwidths=2; 115 | nlambdas=5; 116 | nthetas=4; 117 | mainWinTitle=string("GaborGui: Gabor parameter tuning"); 118 | gaborWinTitle=string("GaborGui: Gabor features"); 119 | } 120 | 121 | /** 122 | * @brief Initialize windows and trackbars 123 | * 124 | * @return Always returns true 125 | * 126 | */ 127 | bool init(){ 128 | namedWindow(mainWinTitle.c_str(), WINDOW_NORMAL); 129 | namedWindow(gaborWinTitle.c_str(), WINDOW_AUTOSIZE); 130 | createTrackbar("isize",mainWinTitle.c_str(), &isize, 250, GaborGui::on_trackbar, this); 131 | createTrackbar("nwidths",mainWinTitle.c_str(), &nwidths, 4, GaborGui::on_trackbar, this); 132 | createTrackbar("nlamdas",mainWinTitle.c_str(), &nlambdas, 12, GaborGui::on_trackbar, this); 133 | createTrackbar("nthetas",mainWinTitle.c_str(), &nthetas, 12, GaborGui::on_trackbar, this); 134 | return true; 135 | } 136 | 137 | 138 | /** 139 | * @brief Capture the next frame and reset the trackbar 140 | * 141 | * @return Returns false if no frame are available 142 | * 143 | */ 144 | bool nextFrame(){ 145 | Mat frame; 146 | Mat featvector; 147 | if (capture->nextFrame(frame)){ 148 | currframe.release(); 149 | frame.copyTo(currframe); 150 | on_trackbar(0, this); 151 | return true; 152 | } else{ 153 | return false; 154 | } 155 | } 156 | 157 | 158 | /** 159 | * @brief Produce a new frame at every keystroke 160 | * 161 | * @return Returns when no more frame are available. The value 162 | * is always true. 163 | * 164 | */ 165 | bool run(){ 166 | init(); 167 | while (nextFrame()){ 168 | waitKey(0); 169 | } 170 | return true; 171 | 172 | } 173 | }; 174 | 175 | } 176 | 177 | #endif /* !GABORGUI_H */ 178 | 179 | -------------------------------------------------------------------------------- /src/gui/ImageCapture.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file ImageCapture.cpp 4 | * @brief Contains the implementation of ImageCapture 5 | * 6 | */ 7 | 8 | #include "ImageCapture.h" 9 | 10 | namespace emotime{ 11 | 12 | ImageCapture::ImageCapture(string infile, bool grayScale): ACapture(grayScale) { 13 | image=imread(infile.c_str(), CV_LOAD_IMAGE_GRAYSCALE); 14 | } 15 | 16 | bool ImageCapture::isReady(){ 17 | return image.cols>0 && image.rows>0; 18 | } 19 | 20 | bool ImageCapture::extractFrame(Mat & frm){ 21 | if (isReady()){ 22 | image.copyTo(frm); 23 | return true; 24 | } else { 25 | return false; 26 | } 27 | } 28 | 29 | } 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/gui/ImageCapture.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file ImageCapture.h 4 | * @brief Defines ImageCapture 5 | * 6 | */ 7 | 8 | 9 | #ifndef IMAGECAPTURE_H 10 | #define IMAGECAPTURE_H 11 | 12 | #include "ACapture.h" 13 | 14 | namespace emotime{ 15 | 16 | /** 17 | * @class ImageCapture 18 | * 19 | * @brief Capture frames from a single image. 20 | * 21 | * @details 22 | * 23 | */ 24 | class ImageCapture: public ACapture { 25 | 26 | public: 27 | 28 | /** 29 | * @brief Load an image from file. 30 | * 31 | * @param[in] infile The file to load 32 | * @param[in] grayScale Determinates if the image should be converted to 33 | * grayscale. 34 | * 35 | */ 36 | ImageCapture(string infile, bool grayScale); 37 | 38 | /** 39 | * @brief Implementation of ACaputre::isReady 40 | * 41 | * @return Returns true if the image was loaded correctly 42 | * 43 | */ 44 | bool isReady(); 45 | 46 | protected: 47 | 48 | /** 49 | * @brief Fill \p frm with the image. 50 | * 51 | * @param[in,out] frm The frame to fill 52 | * 53 | * @return False if the image wasn't successfully loaded. 54 | * 55 | */ 56 | bool extractFrame(Mat & frm); 57 | 58 | private: 59 | 60 | /// Current image 61 | Mat image; 62 | 63 | }; 64 | 65 | } 66 | 67 | #endif /* !IMAGECAPTURE_H */ 68 | 69 | -------------------------------------------------------------------------------- /src/gui/VideoCapture.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * VideoCapture.cpp 3 | * Copyright (C) Luca Mella 4 | * 5 | * Distributed under terms of the CC-BY-NC license. 6 | */ 7 | 8 | #include "VideoCapture.h" 9 | 10 | namespace emotime{ 11 | 12 | VideoCapture::VideoCapture(int deviceID, bool grayScale): ACapture(grayScale) { 13 | cap.open(deviceID); 14 | } 15 | 16 | VideoCapture::VideoCapture(string infile, bool grayScale): ACapture(grayScale) { 17 | cap.open(infile.c_str()); 18 | } 19 | 20 | VideoCapture::~VideoCapture() { 21 | cap.release(); 22 | } 23 | 24 | bool VideoCapture::isReady() { 25 | if (cap.isOpened()) { 26 | return true; 27 | } else { 28 | return true; 29 | } 30 | } 31 | 32 | bool VideoCapture::extractFrame(Mat& frm) { 33 | return cap.read(frm); 34 | } 35 | 36 | } 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/gui/VideoCapture.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file VideoCapture.h 4 | * @brief Contains the implementation of VideoCapture 5 | * 6 | */ 7 | 8 | #ifndef VIDEOCAPTURE_H 9 | #define VIDEOCAPTURE_H 10 | 11 | #include "ACapture.h" 12 | 13 | namespace emotime{ 14 | 15 | /** 16 | * @class VideoCapture 17 | * 18 | * @brief Capture implementation using opencv VideoCapture 19 | * 20 | * @details 21 | * 22 | */ 23 | class VideoCapture: public ACapture { 24 | 25 | public: 26 | 27 | /** 28 | * @brief Open a video from input device (eg. Camera) 29 | * 30 | * @param[in] deviceID opencv device id 31 | * @param[in] grayScale True if the frame should be captured in grayscale. 32 | * 33 | */ 34 | VideoCapture(int deviceID, bool grayScale); 35 | 36 | /** 37 | * @brief Open a video from video file. 38 | * 39 | * @param[in] infile video path 40 | * @param[in] grayScale True if the frame should be captured in grayscale. 41 | */ 42 | VideoCapture(string infile, bool grayScale); 43 | 44 | ~VideoCapture(); 45 | 46 | 47 | /** 48 | * @brief Returns true if the device or file was opened 49 | * 50 | * @return Returns true if the device or file was opened 51 | * 52 | */ 53 | bool isReady(); 54 | 55 | protected: 56 | 57 | /** 58 | * @brief Extract a frame from the device 59 | * 60 | * @param[in,out] frm The frame to fill 61 | * 62 | * @return The result of cap.read 63 | * 64 | */ 65 | bool extractFrame(Mat& frm); 66 | 67 | private: 68 | 69 | /// The opencv video capture 70 | cv::VideoCapture cap; 71 | }; 72 | 73 | 74 | } 75 | 76 | #endif /* !VIDEOCAPTURE_H */ 77 | 78 | -------------------------------------------------------------------------------- /src/gui/WebcamCapture.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file WebcamCapture.h 4 | * @brief Contains the definition of WebcamCapture 5 | * 6 | */ 7 | 8 | 9 | #ifndef WEBCAMCAPTURE_H 10 | #define WEBCAMCAPTURE_H 11 | 12 | #include "VideoCapture.h" 13 | 14 | namespace emotime { 15 | 16 | /** 17 | * @class WebcamCapture 18 | * 19 | * @brief Capture implementation using a Webcam 20 | * 21 | * @details 22 | * 23 | */ 24 | class WebcamCapture : public VideoCapture { 25 | 26 | public: 27 | 28 | /** 29 | * @brief WebcamCapture costructor 30 | * 31 | * @param[in] to_grey If true, each photogram will be converted to greyscale 32 | * 33 | */ 34 | WebcamCapture(bool to_grey) : VideoCapture(0, to_grey){} 35 | 36 | protected: 37 | 38 | private: 39 | 40 | }; 41 | 42 | 43 | } 44 | 45 | 46 | #endif /* !WEBCAMCAPTURE_H */ 47 | 48 | -------------------------------------------------------------------------------- /src/gui/emotimegui_cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file emotimegui_cli.cpp 4 | * @brief Emotime GUI command line interface 5 | * 6 | * @details 7 | * 8 | */ 9 | 10 | #include "WebcamCapture.h" 11 | #include "BoostEmoDetector.h" 12 | #include "SVMEmoDetector.h" 13 | #include "matrix_io.h" 14 | #include "EmotimeGui.h" 15 | #include "FaceDetector.h" 16 | 17 | #include "TrainingParameters.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using std::cout; 28 | using std::cerr; 29 | using std::endl; 30 | using std::string; 31 | 32 | /// Width of the faces used during training 33 | const int kWidth = 48; 34 | /// Height of the faces used during training 35 | const int kHeight = 48; 36 | /// N-Widths used during training 37 | const double kNWidths = 2; 38 | /// N-Lambdas used during training 39 | const double kNLambdas = 5; 40 | /// N-Thetas used during training 41 | const double kNThetas = 4; 42 | 43 | /** 44 | * @brief Prints the CLI banner 45 | * 46 | */ 47 | void banner(); 48 | 49 | /** 50 | * @brief Prints the CLI help 51 | * 52 | */ 53 | void help(); 54 | 55 | void help() { 56 | cout << "Usage:" << endl; 57 | cout << " emotimegui_cli [] {}" << endl; 58 | cout << "Parameters:" << endl; 59 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for face detection" << endl; 60 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for eye detection" << endl; 61 | cout << " - Width of the image, the input image will be scaled"< - Height of the image, the input image will be scaled"< - "< - "< - "< - ada or svm (default ada)" << endl; 67 | cout << " - classificators list" << endl; 68 | cout << endl; 69 | } 70 | 71 | void banner() { 72 | cout << "EmotimeGui Utility:" << endl; 73 | cout << " GUI for emotime" << endl; 74 | } 75 | 76 | 77 | /** 78 | * @brief Main 79 | * 80 | * @param[in] argc 81 | * @param[in] argv 82 | * 83 | * @returns 1 84 | * 85 | */ 86 | int main(int argc, const char* argv[]) { 87 | if (argc < 5) { 88 | banner(); 89 | help(); 90 | cerr << "ERR: missing parameters" << endl; 91 | return -3; 92 | } 93 | 94 | // Intializing the face detector 95 | string faceDetConfig(argv[1]); 96 | string eyeDetConfig(argv[2]); 97 | int width = std::atoi(argv[3]); 98 | int height = std::atoi(argv[4]); 99 | int nwidths = std::atoi(argv[5]); 100 | int nlambdas = std::atoi(argv[6]); 101 | int nthetas = std::atoi(argv[7]); 102 | FacePreProcessor facepreproc(faceDetConfig, eyeDetConfig, width, height, nwidths, nlambdas, nthetas); 103 | 104 | // Setting the mode 105 | int i; 106 | string mode; 107 | mode = string(argv[8]); 108 | 109 | if (mode != "svm" && mode != "ada") { 110 | mode = "ada"; 111 | i = 8; 112 | } else { 113 | i = 9; 114 | } 115 | 116 | // Setting the classifiers 117 | vector cl_paths; 118 | EmoDetector* emodetector; 119 | 120 | for(; i < argc; i++) { 121 | cl_paths.push_back(string(argv[i])); 122 | } 123 | 124 | if (mode == "svm") { 125 | emodetector = new SVMEmoDetector(kCfactor, kMaxIteration, kErrorMargin); 126 | } else { 127 | emodetector = new BoostEmoDetector(kBoostType, kTrimWeight, kMaxDepth); 128 | } 129 | emodetector->init(cl_paths); 130 | 131 | // Creating and starting the EmotimeGUI 132 | int fps = 120; 133 | try { 134 | EmotimeGui gui(&facepreproc, emodetector, fps); 135 | gui.run(); 136 | } catch (int e) { 137 | cerr << "ERR: Exception #" << e << endl; 138 | return -e; 139 | } 140 | 141 | delete emodetector; 142 | return 0; 143 | } 144 | -------------------------------------------------------------------------------- /src/gui/emotimevideo_cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file emotimegui_cli.cpp 4 | * @author Daniele Bellavista (daniele.bellavista@studio.unibo.it) 5 | * @date 12/31/2013 11:05:55 AM 6 | * @brief Emotime GUI command line interface 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include "VideoCapture.h" 13 | #include "BoostEmoDetector.h" 14 | #include "SVMEmoDetector.h" 15 | #include "matrix_io.h" 16 | #include "EmotimeGui.h" 17 | #include "FaceDetector.h" 18 | 19 | #include "TrainingParameters.h" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | using namespace emotime; 30 | 31 | using std::cout; 32 | using std::cerr; 33 | using std::endl; 34 | using std::string; 35 | 36 | /// Width of the faces used during training 37 | const int kWidth = 48; 38 | /// Height of the faces used during training 39 | const int kHeight = 48; 40 | /// N-Widths used during training 41 | const double kNWidths = 2; 42 | /// N-Lambdas used during training 43 | const double kNLambdas = 5; 44 | /// N-Thetas used during training 45 | const double kNThetas = 4; 46 | 47 | void help(){ 48 | cout << "Usage:" << endl; 49 | cout << " emotimegui_cli [] {}" << endl; 50 | cout << "Parameters:" << endl; 51 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for face detection" << endl; 52 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for eye detection" << endl; 53 | cout << " - Width of the image, the input image will be scaled"< - Height of the image, the input image will be scaled"< - "< - "< - "< - ada or svm (default ada)" << endl; 59 | cout << " - classificators list" << endl; 60 | cout << endl; 61 | } 62 | 63 | void banner(){ 64 | cout << "EmotimeGui Utility:" << endl; 65 | cout << " GUI for emotime. Load the video specified in stdin" << endl; 66 | } 67 | 68 | int main(int argc, const char* argv[]){ 69 | if (argc < 5) { 70 | banner(); 71 | help(); 72 | cerr << "ERR: missing parameters" << endl; 73 | return -3; 74 | } 75 | 76 | // Intializing the face detector 77 | string infile; 78 | string faceDetConfig = string(argv[1]); 79 | string eyeDetConfig = string(argv[2]); 80 | int width = std::atoi(argv[3]); 81 | int height = std::atoi(argv[4]); 82 | int nwidths = std::atoi(argv[5]); 83 | int nlambdas = std::atoi(argv[6]); 84 | int nthetas = std::atoi(argv[7]); 85 | FacePreProcessor facepreproc(faceDetConfig, eyeDetConfig, width, height, nwidths, nlambdas, nthetas); 86 | 87 | // Setting the mode 88 | int i; 89 | string mode; 90 | mode = string(argv[8]); 91 | if (mode!="svm" && mode!="ada") { 92 | mode="ada"; 93 | i = 8; 94 | } else { 95 | i = 9; 96 | } 97 | 98 | // Setting the classifiers 99 | vector cl_paths; 100 | EmoDetector* emodetector; 101 | 102 | for(; i < argc; i++) { 103 | cl_paths.push_back(string(argv[i])); 104 | } 105 | 106 | if (mode == "svm") { 107 | emodetector = new SVMEmoDetector(kCfactor, kMaxIteration, kErrorMargin); 108 | } else { 109 | emodetector = new BoostEmoDetector(kBoostType, kTrimWeight, kMaxDepth); 110 | } 111 | 112 | emodetector->init(cl_paths); 113 | 114 | cout<<"Insert the video file path: "; 115 | std::getline(std::cin, infile); 116 | cout<<"Loading '"< 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | using namespace emotime; 30 | 31 | using std::cout; 32 | using std::cerr; 33 | using std::endl; 34 | using std::string; 35 | 36 | /// Width of the faces used during training 37 | const int kWidth = 48; 38 | /// Height of the faces used during training 39 | const int kHeight = 48; 40 | /// N-Widths used during training 41 | const double kNWidths = 2; 42 | /// N-Lambdas used during training 43 | const double kNLambdas = 5; 44 | /// N-Thetas used during training 45 | const double kNThetas = 4; 46 | 47 | void help(){ 48 | cout << "Usage:" << endl; 49 | cout << " emotimegui_cli [] {}" << endl; 50 | cout << "Parameters:" << endl; 51 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for face detection" << endl; 52 | cout << " - OpenCV cascade classifier configuration file (Haar or LBP) for eye detection" << endl; 53 | cout << " - Width of the image, the input image will be scaled"< - Height of the image, the input image will be scaled"< - "< - "< - "< - ada or svm (default ada)" << endl; 59 | cout << " - classificators list" << endl; 60 | cout << endl; 61 | } 62 | 63 | void banner(){ 64 | cout << "EmotimeGui Utility:" << endl; 65 | cout << " GUI for emotime. Load the video specified in stdin" << endl; 66 | } 67 | 68 | int main(int argc, const char* argv[]){ 69 | if (argc < 5) { 70 | banner(); 71 | help(); 72 | cerr << "ERR: missing parameters" << endl; 73 | return -3; 74 | } 75 | 76 | // Intializing the face detector 77 | string infile; 78 | string faceDetConfig = string(argv[1]); 79 | string eyeDetConfig = string(argv[2]); 80 | int width = std::atoi(argv[3]); 81 | int height = std::atoi(argv[4]); 82 | int nwidths = std::atoi(argv[5]); 83 | int nlambdas = std::atoi(argv[6]); 84 | int nthetas = std::atoi(argv[7]); 85 | FacePreProcessor facepreproc(faceDetConfig, eyeDetConfig, width, height, nwidths, nlambdas, nthetas); 86 | 87 | // Setting the mode 88 | int i; 89 | string mode; 90 | mode = string(argv[8]); 91 | if (mode!="svm" && mode!="ada") { 92 | mode="ada"; 93 | i = 8; 94 | } else { 95 | i = 9; 96 | } 97 | 98 | // Setting the classifiers 99 | vector cl_paths; 100 | EmoDetector* emodetector; 101 | 102 | for(; i < argc; i++) { 103 | cl_paths.push_back(string(argv[i])); 104 | } 105 | 106 | if (mode == "svm") { 107 | emodetector = new SVMEmoDetector(kCfactor, kMaxIteration, kErrorMargin); 108 | } else { 109 | emodetector = new BoostEmoDetector(kBoostType, kTrimWeight, kMaxDepth); 110 | } 111 | 112 | emodetector->init(cl_paths); 113 | 114 | cout<<"Insert the video file path: "; 115 | std::getline(std::cin, infile); 116 | cout<<"Loading '"< 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | using namespace std; 17 | using namespace cv; 18 | using namespace emotime; 19 | 20 | void help() { 21 | cout<<"Usage:"< "< - Image file "<params.boost_type = boost_type; 24 | this->params.weak_count = kWeakCount; 25 | this->params.weight_trim_rate = trim_weight; 26 | this->params.max_depth = max_depth; 27 | this->params.use_surrogates = false; 28 | this->params.priors = NULL; 29 | } 30 | 31 | AdaBoostClassifier::~AdaBoostClassifier() { 32 | 33 | } 34 | 35 | bool AdaBoostClassifier::save(std::string outfile) { 36 | this->boost.save(outfile.c_str()); 37 | return true; 38 | } 39 | 40 | bool AdaBoostClassifier::load(std::string infile) { 41 | this->boost.load(infile.c_str()); 42 | return this->boost.get_weak_predictors() != NULL; 43 | } 44 | 45 | float AdaBoostClassifier::predict(cv::Mat& features) { 46 | return this->boost.predict(features, cv::Mat(), cv::Range::all(), false, false); 47 | } 48 | 49 | bool AdaBoostClassifier::doTraining(cv::Mat train_data, cv::Mat train_labels) { 50 | this->params.weak_count = train_data.cols; 51 | return this->boost.train(train_data, CV_ROW_SAMPLE, train_labels, 52 | cv::Mat(), cv::Mat(), cv::Mat(), cv::Mat(), this->params, false); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/training/AdaBoostClassifier.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file AdaBoostClassifier.h 4 | * @brief Definition of AdaBoostClassifier 5 | * 6 | */ 7 | 8 | #ifndef __ADABOOSTCLASSIFIER_INC__ 9 | #define __ADABOOSTCLASSIFIER_INC__ 10 | 11 | #include "Classifier.h" 12 | 13 | #include 14 | 15 | namespace emotime { 16 | 17 | /** 18 | * @class AdaBoostClassifier 19 | * 20 | * @brief Classifier specialization using AdaBoost 21 | * 22 | * @details 23 | * 24 | */ 25 | class AdaBoostClassifier : public Classifier { 26 | 27 | public: 28 | 29 | 30 | /** 31 | * @brief Create a new AdaBoostClassifier specifying opencv 32 | * boost type, trim weight and max depth 33 | * 34 | * @param[in] boost_type Type of the opencv boosting algorithm 35 | * @param[in] trim_weight The opencv trim weight value 36 | * @param[in] max_depth Algorithm max depth 37 | * 38 | * 39 | * @see CvBoostParams::CvBoostParams 40 | */ 41 | AdaBoostClassifier(int boost_type, double trim_weight, int max_depth); 42 | ~AdaBoostClassifier(); 43 | 44 | bool save(std::string outfile); 45 | 46 | bool load(std::string infile); 47 | 48 | float predict(cv::Mat& img); 49 | 50 | protected: 51 | 52 | bool doTraining(cv::Mat train_data, cv::Mat train_labels); 53 | 54 | private: 55 | 56 | /// The opencv implementation of AdaBoost 57 | CvBoost boost; 58 | /// Adaboost params 59 | CvBoostParams params; 60 | 61 | }; 62 | 63 | } 64 | 65 | #endif /* ----- #ifndef __ADABOOSTCLASSIFIER_INC__ ----- */ 66 | -------------------------------------------------------------------------------- /src/training/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | SET(EMOTIME_LIB_FILES "${EMOTIME_LIB_FILES} training/Classifier.cpp training/SVMClassifier.cpp training/AdaBoostClassifier.cpp") 3 | SET(EMOTIME_H_FILES "${EMOTIME_H_FILES} training/Classifier.h training/SVMClassifier.h training/AdaBoostClassifier.h") 4 | 5 | 6 | ADD_EXECUTABLE(train_cli train_cli.cpp Classifier.cpp SVMClassifier.cpp 7 | AdaBoostClassifier.cpp ../utils/matrix_io.cpp) 8 | 9 | FIND_PACKAGE(OpenCV COMPONENTS opencv_core opencv_ml opencv_contrib 10 | opencv_imgproc opencv_highgui REQUIRED) 11 | 12 | TARGET_LINK_LIBRARIES(train_cli ${OpenCV_LIBS}) 13 | 14 | INSTALL(TARGETS train_cli DESTINATION ${ASSETDIR}) 15 | -------------------------------------------------------------------------------- /src/training/Classifier.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file Classifier.cpp 4 | * 5 | * @brief The Classifier implementation 6 | * 7 | * @details 8 | * 9 | */ 10 | 11 | #include "Classifier.h" 12 | #include "matrix_io.h" 13 | 14 | #include 15 | #include 16 | 17 | using std::ifstream; 18 | using std::cerr; 19 | using std::endl; 20 | 21 | using std::string; 22 | using std::vector; 23 | using std::pair; 24 | 25 | using cv::Mat; 26 | 27 | namespace emotime { 28 | 29 | Classifier::Classifier() { 30 | 31 | } 32 | 33 | Classifier::~Classifier() { 34 | 35 | } 36 | 37 | bool Classifier::train(std::string csv_file) { 38 | vector > files_classes; 39 | if (!this->loadTrainData(csv_file, files_classes)) { 40 | return false; 41 | } 42 | if (files_classes.size() == 0){ 43 | cerr << "ERR: empty samples!" << endl; 44 | return false; 45 | } 46 | 47 | Mat sample = matrix_io_load(files_classes.at(0).first); 48 | int nfeatures = sample.rows * sample.cols; 49 | Mat train_data = Mat(0, nfeatures, CV_32FC1); 50 | Mat train_labels = Mat(0, 1, CV_32FC1); 51 | 52 | for(size_t i = 0; i < files_classes.size(); i++) { 53 | string fpath = files_classes.at(i).first; 54 | 55 | sample = matrix_io_load(fpath); 56 | Mat sample_float; 57 | 58 | if (sample.type() != CV_32FC1) { 59 | sample.convertTo(sample_float, CV_32FC1); 60 | } else { 61 | sample_float=sample; 62 | } 63 | train_data.push_back(sample_float.reshape(1, 1)); // reshape(int channels, int rows) 64 | train_labels.push_back((float)files_classes.at(i).second); 65 | } 66 | 67 | return this->doTraining(train_data, train_labels); 68 | } 69 | 70 | 71 | bool Classifier::loadTrainData(string csv_file, 72 | std::vector >& sample_files_classes) { 73 | try { 74 | ifstream csvf(csv_file.c_str()); 75 | string sampleF; 76 | string sampleC; 77 | int sclass; 78 | while (getline(csvf, sampleC, ',')) { 79 | getline(csvf, sampleF, '\n'); 80 | if (sampleC == "N") { 81 | sclass = 0; 82 | } else { 83 | sclass = 1; 84 | } 85 | sample_files_classes.push_back(std::make_pair(sampleF, sclass)); 86 | } 87 | return true; 88 | } catch (int e) { 89 | cerr << "ERR: Something wrong during csv loading #" << e << endl; 90 | return false; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/training/Classifier.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file Classifier.h 4 | * @brief Definition of Classifier 5 | * 6 | */ 7 | 8 | #ifndef __CLASSIFIER_INC__ 9 | #define __CLASSIFIER_INC__ 10 | 11 | #include "TrainingParameters.h" 12 | 13 | #include 14 | #include 15 | 16 | namespace emotime { 17 | 18 | /** 19 | * @class Classifier 20 | * 21 | * @brief Represent a general classifier method, capable of training, saving 22 | * and loading the status and predicting a class. 23 | * 24 | * @details 25 | * 26 | */ 27 | class Classifier { 28 | 29 | public: 30 | Classifier(); 31 | virtual ~Classifier(); 32 | 33 | 34 | /** 35 | * @brief Train the classifier using the given csv file 36 | * 37 | * @param[in] csv_file A csv file in the training format 38 | * 39 | * @return True if the training was successful 40 | * 41 | */ 42 | virtual bool train(std::string csv_file); 43 | 44 | /** 45 | * @brief Save the trained classifier status in the given file 46 | * 47 | * @param[in] outfile Where to save the status 48 | * 49 | * @return True if the save operation succeded 50 | */ 51 | virtual bool save(std::string outfile) = 0; 52 | 53 | /** 54 | * @brief Load a trained classifier status from the given file 55 | * 56 | * @param[in] infile The file to load 57 | * 58 | * @return True if the load operation succeded 59 | */ 60 | virtual bool load(std::string infile) = 0; 61 | 62 | /** 63 | * @brief Predict the given image 64 | * 65 | * @param[in] features The features to predict 66 | * 67 | * @return A value between 0 and 1 indicating the proxximity to 68 | * a class. 69 | */ 70 | virtual float predict(cv::Mat& features) = 0; 71 | 72 | protected: 73 | 74 | 75 | /** 76 | * @brief Load the training data from a CSV 77 | * 78 | * @param[in] csv_file The file to load 79 | * @param[in,out] sample_files_classes Filled with the sample files path and their class 80 | * 81 | * @return True if the parsing was correct 82 | * 83 | * @details 84 | */ 85 | bool loadTrainData(std::string csv_file, std::vector >& 86 | sample_files_classes); 87 | 88 | 89 | /** 90 | * @brief Perform the real training given files and related classes 91 | * 92 | * @param[in] train_data The matrix containing the training data 93 | * @param[in] train_labels The matrix containing the data labels 94 | * 95 | * @return True if the training succeded 96 | * 97 | * @details 98 | */ 99 | virtual bool doTraining(cv::Mat train_data, cv::Mat train_labels) = 0; 100 | 101 | private: 102 | 103 | }; /* ----- end of class Classifier ----- */ 104 | 105 | } 106 | 107 | 108 | #endif /* ----- #ifndef __CLASSIFIER_INC__ ----- */ 109 | -------------------------------------------------------------------------------- /src/training/SVMClassifier.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file SVMClassifier.cpp 4 | * 5 | * @brief Implementation of SVMClassifier 6 | * 7 | */ 8 | 9 | #include "SVMClassifier.h" 10 | 11 | using std::vector; 12 | using std::pair; 13 | using std::string; 14 | using std::cerr; 15 | using std::endl; 16 | 17 | namespace emotime { 18 | 19 | SVMClassifier::SVMClassifier(double C_factor, int max_iteration, double 20 | error_margin) { 21 | 22 | // Set up SVM's parameters 23 | this->params.svm_type = CvSVM::C_SVC; 24 | this->params.kernel_type = CvSVM::LINEAR; 25 | this->params.C = C_factor; 26 | this->params.term_crit = cvTermCriteria(CV_TERMCRIT_ITER, max_iteration, 27 | error_margin); 28 | } 29 | 30 | SVMClassifier::~SVMClassifier() { 31 | 32 | } 33 | 34 | bool SVMClassifier::doTraining(cv::Mat train_data, cv::Mat train_labels) { 35 | return this->svm.train(train_data, train_labels, cv::Mat(), cv::Mat(), this->params); 36 | } 37 | 38 | bool SVMClassifier::save(std::string outfile) { 39 | this->svm.save(outfile.c_str()); 40 | return true; 41 | } 42 | 43 | bool SVMClassifier::load(std::string infile) { 44 | this->svm.load(infile.c_str()); 45 | return svm.get_var_count() > 0; 46 | } 47 | 48 | float SVMClassifier::predict(cv::Mat& features) { 49 | return svm.predict(features); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/training/SVMClassifier.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file SVMClassifier.h 4 | * @brief Definition of SVMClassifier 5 | * 6 | */ 7 | 8 | #ifndef __SVMCLASSIFIER_INC__ 9 | #define __SVMCLASSIFIER_INC__ 10 | 11 | #include "Classifier.h" 12 | 13 | #include 14 | #include 15 | 16 | namespace emotime { 17 | 18 | /** 19 | * @class SVMClassifier 20 | * 21 | * @brief Classifier specialization using SVM. 22 | * 23 | * @details 24 | * 25 | */ 26 | class SVMClassifier : public Classifier { 27 | 28 | public: 29 | 30 | /** 31 | * @brief Create a linear C svm classifier. 32 | * 33 | * @param[in] C_factor The algorithm C factor 34 | * @param[in] max_iteration Maximum number of iteration termination criteria 35 | * @param[in] error_margin Minimum error termination criteria 36 | * 37 | */ 38 | SVMClassifier(double C_factor, int max_iteration, double error_margin); 39 | ~SVMClassifier(); 40 | 41 | bool save(std::string outfile); 42 | 43 | bool load(std::string infile); 44 | 45 | float predict(cv::Mat& features); 46 | 47 | protected: 48 | 49 | bool doTraining(cv::Mat train_data, cv::Mat train_labels); 50 | 51 | private: 52 | 53 | /// The opencv implementation of SVM 54 | CvSVM svm; 55 | /// The SVM parameters 56 | CvSVMParams params; 57 | 58 | }; 59 | 60 | } 61 | 62 | 63 | #endif /* ----- #ifndef __SVMCLASSIFIER_INC__ ----- */ 64 | -------------------------------------------------------------------------------- /src/training/TrainingParameters.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file TrainingParameters.h 4 | * @date 01/10/2014 01:50:29 AM 5 | * @brief Global parameter algorithm 6 | * 7 | */ 8 | 9 | #ifndef __DETECTIONPARAMETERS_INC__ 10 | #define __DETECTIONPARAMETERS_INC__ 11 | 12 | #include 13 | 14 | namespace emotime { 15 | 16 | /// SVM C factor parameter 17 | const double kCfactor = 1.0; 18 | /// SVM max interation termination criteria 19 | const int kMaxIteration = 1000000; 20 | /// SVM error margin termination criteria 21 | const double kErrorMargin = 1e-6; 22 | 23 | /// AdaBoost algorithm type 24 | const int kBoostType = CvBoost::REAL; 25 | /// AdaBoost trim weight value 26 | const double kTrimWeight = 0.95; 27 | /// AdaBoost weak count 28 | const double kWeakCount = 100; 29 | /// AdaBoost max algorithm depth 30 | const int kMaxDepth = 3; 31 | 32 | } 33 | 34 | 35 | #endif /* ----- #ifndef __DETECTIONPARAMETERS_INC__ ----- */ 36 | -------------------------------------------------------------------------------- /src/training/train_cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file train_cli.cpp 4 | * 5 | * @date 12/04/2013 10:33:42 AM 6 | * @brief CLI interface for training a Classifier 7 | * 8 | * @details 9 | * 10 | */ 11 | 12 | #include "Classifier.h" 13 | #include "SVMClassifier.h" 14 | #include "AdaBoostClassifier.h" 15 | 16 | #include "TrainingParameters.h" 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | using namespace cv; 23 | 24 | using std::cout; 25 | using std::cerr; 26 | using std::endl; 27 | 28 | /** 29 | * @brief Prints the CLI banner 30 | * 31 | */ 32 | void banner(); 33 | 34 | /** 35 | * @brief Prints the CLI help 36 | * 37 | */ 38 | void help(); 39 | 40 | void help() { 41 | cout << "Usage:" << endl; 42 | cout << " train_cli (svm|ada) " << endl; 43 | cout << "Parameters:" << endl; 44 | cout << " (svm|ada) - Training method to use" << endl; 45 | cout << " - File containing the training data" << endl; 46 | cout << " (CSV format, first column contains class, features in others)" << endl; 47 | cout << " - Output file where to store classifier" << endl; 48 | cout << endl; 49 | } 50 | 51 | void banner() { 52 | cout << "Train Utility:" << endl; 53 | cout << " Train an classifier." << endl; 54 | cout << " Train a classifier using data specified in a CSV file." << endl; 55 | cout << " NOTE: the CSV file may contain sample data directly (aka read samples from CSV mode)" << endl; 56 | cout << " or also contains path of the file where to find samples (images or OpenCV yml)" << endl; 57 | cout << "" << endl; 58 | } 59 | 60 | /** 61 | * @brief Main 62 | * 63 | * @param[in] argc 64 | * @param[in] argv 65 | * 66 | * @returns 1 67 | * 68 | */ 69 | int main(int argc, const char* argv[]) { 70 | 71 | if (argc < 4) { 72 | banner(); 73 | help(); 74 | cerr << "ERR: missing parameters" << endl; 75 | return 1; 76 | } 77 | string method(argv[1]); 78 | string infile(argv[2]); 79 | string outfile(argv[3]); 80 | 81 | emotime::Classifier* classifier; 82 | if (method == "svm") { 83 | classifier = new emotime::SVMClassifier(emotime::kCfactor, 84 | emotime::kMaxIteration, emotime::kErrorMargin); 85 | } else { 86 | classifier = new emotime::AdaBoostClassifier(emotime::kBoostType, 87 | emotime::kTrimWeight, emotime::kMaxDepth); 88 | } 89 | 90 | int ret = 0; 91 | if (!classifier->train(infile)) { 92 | cerr << "Error training!" << endl; 93 | ret = 1; 94 | } else if (!classifier->save(outfile)) { 95 | cerr << "Error saving!" << endl; 96 | ret = 1; 97 | } 98 | 99 | delete classifier; 100 | return ret; 101 | } 102 | -------------------------------------------------------------------------------- /src/utils/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | ### 3 | 4 | -------------------------------------------------------------------------------- /src/utils/matrix_io.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file matrix_io.cpp 4 | * @brief Implementation of matrix_io.h utils 5 | * 6 | */ 7 | 8 | #include "matrix_io.h" 9 | #include 10 | #include 11 | #include 12 | 13 | using namespace std; 14 | using namespace cv; 15 | 16 | std::string matrix_io_fileExt( std::string & file){ 17 | return file.substr(file.find_last_of(".") + 1); 18 | } 19 | 20 | std::string matrix_io_fileName( std::string & file){ 21 | int nameBegin=std::max( (int) file.find_last_of(string(PATH_SEPARATOR))+1, 0 ); 22 | size_t nameEnd=file.find_last_of("."); 23 | size_t extLen= file.substr(nameEnd, file.length()-nameEnd).length(); 24 | return file.substr( std::max( (int) file.find_last_of(string(PATH_SEPARATOR))+1, 0 ) , file.length()-nameBegin-extLen ); 25 | } 26 | std::string matrix_io_fileBaseName(std::string & file){ 27 | int nameBegin=std::max( (int) file.find_last_of(string(PATH_SEPARATOR))+1, 0 ); 28 | return file.substr(nameBegin, string::npos); 29 | } 30 | 31 | cv::Mat matrix_io_load(std::string & filePath){ 32 | try { 33 | string file = filePath; 34 | string format = matrix_io_fileExt(file); 35 | if(format==XMLEXT || format==YMLEXT) { 36 | string name = matrix_io_fileName(file); 37 | FileStorage fs(file, FileStorage::READ); 38 | Mat * mat = new Mat(); 39 | fs[name] >> *mat; 40 | fs.release(); 41 | return *mat; 42 | } else { 43 | // Otherwise threat it as image 44 | return imread( filePath, CV_LOAD_IMAGE_GRAYSCALE ); 45 | } 46 | } 47 | catch (int e) { 48 | cerr<<"ERR: Exception #" << e << endl; 49 | return *(new Mat(0,0,CV_32FC1)); 50 | } 51 | 52 | } 53 | 54 | bool matrix_io_save( cv::Mat & mat, std::string & filePath){ 55 | try { 56 | string file = filePath; 57 | string format = matrix_io_fileExt(file); 58 | if(format==XMLEXT || format==YMLEXT) { 59 | string name = matrix_io_fileName(file); 60 | FileStorage fs(file, FileStorage::WRITE); 61 | fs << name << mat; 62 | fs.release(); 63 | } else { 64 | // Otherwise threat it as image 65 | if (mat.type()==CV_32FC1){ 66 | double min; 67 | double max; 68 | cv::minMaxIdx(mat, &min, &max); 69 | cv::Mat adjMap; 70 | cv::convertScaleAbs(mat, adjMap, 255/max); 71 | imwrite(file, adjMap); 72 | } else { 73 | imwrite(file, mat); 74 | } 75 | } 76 | return true; 77 | } 78 | catch (int e) { 79 | cerr<<"ERR: Exception #" << e << endl; 80 | return false; 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/utils/matrix_io.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file matrix_io.h 4 | * @brief IO utilities for opencv matrix. 5 | * 6 | */ 7 | 8 | #ifndef _H_MATRIX_IO 9 | #define _H_MATRIX_IO 10 | 11 | #include 12 | #include 13 | 14 | #if defined(WIN32) || defined(_WIN32) 15 | /// System dependent path separator 16 | #define PATH_SEPARATOR "\\" 17 | #else 18 | /// System dependent path separator 19 | #define PATH_SEPARATOR "/" 20 | #endif 21 | 22 | /// YML extension 23 | #define YMLEXT "yml" 24 | /// XML extension 25 | #define XMLEXT "xml" 26 | 27 | /** 28 | * Retrieve file extension 29 | * 30 | * @param[in] file The filename 31 | * 32 | * @return The file extension 33 | * 34 | */ 35 | std::string matrix_io_fileExt(std::string& file); 36 | 37 | /** 38 | * Retrieve file name without extension and path 39 | * 40 | * @param[in] file The filename 41 | * 42 | * @return The file name without extension and path 43 | * 44 | */ 45 | std::string matrix_io_fileName(std::string& file); 46 | 47 | /** 48 | * Retrieve the file basename of a filepath 49 | * 50 | * @param[in] file The filename 51 | * 52 | * @return The base name 53 | * 54 | */ 55 | std::string matrix_io_fileBaseName(std::string& file); 56 | 57 | /** 58 | * Load matrix from an image or from a data file (xml,yml) 59 | * 60 | * @param[in] filePath The filename 61 | * 62 | * @details If \p filePath has as extension YMLEXT or XMLEXT, then the file is 63 | * treated as data file. 64 | * 65 | * @return The loaded matrix 66 | * 67 | */ 68 | cv::Mat matrix_io_load(std::string& filePath); 69 | 70 | /** 71 | * Save matrix to an image or to a data file (xml,yml). 72 | * 73 | * @param[in] mat The matrix to save 74 | * @param[in] filePath The destination file. If its extension is YMLEXT or 75 | * XMLEXT, then the file is saved as data file. 76 | * 77 | * @return True if the save operation was successful 78 | * 79 | */ 80 | bool matrix_io_save(cv::Mat& mat, std::string& filePath); 81 | 82 | #endif // _H_MATRIX_IO 83 | -------------------------------------------------------------------------------- /src/utils/string_utils.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file string_utils.cpp 4 | * @date 12/30/2013 02:26:11 PM 5 | * @brief string_utils.h implementation 6 | * 7 | */ 8 | 9 | #include "string_utils.h" 10 | 11 | using std::string; 12 | using std::vector; 13 | 14 | std::vector split_string(const std::string& str, const std::string& sequence) { 15 | size_t p = 0, p2; 16 | vector res; 17 | res.reserve(str.size()/2); 18 | 19 | while(true) { 20 | p2 = str.find_first_of("_", p); 21 | if(p2 >= str.size()) { 22 | break; 23 | } 24 | res.push_back(str.substr(p, p2 - p)); 25 | p = p2 + 1; 26 | } 27 | if(res.size() > 1) { 28 | res.push_back(str.substr(p)); 29 | } 30 | return res; 31 | } 32 | -------------------------------------------------------------------------------- /src/utils/string_utils.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @file string_utils.h 4 | * @date 12/30/2013 02:24:44 PM 5 | * @brief Utility functions for strings 6 | * 7 | */ 8 | 9 | #ifndef __STRING_UTILS_INC__ 10 | #define __STRING_UTILS_INC__ 11 | 12 | #include 13 | #include 14 | 15 | 16 | /** 17 | * @brief Split the string. 18 | * 19 | * @param[in] string The string to split 20 | * @param[in] sequence The splitter sequence 21 | * 22 | * @return A vector containing the splitted string 23 | * 24 | */ 25 | std::vector split_string(const std::string& string, const 26 | std::string& sequence); 27 | 28 | #endif /* ----- #ifndef __STRING_UTILS_INC__ ----- */ 29 | -------------------------------------------------------------------------------- /test/adaboost_train_error.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FDETECTOR="../resources/haarcascade_frontalface_alt2.xml" 4 | 5 | totHit=0 6 | tot=0 7 | 8 | for emotion in `ls -1 ../dataset/images | xargs`; 9 | do 10 | total=`ls -1 ../dataset/images/$emotion/*|wc -l` 11 | hit=$(echo "`ls -1 ../dataset/images/$emotion/*`"|./boost_emo_detector_cli "$FDETECTOR" 32 32 1 5 8 `ls ../dataset/classifiers/*.xml | xargs` 2>/dev/null| grep "Emotion predicted: $emotion" | wc -l) 12 | echo "$emotion : $hit/$total - `echo "scale=10;$hit/$total"|bc`" 13 | totHit=`expr $totHit + $hit` 14 | tot=`expr $tot + $total` 15 | done 16 | 17 | echo "Train hit rate: $totHit/$tot - `echo "scale=10;$totHit/$tot"|bc`" 18 | -------------------------------------------------------------------------------- /test/boost_emodetector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FDETECTOR="../resources/haarcascade_frontalface_alt2.xml" 4 | 5 | echo "*** BOOSTEMODETECT CLI" 6 | ./boost_emo_detector_cli "$FDETECTOR" 32 32 1 5 8 `ls ../dataset/classifiers/*.xml | xargs` 7 | 8 | -------------------------------------------------------------------------------- /test/clahe.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2015 stk 6 | # 7 | # Distributed under terms of the MIT license. 8 | 9 | """ 10 | 11 | """ 12 | import numpy as np 13 | import cv2 14 | import sys 15 | 16 | imgbig = cv2.imread(sys.argv[1],0) 17 | img=cv2.resize(imgbig,(200,200)) 18 | # create a CLAHE object (Arguments are optional). 19 | clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8)) 20 | cl1 = clahe.apply(img) 21 | eq = cv2.equalizeHist(img) 22 | eq1 = cv2.equalizeHist(cl1) 23 | 24 | 25 | final = np.concatenate((img,eq,cl1,eq1)) 26 | cv2.imshow('equalization %s'% sys.argv[1], final) 27 | cv2.waitKey(0) 28 | cv2.destroyAllWindows() 29 | 30 | -------------------------------------------------------------------------------- /test/face_test.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import cv2.cv as cv 3 | #import time 4 | 5 | 6 | storage=cv.CreateMemStorage() 7 | face_haar=cv.Load('../resources/haarcascade_frontalface_default.xml') 8 | eye_haar=cv.Load('../resources/haarcascade_eye.xml') 9 | #nose_haar=cv.Load('../resources/haarcascade_mcs_nose.xml') 10 | #mouth_haar=cv.Load('workspace/haarcascade_mcs_mouth.xml') 11 | haars=(face_haar,[eye_haar]) 12 | 13 | 14 | def detect1(image , haar_feat, rect=None): 15 | #t0=time.time() 16 | rects=list() 17 | img=cv.fromarray(image) 18 | containers=cv.HaarDetectObjects(img, haar_feat, storage, 1.1, 2, cv.CV_HAAR_DO_CANNY_PRUNING, rect) 19 | if containers: 20 | for f in containers: 21 | rects.append( (f[0][0],f[0][1],f[0][0]+f[0][2],f[0][1]+f[0][3]) ) 22 | return rects 23 | else: 24 | return None 25 | 26 | def detect(image , (haar_container, haar_feats) ): 27 | #t0=time.time() 28 | faces=list() 29 | #img=cv.fromarray(image) 30 | containers=detect1(image,haar_container,(30,60)) 31 | if containers: 32 | print containers 33 | for face in containers: 34 | # Find other features 35 | rectlist=list() 36 | for feat in haar_feats: 37 | r=detect1(image, feat, (face[1]/5,face[1]/10) ) 38 | if r: 39 | rectlist.extend(r) 40 | faces.append( (face, rectlist) ) 41 | return faces 42 | print faces 43 | else: 44 | return None 45 | 46 | def draw_rect(image, feat): 47 | cv.Rectangle(image,(feat[0],feat[1]),(feat[2],feat[3]),cv.RGB(155, 255, 25),2) 48 | 49 | def draw_rects(image, faces): 50 | for face,rectlist in faces: 51 | draw_rect(image, face) 52 | for r in rectlist: 53 | draw_rect(image, r) 54 | 55 | #image=cv2.imread(sys.argv[1], cv2.CV_LOAD_IMAGE_GRAYSCALE) 56 | #image=cv.fromarray(image) 57 | 58 | 59 | def main(): 60 | cv2.namedWindow("preview") 61 | vc = cv2.VideoCapture(0) 62 | 63 | if vc.isOpened(): 64 | rval, frame = vc.read() 65 | else: 66 | rval = False 67 | 68 | while rval: 69 | feats=detect(frame,haars) 70 | print feats 71 | img=cv.fromarray(frame) 72 | if feats: 73 | draw_rects(img,feats) 74 | cv.ShowImage("preview", img) 75 | rval, frame = vc.read() 76 | key = cv2.waitKey(1) 77 | if key == 27: # exit on ESC 78 | break 79 | 80 | if __name__=='__main__': 81 | main() 82 | 83 | 84 | -------------------------------------------------------------------------------- /test/gabor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Usage: 4 | 5 | gabor.py img nlambda ntheta sigma ksize size 6 | 7 | """ 8 | import numpy as np 9 | import cv2 10 | 11 | IMAGES=['training/faces/neutral/S081_008_00000001.png', 12 | 'training/faces/disgust/S005_001_00000011.png', 13 | 'training/faces/anger/S037_003_00000022.png', 14 | 'training/faces/happy/S011_006_00000013.png', 15 | 'training/faces/surprise/S073_001_00000013.png', 16 | 'training/faces/sadness/S042_002_00000016.png', 17 | 'training/faces/contempt/S157_002_00000011.png', 18 | 'training/faces/fear/S011_003_00000014.png' 19 | ] 20 | SIZE = 24 21 | KSIZE = 7 22 | SIGMA = 2.2 23 | LAMBDA = 10 24 | LAMBDAS = {"beg":8,"end":16} 25 | LAMBDAS_P = [ 26 | 3, 4, 6, 27 | 8, 12, 16, 28 | 24, 36 29 | ] 30 | THETAS = {"beg":0,"end":np.pi} 31 | 32 | def build_filters(nlambda,ntheta): 33 | filters = [] 34 | ksize = KSIZE 35 | lambd=LAMBDA 36 | print(np.arange(LAMBDAS['beg'],LAMBDAS['end'], (LAMBDAS['end']-LAMBDAS['beg'])/float(nlambda))) 37 | for lambd in np.arange(LAMBDAS['beg'],LAMBDAS['end'], (LAMBDAS['end']-LAMBDAS['beg'])/float(nlambda)): 38 | for theta in np.arange(THETAS['beg'], THETAS['end'], (THETAS['end']-THETAS['beg']) / float(ntheta)): 39 | kern = cv2.getGaborKernel((ksize, ksize), SIGMA, 40 | theta, lambd, 0.5, 0, ktype=cv2.CV_32F) 41 | kern /= 1.5*kern.sum() 42 | filters.append(kern) 43 | return filters 44 | 45 | def process(img2, filters): 46 | feats=[] 47 | img=cv2.resize(img2,(SIZE,SIZE)) 48 | accum = np.zeros_like(img) 49 | void = np.zeros_like(img) 50 | for kern in filters: 51 | fimg = cv2.filter2D(img, cv2.CV_8U, kern) 52 | feats.append(fimg) 53 | np.maximum(accum, fimg, accum) 54 | acc_2=accum.astype(np.uint8) 55 | thr,accum_b=cv2.threshold(acc_2, 170, 255 ,cv2.THRESH_BINARY|cv2.THRESH_OTSU) 56 | #accum_b=cv2.erode(accum_b, np.ones((2,2),dtype=np.uint8)) 57 | #accum_b=cv2.erode(accum_b, np.ones((3,3),dtype=np.uint8)) 58 | #accum_b=cv2.morphologyEx(accum_b, cv2.MORPH_OPEN, np.ones((2,2),dtype=np.uint8)) 59 | accum_b=cv2.morphologyEx(accum_b, cv2.MORPH_OPEN, np.ones((3,3),dtype=np.uint8)) 60 | return (acc_2,accum_b.astype(np.uint8),feats) 61 | 62 | if __name__ == '__main__': 63 | import sys 64 | global SIGMA 65 | global KSIZE 66 | global SIZE 67 | global LAMBDAS 68 | SIGMA = float(sys.argv[3]) 69 | KSIZE= int(sys.argv[4]) 70 | SIZE= int(sys.argv[5]) 71 | LAMBDA=int(sys.argv[1]) 72 | filters = build_filters(int(sys.argv[1]),int(sys.argv[2])) 73 | 74 | res = [] 75 | for img_fn in IMAGES: 76 | rr=[] 77 | img = cv2.imread(img_fn, cv2.CV_LOAD_IMAGE_GRAYSCALE) 78 | res2,res_b,bnk = process(img, filters) 79 | res1 = cv2.resize(res2,(72,72)) 80 | res_b2 = cv2.resize(res_b,(72,72)) 81 | img1 = cv2.resize(img,(72,72)) 82 | rr.append(np.rot90(img1)) 83 | rr.append(np.rot90(res1)) 84 | rr.append(np.rot90(res_b2)) 85 | for b in bnk: 86 | print b.dtype 87 | bb = cv2.resize(b,(72,72)) 88 | rr.append(np.rot90(bb)) 89 | res.append(np.concatenate(rr).transpose()) 90 | 91 | final = np.concatenate(res) 92 | cv2.imshow('result', final) 93 | key='' 94 | while key!=ord('q'): 95 | key=cv2.waitKey(0) 96 | cv2.destroyAllWindows() 97 | -------------------------------------------------------------------------------- /test/gaborpng.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | H=32 4 | W=32 5 | NW=1 6 | NL=5 7 | NT=8 8 | 9 | DSFOLDER="../dataset" 10 | 11 | for emotion in `ls -1 $DSFOLDER/faces|xargs`; 12 | do 13 | for face in `ls -1 $DSFOLDER/faces/$emotion/|xargs`; 14 | do 15 | ./gaborbank_cli $W $H $NW $NL $NT "$DSFOLDER/faces/$emotion/$face" "$DSFOLDER/features/$emotion/$face" 16 | done 17 | done 18 | -------------------------------------------------------------------------------- /test/test_facerotation.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Distributed under terms of the MIT license. 6 | 7 | """ 8 | 9 | """ 10 | import cv 11 | import cv2 12 | import sys 13 | import math 14 | from math import sqrt 15 | import numpy as np 16 | 17 | def search_eyes(image,eye_cascade): 18 | detectedEyes = eye_cascade.detectMultiScale(image,1.3,5) 19 | eyepos=[] 20 | if detectedEyes is not None: 21 | eye1=(0,0) 22 | eye2=(0,0) 23 | val1=-1 24 | val2=-1 25 | for (x,y,w,h) in detectedEyes: 26 | area=w*h 27 | if area>val1 and val1>val2: 28 | tmp=val1 29 | tmpe=eye1 30 | val1=area 31 | eye1=(x+w/2, y+h/2) 32 | val2=tmp 33 | eye2=tmpe 34 | elif area>val2 and val2>val1: 35 | tmp=val2 36 | tmpe=eye2 37 | val2=area 38 | eye2=(x+w/2, y+h/2) 39 | val1=tmp 40 | eye1=tmpe 41 | elif area>val1 : 42 | # second 43 | val1=area 44 | eye1=(x+w/2, y+h/2) 45 | elif area>val2 : 46 | # second 47 | val2=area 48 | eye2=(x+w/2, y+h/2) 49 | eyepos.append(eye1) 50 | eyepos.append(eye2) 51 | print "eyepos=%s"%str(eyepos) 52 | left=(0,0) 53 | right=(0,0) 54 | upper=(0,0) 55 | lower=(0,0) 56 | if len(eyepos)>1: 57 | if eye1[0] 6 | # 7 | # Distributed under terms of the MIT license. 8 | import cv2 9 | import cv2 10 | import numpy 11 | import sys 12 | 13 | imgFile= numpy.asarray(cv2.cv.Load(sys.argv[1])) 14 | img8=imgFile.astype(numpy.uint8) 15 | #cv2.equalizeHist(img8,img8) 16 | cv2.imshow('gabor %s'% sys.argv[1], img8) 17 | while cv2.waitKey(0) != ord('q'): 18 | pass 19 | cv2.destroyAllWindows() 20 | --------------------------------------------------------------------------------