├── .gitignore ├── Makefile ├── README.markdown ├── bib └── facerec.bib ├── definitions.tex ├── facerec.tex ├── facerec_octave.pdf ├── facerec_python.pdf ├── img ├── eigenfaces │ ├── octave_pca_eigenfaces.pdf │ ├── octave_pca_reconstruction.pdf │ ├── python_pca_eigenfaces.pdf │ └── python_pca_reconstruction.pdf ├── fisherfaces │ ├── multiclasslda.pdf │ ├── octave_fisherfaces_fisherfaces.pdf │ ├── octave_fisherfaces_reconstruction.pdf │ ├── python_fisherfaces_fisherfaces.pdf │ └── python_fisherfaces_reconstruction.pdf └── libfacerec │ ├── colormap │ ├── lena.jpg │ ├── lena_autumn.jpg │ ├── lena_bone.jpg │ ├── lena_cool.jpg │ ├── lena_hot.jpg │ ├── lena_hsv.jpg │ ├── lena_imshow_jet.jpg │ ├── lena_jet.jpg │ ├── lena_mkpj1.jpg │ ├── lena_mkpj2.jpg │ ├── lena_ocean.jpg │ ├── lena_pink.jpg │ ├── lena_rainbow.jpg │ ├── lena_spring.jpg │ ├── lena_summer.jpg │ └── lena_winter.jpg │ ├── eigenfaces_at.png │ ├── fisherfaces_yale_a.png │ ├── gender_classification │ ├── clooney.png │ └── fisherface_0.png │ └── stored_loaded_eigenfaces_at.png ├── section ├── conclusion.tex ├── facerecognition.tex └── introduction.tex └── src ├── m ├── example_eigenfaces.m ├── example_fisherfaces.m ├── fisherfaces.m ├── lda.m ├── list_files.m ├── normalize.m ├── pca.m ├── project.m ├── read_images.m ├── reconstruct.m └── toGrayscale.m └── py ├── crop_face.py ├── scripts ├── example_eigenfaces.py ├── example_fisherfaces.py ├── example_model_eigenfaces.py ├── example_model_fisherfaces.py └── example_pca.py └── tinyfacerec ├── __init__.py ├── distance.py ├── model.py ├── subspace.py ├── util.py └── visual.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.aux 2 | *.bbl 3 | *.blg 4 | *.log 5 | *.out 6 | *.pyc 7 | *.xml 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | python: 2 | echo "\def\python{}" > definitions.tex 3 | pdflatex facerec.tex 4 | bibtex facerec.aux 5 | pdflatex facerec.tex 6 | pdflatex facerec.tex 7 | mv facerec.pdf facerec_python.pdf 8 | make clean 9 | 10 | octave: 11 | echo "%\def\python{}" > definitions.tex 12 | pdflatex facerec.tex 13 | bibtex facerec.aux 14 | pdflatex facerec.tex 15 | pdflatex facerec.tex 16 | mv facerec.pdf facerec_octave.pdf 17 | make clean 18 | 19 | clean: 20 | rm *.log 21 | rm *.bbl 22 | rm *.out 23 | rm *.blg 24 | rm *.aux 25 | rm *.toc 26 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # bytefish/facerecognition_guide # 2 | 3 | ## description ## 4 | 5 | This is my guide to face recognition with OpenCV2 C++ and GNU Octave/MATLAB. If you research on face recognition, you'll soon notice there's a [gigantic number of publications](http://scholar.google.de/scholar?q=face+recognition), but source code is very sparse. So this guide is here to change that. Two algorithms are explained and implemented with GNU Octave/MATLAB and OpenCV2 C++ namely [Eigenfaces](http://www.bytefish.de/blog/eigenfaces) and [Fisherfaces](http://www.bytefish.de/blog/fisherfaces). 6 | 7 | ## build the project ## 8 | 9 | To build the Python version of this document simply run `make python`, to build the Octave version of this document run `make octave`. 10 | 11 | ## further reading ## 12 | If you are looking for a complete Python or GNU Octave/MATLAB implementation of various algorithms, please use [github.com/bytefish/facerec](https://github.com/bytefish/facerec) instead. 13 | 14 | More (maybe) here: [http://www.bytefish.de](http://www.bytefish.de). 15 | -------------------------------------------------------------------------------- /bib/facerec.bib: -------------------------------------------------------------------------------- 1 | @Comment { 2 | 3 | Typ :: Verwendung :: notwendige Eigenschaften :: Optionale Eigenschaften 4 | 5 | article :: Zeitungs- oder Zeitschriftenartikel :: author, title, journal, year :: volume, number, pages, month, note 6 | 7 | book :: Buch :: author or editor, title, publisher, year :: volume or number, series, address, edition, month, note 8 | 9 | phdthesis :: Doktor- oder andere Promotionsarbeit :: author, title, school, year :: type, address, month, note 10 | 11 | misc :: Wenn nichts anderes passt. :: - :: author, title, howpublished, month, year, note 12 | 13 | techreport :: veröffentlichter Bericht einer Hochschule oder anderen Institution :: author, title, institution, year :: type, note, number, address, month 14 | 15 | BEISPIELE FÜR URL 16 | ----------------- 17 | 18 | manual{KEITH, 19 | author = "Keith Reckdahl", 20 | title = "Using Imported Graphics in \LaTeXe\ Documents", 21 | year = "1997", 22 | month = dec, 23 | note = "CTAN: \url{tex-archive/info/epslatex.ps}"} 24 | 25 | misc{GRANT, 26 | author = "Michael C. Grant and David Carlisle", 27 | title = "The PSFrag system, version~3", 28 | year = "1996", 29 | month = nov, 30 | howpublished = "CTAN: \url{tex-archive/macros/latex/contrib/% 31 | supported/psfrag/pfgguide.tex}"} 32 | 33 | } 34 | 35 | @article{KKPCA, 36 | AUTHOR = "Kim, K.I. and Jung, K.C. and Kim, H.J.", 37 | TITLE = "Face recognition using kernel principal component analysis", 38 | JOURNAL = SPLetters, 39 | VOLUME = "9", 40 | YEAR = "2002", 41 | NUMBER = "2", 42 | MONTH = "February", 43 | PAGES = "40-42" 44 | } 45 | 46 | @article{Gottumukkal2004429, 47 | title = "An improved face recognition technique based on modular PCA approach", 48 | journal = "Pattern Recognition Letters", 49 | volume = "25", 50 | number = "4", 51 | pages = "429 - 436", 52 | year = "2004", 53 | note = "", 54 | issn = "0167-8655", 55 | doi = "DOI: 10.1016/j.patrec.2003.11.005", 56 | author = "Rajkiran Gottumukkal and Vijayan K. Asari" 57 | } 58 | 59 | @book{haykin08neural, 60 | author = {Haykin, Simon}, 61 | edition = {3}, 62 | howpublished = {Hardcover}, 63 | isbn = {0131471392}, 64 | keywords = {neural\_networks}, 65 | month = {November}, 66 | posted-at = {2010-01-21 14:20:04}, 67 | priority = {2}, 68 | publisher = {Prentice Hall}, 69 | title = {Neural Networks and Learning Machines (3rd Edition)}, 70 | year = {2008}, 71 | } 72 | 73 | @article{Wiskott97, 74 | author = {Laurenz Wiskott and Jean-Marc Fellous and Norbert Krüger and Christoph Von Der Malsburg}, 75 | title = {Face Recognition By Elastic Bunch Graph Matching}, 76 | journal = {IEEE TRANSACTIONS ON PATTERN ANALYSIS AND MACHINE INTELLIGENCE}, 77 | year = {1997}, 78 | volume = {19}, 79 | pages = {775--779} 80 | } 81 | 82 | @article{Raudys1991, 83 | title = {Small sample size effects in statistical pattern recognition: Recommendations for practitioners.}, 84 | author = {S. J. Raudys \and A.K. Jain}, 85 | year = {1991}, 86 | journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence} , 87 | volume= {13}, 88 | number = {3}, 89 | pages = {252-264} 90 | } 91 | 92 | @PhDThesis{Kanade73, 93 | title = {Picture processing system by computer complex and recognition of human faces}, 94 | author = {T. Kanade}, 95 | school = {Kyoto University}, 96 | month = {November}, 97 | year = {1973} 98 | } 99 | 100 | @PhDThesis{Wang04, 101 | title = {Studies On Sensitivy Of Face Recognition Performance To Eye Location Accuracy}, 102 | author = {Haoshu Wang}, 103 | school = {Graduate School of the University of Notre Dame}, 104 | month = {November}, 105 | year = {2004}, 106 | address = {Notre Dame, Indiana, USA}, 107 | } 108 | 109 | @article{Belhumeur2001, 110 | author = {Athinodoros Georghiades and Peter Belhumeur and David Kriegman}, 111 | title = {From Few to Many: Illumination Cone Models for Face Recognition under Variable Lighting and Pose}, 112 | journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, 113 | volume = {23}, 114 | number = {6}, 115 | year = {2001}, 116 | pages = {643-660}, 117 | publisher = {IEEE Computer Society}, 118 | } 119 | 120 | @Article{Lee2005, 121 | title = {Acquiring Linear Subspaces for Face Recognition under Variable Lighting}, 122 | author = {Kuang-Chih Lee and Jeffrey Ho and David Kriegman}, 123 | journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)}, 124 | year = {2005}, 125 | volume = {27}, 126 | number = {5} 127 | } 128 | 129 | @Techreport{Martinovsky2011, 130 | title = {Gesichtserkennung mit Eigenfaces}, 131 | author = {Filip Martinovský and Philipp Wagner}, 132 | year = {2010} 133 | } 134 | 135 | @article{PT91, 136 | author = {M. Turk and A. Pentland}, 137 | title={Eigenfaces for recognition.}, 138 | year={1991}, 139 | journal={Journal of Cognitive Neuroscience}, 140 | volume={3}, 141 | pages={71-86} 142 | } 143 | 144 | @book{Koecher03, 145 | author={Max Koecher}, 146 | title={Lineare Algebra und Analytische Geometrie}, 147 | publisher={Springer-Verlag Berlin Heidelberg}, 148 | edition={4. Auflage}, 149 | year={2003} 150 | } 151 | 152 | @misc{bsi03, 153 | title = {BioFace: Vergleichende Untersuchung von Gesichtserkennungssystemen}, 154 | author={BSI, Bundesamt für Sicherheit in der Informationstechnik}, 155 | year={2003}, 156 | note={Öffentlicher Abschlussbericht BioFace I und II} 157 | } 158 | 159 | @article{belhumeru97, 160 | author = {Peter N. Belhumeur and João Hespanha and David Kriegman}, 161 | title = {Eigenfaces vs. Fisherfaces: Recognition Using Class Specific Linear Projection}, 162 | journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 163 | volume={19}, 164 | number={7}, 165 | year={1997}, 166 | pages={711-720} 167 | } 168 | 169 | @book{myers08, 170 | author = {David G. Myers}, 171 | edition = {1}, 172 | howpublished = {Hardcover}, 173 | isbn = {9783540790327}, 174 | publisher = {Springer-Verlag Berlin Heidelberg}, 175 | title = {Psychologie}, 176 | year = {2008}, 177 | } 178 | 179 | @article{ zhao2003frl, 180 | title={{Face recognition: A literature survey}}, 181 | author={Zhao, W. and Chellappa, R. and Phillips, PJ and Rosenfeld, A.}, 182 | journal={Acm Computing Surveys (CSUR)}, 183 | volume={35}, 184 | number={4}, 185 | pages={399--458}, 186 | year={2003}, 187 | publisher={ACM New York, NY, USA} 188 | } 189 | 190 | @article{Tu06, 191 | author = {Chiara Turati, Viola Macchi Cassia, Francesca Simion and Irene Leo}, 192 | title = {Newborns Face Recognition: Role of Inner and Outer Facial Features}, 193 | journal={Child Development}, 194 | volume={77}, 195 | number={2}, 196 | year={2006}, 197 | pages={297-311} 198 | } 199 | 200 | @ARTICLE{Ojala02, 201 | author = {Timo Ojala and Matti Pietikäinen and Topi Mäenpää}, 202 | title = {Multiresolution gray-scale and rotation invariant texture classification with local binary patterns}, 203 | journal = {IEEE TRANSACTIONS ON PATTERN ANALYSIS AND MACHINE INTELLIGENCE}, 204 | year = {2002}, 205 | volume = {24}, 206 | number = {7}, 207 | pages = {971--987} 208 | } 209 | 210 | @INPROCEEDINGS{Bru92, 211 | author = {R. Brunelli and T. Poggio}, 212 | title = {Face Recognition through Geometrical Features}, 213 | booktitle = {European Conference on Computer Vision (ECCV)}, 214 | year = {1992}, 215 | pages = {792--800} 216 | } 217 | 218 | @Book{VC74, 219 | author = "V. N. Vapnik and A. Ya. Chervonenkis", 220 | title = "Theory of Pattern Recognition [in Russian]", 221 | publisher = "Nauka", 222 | year = 1974, 223 | address = "USSR", 224 | abstract = "Basis of vapnik's work" 225 | } 226 | 227 | @inproceedings{VC95, 228 | author = {Cortes, Corinna and Vapnik, Vladimir}, 229 | booktitle = {Machine Learning}, 230 | pages = {273--297}, 231 | priority = {2}, 232 | title = {Support-Vector Networks}, 233 | volume = {20}, 234 | year = {1995} 235 | } 236 | 237 | @article{culloch1943, 238 | author={W. McCulloch and W. Pitts}, 239 | title={A logical calculus of the ideas immanent in nervous activity.}, 240 | journal={Bulletin of Mathematical Biophysics}, 241 | volume={5}, 242 | year={1943}, 243 | pages={115-133} 244 | } 245 | 246 | @book{werbos1994, 247 | author={P.J. Werbos}, 248 | title = {The Roots of Backpropagation: From ordered derivatives to Neural Networks and Political Forecasting}, 249 | publisher={John Wiley and Sons}, 250 | year={1994}, 251 | address={New York} 252 | } 253 | 254 | @article{Viola2001, 255 | author={Paul Viola and Michael Jones}, 256 | title={Robust Real-time Object Detection}, 257 | journal={International Journal of Computer Vision}, 258 | volume={57}, 259 | number={2}, 260 | year={2001}, 261 | pages={137-154} 262 | } 263 | 264 | @phdthesis{Rodriguez2006, 265 | author={Yann Rodriguez}, 266 | title={Face Detection and Verification using Local Binary Patterns}, 267 | school={École Polytechnique Fédérale De Lausanne}, 268 | year={2006}, 269 | month={October} 270 | } 271 | 272 | @article{Lienhart2002, 273 | author={Rainer Lienhart and Jochen Maydt}, 274 | title={An Extended Set of Haar-like Features for Rapid Object Detection}, 275 | journal={IEEE ICIP 2002}, 276 | volume={1}, 277 | year={2002}, 278 | pages={900-903} 279 | } 280 | 281 | @article{frvt2010, 282 | title={FRVT 2006 and ICE 2006 large-scale experimental results.}, 283 | author={Phillips, P.J. and Scruggs, W.T. and O'Toole, A.J. and Flynn, P.J. and Bowyer, K.W. and Schott, C.L. and Sharpe, M.}, 284 | journal={IEEE Trans Pattern Anal Mach Intell}, 285 | volume={32}, 286 | number={5}, 287 | pages={831-46}, 288 | year={2010} 289 | } 290 | 291 | @article{haar1910, 292 | title={Zur Theorie der orthogonalen Funktionensysteme}, 293 | author={A. Haar}, 294 | journal={Mathematische Annalen}, 295 | volume={69}, 296 | pages={331-371}, 297 | year={1910} 298 | } 299 | 300 | @InProceedings{Schapire01, 301 | author = {R. Schapire}, 302 | title = {The boosting approach to machine learning: An overview}, 303 | booktitle = {MSRI Workshop on Nonlinear Estimation and Classification}, 304 | publiher = {Berkeley, CA}, 305 | year = 2001 306 | } 307 | 308 | 309 | @article{Quinlan1986, 310 | author = {Quinlan, J. R.}, 311 | title = {Induction of Decision Trees}, 312 | journal = {Mach. Learn.}, 313 | volume = {1}, 314 | issue = {1}, 315 | month = {March}, 316 | year = {1986}, 317 | issn = {0885-6125}, 318 | pages = {81--106}, 319 | numpages = {26}, 320 | url = {http://dl.acm.org/citation.cfm?id=637962.637969}, 321 | doi = {10.1023/A:1022643204877}, 322 | acmid = {637969}, 323 | publisher = {Kluwer Academic Publishers}, 324 | address = {Hingham, MA, USA} 325 | } 326 | 327 | 328 | @INPROCEEDINGS{Messer06, 329 | author = {Kieron Messer and Josef Kittler and James Short and G. Heusch and Fabien Cardinaux and Sebastien Marcel and Yann Rodriguez and Shiguang Shan and Y. Su and X. Chen}, 330 | title = {Performance Characterisation of Face Recognition Algorithms and Their Sensitivity to Severe Illumination Changes}, 331 | booktitle = {In: ICB}, 332 | year = {2006}, 333 | pages = {1--11} 334 | } 335 | 336 | @INPROCEEDINGS{Chen2000, 337 | title={In search of illumination invariants}, 338 | author={H. F. Chen and P. N. Belhumeur and D. W. Jacobs}, 339 | booktitle={IEEE Computer Vision and Pattern Recognition}, 340 | year={2000}, 341 | pages={254-261} 342 | } 343 | 344 | @article{Chen2006, 345 | author={T. Chen and W. Yin and X. Zhou and D. Comaniciu and T. Huang}, 346 | title={Total variation models for variable lighting face recognition}, 347 | year={2006}, 348 | journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 349 | pages={1519–1524}, 350 | volume = {28}, 351 | issue = {9}, 352 | } 353 | 354 | @article{Tan10, 355 | author={Xiaoyang Tan and Bill Triggs}, 356 | title={Enhanced Local Texture Feature Sets for Face Recognition Under Difficult Lighting Conditions}, 357 | year={2010}, 358 | journal={IEEE Transactions on Image Processing}, 359 | volume={19}, 360 | issue={6}, 361 | pages={1635-650 } 362 | } 363 | 364 | @article{Zheng2011, 365 | author={Ji Zheng and Bao-Liang Lu}, 366 | title={A support vector machine classifier with automatic confidence and its application to gender classification}, 367 | journal={Elsevier Neurocomputing}, 368 | year={2011}, 369 | volume={74}, 370 | pages={1926–1935} 371 | } 372 | 373 | @Article{Ahonen04, 374 | author = {Timo Ahonen and Abdenour Hadid and Matti Pietikainen}, 375 | journal = {Computer Vision - ECCV 2004}, 376 | pages = {469--481}, 377 | title = {{Face Recognition with Local Binary Patterns}}, 378 | year = {2004} 379 | } 380 | 381 | @Article{Maturana09, 382 | title = {Face Recognition with Local Binary Patterns, Spatial Pyramid Histograms and Naive Bayes Nearest Neighbor Classification}, 383 | author = {Daniel Maturana and Domingo Mery and Alvaro Soto}, 384 | journal = {2009 International Conference of the Chilean Computer Science Society (SCCC)}, 385 | year = {2009}, 386 | pages={125-132} 387 | } 388 | 389 | @Article{Huang2011, 390 | title = {Local Binary Patterns and Its Application to Facial Image Analysis: A Survey}, 391 | author = {Di Huang and Caifeng Shan and Mohsen Ardabilian and Yunhong Wang and Liming Chen}, 392 | journal = {IEEE Transactions on Systems, Man, and Cybernetics, Part C: Applications and Reviews}, 393 | year = {2010} 394 | } 395 | 396 | @Article{Ojala1996, 397 | title = {A comparative study of texture measures with classification based on featured distribution.}, 398 | author = {T. Ojala and M. Pietikäinen and D. Harwood}, 399 | journal = {IEEE Transactions on Pattern Recognition}, 400 | volume={29}, 401 | issue={1}, 402 | pages={51-59}, 403 | year = {1996} 404 | } 405 | 406 | @Article{Bianconi2011, 407 | title = {On the Occurrence Probability of Local Binary Patterns: A Theoretical Study}, 408 | author = {Francesco Bianconi and Antonio Fernández}, 409 | journal = {Journal of Mathematical Imaging and Vision}, 410 | volume = {40}, 411 | issue = {3}, 412 | month = {July}, 413 | year = {2011} 414 | } 415 | 416 | @Article{Cardinaux2006, 417 | title = {User Authentication via Adapted Statistical Models of Face Images}, 418 | author = {F. Cardinaux and C. Sanderson and S. Bengio}, 419 | journal = {IEEE Transactions on Signal Processing}, 420 | volume = {54}, 421 | issue = {1}, 422 | pages= {361-373}, 423 | month = {January}, 424 | year = {2006} 425 | } 426 | 427 | @Book{Vapnik1998, 428 | author={V. Vapnik}, 429 | title = {Statistical Learning Theory}, 430 | year = {1998}, 431 | publisher = {John Wiley and Sons}, 432 | address={New York} 433 | } 434 | 435 | @Article{Veres2009, 436 | title = {Recognition Of Briefly Presented Familiar And Unfamiliar Faces}, 437 | author = {Bozana Veres-Injac and Malte Persike}, 438 | journal = {Psihologija}, 439 | year = {2009}, 440 | volume={42}, 441 | issue={1}, 442 | pages={47-66} 443 | } 444 | 445 | @journal{Preprocessing10, 446 | title={Robust face recognition based on illumination invariant in nonsubsampled contourlet transform domain}, 447 | author={Yong Cheng and Yingkun Hou and Chunxia Zhao and Zuoyong Li and Yong Hu and Cailing Wang}, 448 | journal={Neurocomputing}, 449 | volume={73}, 450 | year={2010}, 451 | pages={2217–2224} 452 | } 453 | 454 | @misc{Preprocessing04, 455 | author={H. Ando and N. Fuchigami and M. Sasaki and A. Iwata}, 456 | title={Robust face recognition models under illumination variations toward hardware implementation on 3dcss}, 457 | year={2004} 458 | } 459 | 460 | @PhDThesis{Chan2008, 461 | title = {Multi-scale Local Binary Pattern Histogram for Face Recognition}, 462 | author = {Chi Ho Chan}, 463 | school = {University of Surrey, Guildford, Surrey, England}, 464 | year = {2008} 465 | } 466 | 467 | @Book{Klinke1996, 468 | title = {Lehrbuch der Physiologie}, 469 | author = {R. Klinke and S. Silbernagel}, 470 | year = {1996}, 471 | publisher = {Georg Thieme Verlag, Stuttgart, New York} 472 | } 473 | 474 | \@Article{Kohavi1995, 475 | title = {A study of cross-validation and bootstrap for accuracy estimation and model selection.}, 476 | author = {R. Kohavi}, 477 | year = {1995}, 478 | journal = {Proceedings of International Joint Conference on AI}, 479 | pages = {1137–1145} 480 | } 481 | 482 | @article{Hart1967, 483 | author = {T.M. Cover and P.E. Hart}, 484 | title = {Nearest neighbor pattern classification}, 485 | journal = {IEEE Trans. Inform. Theory}, 486 | volume = {IT-13(1)}, 487 | pages = {21–27}, 488 | year = {1967} 489 | } 490 | 491 | @article{Fix1951, 492 | author = {E. Fix and J.L. Hodges}, 493 | title={Discriminatory analysis, nonparametric discrimination: Consistency properties}, 494 | journal={USAF School of Aviation Medicine, Randolph Field}, 495 | volume={Technical Report 4}, 496 | year={1951} 497 | } 498 | 499 | @article {Ca2000, 500 | author = {S. Bermejo and J. Cabestany}, 501 | title = {Adaptive soft k-nearest-neighbour classifiers}, 502 | journal = {Pattern Recognition}, 503 | volume = {33}, 504 | pages = {1999-2005}, 505 | year = {2000} 506 | } 507 | 508 | @Techreport{Arya95, 509 | author = {Arya, Sunil and Mount, David M. and Netanyahu, Nathan S. and Silverman, Ruth and Wu, Angela Y.}, 510 | title = {An optimal algorithm for approximate nearest neighbor searching in fixed dimensions}, 511 | year = {1995}, 512 | source = {Center for Automation Research Report No. CAR-TR-802}, 513 | publisher = {University of Maryland at College Park}, 514 | address = {College Park, MD, USA}, 515 | } 516 | 517 | @Article{He2002, 518 | author={Yunshuang He and Guojun Lu and ShyhWei Teng}, 519 | title={An Investigation of Using K-d Tree to Improve Image Retrieval Efficiency}, 520 | year={2002}, 521 | journal={Digital Image Computing Techniques and Application}, 522 | pages={21--22} 523 | } 524 | 525 | 526 | @Inproceedings{Xie2011, 527 | title={Use Bin-Ratio Information for Category and Scene Classification}, 528 | author={Nianhua Xie and Haibin Ling and Weiming Hu and Xiaoqin Zhang}, 529 | journal={IEEE Conference on Computer Vision and Pattern Recognition}, 530 | pages = {2313-2319}, 531 | year = {2010} 532 | } 533 | 534 | @article{Fisher36, 535 | author = {Fisher, Ronald A.}, 536 | journal = {Annals Eugen.}, 537 | pages = {179--188}, 538 | title = {{The use of multiple measurements in taxonomic problems}}, 539 | volume = {7}, 540 | year = {1936} 541 | } 542 | 543 | @book{Duda2001, 544 | author = {Duda, Richard O. and Hart, Peter E. and Stork, David G.}, 545 | howpublished = {Hardcover}, 546 | edition = {2}, 547 | isbn = {0471056693}, 548 | month = {November}, 549 | title = {{Pattern Classification (2nd Edition)}}, 550 | year = {2001} 551 | } 552 | 553 | 554 | -------------------------------------------------------------------------------- /definitions.tex: -------------------------------------------------------------------------------- 1 | \def\python{} 2 | -------------------------------------------------------------------------------- /facerec.tex: -------------------------------------------------------------------------------- 1 | \documentclass[10pt]{article} 2 | 3 | % compile switches definitions 4 | \input{definitions} 5 | 6 | 7 | \usepackage[utf8]{inputenc} 8 | 9 | \usepackage{tabularx} 10 | \usepackage{amsmath} 11 | \usepackage{amssymb} 12 | \usepackage{geometry} 13 | 14 | \geometry{a4paper, top=25mm, left=30mm, right=25mm, bottom=30mm, 15 | headsep=10mm, footskip=12mm} 16 | 17 | \usepackage{graphicx} 18 | \usepackage{url} 19 | \usepackage[pdftex,colorlinks=true,citecolor=blue,urlcolor=blue,linkcolor=blue]{hyperref} 20 | \usepackage{color} 21 | \usepackage{xcolor} 22 | \usepackage{tabularx} 23 | \usepackage{listings} 24 | \usepackage{parcolumns} 25 | \usepackage{longtable} 26 | \usepackage[margin=10pt]{caption} 27 | 28 | % set the title 29 | 30 | \title{\begin{tabular}{p{11cm}}\centering 31 | Face Recognition with \ifx\python\undefined GNU Octave/MATLAB \else Python\fi 32 | \end{tabular}} 33 | 34 | % contact details 35 | \author{Philipp Wagner\\\href{http://www.bytefish.de}{http://www.bytefish.de}} 36 | 37 | % colors for syntax highlightning 38 | \definecolor{darkblue}{rgb}{0,0,.6} 39 | \definecolor{darkred}{rgb}{.6,0,0} 40 | \definecolor{darkgreen}{rgb}{0,.6,0} 41 | \definecolor{red}{rgb}{.98,0,0} 42 | 43 | 44 | % Listings Style 45 | \lstset{ 46 | numberstyle=\footnotesize, 47 | basicstyle=\footnotesize\ttfamily 48 | numbers=left, 49 | numbersep=5pt, 50 | frame=single 51 | } 52 | 53 | % Add basic syntax highlightning to C++ 54 | \lstset{ 55 | numberstyle=\footnotesize, 56 | basicstyle=\footnotesize\ttfamily, 57 | numbers=none, 58 | numbersep=5pt, 59 | frame=single, 60 | breaklines=true, 61 | showspaces=false, 62 | showstringspaces=false, 63 | showtabs=false, 64 | tabsize=2, 65 | } 66 | 67 | \lstloadlanguages{C++} 68 | \lstset{ 69 | language=C++, 70 | commentstyle=\footnotesize\ttfamily\color{darkgreen}, 71 | keywordstyle=\footnotesize\ttfamily\color{darkblue}, 72 | stringstyle=\footnotesize\ttfamily\color{darkred}, 73 | } 74 | 75 | \lstdefinelanguage{cmake}{ 76 | morekeywords={cmake_minimum_required, project, find\_package, add\_executable, target\_link\_libraries}, 77 | sensitive=false, 78 | morecomment=[l]{\#}, 79 | } 80 | 81 | % change vertical space of itemize, description and enumeration environments 82 | \let\olditemize=\itemize 83 | \def\itemize{\olditemize\setlength{\itemsep}{-0.5ex}} 84 | \let\oldenumerate=\enumerate 85 | \def\enumerate{\oldenumerate\setlength{\itemsep}{-0.5ex}} 86 | \let\olddescription=\description 87 | \def\description{\olddescription\setlength{\itemsep}{-0.5ex}} 88 | 89 | % don't indent sections 90 | \setlength{\parindent}{0pt} 91 | 92 | \begin{document} 93 | 94 | \maketitle 95 | \tableofcontents 96 | 97 | \input{section/introduction} 98 | \input{section/facerecognition} 99 | \input{section/conclusion} 100 | 101 | \bibliography{bib/facerec} 102 | \bibliographystyle{acm} 103 | 104 | \end{document} 105 | 106 | 107 | -------------------------------------------------------------------------------- /facerec_octave.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/facerec_octave.pdf -------------------------------------------------------------------------------- /facerec_python.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/facerec_python.pdf -------------------------------------------------------------------------------- /img/eigenfaces/octave_pca_eigenfaces.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/eigenfaces/octave_pca_eigenfaces.pdf -------------------------------------------------------------------------------- /img/eigenfaces/octave_pca_reconstruction.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/eigenfaces/octave_pca_reconstruction.pdf -------------------------------------------------------------------------------- /img/eigenfaces/python_pca_eigenfaces.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/eigenfaces/python_pca_eigenfaces.pdf -------------------------------------------------------------------------------- /img/eigenfaces/python_pca_reconstruction.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/eigenfaces/python_pca_reconstruction.pdf -------------------------------------------------------------------------------- /img/fisherfaces/multiclasslda.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/fisherfaces/multiclasslda.pdf -------------------------------------------------------------------------------- /img/fisherfaces/octave_fisherfaces_fisherfaces.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/fisherfaces/octave_fisherfaces_fisherfaces.pdf -------------------------------------------------------------------------------- /img/fisherfaces/octave_fisherfaces_reconstruction.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/fisherfaces/octave_fisherfaces_reconstruction.pdf -------------------------------------------------------------------------------- /img/fisherfaces/python_fisherfaces_fisherfaces.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/fisherfaces/python_fisherfaces_fisherfaces.pdf -------------------------------------------------------------------------------- /img/fisherfaces/python_fisherfaces_reconstruction.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/fisherfaces/python_fisherfaces_reconstruction.pdf -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_autumn.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_autumn.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_bone.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_bone.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_cool.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_cool.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_hot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_hot.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_hsv.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_hsv.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_imshow_jet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_imshow_jet.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_jet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_jet.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_mkpj1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_mkpj1.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_mkpj2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_mkpj2.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_ocean.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_ocean.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_pink.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_pink.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_rainbow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_rainbow.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_spring.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_spring.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_summer.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_summer.jpg -------------------------------------------------------------------------------- /img/libfacerec/colormap/lena_winter.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/colormap/lena_winter.jpg -------------------------------------------------------------------------------- /img/libfacerec/eigenfaces_at.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/eigenfaces_at.png -------------------------------------------------------------------------------- /img/libfacerec/fisherfaces_yale_a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/fisherfaces_yale_a.png -------------------------------------------------------------------------------- /img/libfacerec/gender_classification/clooney.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/gender_classification/clooney.png -------------------------------------------------------------------------------- /img/libfacerec/gender_classification/fisherface_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/gender_classification/fisherface_0.png -------------------------------------------------------------------------------- /img/libfacerec/stored_loaded_eigenfaces_at.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/img/libfacerec/stored_loaded_eigenfaces_at.png -------------------------------------------------------------------------------- /section/conclusion.tex: -------------------------------------------------------------------------------- 1 | \section{Conclusion} 2 | 3 | This document explained and implemented the Eigenfaces \cite{PT91} and the Fisherfaces \cite{belhumeru97} method with \href{http://www.gnu.org/software/octave}{GNU Octave}/\href{http://mathworks.com}{MATLAB}, \href{http://www.python.org}{Python}. It gave you some ideas to get started and research this highly active topic. I hope you had fun reading and I hope you think \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/facerec_api.html}{cv::FaceRecognizer} is a useful addition to OpenCV. 4 | 5 | More maybe here: 6 | 7 | \begin{itemize} 8 | \item \url{http://www.opencv.org} 9 | \item \url{http://www.bytefish.de/blog} 10 | \item \url{http://www.github.com/bytefish} 11 | \end{itemize} 12 | 13 | -------------------------------------------------------------------------------- /section/facerecognition.tex: -------------------------------------------------------------------------------- 1 | \section{Face Recognition} 2 | 3 | Face recognition is an easy task for humans. Experiments in \cite{Tu06} have shown, that even one to three day old babies are able to distinguish between known faces. So how hard could it be for a computer? It turns out we know little about human recognition to date. Are inner features (eyes, nose, mouth) or outer features (head shape, hairline) used for a successful face recognition? How do we analyze an image and how does the brain encode it? It was shown by \href{http://en.wikipedia.org/wiki/David_H._Hubel}{David Hubel} and \href{http://en.wikipedia.org/wiki/Torsten_Wiesel}{Torsten Wiesel}, that our brain has specialized nerve cells responding to specific local features of a scene, such as lines, edges, angles or movement. Since we don't see the world as scattered pieces, our visual cortex must somehow combine the different sources of information into useful patterns. Automatic face recognition is all about extracting those meaningful features from an image, putting them into a useful representation and performing some kind of classification on them. 4 | 5 | Face recognition based on the geometric features of a face is probably the most intuitive approach to face recognition. One of the first automated face recognition systems was described in \cite{Kanade73}: marker points (position of eyes, ears, nose, ...) were used to build a feature vector (distance between the points, angle between them, ...). The recognition was performed by calculating the euclidean distance between feature vectors of a probe and reference image. Such a method is robust against changes in illumination by its nature, but has a huge drawback: the accurate registration of the marker points is complicated, even with state of the art algorithms. Some of the latest work on geometric face recognition was carried out in \cite{Bru92}. A 22-dimensional feature vector was used and experiments on large datasets have shown, that geometrical features alone don't carry enough information for face recognition. 6 | 7 | The Eigenfaces method described in \cite{PT91} took a holistic approach to face recognition: A facial image is a point from a high-dimensional image space and a lower-dimensional representation is found, where classification becomes easy. The lower-dimensional subspace is found with Principal Component Analysis, which identifies the axes with maximum variance. While this kind of transformation is optimal from a reconstruction standpoint, it doesn't take any class labels into account. Imagine a situation where the variance is generated from external sources, let it be light. The axes with maximum variance do not necessarily contain any discriminative information at all, hence a classification becomes impossible. So a class-specific projection with a Linear Discriminant Analysis was applied to face recognition in \cite{belhumeru97}. The basic idea is to minimize the variance within a class, while maximizing the variance between the classes at the same time (Figure \ref{fig:scatter_matrices}). 8 | 9 | Recently various methods for a local feature extraction emerged. To avoid the high-dimensionality of the input data only local regions of an image are described, the extracted features are (hopefully) more robust against partial occlusion, illumation and small sample size. Algorithms used for a local feature extraction are Gabor Wavelets (\cite{Wiskott97}), Discrete Cosinus Transform (\cite{Cardinaux2006}) and Local Binary Patterns (\cite{Ahonen04,Maturana09, Rodriguez2006}). It's still an open research question how to preserve spatial information when applying a local feature extraction, because spatial information is potentially useful information. 10 | 11 | \subsection{Face Database} 12 | 13 | I don't want to do a toy example here. We are doing face recognition, so you'll need some face images! You can either create your own database or start with one of the available databases, \href{http://face-rec.org/databases/}{face-rec.org/databases} gives an up-to-date overview. Three interesting databases are\footnote{Parts of the description are quoted from \href{http://face-rec.org}{face-rec.org}.}: 14 | 15 | \begin{description} 16 | 17 | \item[\href{http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html}{AT\&T Facedatabase}] The AT\&T Facedatabase, sometimes also known as \textit{ORL Database of Faces}, contains ten different images of each of 40 distinct subjects. For some subjects, the images were taken at different times, varying the lighting, facial expressions (open / closed eyes, smiling / not smiling) and facial details (glasses / no glasses). All the images were taken against a dark homogeneous background with the subjects in an upright, frontal position (with tolerance for some side movement). 18 | 19 | \item[\href{http://vision.ucsd.edu/content/yale-face-database}{Yale Facedatabase A}] The AT\&T Facedatabase is good for initial tests, but it's a fairly easy database. The Eigenfaces method already has a 97\% recognition rate, so you won't see any improvements with other algorithms. The Yale Facedatabase A is a more appropriate dataset for initial experiments, because the recognition problem is harder. The database consists of 15 people (14 male, 1 female) each with 11 grayscale images sized $320 \times 243$ pixel. There are changes in the light conditions (center light, left light, right light), facial expressions (happy, normal, sad, sleepy, surprised, wink) and glasses (glasses, no-glasses). 20 | 21 | The original images are not cropped or aligned. I've prepared a Python script available in \lstinline|src/py/crop_face.py|, that does the job for you. 22 | 23 | \item[\href{http://vision.ucsd.edu/~leekc/ExtYaleDatabase/ExtYaleB.html}{Extended Yale Facedatabase B}] The Extended Yale Facedatabase B contains 2414 images of 38 different people in its cropped version. The focus is on extracting features that are robust to illumination, the images have almost no variation in emotion/occlusion/$\ldots$. I personally think, that this dataset is too large for the experiments I perform in this document, you better use the \href{http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html}{AT\&T Facedatabase}. A first version of the Yale Facedatabase B was used in \cite{belhumeru97} to see how the Eigenfaces and Fisherfaces method (section \ref{ssection:fisherfaces}) perform under heavy illumination changes. \cite{Lee2005} used the same setup to take 16128 images of 28 people. The Extended Yale Facedatabase B is the merge of the two databases, which is now known as Extended Yalefacedatabase B. 24 | 25 | \end{description} 26 | 27 | The face images need to be stored in a folder hierachy similar to \lstinline|//.|. The \href{http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html}{AT\&T Facedatabase} for example comes in such a hierarchy, see Listing \ref{lst:att}. 28 | 29 | \begin{lstlisting}[caption={\label{lst:att}}, language=,] 30 | philipp@mango:~/facerec/data/at$ tree 31 | . 32 | |-- README 33 | |-- s1 34 | | |-- 1.pgm 35 | | |-- ... 36 | | |-- 10.pgm 37 | |-- s2 38 | | |-- 1.pgm 39 | | |-- ... 40 | | |-- 10.pgm 41 | ... 42 | |-- s40 43 | | |-- 1.pgm 44 | | |-- ... 45 | | |-- 10.pgm 46 | \end{lstlisting} 47 | 48 | \ifx\python\undefined 49 | \subsubsection{Reading the images with GNU Octave/MATLAB} 50 | \else 51 | \subsubsection{Reading the images with Python} 52 | \fi 53 | 54 | 55 | \ifx\python\undefined 56 | First we'll need a function to list the files for a given path. This can be done with a function similar to Listing \ref{lst:listfiles}. 57 | 58 | \lstinputlisting[caption={\href{src/m/list\_files.m}{src/m/list\_files.m} \label{lst:listfiles}}, language=matlab]{src/m/list_files.m} 59 | \fi 60 | 61 | The function in Listing \ref{lst:readimages} can be used to read in the images for each subfolder of a given directory. Each directory is given a unique (integer) label, you probably want to store the folder name as well. The function returns the images\ifx\python\undefined{} as a data matrix\fi{} and the corresponding classes\ifx\python\undefined, the width and height of the images (we'll need this in later code)\fi{}. This function is really basic and there's much to enhance, but it does its job. 62 | 63 | \ifx\python\undefined 64 | \lstinputlisting[caption={\href{src/m/read\_images.m}{src/m/read\_images.m} \label{lst:readimages}}, language=matlab]{src/m/read_images.m} 65 | \else 66 | \lstinputlisting[caption={\href{src/py/tinyfacerec/util.py}{src/py/tinyfacerec/util.py} \label{lst:readimages}}, language=python, linerange={18-39}]{src/py/tinyfacerec/util.py} 67 | \fi 68 | 69 | 70 | \subsection{Eigenfaces} 71 | 72 | \lstset{language=matlab} 73 | 74 | The problem with the image representation we are given is its high dimensionality. Two-dimensional $p \times q$ grayscale images span a $m = pq$-dimensional vector space, so an image with $100 \times 100$ pixels lies in a $10,000$-dimensional image space already. That's way too much for any computations, but are all dimensions really useful for us? We can only make a decision if there's any variance in data, so what we are looking for are the components that account for most of the information. The Principal Component Analysis (PCA) was independently proposed by \href{http://en.wikipedia.org/wiki/Karl_Pearson}{Karl Pearson} (1901) and \href{http://en.wikipedia.org/wiki/Harold_Hotelling}{Harold Hotelling} (1933) to turn a set of possibly correlated variables into a smaller set of uncorrelated variables. The idea is that a high-dimensional dataset is often described by correlated variables and therefore only a few meaningful dimensions account for most of the information. The PCA method finds the directions with the greatest variance in the data, called principal components. 75 | 76 | \subsubsection{Algorithmic Description} 77 | 78 | \label{ssection:pca_algorithm} 79 | 80 | Let $X = \{ x_{1}, x_{2}, \ldots, x_{n} \}$ be a random vector with observations $x_i \in \mathbb{R}^{d}$. 81 | 82 | \begin{enumerate} 83 | \item Compute the mean $\mu$ 84 | \begin{equation} \label{eqn:pca_mean} 85 | \mu = \frac{1}{n} \sum_{i=1}^{n} x_{i} 86 | \end{equation} 87 | \item Compute the the Covariance Matrix $S$ 88 | \begin{equation} \label{eqn:pca_cov} 89 | S = \frac{1}{n} \sum_{i=1}^{n} (x_{i} - \mu) (x_{i} - \mu)^{T} 90 | \end{equation} 91 | \item Compute the eigenvalues $\lambda_{i}$ and eigenvectors $v_{i}$ of $S$ 92 | \begin{equation} \label{eqn:pca_eigenvalues} 93 | S v_{i} = \lambda_{i} v_{i}, i=1,2,\ldots,n 94 | \end{equation} 95 | \item Order the eigenvectors descending by their eigenvalue. The $k$ principal components are the eigenvectors corresponding to the $k$ largest eigenvalues. 96 | \end{enumerate} 97 | 98 | The $k$ principal components of the observed vector $x$ are then given by: 99 | 100 | \begin{equation} \label{eqn:pca_projection} 101 | y = W^{T} (x - \mu) 102 | \end{equation} 103 | 104 | where $W = (v_{1}, v_{2}, \ldots, v_{k})$. The reconstruction from the PCA basis is given by: 105 | 106 | \begin{equation} \label{eqn:pca_reconstruction} 107 | x = W y + \mu 108 | \end{equation} 109 | 110 | The Eigenfaces method then performs face recognition by: 111 | 112 | \begin{enumerate} 113 | \item Projecting all training samples into the PCA subspace (using Equation \ref{eqn:pca_projection}). 114 | \item Projecting the query image into the PCA subspace (using Listing \ref{eqn:pca_reconstruction}). 115 | \item Finding the nearest neighbor between the projected training images and the projected query image. 116 | \end{enumerate} 117 | 118 | Still there's one problem left to solve. Imagine we are given $400$ images sized $100 \times 100$ pixel. The Principal Component Analysis solves the covariance matrix $S = X X^{T}$, where ${size}(X) = 10000 \times 400$ in our example. You would end up with a $10000 \times 10000$ matrix, roughly $0.8 GB$. Solving this problem isn't feasible, so we'll need to apply a trick. From your linear algebra lessons you know that a $M \times N$ matrix with $M > N$ can only have $N - 1$ non-zero eigenvalues. So it's possible to take the eigenvalue decomposition $S = X^{T} X$ of size $N x N$ instead: 119 | 120 | \begin{equation} 121 | X^{T} X v_{i} = \lambda_{i} v{i} 122 | \end{equation} 123 | 124 | and get the original eigenvectors of $S = X X^{T}$ with a left multiplication of the data matrix: 125 | 126 | \begin{equation} 127 | X X^{T} (X v_{i}) = \lambda_{i} (X v_{i}) 128 | \end{equation} 129 | 130 | The resulting eigenvectors are orthogonal, to get orthonormal eigenvectors they need to be normalized to unit length. I don't want to turn this into a publication, so please look into \cite{Duda2001} for the derivation and proof of the equations. 131 | 132 | \ifx\python\undefined 133 | \subsubsection{Eigenfaces in GNU Octave/MATLAB} 134 | \else 135 | \subsubsection{Eigenfaces in Python} 136 | \fi 137 | 138 | \ifx\python\undefined 139 | \else 140 | We've already seen, that the Eigenfaces and Fisherfaces method expect a data matrix with observations by row (or column if you prefer it). Listing \ref{lst:matrices} defines two functions to reshape a list of multi-dimensional data into a data matrix. Note, that all samples are assumed to be of equal size. 141 | 142 | \lstinputlisting[caption={\href{src/py/tinyfacerec/util.py}{src/py/tinyfacerec/util.py} \label{lst:matrices}}, language=python, linerange={41-55}]{src/py/tinyfacerec/util.py} 143 | \fi 144 | 145 | Translating the PCA from the algorithmic description of section \ref{ssection:pca_algorithm} to \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://www.python.org}{Python} \fi is almost trivial. Don't copy and paste from this document, the source code is available in folder \ifx\python\undefined \lstinline|src/m|\else \lstinline|src/py/tinyfacerec|\fi. Listing \ref{lst:pca} implements the Principal Component Analysis given by Equation \ref{eqn:pca_mean}, \ref{eqn:pca_cov} and \ref{eqn:pca_eigenvalues}. It also implements the inner-product PCA formulation, which occurs if there are more dimensions than samples. You can shorten this code, I just wanted to point out how it works. 146 | 147 | \ifx\python\undefined 148 | \lstinputlisting[caption={\href{src/m/pca.m}{src/m/pca.m} \label{lst:pca}}, language=matlab]{src/m/pca.m} 149 | \else 150 | \lstinputlisting[caption={\href{src/py/tinyfacerec/subspace.py}{src/py/tinyfacerec/subspace.py} \label{lst:pca}}, language=python, linerange={13-37}]{src/py/tinyfacerec/subspace.py} 151 | \fi 152 | 153 | The observations are given by row, so the projection in Equation \ref{eqn:pca_projection} needs to be rearranged a little: 154 | 155 | \ifx\python\undefined 156 | \lstinputlisting[caption={\href{src/m/project.m}{src/m/project.m} \label{lst:project}}, language=matlab]{src/m/project.m} 157 | \else 158 | \lstinputlisting[caption={\href{src/py/tinyfacerec/subspace.py}{src/py/tinyfacerec/subspace.py} \label{lst:project}}, language=python, linerange={3-6}]{src/py/tinyfacerec/subspace.py} 159 | \fi 160 | 161 | The same applies to the reconstruction in Equation \ref{eqn:pca_reconstruction}: 162 | 163 | \ifx\python\undefined 164 | \lstinputlisting[caption={\href{src/m/reconstruct.m}{src/m/reconstruct.m} \label{lst:reconstruct}}, language=matlab]{src/m/reconstruct.m} 165 | \else 166 | \lstinputlisting[caption={\href{src/py/tinyfacerec/subspace.py}{src/py/tinyfacerec/subspace.py} \label{lst:reconstruct}}, language=python, linerange={8-11}]{src/py/tinyfacerec/subspace.py} 167 | \fi 168 | 169 | Now that everything is defined it's time for the fun stuff. The face images are read with Listing \ref{lst:readimages} and then a full PCA (see Listing \ref{lst:pca}) is performed. \ifx\python\undefined \else I'll use the great \href{http://matplotlib.sourceforge.net/}{matplotlib} library for plotting in \href{http://www.python.org}{Python}, please install it if you haven't done already.\fi 170 | 171 | \ifx\python\undefined 172 | \lstinputlisting[caption={\href{src/m/example\_eigenfaces.m}{src/m/example\_eigenfaces.m} \label{lst:example_pca1}}, language=matlab, linerange={1-9}]{src/m/example_eigenfaces.m} 173 | \else 174 | \lstinputlisting[caption={\href{src/py/scripts/example\_pca.py}{src/py/scripts/example\_pca.py} \label{lst:example_pca1}}, language=python, linerange={1-14}]{src/py/scripts/example_pca.py} 175 | \fi 176 | 177 | That's it already. Pretty easy, no? Each principal component has the same length as the original image, thus it can be displayed as an image. \cite{PT91} referred to these ghostly looking faces as \textit{Eigenfaces}, that's where the Eigenfaces method got its name from. We'll now want to look at the Eigenfaces, but first of all we need a method to turn the data into a representation \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://matplotlib.sourceforge.net/}{matplotlib} \fi understands. The eigenvectors we have calculated can contain negative values, but the image data is excepted as unsigned integer values in the range of $0$ to $255$. So we need a function to normalize the data first (Listing \ref{lst:normalize}): 178 | 179 | \ifx\python\undefined 180 | \lstinputlisting[caption={\href{src/m/normalize.m}{src/m/normalize.m} \label{lst:normalize}}, language=matlab]{src/m/normalize.m} 181 | 182 | Listing \ref{lst:toGrayscale} then turns the image into the expected representation, by first normalizing the values to a range between $[0,255]$ and then casting the matrix to unsigned integer values. 183 | 184 | \lstinputlisting[caption={\href{src/m/toGrayscale.m}{src/m/toGrayscale.m} \label{lst:toGrayscale}}, language=matlab]{src/m/toGrayscale.m} 185 | 186 | \else 187 | \lstinputlisting[caption={\href{src/py/tinyfacerec/util.py}{src/py/tinyfacerec/util.py} \label{lst:normalize}}, language=python, linerange={5-16}]{src/py/tinyfacerec/util.py} 188 | \fi 189 | 190 | \ifx\python\undefined 191 | 192 | Listing \ref{lst:example_pca2} then does a subplot for the first (at most) 16 Eigenfaces. 193 | 194 | \else 195 | 196 | In Python we'll then define a \lstinline|subplot| method (see \url{src/py/tinyfacerec/visual.py}) to simplify the plotting. The method takes a list of images, a title, color scale and finally generates a subplot: 197 | 198 | \lstinputlisting[caption={\href{src/py/tinyfacerec/visual.py}{src/py/tinyfacerec/visual.py} \label{lst:subplot}}, language=python, linerange={1-24}]{src/py/tinyfacerec/visual.py} 199 | 200 | This simplified the \href{http://www.python.org}{Python} script in Listing \ref{lst:example_pca2} to: 201 | \fi 202 | 203 | \ifx\python\undefined 204 | \lstinputlisting[caption={\href{src/m/example\_eigenfaces.m}{src/m/example\_eigenfaces.m} \label{lst:example_pca2}}, language=matlab, linerange={10-19}]{src/m/example_eigenfaces.m} 205 | \else 206 | \lstinputlisting[caption={\href{src/py/scripts/example\_pca.py}{src/py/scripts/example\_pca.py} \label{lst:example_pca2}}, language=python, linerange={16-25}]{src/py/scripts/example_pca.py} 207 | \fi 208 | 209 | I've used the jet colormap, so you can see how the grayscale values are distributed within the specific Eigenfaces. You can see, that the Eigenfaces do not only encode facial features, but also the illumination in the images (see the left light in Eigenface \#4, right light in Eigenfaces \#5): 210 | 211 | \ifx\python\undefined 212 | \begin{center} 213 | \includegraphics[scale=0.6]{img/eigenfaces/octave_pca_eigenfaces} 214 | \end{center} 215 | \else 216 | \begin{center} 217 | \includegraphics[scale=0.6]{img/eigenfaces/python_pca_eigenfaces} 218 | \end{center} 219 | \fi 220 | 221 | We've already seen in Equation \ref{eqn:pca_reconstruction}, that we can reconstruct a face from its lower dimensional approximation. So let's see how many Eigenfaces are needed for a good reconstruction. I'll do a subplot with $10,30,\ldots,310$ Eigenfaces: 222 | 223 | \ifx\python\undefined 224 | \lstinputlisting[caption={\href{src/m/example\_eigenfaces.m}{src/m/example\_eigenfaces.m} \label{lst:example_pca3}}, language=matlab, linerange={23-36}]{src/m/example_eigenfaces.m} 225 | \else 226 | \lstinputlisting[caption={\href{src/py/scripts/example\_pca.py}{src/py/scripts/example\_pca.py} \label{lst:example_pca3}}, language=python, linerange={27-40}]{src/py/scripts/example_pca.py} 227 | \fi 228 | 229 | 10 Eigenvectors are obviously not sufficient for a good image reconstruction, 50 Eigenvectors may already be sufficient to encode important facial features. You'll get a good reconstruction with approximately 300 Eigenvectors for the AT\&T Facedatabase. There are rule of thumbs how many Eigenfaces you should choose for a successful face recognition, but it heavily depends on the input data. \cite{zhao2003frl} is the perfect point to start researching for this. 230 | 231 | \ifx\python\undefined 232 | \begin{center} 233 | \includegraphics[scale=0.6]{img/eigenfaces/octave_pca_reconstruction} 234 | \end{center} 235 | \else 236 | \begin{center} 237 | \includegraphics[scale=0.6]{img/eigenfaces/python_pca_reconstruction} 238 | \end{center} 239 | \fi 240 | 241 | \ifx\python\undefined 242 | The k-Nearest Neighbor matching is left out for this example. Please see the GNU Octave/MATLAB code at \url{https://github.com/bytefish/facerec/tree/master/m} to see how it is implemented, it's all there. 243 | \else 244 | Now we have got everything to implement the Eigenfaces method. \href{http://www.python.org}{Python} is object oriented and so is our Eigenfaces model. Let's recap: The Eigenfaces method is basically a Pricipal Component Analysis with a Nearest Neighbor model. Some publications report about the influence of the distance metric (I can't support these claims with my research), so various distance metrics for the Nearest Neighbor should be supported. Listing \ref{lst:distance_base} defines an \lstinline|AbstractDistance| as the abstract base class for each distance metric. Every subclass overrides the call operator \lstinline|__call__| as shown for the Euclidean Distance and the Negated Cosine Distance. If you need more distance metrics, please have a look at the distance metrics implemented in \url{https://www.github.com/bytefish/facerec}. 245 | 246 | \lstinputlisting[caption={\href{src/py/tinyfacerec/distance.py}{src/py/tinyfacerec/distance.py} \label{lst:distance_base}}, language=python]{src/py/tinyfacerec/distance.py} 247 | 248 | The Eigenfaces and Fisherfaces method both share common methods, so we'll define a base prediction model in Listing \ref{lst:eigenfaces_base_model}. I don't want to do a full k-Nearest Neighbor implementation here, because (1) the number of neighbors doesn't really matter for both methods and (2) it would confuse people. If you are implementing it in a language of your choice, you should really separate the feature extraction and classification from the model itself. A real generic approach is given in my \href{https://www.github.com/bytefish/facerec}{facerec} framework. However, feel free to extend these basic classes for your needs. 249 | 250 | \lstinputlisting[caption={\href{src/py/tinyfacerec/model.py}{src/py/tinyfacerec/model.py} \label{lst:eigenfaces_base_model}}, language=python, linerange={1-28}]{src/py/tinyfacerec/model.py} 251 | 252 | Listing \ref{lst:eigenfaces_eigenfaces_model} then subclasses the \lstinline|EigenfacesModel| from the \lstinline|BaseModel|, so only the \lstinline|compute| method needs to be overriden with our specific feature extraction. The prediction is a 1-Nearest Neighbor search with a distance metric. 253 | 254 | \lstinputlisting[caption={\href{src/py/tinyfacerec/model.py}{src/py/tinyfacerec/model.py} \label{lst:eigenfaces_eigenfaces_model}}, language=python, linerange={30-41}]{src/py/tinyfacerec/model.py} 255 | 256 | Now that the \lstinline|EigenfacesModel| is defined, it can be used to learn the Eigenfaces and generate predictions. In the following Listing \ref{lst:example_model_eigenfaces} we'll load the Yale Facedatabase A and perform a prediction on the first image. 257 | 258 | \lstinputlisting[caption={\href{src/py/scripts/example\_model\_eigenfaces.py}{src/py/scripts/example\_model\_eigenfaces.py} \label{lst:example_model_eigenfaces}}, language=python]{src/py/scripts/example_model_eigenfaces.py} 259 | 260 | \fi 261 | 262 | \subsection{Fisherfaces} 263 | 264 | \label{ssection:fisherfaces} 265 | 266 | The Linear Discriminant Analysis was invented by the great statistician \href{http://en.wikipedia.org/wiki/Ronald_Fisher}{Sir R. A. Fisher}, who successfully used it for classifying flowers in his 1936 paper \textit{The use of multiple measurements in taxonomic problems} \cite{Fisher36}. But why do we need another dimensionality reduction method, if the Principal Component Analysis (PCA) did such a good job? 267 | 268 | The PCA finds a linear combination of features that maximizes the total variance in data. While this is clearly a powerful way to represuccsent data, it doesn't consider any classes and so a lot of discriminative information may be lost when throwing components away. Imagine a situation where the variance is generated by an external source, let it be the light. The components identified by a PCA do not necessarily contain any discriminative information at all, so the projected samples are smeared together and a classification becomes impossible. 269 | 270 | In order to find the combination of features that separates best between classes the Linear Discriminant Analysis maximizes the ratio of between-classes to within-classes scatter. The idea is simple: same classes should cluster tightly together, while different classes are as far away as possible from each other. This was also recognized by \href{http://www.cs.columbia.edu/~belhumeur/}{Belhumeur}, \href{http://www.ece.ucsb.edu/~hespanha/}{Hespanha} and \href{http://cseweb.ucsd.edu/~kriegman/}{Kriegman} and so they applied a Discriminant Analysis to face recognition in \cite{belhumeru97}. 271 | 272 | \begin{figure} 273 | \begin{center} 274 | \includegraphics[scale=1.70]{img/fisherfaces/multiclasslda} 275 | \captionof{figure}{This figure shows the scatter matrices $S_{B}$ and $S_{W}$ for a 3 class problem. $\mu$ represents the total mean and $[\mu_{1},\mu_{2},\mu_{3}]$ are the class means.} 276 | \label{fig:scatter_matrices} 277 | \end{center} 278 | \end{figure} 279 | 280 | \subsubsection{Algorithmic Description} 281 | 282 | Let $X$ be a random vector with samples drawn from $c$ classes: 283 | 284 | \begin{eqnarray} 285 | X & = & \{X_1,X_2,\ldots,X_c\} \\ 286 | X_i & = & \{x_1, x_2, \ldots, x_n\} 287 | \end{eqnarray} 288 | 289 | The scatter matrices $S_{B}$ and $S_{W}$ are calculated as: 290 | 291 | % between-classes scatter 292 | \begin{eqnarray} 293 | \label{eqn:scatter_matrices} 294 | S_{B} & = & \sum_{i=1}^{c} N_{i} (\mu_i - \mu)(\mu_i - \mu)^{T} \\ 295 | S_{W} & = & \sum_{i=1}^{c} \sum_{x_{j} \in X_{i}} (x_j - \mu_i)(x_j - \mu_i)^{T} 296 | \end{eqnarray} 297 | 298 | , where $\mu$ is the total mean: 299 | 300 | \begin{equation} 301 | \mu = \frac{1}{N} \sum_{i=1}^{N} x_i 302 | \end{equation} 303 | 304 | And $\mu_i$ is the mean of class $i \in \{1,\ldots,c\}$: 305 | 306 | % Class-Average 307 | \begin{equation} 308 | \mu_i = \frac{1}{|X_i|} \sum_{x_j \in X_i} x_j 309 | \end{equation} 310 | 311 | Fisher's classic algorithm now looks for a projection $W$, that maximizes the class separability criterion: 312 | 313 | \begin{equation} 314 | W_{opt} = \operatorname{arg\,max}_{W} \frac{|W^T S_B W|}{|W^T S_W W|} 315 | \end{equation} 316 | 317 | Following \cite{belhumeru97}, a solution for this optimization problem is given by solving the General Eigenvalue Problem: 318 | 319 | \begin{eqnarray} 320 | \label{eqn:general_eigenwert} 321 | S_{B} v_{i} & = & \lambda_{i} S_w v_{i} \nonumber \\ 322 | S_{W}^{-1} S_{B} v_{i} & = & \lambda_{i} v_{i} 323 | \end{eqnarray} 324 | 325 | There's one problem left to solve: The rank of $S_{W}$ is at most $(N-c)$, with $N$ samples and $c$ classes. In pattern recognition problems the number of samples $N$ is almost always smaller than the dimension of the input data (the number of pixels), so the scatter matrix $S_{W}$ becomes singular (see \cite{Raudys1991}). In \cite{belhumeru97} this was solved by performing a Principal Component Analysis on the data and projecting the samples into the $(N-c)$-dimensional space. A Linear Discriminant Analysis was then performed on the reduced data, because $S_{W}$ isn't singular anymore. 326 | 327 | The optimization problem can be rewritten as: 328 | 329 | \begin{eqnarray} 330 | W_{pca} & = & \operatorname{arg\,max}_{W} |W^T S_T W| \\ 331 | W_{fld} & = & \operatorname{arg\,max}_{W} \frac{|W^T W_{pca}^T S_{B} W_{pca} W|}{|W^T W_{pca}^T S_{W} W_{pca} W|} 332 | \end{eqnarray} 333 | 334 | The transformation matrix $W$, that projects a sample into the $(c-1)$-dimensional space is then given by: 335 | 336 | \begin{equation} \label{eqn:fisherfaces} 337 | W = W_{fld}^{T} W_{pca}^{T} 338 | \end{equation} 339 | 340 | One final note: Although $S_{W}$ and $S_{B}$ are symmetric matrices, the product of two symmetric matrices is not necessarily symmetric. so you have to use an eigenvalue solver for general matrices. OpenCV's \href{http://opencv.willowgarage.com/documentation/cpp/operations_on_arrays.html#cv-eigen}{cv::eigen} only works for symmetric matrices in its current version; since eigenvalues and singular values aren't equivalent for non-symmetric matrices you can't use a Singular Value Decomposition (SVD) either. 341 | 342 | \ifx\python\undefined 343 | \subsubsection{Fisherfaces in GNU Octave/MATLAB} 344 | \else 345 | \subsubsection{Fisherfaces in Python} 346 | \fi 347 | 348 | \label{ssection:example_fisherfaces} 349 | 350 | Translating the Linear Discriminant Analysis to \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://www.python.org}{Python} \fi is almost trivial again, see Listing \ref{lst:lda}. For projecting and reconstructing from the basis you can use the functions from Listing \ref{lst:project} and \ref{lst:reconstruct}. 351 | 352 | \ifx\python\undefined 353 | \lstinputlisting[caption={\href{src/m/lda.m}{src/m/lda.m} \label{lst:lda}}, language=matlab]{src/m/lda.m} 354 | \else 355 | \lstinputlisting[caption={\href{src/py/tinyfacerec/subspace.py}{src/py/tinyfacerec/subspace.py} \label{lst:lda}}, language=python, linerange={39-58}]{src/py/tinyfacerec/subspace.py} 356 | \fi 357 | 358 | The functions to perform a PCA (Listing \ref{lst:pca}) and LDA (Listing \ref{lst:lda}) are now defined, so we can go ahead and implement the Fisherfaces from Equation \ref{eqn:fisherfaces}. 359 | 360 | \ifx\python\undefined 361 | \lstinputlisting[caption={\href{src/m/fisherfaces.m}{src/m/fisherfaces.m} \label{lst:fisherfaces}}, language=matlab]{src/m/fisherfaces.m} 362 | \else 363 | \lstinputlisting[caption={\href{src/py/tinyfacerec/subspace.py}{src/py/tinyfacerec/subspace.py} \label{lst:fisherfaces}}, language=python, linerange={60-67}]{src/py/tinyfacerec/subspace.py} 364 | \fi 365 | 366 | For this example I am going to use the Yale Facedatabase A, just because the plots are nicer. Each Fisherface has the same length as an original image, thus it can be displayed as an image. We'll again load the data, learn the Fisherfaces and make a subplot of the first 16 Fisherfaces. 367 | 368 | \ifx\python\undefined 369 | \lstinputlisting[caption={\href{src/m/example\_fisherfaces.m}{src/m/example\_fisherfaces.m} \label{lst:example_fisherfaces_fisherfaces}}, language=matlab, linerange={1-23}]{src/m/example_fisherfaces.m} 370 | \else 371 | \lstinputlisting[caption={\href{src/py/scripts/example\_fisherfaces.py}{src/py/scripts/example\_fisherfaces.py} \label{lst:example_fisherfaces_fisherfaces}}, language=python, linerange={1-23}]{src/py/scripts/example_fisherfaces.py} 372 | \fi 373 | 374 | The Fisherfaces method learns a class-specific transformation matrix, so the they do not capture illumination as obviously as the Eigenfaces method. The Discriminant Analysis instead finds the facial features to discriminate between the persons. It's important to mention, that the performance of the Fisherfaces heavily depends on the input data as well. Practically said: if you learn the Fisherfaces for well-illuminated pictures only and you try to recognize faces in bad-illuminated scenes, then method is likely to find the wrong components (just because those features may not be predominant on bad illuminated images). This is somewhat logical, since the method had no chance to learn the illumination. 375 | 376 | \ifx\python\undefined 377 | \begin{center} 378 | \includegraphics[scale=0.6]{img/fisherfaces/octave_fisherfaces_fisherfaces} 379 | \end{center} 380 | \else 381 | \begin{center} 382 | \includegraphics[scale=0.6]{img/fisherfaces/python_fisherfaces_fisherfaces} 383 | \end{center} 384 | \fi 385 | 386 | The Fisherfaces allow a reconstruction of the projected image, just like the Eigenfaces did. But since we only identified the features to distinguish between subjects, you can't expect a nice approximation of the original image. We can rewrite Listing \ref{lst:example_pca3} for the Fisherfaces method into Listing \ref{lst:example_fisherfaces_reconstruction}, but this time we'll project the sample image onto each of the Fisherfaces instead. So you'll have a visualization, which features each Fisherface describes. 387 | 388 | \ifx\python\undefined 389 | \lstinputlisting[caption={\href{src/m/example\_fisherfaces.m}{src/m/example\_fisherfaces.m} \label{lst:example_fisherfaces_reconstruction}}, language=matlab, linerange={28-40}]{src/m/example_fisherfaces.m} 390 | \else 391 | \lstinputlisting[caption={\href{src/py/scripts/example\_fisherfaces.py}{src/py/scripts/example\_fisherfaces.py} \label{lst:example_fisherfaces_reconstruction}}, language=python, linerange={25-36}]{src/py/scripts/example_fisherfaces.py} 392 | \fi 393 | 394 | 395 | \ifx\python\undefined 396 | \begin{center} 397 | \includegraphics[scale=0.6]{img/fisherfaces/octave_fisherfaces_reconstruction} 398 | \end{center} 399 | \else 400 | \begin{center} 401 | \includegraphics[scale=0.6]{img/fisherfaces/python_fisherfaces_reconstruction} 402 | \end{center} 403 | \fi 404 | 405 | \ifx\python\undefined 406 | The k-Nearest Neighbor matching is left out for this example. Please see the GNU Octave/MATLAB code at \url{https://github.com/bytefish/facerec/tree/master/m} to see how it is implemented, it's all there. 407 | \else 408 | The implementation details are not repeated in this section. For the Fisherfaces method a similar model to the \lstinline|EigenfacesModel| in Listing \ref{lst:eigenfaces_eigenfaces_model} must be defined. 409 | 410 | \lstinputlisting[caption={\href{src/py/tinyfacerec/model.py}{src/py/tinyfacerec/model.py} \label{lst:eigenfaces_eigenfaces_model}}, language=python, linerange={43-54}]{src/py/tinyfacerec/model.py} 411 | 412 | Once the \lstinline|FisherfacesModel| is defined, it can be used to learn the Fisherfaces and generate predictions. In the following Listing \ref{lst:example_model_fisherfaces} we'll load the Yale Facedatabase A and perform a prediction on the first image. 413 | 414 | \lstinputlisting[caption={\href{src/py/scripts/example\_model\_fisherfaces.py}{src/py/scripts/example\_model\_fisherfaces.py} \label{lst:example_model_fisherfaces}}, language=python]{src/py/scripts/example_model_fisherfaces.py} 415 | 416 | \fi 417 | -------------------------------------------------------------------------------- /section/introduction.tex: -------------------------------------------------------------------------------- 1 | \section{Introduction} 2 | 3 | In this document I'll show you how to implement the Eigenfaces \cite{PT91} and Fisherfaces \cite{belhumeru97} method with \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://www.python.org}{Python}\fi{}, so you'll understand the basics of Face Recognition. All concepts are explained in detail, but a basic knowledge of \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://www.python.org}{Python}\fi{} is assumed. Originally this document was a Guide to Face Recognition with OpenCV. Since \href{http://www.opencv.org}{OpenCV} now comes with the \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/facerec_api.html}{cv::FaceRecognizer}, this document has been reworked into the official OpenCV documentation at: 4 | 5 | \begin{itemize} 6 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/index.html}{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/index.html} 7 | \end{itemize} 8 | 9 | I am doing all this in my spare time and I simply can't maintain two separate documents on the same topic any more. So I have decided to turn this document into a guide on Face Recognition with \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://www.python.org}{Python}\fi{} only. You'll find the very detailed documentation on the OpenCV \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/facerec_api.html}{cv::FaceRecognizer} at: 10 | 11 | \begin{itemize} 12 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/index.html}{FaceRecognizer - Face Recognition with OpenCV} 13 | \begin{itemize} 14 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/facerec_api.html}{FaceRecognizer API} 15 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/facerec_tutorial.html}{Guide to Face Recognition with OpenCV} 16 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/tutorial/facerec_gender_classification.html}{Tutorial on Gender Classification} 17 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/tutorial/facerec_video_recognition.html}{Tutorial on Face Recognition in Videos} 18 | \item \href{http://docs.opencv.org/trunk/modules/contrib/doc/facerec/tutorial/facerec_save_load.html}{Tutorial On Saving \& Loading a FaceRecognizer} 19 | \end{itemize} 20 | \end{itemize} 21 | 22 | By the way you don't need to copy and paste the code snippets, all code has been pushed into my github repository: 23 | 24 | \begin{itemize} 25 | \item \href{http://www.github.com/bytefish}{github.com/bytefish} 26 | \item \href{http://www.github.com/bytefish/facerecognition_guide}{github.com/bytefish/facerecognition\_guide} 27 | \end{itemize} 28 | 29 | Everything in here is released under a \href{http://www.opensource.org/licenses/bsd-license}{BSD license}, so feel free to use it for your projects. You are currently reading the \ifx\python\undefined \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB} \else \href{http://www.python.org}{Python}\fi{} version of the Face Recognition Guide, you can compile the \ifx\python\undefined \href{http://www.python.org}{Python} \else \href{http://www.gnu.org/software/octave/}{GNU Octave}/\href{http://www.mathworks.com}{MATLAB}\fi{} version with \ifx\python\undefined \lstinline|make python| \else \lstinline|make octave|\fi{}. 30 | -------------------------------------------------------------------------------- /src/m/example_eigenfaces.m: -------------------------------------------------------------------------------- 1 | % load function files from subfolders aswell 2 | addpath (genpath ('.')); 3 | % read images 4 | [X,y,w,h] = read_images('/home/philipp/facerec/data/at'); 5 | % n - number of samples 6 | % d - dimensionality 7 | [n,d] = size(X); 8 | % perform a full pca 9 | [W,mu] = pca(X,y,n); 10 | % plot eigenfaces 11 | figure; hold on; 12 | title('Eigenfaces (AT&T Facedatabase)'); 13 | for i=1:min(16,n) 14 | subplot(4,4,i); 15 | eigenface_i = toGrayscale(W(:,i), w, h); 16 | imshow(eigenface_i); 17 | colormap(jet(256)); 18 | title(sprintf('Eigenface #%i', i)); 19 | end 20 | % print the plot 21 | %set (gca,'fontsize',20) 22 | %print('pca_eigenfaces.eps','-depsc','-F:20') 23 | %plot eigenfaces reconstruction 24 | steps = 10:20:min(n,320); 25 | Q = X(1,:); % first image to reconstruct 26 | figure; hold on; 27 | title('Reconstruction (AT&T Facedatabase)'); 28 | for i=1:min(16, length(steps)) 29 | subplot(4,4,i); 30 | numEvs = steps(i); 31 | P = project(W(:,1:numEvs), X(1,:), mu); 32 | R = reconstruct(W(:,1:numEvs),P,mu); 33 | comp = toGrayscale(R, w, h); 34 | imshow(comp); 35 | title(sprintf('%i Eigenvectors', numEvs)); 36 | end 37 | % print the plot 38 | %set (gca,'fontsize',20) 39 | %print('pca_reconstruction.eps','-depsc','-F:20') 40 | 41 | pause; 42 | -------------------------------------------------------------------------------- /src/m/example_fisherfaces.m: -------------------------------------------------------------------------------- 1 | % load function files from subfolders aswell 2 | addpath (genpath ('.')); 3 | % for plotting 4 | db_name = 'Yale Facedatabase'; 5 | % read images 6 | [X,y,w,h] = read_images('/home/philipp/facerec/data/yalefaces_recognition'); 7 | % n - number of samples 8 | % d - dimensionality 9 | [n,d] = size(X); 10 | % get the unique classes 11 | c = unique(y); 12 | % compute the fisherfaces 13 | [W,mu] = fisherfaces(X,y); 14 | % plot fisherfaces 15 | figure; hold on; 16 | title(sprintf('Fisherfaces %s', db_name)); 17 | for i=1:min(16,length(c)-1) 18 | subplot(4,4,i); 19 | fisherface_i = toGrayscale(W(:,i), w, h); 20 | imshow(fisherface_i); 21 | colormap(jet(256)); 22 | title(sprintf('Fisherface #%i', i)); 23 | end 24 | % print the plot 25 | %set (gca,'fontsize',20) 26 | %print('fisherfaces_fisherfaces.eps','-depsc','-F:20') 27 | % plot fisherfaces reconstruction 28 | steps = 1:min(16,length(c)-1); 29 | Q = X(1,:); % first image to reconstruct 30 | figure; hold on; 31 | title(sprintf('Fisherfaces Reconstruction %s', db_name)); 32 | for i=1:min(16, length(steps)) 33 | subplot(4,4,i); 34 | numEv = steps(i); 35 | P = project(W(:,numEv), X(1,:), mu); 36 | R = reconstruct(W(:,numEv),P,mu); 37 | comp = toGrayscale(R, w, h); 38 | imshow(comp); 39 | title(sprintf('Fisherface #%i', numEv)); 40 | end 41 | % print the plot 42 | %set (gca,'fontsize',20) 43 | %print('fisherfaces_reconstruction.eps','-depsc','-F:20') 44 | 45 | pause; 46 | -------------------------------------------------------------------------------- /src/m/fisherfaces.m: -------------------------------------------------------------------------------- 1 | function [W, mu] = fisherfaces(X,y,k) 2 | % number of samples 3 | N = rows(X); 4 | % number of classes 5 | labels = unique(y); 6 | c = length(labels); 7 | if(nargin < 3) 8 | k = c-1; 9 | end 10 | k = min(k,(c-1)); 11 | % get (N-c) principal components 12 | [Wpca, mu] = pca(X, y, (N-c)); 13 | [Wlda, mu_lda] = lda(project(Wpca, X, mu), y, k); 14 | W = Wpca*Wlda; 15 | end 16 | -------------------------------------------------------------------------------- /src/m/lda.m: -------------------------------------------------------------------------------- 1 | function [W, mu] = lda(X,y,k) 2 | % dimension of observation 3 | [n,d] = size(X); 4 | % number of classes 5 | labels = unique(y); 6 | C = length(labels); 7 | % allocate scatter matrices 8 | Sw = zeros(d,d); 9 | Sb = zeros(d,d); 10 | % total mean 11 | mu = mean(X); 12 | % calculate scatter matrices 13 | for i = 1:C 14 | Xi = X(find(y == labels(i)),:); % samples for current class 15 | n = rows(Xi); 16 | mu_i = mean(Xi); % mean vector for current class 17 | Xi = Xi - repmat(mu_i, n, 1); 18 | Sw = Sw + Xi'*Xi; 19 | Sb = Sb + n * (mu_i - mu)'*(mu_i - mu); 20 | end 21 | % solve general eigenvalue problem 22 | [W, D] = eig(Sb, Sw); 23 | % sort eigenvectors 24 | [D, i] = sort(diag(D), 'descend'); 25 | W = W(:,i); 26 | % keep at most (c-1) eigenvectors 27 | W = W(:,1:k); 28 | end 29 | -------------------------------------------------------------------------------- /src/m/list_files.m: -------------------------------------------------------------------------------- 1 | function L = list_files(path_fn) 2 | % get information about given path_fn 3 | L = dir(path_fn); 4 | % ... ignore . and .. 5 | L = L(3:length(L)); 6 | % ... turn into a cell array 7 | L = struct2cell(L); 8 | % ... and only keep the filenames. 9 | L = L(1,:); 10 | end 11 | -------------------------------------------------------------------------------- /src/m/normalize.m: -------------------------------------------------------------------------------- 1 | function X = normalize(X, l, h) 2 | minX = min(X(:)); 3 | maxX = max(X(:)); 4 | %% Normalize to [0...1]. 5 | X = X - minX; 6 | X = X ./ (maxX - minX); 7 | %% Scale to [low...high]. 8 | X = X .* (h-l); 9 | X = X + l; 10 | end 11 | -------------------------------------------------------------------------------- /src/m/pca.m: -------------------------------------------------------------------------------- 1 | function [W, mu] = pca(X, y, k) 2 | [n,d] = size(X); 3 | mu = mean(X); 4 | Xm = X - repmat(mu, rows(X), 1); 5 | if(n>d) 6 | C = Xm'*Xm; 7 | [W,D] = eig(C); 8 | % sort eigenvalues and eigenvectors 9 | [D, i] = sort(diag(D), 'descend'); 10 | W = W(:,i); 11 | % keep k components 12 | W = W(:,1:k); 13 | else 14 | C = Xm*Xm'; 15 | %C = cov(Xm'); 16 | [W,D] = eig(C); 17 | % multiply with data matrix 18 | W = Xm'*W; 19 | % normalize eigenvectors 20 | for i=1:n 21 | W(:,i) = W(:,i)/norm(W(:,i)); 22 | end 23 | % sort eigenvalues and eigenvectors 24 | [D, i] = sort(diag(D), 'descend'); 25 | W = W(:,i); 26 | % keep k components 27 | W = W(:,1:k); 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /src/m/project.m: -------------------------------------------------------------------------------- 1 | function Y = project(W, X, mu) 2 | if(nargin<3) 3 | Y = X*W; 4 | else 5 | Y = (X-repmat(mu, rows(X), 1))*W; 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /src/m/read_images.m: -------------------------------------------------------------------------------- 1 | function [X y width height] = read_images(path_fn) 2 | % get files for a given path 3 | folder = list_files(path_fn); 4 | % initialize the empty return values 5 | X=[]; 6 | y=[]; 7 | width=0; 8 | height=0; 9 | % start counting with class index 1 10 | classIdx = 1; 11 | % for each file... 12 | for i=1:length(folder) 13 | subject = folder{i}; 14 | % ... get files in this subdir 15 | images = list_files([path_fn, filesep, subject]); 16 | % ... ignore a file or empty folder 17 | if(length(images) == 0) 18 | continue; 19 | end 20 | % ... for each image 21 | for j=1:length(images) 22 | % ... get the absolute path 23 | filename = [path_fn, filesep, subject, filesep, images{j}]; 24 | % ... read the image 25 | T = double(imread(filename)); 26 | % ... get the image information 27 | [height width channels] = size(T); 28 | % ... and grayscale if it's a color image 29 | if(channels == 3) 30 | T = 0.2989 * T(:,:,1) + 0.5870* T(:,:,2) + 0.1140 * T(:,:,3); 31 | end 32 | % ... reshape into a row vector and append to data matrix 33 | X = [X; reshape(T,1,width*height)]; 34 | % ... append the corresponding class to the class vector 35 | y = [y, classIdx]; 36 | end 37 | % ... increase the class index 38 | classIdx = classIdx + 1; 39 | end % ... for-each folder. 40 | end 41 | -------------------------------------------------------------------------------- /src/m/reconstruct.m: -------------------------------------------------------------------------------- 1 | function X = reconstruct(W, Y, mu) 2 | if(nargin<3) 3 | X = Y * W'; 4 | else 5 | X = Y * W' + repmat(mu, rows(Y), 1); 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /src/m/toGrayscale.m: -------------------------------------------------------------------------------- 1 | function Y = toGrayscale(X, width, height) 2 | Y = normalize(X, 0, 255); 3 | if(nargin==3) 4 | Y = reshape(Y, height, width); 5 | end 6 | Y = uint8(Y); 7 | end 8 | -------------------------------------------------------------------------------- /src/py/crop_face.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Software License Agreement (BSD License) 3 | # 4 | # Copyright (c) 2012, Philipp Wagner 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions 9 | # are met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of its 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | # POSSIBILITY OF SUCH DAMAGE. 33 | 34 | import sys, math, Image 35 | 36 | def Distance(p1,p2): 37 | dx = p2[0] - p1[0] 38 | dy = p2[1] - p1[1] 39 | return math.sqrt(dx*dx+dy*dy) 40 | 41 | def ScaleRotateTranslate(image, angle, center = None, new_center = None, scale = None, resample=Image.BICUBIC): 42 | if (scale is None) and (center is None): 43 | return image.rotate(angle=angle, resample=resample) 44 | nx,ny = x,y = center 45 | sx=sy=1.0 46 | if new_center: 47 | (nx,ny) = new_center 48 | if scale: 49 | (sx,sy) = (scale, scale) 50 | cosine = math.cos(angle) 51 | sine = math.sin(angle) 52 | a = cosine/sx 53 | b = sine/sx 54 | c = x-nx*a-ny*b 55 | d = -sine/sy 56 | e = cosine/sy 57 | f = y-nx*d-ny*e 58 | return image.transform(image.size, Image.AFFINE, (a,b,c,d,e,f), resample=resample) 59 | 60 | def CropFace(image, eye_left=(0,0), eye_right=(0,0), offset_pct=(0.2,0.2), dest_sz = (70,70)): 61 | # calculate offsets in original image 62 | offset_h = math.floor(float(offset_pct[0])*dest_sz[0]) 63 | offset_v = math.floor(float(offset_pct[1])*dest_sz[1]) 64 | # get the direction 65 | eye_direction = (eye_right[0] - eye_left[0], eye_right[1] - eye_left[1]) 66 | # calc rotation angle in radians 67 | rotation = -math.atan2(float(eye_direction[1]),float(eye_direction[0])) 68 | # distance between them 69 | dist = Distance(eye_left, eye_right) 70 | # calculate the reference eye-width 71 | reference = dest_sz[0] - 2.0*offset_h 72 | # scale factor 73 | scale = float(dist)/float(reference) 74 | # rotate original around the left eye 75 | image = ScaleRotateTranslate(image, center=eye_left, angle=rotation) 76 | # crop the rotated image 77 | crop_xy = (eye_left[0] - scale*offset_h, eye_left[1] - scale*offset_v) 78 | crop_size = (dest_sz[0]*scale, dest_sz[1]*scale) 79 | image = image.crop((int(crop_xy[0]), int(crop_xy[1]), int(crop_xy[0]+crop_size[0]), int(crop_xy[1]+crop_size[1]))) 80 | # resize it 81 | image = image.resize(dest_sz, Image.ANTIALIAS) 82 | return image 83 | 84 | if __name__ == "__main__": 85 | image = Image.open("arnie.jpg") 86 | CropFace(image, eye_left=(252,364), eye_right=(420,366), offset_pct=(0.1,0.1), dest_sz=(200,200)).save("arnie_10_10_200_200.jpg") 87 | CropFace(image, eye_left=(252,364), eye_right=(420,366), offset_pct=(0.2,0.2), dest_sz=(200,200)).save("arnie_20_20_200_200.jpg") 88 | CropFace(image, eye_left=(252,364), eye_right=(420,366), offset_pct=(0.3,0.3), dest_sz=(200,200)).save("arnie_30_30_200_200.jpg") 89 | CropFace(image, eye_left=(252,364), eye_right=(420,366), offset_pct=(0.2,0.2)).save("arnie_20_20_70_70.jpg") 90 | -------------------------------------------------------------------------------- /src/py/scripts/example_eigenfaces.py: -------------------------------------------------------------------------------- 1 | import sys 2 | # append tinyfacerec to module search path 3 | sys.path.append("..") 4 | # import numpy and matplotlib colormaps 5 | import numpy as np 6 | # import tinyfacerec modules 7 | from tinyfacerec.subspace import pca 8 | from tinyfacerec.util import normalize, asRowMatrix, read_images 9 | from tinyfacerec.visual import subplot 10 | 11 | if __name__ == '__main__': 12 | 13 | if len(sys.argv) != 2: 14 | print "USAGE: example_eigenfaces.py " 15 | sys.exit() 16 | 17 | # read images 18 | [X,y] = read_images(sys.argv[1]) 19 | 20 | # perform a full pca 21 | [D, W, mu] = pca(asRowMatrix(X), y) 22 | 23 | import matplotlib.cm as cm 24 | 25 | # turn the first (at most) 16 eigenvectors into grayscale 26 | # images (note: eigenvectors are stored by column!) 27 | E = [] 28 | for i in xrange(min(len(X), 16)): 29 | e = W[:,i].reshape(X[0].shape) 30 | E.append(normalize(e,0,255)) 31 | # plot them and store the plot to "python_eigenfaces.pdf" 32 | subplot(title="Eigenfaces AT&T Facedatabase", images=E, rows=4, cols=4, sptitle="Eigenface", colormap=cm.jet, filename="python_pca_eigenfaces.png") 33 | 34 | from tinyfacerec.subspace import project, reconstruct 35 | 36 | # reconstruction steps 37 | steps=[i for i in xrange(10, min(len(X), 320), 20)] 38 | E = [] 39 | for i in xrange(min(len(steps), 16)): 40 | numEvs = steps[i] 41 | P = project(W[:,0:numEvs], X[0].reshape(1,-1), mu) 42 | R = reconstruct(W[:,0:numEvs], P, mu) 43 | # reshape and append to plots 44 | R = R.reshape(X[0].shape) 45 | E.append(normalize(R,0,255)) 46 | # plot them and store the plot to "python_reconstruction.pdf" 47 | subplot(title="Reconstruction AT&T Facedatabase", images=E, rows=4, cols=4, sptitle="Eigenvectors", sptitles=steps, colormap=cm.gray, filename="python_pca_reconstruction.png") 48 | -------------------------------------------------------------------------------- /src/py/scripts/example_fisherfaces.py: -------------------------------------------------------------------------------- 1 | import sys 2 | # append tinyfacerec to module search path 3 | sys.path.append("..") 4 | # import numpy and matplotlib colormaps 5 | import numpy as np 6 | # import tinyfacerec modules 7 | from tinyfacerec.subspace import fisherfaces 8 | from tinyfacerec.util import normalize, asRowMatrix, read_images 9 | from tinyfacerec.visual import subplot 10 | 11 | if __name__ == '__main__': 12 | 13 | if len(sys.argv) != 2: 14 | print "USAGE: example_fisherfaces.py " 15 | sys.exit() 16 | 17 | # read images 18 | [X,y] = read_images(sys.argv[1]) 19 | # perform a full pca 20 | [D, W, mu] = fisherfaces(asRowMatrix(X), y) 21 | #import colormaps 22 | import matplotlib.cm as cm 23 | # turn the first (at most) 16 eigenvectors into grayscale 24 | # images (note: eigenvectors are stored by column!) 25 | E = [] 26 | for i in xrange(min(W.shape[1], 16)): 27 | e = W[:,i].reshape(X[0].shape) 28 | E.append(normalize(e,0,255)) 29 | # plot them and store the plot to "python_fisherfaces_fisherfaces.pdf" 30 | subplot(title="Fisherfaces AT&T Facedatabase", images=E, rows=4, cols=4, sptitle="Fisherface", colormap=cm.jet, filename="python_fisherfaces_fisherfaces.png") 31 | 32 | from tinyfacerec.subspace import project, reconstruct 33 | 34 | E = [] 35 | for i in xrange(min(W.shape[1], 16)): 36 | e = W[:,i].reshape(-1,1) 37 | P = project(e, X[0].reshape(1,-1), mu) 38 | R = reconstruct(e, P, mu) 39 | # reshape and append to plots 40 | R = R.reshape(X[0].shape) 41 | E.append(normalize(R,0,255)) 42 | # plot them and store the plot to "python_reconstruction.pdf" 43 | subplot(title="Fisherfaces Reconstruction Yale FDB", images=E, rows=4, cols=4, sptitle="Fisherface", colormap=cm.gray, filename="python_fisherfaces_reconstruction.png") 44 | -------------------------------------------------------------------------------- /src/py/scripts/example_model_eigenfaces.py: -------------------------------------------------------------------------------- 1 | import sys 2 | # append tinyfacerec to module search path 3 | sys.path.append("..") 4 | # import numpy and matplotlib colormaps 5 | import numpy as np 6 | # import tinyfacerec modules 7 | from tinyfacerec.util import read_images 8 | from tinyfacerec.model import EigenfacesModel 9 | 10 | if __name__ == '__main__': 11 | 12 | if len(sys.argv) != 2: 13 | print "USAGE: example_model_eigenfaces.py " 14 | sys.exit() 15 | 16 | # read images 17 | [X,y] = read_images(sys.argv[1]) 18 | # compute the eigenfaces model 19 | model = EigenfacesModel(X[1:], y[1:]) 20 | # get a prediction for the first observation 21 | print "expected =", y[0], "/", "predicted =", model.predict(X[0]) 22 | -------------------------------------------------------------------------------- /src/py/scripts/example_model_fisherfaces.py: -------------------------------------------------------------------------------- 1 | import sys 2 | # append tinyfacerec to module search path 3 | sys.path.append("..") 4 | # import numpy and matplotlib colormaps 5 | import numpy as np 6 | # import tinyfacerec modules 7 | from tinyfacerec.util import read_images 8 | from tinyfacerec.model import FisherfacesModel 9 | 10 | if __name__ == '__main__': 11 | 12 | if len(sys.argv) != 2: 13 | print "USAGE: example_model_fisherfaces.py " 14 | sys.exit() 15 | 16 | # read images 17 | [X,y] = read_images(sys.argv[1]) 18 | # compute the eigenfaces model 19 | model = FisherfacesModel(X[1:], y[1:]) 20 | # get a prediction for the first observation 21 | print "expected =", y[0], "/", "predicted =", model.predict(X[0]) 22 | -------------------------------------------------------------------------------- /src/py/scripts/example_pca.py: -------------------------------------------------------------------------------- 1 | import sys 2 | # append tinyfacerec to module search path 3 | sys.path.append("..") 4 | # import numpy and matplotlib colormaps 5 | import numpy as np 6 | # import tinyfacerec modules 7 | from tinyfacerec.subspace import pca 8 | from tinyfacerec.util import normalize, asRowMatrix, read_images 9 | from tinyfacerec.visual import subplot 10 | 11 | # read images 12 | [X,y] = read_images("/home/philipp/facerec/data/at") 13 | # perform a full pca 14 | [D, W, mu] = pca(asRowMatrix(X), y) 15 | 16 | import matplotlib.cm as cm 17 | 18 | # turn the first (at most) 16 eigenvectors into grayscale 19 | # images (note: eigenvectors are stored by column!) 20 | E = [] 21 | for i in xrange(min(len(X), 16)): 22 | e = W[:,i].reshape(X[0].shape) 23 | E.append(normalize(e,0,255)) 24 | # plot them and store the plot to "python_eigenfaces.pdf" 25 | subplot(title="Eigenfaces AT&T Facedatabase", images=E, rows=4, cols=4, sptitle="Eigenface", colormap=cm.jet, filename="python_pca_eigenfaces.pdf") 26 | 27 | from tinyfacerec.subspace import project, reconstruct 28 | 29 | # reconstruction steps 30 | steps=[i for i in xrange(10, min(len(X), 320), 20)] 31 | E = [] 32 | for i in xrange(min(len(steps), 16)): 33 | numEvs = steps[i] 34 | P = project(W[:,0:numEvs], X[0].reshape(1,-1), mu) 35 | R = reconstruct(W[:,0:numEvs], P, mu) 36 | # reshape and append to plots 37 | R = R.reshape(X[0].shape) 38 | E.append(normalize(R,0,255)) 39 | # plot them and store the plot to "python_reconstruction.pdf" 40 | subplot(title="Reconstruction AT&T Facedatabase", images=E, rows=4, cols=4, sptitle="Eigenvectors", sptitles=steps, colormap=cm.gray, filename="python_pca_reconstruction.pdf") 41 | -------------------------------------------------------------------------------- /src/py/tinyfacerec/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytefish/facerecognition_guide/9defdbbf4d269e3ad7e67c4e8ee0facf75b987b6/src/py/tinyfacerec/__init__.py -------------------------------------------------------------------------------- /src/py/tinyfacerec/distance.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class AbstractDistance(object): 4 | def __init__(self, name): 5 | self._name = name 6 | 7 | def __call__(self,p,q): 8 | raise NotImplementedError("Every AbstractDistance must implement the __call__ method.") 9 | 10 | @property 11 | def name(self): 12 | return self._name 13 | 14 | def __repr__(self): 15 | return self._name 16 | 17 | class EuclideanDistance(AbstractDistance): 18 | 19 | def __init__(self): 20 | AbstractDistance.__init__(self,"EuclideanDistance") 21 | 22 | def __call__(self, p, q): 23 | p = np.asarray(p).flatten() 24 | q = np.asarray(q).flatten() 25 | return np.sqrt(np.sum(np.power((p-q),2))) 26 | 27 | class CosineDistance(AbstractDistance): 28 | 29 | def __init__(self): 30 | AbstractDistance.__init__(self,"CosineDistance") 31 | 32 | def __call__(self, p, q): 33 | p = np.asarray(p).flatten() 34 | q = np.asarray(q).flatten() 35 | return -np.dot(p.T,q) / (np.sqrt(np.dot(p,p.T)*np.dot(q,q.T))) 36 | 37 | -------------------------------------------------------------------------------- /src/py/tinyfacerec/model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from util import asRowMatrix 3 | from subspace import pca, lda, fisherfaces, project 4 | from distance import EuclideanDistance 5 | 6 | class BaseModel(object): 7 | def __init__(self, X=None, y=None, dist_metric=EuclideanDistance(), num_components=0): 8 | self.dist_metric = dist_metric 9 | self.num_components = 0 10 | self.projections = [] 11 | self.W = [] 12 | self.mu = [] 13 | if (X is not None) and (y is not None): 14 | self.compute(X,y) 15 | 16 | def compute(self, X, y): 17 | raise NotImplementedError("Every BaseModel must implement the compute method.") 18 | 19 | def predict(self, X): 20 | minDist = np.finfo('float').max 21 | minClass = -1 22 | Q = project(self.W, X.reshape(1,-1), self.mu) 23 | for i in xrange(len(self.projections)): 24 | dist = self.dist_metric(self.projections[i], Q) 25 | if dist < minDist: 26 | minDist = dist 27 | minClass = self.y[i] 28 | return minClass 29 | 30 | class EigenfacesModel(BaseModel): 31 | 32 | def __init__(self, X=None, y=None, dist_metric=EuclideanDistance(), num_components=0): 33 | super(EigenfacesModel, self).__init__(X=X,y=y,dist_metric=dist_metric,num_components=num_components) 34 | 35 | def compute(self, X, y): 36 | [D, self.W, self.mu] = pca(asRowMatrix(X),y, self.num_components) 37 | # store labels 38 | self.y = y 39 | # store projections 40 | for xi in X: 41 | self.projections.append(project(self.W, xi.reshape(1,-1), self.mu)) 42 | 43 | class FisherfacesModel(BaseModel): 44 | 45 | def __init__(self, X=None, y=None, dist_metric=EuclideanDistance(), num_components=0): 46 | super(FisherfacesModel, self).__init__(X=X,y=y,dist_metric=dist_metric,num_components=num_components) 47 | 48 | def compute(self, X, y): 49 | [D, self.W, self.mu] = fisherfaces(asRowMatrix(X),y, self.num_components) 50 | # store labels 51 | self.y = y 52 | # store projections 53 | for xi in X: 54 | self.projections.append(project(self.W, xi.reshape(1,-1), self.mu)) 55 | -------------------------------------------------------------------------------- /src/py/tinyfacerec/subspace.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def project(W, X, mu=None): 4 | if mu is None: 5 | return np.dot(X,W) 6 | return np.dot(X - mu, W) 7 | 8 | def reconstruct(W, Y, mu=None): 9 | if mu is None: 10 | return np.dot(Y,W.T) 11 | return np.dot(Y, W.T) + mu 12 | 13 | def pca(X, y, num_components=0): 14 | [n,d] = X.shape 15 | if (num_components <= 0) or (num_components>n): 16 | num_components = n 17 | mu = X.mean(axis=0) 18 | X = X - mu 19 | if n>d: 20 | C = np.dot(X.T,X) 21 | [eigenvalues,eigenvectors] = np.linalg.eigh(C) 22 | else: 23 | C = np.dot(X,X.T) 24 | [eigenvalues,eigenvectors] = np.linalg.eigh(C) 25 | eigenvectors = np.dot(X.T,eigenvectors) 26 | for i in xrange(n): 27 | eigenvectors[:,i] = eigenvectors[:,i]/np.linalg.norm(eigenvectors[:,i]) 28 | # or simply perform an economy size decomposition 29 | # eigenvectors, eigenvalues, variance = np.linalg.svd(X.T, full_matrices=False) 30 | # sort eigenvectors descending by their eigenvalue 31 | idx = np.argsort(-eigenvalues) 32 | eigenvalues = eigenvalues[idx] 33 | eigenvectors = eigenvectors[:,idx] 34 | # select only num_components 35 | eigenvalues = eigenvalues[0:num_components].copy() 36 | eigenvectors = eigenvectors[:,0:num_components].copy() 37 | return [eigenvalues, eigenvectors, mu] 38 | 39 | def lda(X, y, num_components=0): 40 | y = np.asarray(y) 41 | [n,d] = X.shape 42 | c = np.unique(y) 43 | if (num_components <= 0) or (num_component>(len(c)-1)): 44 | num_components = (len(c)-1) 45 | meanTotal = X.mean(axis=0) 46 | Sw = np.zeros((d, d), dtype=np.float32) 47 | Sb = np.zeros((d, d), dtype=np.float32) 48 | for i in c: 49 | Xi = X[np.where(y==i)[0],:] 50 | meanClass = Xi.mean(axis=0) 51 | Sw = Sw + np.dot((Xi-meanClass).T, (Xi-meanClass)) 52 | Sb = Sb + n * np.dot((meanClass - meanTotal).T, (meanClass - meanTotal)) 53 | eigenvalues, eigenvectors = np.linalg.eig(np.linalg.inv(Sw)*Sb) 54 | idx = np.argsort(-eigenvalues.real) 55 | eigenvalues, eigenvectors = eigenvalues[idx], eigenvectors[:,idx] 56 | eigenvalues = np.array(eigenvalues[0:num_components].real, dtype=np.float32, copy=True) 57 | eigenvectors = np.array(eigenvectors[0:,0:num_components].real, dtype=np.float32, copy=True) 58 | return [eigenvalues, eigenvectors] 59 | 60 | def fisherfaces(X,y,num_components=0): 61 | y = np.asarray(y) 62 | [n,d] = X.shape 63 | c = len(np.unique(y)) 64 | [eigenvalues_pca, eigenvectors_pca, mu_pca] = pca(X, y, (n-c)) 65 | [eigenvalues_lda, eigenvectors_lda] = lda(project(eigenvectors_pca, X, mu_pca), y, num_components) 66 | eigenvectors = np.dot(eigenvectors_pca,eigenvectors_lda) 67 | return [eigenvalues_lda, eigenvectors, mu_pca] 68 | -------------------------------------------------------------------------------- /src/py/tinyfacerec/util.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import numpy as np 3 | import PIL.Image as Image 4 | 5 | def normalize(X, low, high, dtype=None): 6 | X = np.asarray(X) 7 | minX, maxX = np.min(X), np.max(X) 8 | # normalize to [0...1]. 9 | X = X - float(minX) 10 | X = X / float((maxX - minX)) 11 | # scale to [low...high]. 12 | X = X * (high-low) 13 | X = X + low 14 | if dtype is None: 15 | return np.asarray(X) 16 | return np.asarray(X, dtype=dtype) 17 | 18 | def read_images(path, sz=None): 19 | c = 0 20 | X,y = [], [] 21 | for dirname, dirnames, filenames in os.walk(path): 22 | for subdirname in dirnames: 23 | subject_path = os.path.join(dirname, subdirname) 24 | for filename in os.listdir(subject_path): 25 | try: 26 | im = Image.open(os.path.join(subject_path, filename)) 27 | im = im.convert("L") 28 | # resize to given size (if given) 29 | if (sz is not None): 30 | im = im.resize(sz, Image.ANTIALIAS) 31 | X.append(np.asarray(im, dtype=np.uint8)) 32 | y.append(c) 33 | except IOError: 34 | print "I/O error({0}): {1}".format(errno, strerror) 35 | except: 36 | print "Unexpected error:", sys.exc_info()[0] 37 | raise 38 | c = c+1 39 | return [X,y] 40 | 41 | def asRowMatrix(X): 42 | if len(X) == 0: 43 | return np.array([]) 44 | mat = np.empty((0, X[0].size), dtype=X[0].dtype) 45 | for row in X: 46 | mat = np.vstack((mat, np.asarray(row).reshape(1,-1))) 47 | return mat 48 | 49 | def asColumnMatrix(X): 50 | if len(X) == 0: 51 | return np.array([]) 52 | mat = np.empty((X[0].size, 0), dtype=X[0].dtype) 53 | for col in X: 54 | mat = np.hstack((mat, np.asarray(col).reshape(-1,1))) 55 | return mat 56 | -------------------------------------------------------------------------------- /src/py/tinyfacerec/visual.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import matplotlib.cm as cm 4 | 5 | def create_font(fontname='Tahoma', fontsize=10): 6 | return { 'fontname': fontname, 'fontsize':fontsize } 7 | 8 | def subplot(title, images, rows, cols, sptitle="subplot", sptitles=[], colormap=cm.gray, ticks_visible=True, filename=None): 9 | fig = plt.figure() 10 | # main title 11 | fig.text(.5, .95, title, horizontalalignment='center') 12 | for i in xrange(len(images)): 13 | ax0 = fig.add_subplot(rows,cols,(i+1)) 14 | plt.setp(ax0.get_xticklabels(), visible=False) 15 | plt.setp(ax0.get_yticklabels(), visible=False) 16 | if len(sptitles) == len(images): 17 | plt.title("%s #%s" % (sptitle, str(sptitles[i])), create_font('Tahoma',10)) 18 | else: 19 | plt.title("%s #%d" % (sptitle, (i+1)), create_font('Tahoma',10)) 20 | plt.imshow(np.asarray(images[i]), cmap=colormap) 21 | if filename is None: 22 | plt.show() 23 | else: 24 | fig.savefig(filename) 25 | 26 | def imsave(image, title="", filename=None): 27 | plt.figure() 28 | plt.imshow(np.asarray(image)) 29 | plt.title(title, create_font('Tahoma',10)) 30 | if filename is None: 31 | plt.show() 32 | else: 33 | fig.savefig(filename) 34 | --------------------------------------------------------------------------------