├── CMakeLists.txt ├── LICENSE ├── README.md ├── data ├── 300w_private.pb ├── 300w_private │ └── 300w_private_6.bin ├── 300w_public.pb ├── 300w_public │ └── 300w_public_6.bin ├── aflw.pb ├── aflw │ └── aflw_6.bin ├── cofw.pb ├── cofw │ └── cofw_6.bin ├── wflw.pb └── wflw │ └── wflw_6.bin ├── include ├── FaceAlignment3dde.hpp ├── GaussianChannelFeatures.hpp ├── HonariChannelFeatures.hpp └── ShapeCascade.hpp ├── src ├── FaceAlignment3dde.cpp ├── GaussianChannelFeatures.cpp └── HonariChannelFeatures.cpp └── test ├── face_alignment_bobetocalo_eccv18_test.cpp └── image_070.jpg /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Test face alignment algorithms 3 | #------------------------------------------------------------------------------ 4 | project(bobetocalo_eccv18) 5 | cmake_minimum_required(VERSION 3.2) 6 | 7 | include(${CMAKE_CURRENT_LIST_DIR}/../../CMakeLists.txt) 8 | include(${CMAKE_CURRENT_LIST_DIR}/../ert_simple/CMakeLists.txt) 9 | 10 | #-- Setup required libraries 11 | find_package(OpenMP) 12 | if(OPENMP_FOUND) 13 | message(STATUS FOUND_OPENMP) 14 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 15 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -std=c++11") 16 | else() 17 | message(STATUS NOT_FOUND_OPENMP) 18 | endif() 19 | 20 | #-- Find include files 21 | set(face_alignment_bobetocalo_eccv18_include 22 | ${faces_framework_include} 23 | ${face_alignment_ert_simple_include} 24 | ${CMAKE_CURRENT_LIST_DIR}/tensorflow/ 25 | ${CMAKE_CURRENT_LIST_DIR}/tensorflow/bazel-genfiles 26 | ${CMAKE_CURRENT_LIST_DIR}/tensorflow/tensorflow/contrib/makefile/downloads/eigen 27 | ${CMAKE_CURRENT_LIST_DIR}/tensorflow/tensorflow/contrib/makefile/downloads/nsync/public 28 | ${CMAKE_CURRENT_LIST_DIR}/include 29 | ) 30 | 31 | set(face_alignment_bobetocalo_eccv18_src 32 | ${faces_framework_src} 33 | ${face_alignment_ert_simple_src} 34 | ${CMAKE_CURRENT_LIST_DIR}/src/FaceAlignment3dde.cpp 35 | ${CMAKE_CURRENT_LIST_DIR}/src/HonariChannelFeatures.cpp 36 | ${CMAKE_CURRENT_LIST_DIR}/src/GaussianChannelFeatures.cpp 37 | ) 38 | 39 | set(face_alignment_bobetocalo_eccv18_test 40 | ${CMAKE_CURRENT_LIST_DIR}/test/face_alignment_bobetocalo_eccv18_test.cpp 41 | ) 42 | 43 | set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) 44 | set(CMAKE_INSTALL_RPATH ${CMAKE_CURRENT_LIST_DIR}/tensorflow/bazel-bin/tensorflow/) 45 | 46 | set(face_alignment_bobetocalo_eccv18_libs 47 | ${faces_framework_libs} 48 | ${face_alignment_ert_simple_libs} 49 | ${CMAKE_CURRENT_LIST_DIR}/tensorflow/bazel-bin/tensorflow/libtensorflow_cc.so 50 | dl 51 | pthread 52 | ) 53 | 54 | #-- Setup CMake to run tests 55 | enable_testing() 56 | 57 | include_directories(${face_alignment_bobetocalo_eccv18_include}) 58 | 59 | foreach(test ${face_alignment_bobetocalo_eccv18_test}) 60 | get_filename_component(test_name ${test} NAME_WE) 61 | add_executable(${test_name} 62 | ${face_alignment_bobetocalo_eccv18_src} 63 | ${test} 64 | ) 65 | target_link_libraries(${test_name} ${face_alignment_bobetocalo_eccv18_libs}) 66 | add_test(NAME ${test_name} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND ${test_name} --database 300w_public --measure pupils) 67 | endforeach() 68 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2021, Roberto Valle 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face Alignment using a Deeply-initialized Coarse-to-fine Ensemble of Regression Trees 2 | 3 | [![Youtube Video](https://img.youtube.com/vi/Nkv-6nB2yPg/0.jpg)](https://youtu.be/Nkv-6nB2yPg) 4 | 5 | 6 | We provide C++ code in order to replicate the face alignment experiments in our paper 7 | http://openaccess.thecvf.com/content_ECCV_2018/papers/Roberto_Valle_A_Deeply-initialized_Coarse-to-fine_ECCV_2018_paper.pdf 8 | 9 | If you use this code for your own research, you must reference our conference and journal papers: 10 | 11 | ``` 12 | A Deeply-initialized Coarse-to-fine Ensemble of Regression Trees for Face Alignment 13 | Roberto Valle, José M. Buenaposada, Antonio Valdés, Luis Baumela. 14 | European Conference on Computer Vision, ECCV 2018, Munich, Germany, September 8-14, 2018. 15 | ``` 16 | 17 | ``` 18 | Face Alignment using a 3D Deeply-initialized Ensemble of Regression Trees 19 | Roberto Valle, José M. Buenaposada, Antonio Valdés, Luis Baumela. 20 | Computer Vision and Image Understanding, CVIU 2019. 21 | https://doi.org/10.1016/j.cviu.2019.102846 22 | ``` 23 | 24 | #### Requisites 25 | - faces_framework https://github.com/bobetocalo/faces_framework 26 | - ert_simple https://github.com/bobetocalo/ert_simple 27 | - Tensorflow (v1.8.0) 28 | 29 | #### Installation 30 | This repository must be located inside the following directory: 31 | ``` 32 | faces_framework 33 | └── alignment 34 | └── ert_simple 35 | └── bobetocalo_eccv18 36 | ``` 37 | You need to have a C++ compiler (supporting C++11): 38 | ``` 39 | > mkdir release 40 | > cd release 41 | > cmake .. 42 | > make -j$(nproc) 43 | > cd .. 44 | ``` 45 | #### Usage 46 | Use the --measure option to set the face alignment normalization. 47 | 48 | Use the --database option to load the proper trained model. 49 | ``` 50 | > ./release/face_alignment_bobetocalo_eccv18_test --measure pupils --database 300w_public 51 | ``` 52 | -------------------------------------------------------------------------------- /data/300w_private.pb: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9f6e32a70e9e7f5af89b79d7dec105b3affd7db6207d8561cd2b0dda7356e270 3 | size 5754435 4 | -------------------------------------------------------------------------------- /data/300w_private/300w_private_6.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:96ae70e0f8ed0bcb4b0c5a7820037651c08ed223cda0538f6963057d6058ba98 3 | size 52973232 4 | -------------------------------------------------------------------------------- /data/300w_public.pb: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0ba0db7dab3c2497a1915d9c60721070342b04c6c6f047847e99afb4e8f04333 3 | size 5754435 4 | -------------------------------------------------------------------------------- /data/300w_public/300w_public_6.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:16af9b98596c47ba5e1d3a3d19800bbb1b55f6b2b95d4a754c597bbd003731fd 3 | size 65874638 4 | -------------------------------------------------------------------------------- /data/aflw.pb: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:2cc71b81e0821290083f40dac1087b9e8232cdf517485ec712e8e6813c88c08a 3 | size 5741454 4 | -------------------------------------------------------------------------------- /data/aflw/aflw_6.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:fc557bbc50b53cdb81449238aeb7e09ce0b63f8335d3dbc2ee8cdf272f91e9c0 3 | size 14588180 4 | -------------------------------------------------------------------------------- /data/cofw.pb: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c72aec267caf635b2080e6611f0134a1212e4347a36a3ffdb0c6f747696bcab4 3 | size 5743664 4 | -------------------------------------------------------------------------------- /data/cofw/cofw_6.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f852bb5518542d54ba119b44a98fc378ad9e2552f9c183463a23d44fe4d11b59 3 | size 13103952 4 | -------------------------------------------------------------------------------- /data/wflw.pb: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:313bad1e4f303066f7c6a7eecf7dbe36d3d8d043e5588fc01710574ad7597423 3 | size 5762715 4 | -------------------------------------------------------------------------------- /data/wflw/wflw_6.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a23fc87c0945e89cf874b626f4934516b15c0df1f9212f23decd0d6562691d57 3 | size 72536672 4 | -------------------------------------------------------------------------------- /include/FaceAlignment3dde.hpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file FaceAlignment3dde.hpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2018/10 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ------------------ RECURSION PROTECTION ------------------------------------- 11 | #ifndef FACE_ALIGNMENT_3DDE_HPP 12 | #define FACE_ALIGNMENT_3DDE_HPP 13 | 14 | // ----------------------- INCLUDES -------------------------------------------- 15 | #include 16 | #include 17 | #include 18 | 19 | namespace upm { 20 | 21 | /** **************************************************************************** 22 | * @class FaceAlignment3dde 23 | * @brief Class used for facial feature point detection. 24 | ******************************************************************************/ 25 | class FaceAlignment3dde: public FaceAlignment 26 | { 27 | public: 28 | FaceAlignment3dde(std::string path) : _path(path) {}; 29 | 30 | ~FaceAlignment3dde() {}; 31 | 32 | void 33 | parseOptions 34 | ( 35 | int argc, 36 | char **argv 37 | ); 38 | 39 | void 40 | train 41 | ( 42 | const std::vector &anns_train, 43 | const std::vector &anns_valid 44 | ); 45 | 46 | void 47 | load(); 48 | 49 | void 50 | process 51 | ( 52 | cv::Mat frame, 53 | std::vector &faces, 54 | const FaceAnnotation &ann 55 | ); 56 | 57 | private: 58 | std::string _path; 59 | std::vector _sp; 60 | }; 61 | 62 | } // namespace upm 63 | 64 | #endif /* FACE_ALIGNMENT_3DDE_HPP */ 65 | -------------------------------------------------------------------------------- /include/GaussianChannelFeatures.hpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file GaussianChannelFeatures.hpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2015/06 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ------------------ RECURSION PROTECTION ------------------------------------- 11 | #ifndef GAUSSIAN_CHANNEL_FEATURES_HPP 12 | #define GAUSSIAN_CHANNEL_FEATURES_HPP 13 | 14 | // ----------------------- INCLUDES -------------------------------------------- 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | namespace upm { 25 | 26 | /** **************************************************************************** 27 | * @class GaussianChannelFeatures 28 | * @brief Extract pixel features from several gaussian channels. 29 | ******************************************************************************/ 30 | class GaussianChannelFeatures : public ChannelFeatures 31 | { 32 | public: 33 | GaussianChannelFeatures() {}; 34 | 35 | GaussianChannelFeatures 36 | ( 37 | const cv::Mat &shape, 38 | const cv::Mat &label 39 | ); 40 | 41 | virtual 42 | ~GaussianChannelFeatures() {}; 43 | 44 | cv::Rect_ 45 | enlargeBbox 46 | ( 47 | const cv::Rect_ &bbox 48 | ); 49 | 50 | void 51 | loadChannelsGenerator() {}; 52 | 53 | std::vector 54 | generateChannels 55 | ( 56 | const cv::Mat &img, 57 | const cv::Rect_ &bbox 58 | ); 59 | 60 | void 61 | loadFeaturesDescriptor() {}; 62 | 63 | cv::Mat 64 | extractFeatures 65 | ( 66 | const std::vector &img_channels, 67 | const float face_height, 68 | const cv::Mat &rigid, 69 | const cv::Mat &tform, 70 | const cv::Mat &shape, 71 | float level 72 | ); 73 | 74 | friend class cereal::access; 75 | template 76 | void serialize(Archive &ar, const unsigned version) 77 | { 78 | ar & _max_diameter & _min_diameter & _robust_shape & _robust_label & _channel_sigmas & _sampling_pattern & _channel_per_sampling & _encoder; 79 | }; 80 | 81 | protected: 82 | float _max_diameter, _min_diameter; 83 | cv::Mat _robust_shape, _robust_label; 84 | std::vector _channel_sigmas; 85 | std::vector _sampling_pattern; 86 | std::vector _channel_per_sampling; 87 | std::shared_ptr _encoder; 88 | }; 89 | 90 | } // namespace upm 91 | 92 | CEREAL_REGISTER_TYPE(upm::GaussianChannelFeatures); 93 | CEREAL_REGISTER_POLYMORPHIC_RELATION(upm::ChannelFeatures, upm::GaussianChannelFeatures); 94 | 95 | #endif /* GAUSSIAN_CHANNEL_FEATURES_HPP */ 96 | -------------------------------------------------------------------------------- /include/HonariChannelFeatures.hpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file HonariChannelFeatures.hpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2017/12 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ------------------ RECURSION PROTECTION ------------------------------------- 11 | #ifndef HONARI_CHANNEL_FEATURES_HPP 12 | #define HONARI_CHANNEL_FEATURES_HPP 13 | 14 | // ----------------------- INCLUDES -------------------------------------------- 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "tensorflow/core/public/session.h" 22 | #include "tensorflow/cc/ops/standard_ops.h" 23 | 24 | namespace upm { 25 | 26 | /** **************************************************************************** 27 | * @class HonariChannelFeatures 28 | * @brief Extract maximum probability from the Recombinator Network channel. 29 | ******************************************************************************/ 30 | class HonariChannelFeatures : public ChannelFeatures 31 | { 32 | public: 33 | HonariChannelFeatures() {}; 34 | 35 | HonariChannelFeatures 36 | ( 37 | const cv::Mat &shape, 38 | const cv::Mat &label, 39 | const std::string &path, 40 | const std::string &database 41 | ); 42 | 43 | virtual 44 | ~HonariChannelFeatures() {}; 45 | 46 | cv::Rect_ 47 | enlargeBbox 48 | ( 49 | const cv::Rect_ &bbox 50 | ); 51 | 52 | tensorflow::Status 53 | imageToTensor 54 | ( 55 | const cv::Mat &img, 56 | std::vector* output_tensors 57 | ); 58 | 59 | std::vector 60 | tensorToMaps 61 | ( 62 | const tensorflow::Tensor &img_tensor, 63 | const cv::Size &face_size 64 | ); 65 | 66 | void 67 | loadChannelsGenerator(); 68 | 69 | std::vector 70 | generateChannels 71 | ( 72 | const cv::Mat &img, 73 | const cv::Rect_ &bbox 74 | ); 75 | 76 | void 77 | loadFeaturesDescriptor() {}; 78 | 79 | cv::Mat 80 | extractFeatures 81 | ( 82 | const std::vector &img_channels, 83 | const float face_height, 84 | const cv::Mat &rigid, 85 | const cv::Mat &tform, 86 | const cv::Mat &shape, 87 | float level 88 | ); 89 | 90 | friend class cereal::access; 91 | template 92 | void serialize(Archive &ar, const unsigned version) 93 | { 94 | ar & map_scale & _max_diameter & _min_diameter & _robust_shape & _robust_label & _sampling_pattern & _encoder & _path & _database; 95 | }; 96 | 97 | float map_scale; 98 | 99 | private: 100 | float _max_diameter, _min_diameter; 101 | cv::Mat _robust_shape, _robust_label; 102 | std::vector _sampling_pattern; 103 | std::shared_ptr _encoder; 104 | std::string _path; 105 | std::string _database; 106 | std::unique_ptr _session; 107 | std::map< FacePartLabel,std::vector > _cnn_parts; 108 | std::vector _cnn_landmarks; 109 | }; 110 | 111 | } // namespace upm 112 | 113 | CEREAL_REGISTER_TYPE(upm::HonariChannelFeatures); 114 | CEREAL_REGISTER_POLYMORPHIC_RELATION(upm::ChannelFeatures, upm::HonariChannelFeatures); 115 | 116 | #endif /* HONARI_CHANNEL_FEATURES_HPP */ 117 | -------------------------------------------------------------------------------- /include/ShapeCascade.hpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file ShapeCascade.hpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2015/06 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ------------------ RECURSION PROTECTION ------------------------------------- 11 | #ifndef SHAPE_CASCADE_HPP 12 | #define SHAPE_CASCADE_HPP 13 | 14 | // ----------------------- INCLUDES -------------------------------------------- 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace upm { 31 | 32 | enum class ChannelFeature { honari }; 33 | enum class InitialShape { honari }; 34 | enum class InitialHonariMode { maps }; 35 | enum class RuntimeMode { train, test }; 36 | const unsigned int MAX_NUM_LEVELS = 20; 37 | 38 | /** **************************************************************************** 39 | * @class ShapeCascade 40 | * @brief Use cascade of regression trees that can localize facial landmarks. 41 | * Predict several update vectors that best agrees with the image data. 42 | ******************************************************************************/ 43 | class ShapeCascade 44 | { 45 | public: 46 | /// Test predictor 47 | ShapeCascade() {}; 48 | 49 | ~ShapeCascade() {}; 50 | 51 | void 52 | process 53 | ( 54 | const cv::Mat &img, 55 | const float &scale, 56 | FaceAnnotation &face, 57 | const FaceAnnotation &ann, 58 | const ErrorMeasure &measure, 59 | const std::string &path 60 | ) 61 | { 62 | /// Resize bounding box annotation 63 | FaceBox box_scaled = face.bbox; 64 | box_scaled.pos.x *= scale; 65 | box_scaled.pos.y *= scale; 66 | box_scaled.pos.width *= scale; 67 | box_scaled.pos.height *= scale; 68 | 69 | /// Map shape from normalized space into the image dimension 70 | const unsigned int num_landmarks = _robust_shape.rows; 71 | cv::Mat utform = unnormalizingTransform(box_scaled.pos, _shape_size); 72 | 73 | /// Load Honari channels required to obtain probability metric 74 | std::shared_ptr hcf(_hcf); 75 | hcf->loadChannelsGenerator(); 76 | cv::Rect_ prob_bbox = hcf->enlargeBbox(box_scaled.pos); 77 | std::vector prob_channels = hcf->generateChannels(img, prob_bbox); 78 | /// Load feature channels into memory 79 | std::shared_ptr cf(_cf); 80 | cf->loadChannelsGenerator(); 81 | cf->loadFeaturesDescriptor(); 82 | cv::Rect_ feat_bbox = cf->enlargeBbox(box_scaled.pos); 83 | std::vector feat_channels; 84 | for (const cv::Mat &channel : prob_channels) 85 | feat_channels.push_back(channel.clone()); 86 | /// Set bbox according to cropped channel features 87 | cv::Point2f feat_scale = cv::Point2f(feat_channels[0].cols/feat_bbox.width, feat_channels[0].rows/feat_bbox.height); 88 | feat_scale *= hcf->map_scale; 89 | feat_bbox = cv::Rect_(box_scaled.pos.x-feat_bbox.x, box_scaled.pos.y-feat_bbox.y, box_scaled.pos.width, box_scaled.pos.height); 90 | feat_bbox.x *= feat_scale.x; 91 | feat_bbox.y *= feat_scale.y; 92 | feat_bbox.width *= feat_scale.x; 93 | feat_bbox.height *= feat_scale.y; 94 | cv::Mat feat_utform = unnormalizingTransform(feat_bbox, _shape_size); 95 | 96 | /// Run algorithm several times using different initializations 97 | const unsigned int num_initial_shapes = static_cast(_initial_shapes.size()); 98 | std::vector current_shapes(num_initial_shapes), current_labels(num_initial_shapes), current_rigids(num_initial_shapes); 99 | for (unsigned int i=0; i < num_initial_shapes; i++) 100 | { 101 | cv::RNG rnd = cv::RNG(); 102 | FaceAnnotation initial_face = generateInitialHonariShape(path, RuntimeMode::test, prob_bbox, scale, hcf->map_scale, prob_channels, box_scaled.pos, rnd); 103 | const cv::Mat ntform = normalizingTransform(box_scaled.pos, _shape_size); 104 | current_shapes[i] = cv::Mat::zeros(num_landmarks,3,CV_32FC1); 105 | current_labels[i] = cv::Mat::zeros(num_landmarks,1,CV_32FC1); 106 | facePartsToShape(initial_face.parts, ntform, scale, current_shapes[i], current_labels[i]); 107 | } 108 | 109 | for (unsigned int i=0; i < _forests.size(); i++) 110 | { 111 | for (unsigned int j=0; j < num_initial_shapes; j++) 112 | { 113 | /// Global similarity transform that maps 'robust_shape' to 'current_shape' 114 | float level = static_cast(i) / static_cast(MAX_NUM_LEVELS); 115 | current_rigids[j] = findSimilarityTransform(_robust_shape, current_shapes[j], current_labels[j]); 116 | cv::Mat features = cf->extractFeatures(feat_channels, box_scaled.pos.height, current_rigids[j], feat_utform, current_shapes[j], level); 117 | /// Compute residuals according to feature values 118 | for (const std::vector &forest : _forests[i]) 119 | for (const impl::RegressionTree &tree : forest) 120 | addResidualToShape(tree.leafs[tree.predict(features)].residual, current_shapes[j]); 121 | } 122 | if (_feats_convergence_iter == i) 123 | { 124 | cf.reset(new GaussianChannelFeatures(_robust_shape, _robust_label)); 125 | feat_bbox = cf->enlargeBbox(box_scaled.pos); 126 | feat_channels = cf->generateChannels(img, feat_bbox); 127 | cv::Point2f feat_scale = cv::Point2f(feat_channels[0].cols/feat_bbox.width, feat_channels[0].rows/feat_bbox.height); 128 | feat_bbox = cv::Rect_(box_scaled.pos.x-feat_bbox.x, box_scaled.pos.y-feat_bbox.y, box_scaled.pos.width, box_scaled.pos.height); 129 | feat_bbox.x *= feat_scale.x; 130 | feat_bbox.y *= feat_scale.y; 131 | feat_bbox.width *= feat_scale.x; 132 | feat_bbox.height *= feat_scale.y; 133 | feat_utform = unnormalizingTransform(feat_bbox, _shape_size); 134 | } 135 | } 136 | /// Facial feature location obtained 137 | bestEstimation(current_shapes, current_labels, utform, scale, ann, measure, face); 138 | }; 139 | 140 | static FaceAnnotation 141 | generateInitialHonariShape 142 | ( 143 | const std::string &path, 144 | const RuntimeMode &runtime_mode, 145 | const cv::Rect_ &prob_bbox, 146 | const float &scale, 147 | const float &map_scale, 148 | const std::vector &prob_channels, 149 | const cv::Rect_ &bbox, 150 | cv::RNG &rnd 151 | ) 152 | { 153 | /// Robust correspondences 154 | const std::vector mask = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; 155 | /// Load 3D face shape 156 | std::vector world_all; 157 | std::vector index_all; 158 | ModernPosit::loadWorldShape(path + "../../../headpose/posit/data/", DB_LANDMARKS, world_all, index_all); 159 | /// Intrinsic parameters 160 | cv::Rect_ bbox_cnn = cv::Rect_(0, 0, prob_bbox.width, prob_bbox.height); 161 | double focal_length = static_cast(bbox_cnn.width) * 1.5; 162 | cv::Point2f face_center = (bbox_cnn.tl() + bbox_cnn.br()) * 0.5f; 163 | cv::Mat cam_matrix = (cv::Mat_(3,3) << focal_length,0,face_center.x, 0,focal_length,face_center.y, 0,0,1); 164 | /// Extrinsic parameters 165 | cv::Mat rot_matrix, trl_matrix; 166 | FaceAnnotation face_cnn; 167 | unsigned int shape_idx = 0; 168 | for (const auto &db_part: DB_PARTS) 169 | for (int feature_idx : db_part.second) 170 | { 171 | cv::Mat channel; 172 | cv::resize(prob_channels[shape_idx], channel, cv::Size(), map_scale, map_scale, cv::INTER_LINEAR); 173 | cv::Point lnd; 174 | float gaussian_kernel = (32.0f/map_scale)+1; 175 | cv::GaussianBlur(channel, channel, cv::Size(gaussian_kernel,gaussian_kernel), 0, 0, cv::BORDER_REFLECT_101); 176 | cv::minMaxLoc(channel, NULL, NULL, NULL, &lnd); 177 | FaceLandmark landmark; 178 | landmark.feature_idx = static_cast(feature_idx); 179 | landmark.pos.x = lnd.x*(prob_bbox.width/channel.cols); 180 | landmark.pos.y = lnd.y*(prob_bbox.height/channel.rows); 181 | landmark.occluded = 0.0f; 182 | face_cnn.parts[db_part.first].landmarks.push_back(landmark); 183 | shape_idx++; 184 | } 185 | /// Avoid outliers using RANSAC + POSIT 186 | const float MIN_METRIC = 0.3f; 187 | const unsigned int MAX_RANSAC_ITERS = 25; 188 | std::vector rot_matrices(MAX_RANSAC_ITERS), trl_matrices(MAX_RANSAC_ITERS); 189 | std::vector metrics(MAX_RANSAC_ITERS, std::numeric_limits::min()); 190 | for (unsigned int i=0; i < MAX_RANSAC_ITERS; i++) 191 | { 192 | std::vector inliers(mask); 193 | if (i > 0) 194 | inliers.erase(inliers.begin() + static_cast(i%mask.size())); 195 | std::vector world_pts; 196 | std::vector image_pts; 197 | ModernPosit::setCorrespondences(world_all, index_all, face_cnn, inliers, world_pts, image_pts); 198 | ModernPosit::run(world_pts, image_pts, cam_matrix, 100, rot_matrices[i], trl_matrices[i]); 199 | cv::Mat rot_vector; 200 | cv::Rodrigues(rot_matrices[i], rot_vector); 201 | std::vector image_all_proj; 202 | cv::projectPoints(world_all, rot_vector, trl_matrices[i], cam_matrix, cv::Mat(), image_all_proj); 203 | FaceAnnotation initial_face; 204 | for (const auto &db_part : DB_PARTS) 205 | for (int feature_idx : db_part.second) 206 | { 207 | FaceLandmark landmark; 208 | landmark.feature_idx = feature_idx; 209 | unsigned int shape_idx = std::distance(index_all.begin(), std::find(index_all.begin(),index_all.end(),feature_idx)); 210 | landmark.pos.x = (prob_bbox.x + image_all_proj[shape_idx].x) / scale; 211 | landmark.pos.y = (prob_bbox.y + image_all_proj[shape_idx].y) / scale; 212 | landmark.occluded = 0.0f; 213 | initial_face.parts[db_part.first].landmarks.push_back(landmark); 214 | } 215 | std::vector values = getProbabilityMetric(initial_face.parts, prob_channels, prob_bbox, scale, map_scale); 216 | metrics[i] = 100.0f * static_cast(std::accumulate(values.begin(),values.end(),0.0)) / values.size(); 217 | if ((i == 0) and (metrics[i] > MIN_METRIC)) 218 | break; 219 | } 220 | unsigned int best_i = std::distance(metrics.begin(), std::max_element(metrics.begin(),metrics.end())); 221 | rot_matrix = rot_matrices[best_i].clone(); 222 | trl_matrix = trl_matrices[best_i].clone(); 223 | if (runtime_mode == RuntimeMode::train) 224 | { 225 | cv::Point3f headpose = ModernPosit::rotationMatrixToEuler(rot_matrix); 226 | headpose += cv::Point3f(rnd.uniform(-20.0f,20.0f),rnd.uniform(-10.0f,10.0f),rnd.uniform(-10.0f,10.0f)); 227 | rot_matrix = ModernPosit::eulerToRotationMatrix(headpose); 228 | } 229 | /// Project 3D shape into 2D landmarks 230 | cv::Mat rot_vector; 231 | cv::Rodrigues(rot_matrix, rot_vector); 232 | std::vector image_all_proj; 233 | cv::projectPoints(world_all, rot_vector, trl_matrix, cam_matrix, cv::Mat(), image_all_proj); 234 | cv::Point3f headpose = ModernPosit::rotationMatrixToEuler(rot_matrix); 235 | FaceAnnotation initial_face; 236 | for (const auto &db_part : DB_PARTS) 237 | for (int feature_idx : db_part.second) 238 | { 239 | FaceLandmark landmark; 240 | landmark.feature_idx = feature_idx; 241 | unsigned int shape_idx = std::distance(index_all.begin(),std::find(index_all.begin(), index_all.end(), feature_idx)); 242 | landmark.pos.x = (prob_bbox.x + image_all_proj[shape_idx].x) / scale; 243 | landmark.pos.y = (prob_bbox.y + image_all_proj[shape_idx].y) / scale; 244 | landmark.occluded = 0.0f; 245 | initial_face.parts[db_part.first].landmarks.push_back(landmark); 246 | } 247 | return initial_face; 248 | }; 249 | 250 | static void 251 | bestEstimation 252 | ( 253 | const std::vector &shapes, 254 | const std::vector &labels, 255 | const cv::Mat &tform, 256 | const float scale, 257 | const FaceAnnotation &ann, 258 | const ErrorMeasure &measure, 259 | FaceAnnotation &face 260 | ) 261 | { 262 | unsigned int best_idx = 0; 263 | float err, best_err = std::numeric_limits::max(); 264 | const unsigned int num_initials = shapes.size(); 265 | for (unsigned int i=0; i < num_initials; i++) 266 | { 267 | FaceAnnotation current; 268 | shapeToFaceParts(shapes[i], labels[i], tform, scale, current.parts); 269 | std::vector indices; 270 | std::vector errors; 271 | getNormalizedErrors(current, ann, measure, indices, errors); 272 | err = std::accumulate(errors.begin(),errors.end(),0.0) / static_cast(errors.size()); 273 | if (err < best_err) 274 | { 275 | best_err = err; 276 | best_idx = i; 277 | } 278 | } 279 | shapeToFaceParts(shapes[best_idx], labels[best_idx], tform, scale, face.parts); 280 | }; 281 | 282 | static std::vector 283 | getProbabilityMetric 284 | ( 285 | const std::vector &parts, 286 | const std::vector &prob_channels, 287 | const cv::Rect_ &prob_bbox, 288 | const float &scale, 289 | const float &map_scale 290 | ) 291 | { 292 | const unsigned int num_landmarks = prob_channels.size(); 293 | std::vector img_channels(num_landmarks); 294 | for (unsigned int i=0; i < num_landmarks; i++) 295 | cv::resize(prob_channels[i], img_channels[i], cv::Size(), map_scale, map_scale, cv::INTER_LINEAR); 296 | 297 | /// Transform current shape coordinates to CNN channel size 298 | cv::Mat tform = normalizingTransform(prob_bbox, img_channels[0].size()); 299 | cv::Mat shape = cv::Mat::zeros(num_landmarks,3,CV_32FC1); 300 | cv::Mat label = cv::Mat::zeros(num_landmarks,1,CV_32FC1); 301 | facePartsToShape(parts, tform, scale, shape, label); 302 | 303 | /// Return probability value for each landmark 304 | std::vector values; 305 | for (unsigned int i=0; i < shape.rows; i++) 306 | if (label.at(i,0) == 1.0f) 307 | { 308 | int x = static_cast(shape.at(i,0) + 0.5f); 309 | int y = static_cast(shape.at(i,1) + 0.5f); 310 | x = x < 0 ? 0 : x > img_channels[i].cols-1 ? img_channels[i].cols-1 : x; 311 | y = y < 0 ? 0 : y > img_channels[i].rows-1 ? img_channels[i].rows-1 : y; 312 | values.push_back(img_channels[i].at(y,x)); 313 | } 314 | return values; 315 | }; 316 | 317 | friend class cereal::access; 318 | template 319 | void serialize(Archive &ar, const unsigned version) 320 | { 321 | ar & _shape_size & _robust_shape & _robust_label & _initial_shapes & _initial_labels & _forests & _initial_mode & _feature_mode & _feats_convergence_iter & _cf & _hcf; 322 | }; 323 | 324 | private: 325 | cv::Size2f _shape_size; 326 | cv::Mat _robust_shape; 327 | cv::Mat _robust_label; 328 | std::vector _initial_shapes; 329 | std::vector _initial_labels; 330 | std::vector _forests; 331 | InitialShape _initial_mode; 332 | ChannelFeature _feature_mode; 333 | int _feats_convergence_iter; 334 | std::shared_ptr _cf; 335 | std::shared_ptr _hcf; 336 | }; 337 | 338 | } // namespace upm 339 | 340 | #endif /* SHAPE_CASCADE_HPP */ 341 | -------------------------------------------------------------------------------- /src/FaceAlignment3dde.cpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file FaceAlignment3dde.cpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2018/10 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ----------------------- INCLUDES -------------------------------------------- 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace upm { 24 | 25 | const float FACE_HEIGHT = 160.0f; 26 | 27 | // ----------------------------------------------------------------------------- 28 | // 29 | // Purpose and Method: 30 | // Inputs: 31 | // Outputs: 32 | // Dependencies: 33 | // Restrictions and Caveats: 34 | // 35 | // ----------------------------------------------------------------------------- 36 | void 37 | FaceAlignment3dde::parseOptions 38 | ( 39 | int argc, 40 | char **argv 41 | ) 42 | { 43 | /// Declare the supported program options 44 | FaceAlignment::parseOptions(argc, argv); 45 | namespace po = boost::program_options; 46 | po::options_description desc("FaceAlignment3dde options"); 47 | UPM_PRINT(desc); 48 | }; 49 | 50 | // ----------------------------------------------------------------------------- 51 | // 52 | // Purpose and Method: 53 | // Inputs: 54 | // Outputs: 55 | // Dependencies: 56 | // Restrictions and Caveats: 57 | // 58 | // ----------------------------------------------------------------------------- 59 | void 60 | FaceAlignment3dde::train 61 | ( 62 | const std::vector &anns_train, 63 | const std::vector &anns_valid 64 | ) 65 | { 66 | /// Training ERT model 67 | UPM_PRINT("Training ERT will be released soon ..."); 68 | }; 69 | 70 | // ----------------------------------------------------------------------------- 71 | // 72 | // Purpose and Method: 73 | // Inputs: 74 | // Outputs: 75 | // Dependencies: 76 | // Restrictions and Caveats: 77 | // 78 | // ----------------------------------------------------------------------------- 79 | void 80 | FaceAlignment3dde::load() 81 | { 82 | /// Loading shape predictors 83 | UPM_PRINT("Loading facial-feature predictors"); 84 | std::vector paths; 85 | boost::filesystem::path dir_path(_path + _database + "/"); 86 | boost::filesystem::directory_iterator end_it; 87 | for (boost::filesystem::directory_iterator it(dir_path); it != end_it; ++it) 88 | paths.push_back(it->path().string()); 89 | sort(paths.begin(), paths.end()); 90 | UPM_PRINT("> Number of predictors found: " << paths.size()); 91 | 92 | _sp.resize(HP_LABELS.size()); 93 | for (const std::string &path : paths) 94 | { 95 | try 96 | { 97 | std::ifstream ifs(path); 98 | cereal::BinaryInputArchive ia(ifs); 99 | ia >> _sp[boost::lexical_cast(path.substr(0,path.size()-4).substr(path.find_last_of('_')+1))] >> DB_PARTS >> DB_LANDMARKS; 100 | ifs.close(); 101 | } 102 | catch (cereal::Exception &ex) 103 | { 104 | UPM_ERROR("Exception during predictor deserialization: " << ex.what()); 105 | } 106 | } 107 | }; 108 | 109 | // ----------------------------------------------------------------------------- 110 | // 111 | // Purpose and Method: 112 | // Inputs: 113 | // Outputs: 114 | // Dependencies: 115 | // Restrictions and Caveats: 116 | // 117 | // ----------------------------------------------------------------------------- 118 | void 119 | FaceAlignment3dde::process 120 | ( 121 | cv::Mat frame, 122 | std::vector &faces, 123 | const FaceAnnotation &ann 124 | ) 125 | { 126 | /// Analyze each detected face to tell us the face part locations 127 | for (FaceAnnotation &face : faces) 128 | { 129 | int yaw_idx = getHeadposeIdx(FaceAnnotation().headpose.x); 130 | cv::Mat img; 131 | float scale = FACE_HEIGHT/face.bbox.pos.height; 132 | cv::resize(frame, img, cv::Size(), scale, scale); 133 | _sp[yaw_idx].process(img, scale, face, ann, _measure, _path); 134 | } 135 | }; 136 | 137 | } // namespace upm 138 | -------------------------------------------------------------------------------- /src/GaussianChannelFeatures.cpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file GaussianChannelFeatures.cpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2015/06 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ----------------------- INCLUDES -------------------------------------------- 11 | #include 12 | #include 13 | 14 | namespace upm { 15 | 16 | const float BBOX_SCALE = 0.5f; 17 | 18 | // ----------------------------------------------------------------------------- 19 | // 20 | // Purpose and Method: 21 | // Inputs: 22 | // Outputs: 23 | // Dependencies: 24 | // Restrictions and Caveats: 25 | // 26 | // ----------------------------------------------------------------------------- 27 | GaussianChannelFeatures::GaussianChannelFeatures 28 | ( 29 | const cv::Mat &shape, 30 | const cv::Mat &label 31 | ) 32 | { 33 | _max_diameter = 0.20f; 34 | _min_diameter = 0.05f; 35 | _robust_shape = shape.clone(); 36 | _robust_label = label.clone(); 37 | _channel_sigmas = {0.0000000f, 0.0017813f, 0.0023753f, 0.0035633f, 0.0053446f, 0.0077206f, 0.0106900f, 0.0142533f}; 38 | _sampling_pattern = { 39 | {0.667f,0.000f}, {0.333f,0.577f}, {-0.333f,0.577f}, {-0.667f,0.000f}, {-0.333f,-0.577f}, {0.333f,-0.577f}, 40 | {0.433f,0.250f}, {0.000f,0.500f}, {-0.433f,0.250f}, {-0.433f,-0.250f}, {0.000f,-0.500f}, {0.433f,-0.250f}, 41 | {0.361f,0.000f}, {0.180f,0.313f}, {-0.180f,0.313f}, {-0.361f,0.000f}, {-0.180f,-0.313f}, {0.180f,-0.313f}, 42 | {0.216f,0.125f}, {0.000f,0.250f}, {-0.216f,0.125f}, {-0.216f,-0.125f}, {0.000f,-0.250f}, {0.216f,-0.125f}, 43 | {0.167f,0.000f}, {0.083f,0.144f}, {-0.083f,0.144f}, {-0.167f,0.000f}, {-0.083f,-0.144f}, {0.083f,-0.144f}, 44 | {0.096f,0.055f}, {0.000f,0.111f}, {-0.096f,0.055f}, {-0.096f,-0.055f}, {0.000f,-0.111f}, {0.096f,-0.055f}, 45 | {0.083f,0.000f}, {0.042f,0.072f}, {-0.042f,0.072f}, {-0.083f,0.000f}, {-0.042f,-0.072f}, {0.042f,-0.072f}, 46 | {0.000f,0.000f}}; 47 | _channel_per_sampling = { 48 | 7, 7, 7, 7, 7, 7, 49 | 6, 6, 6, 6, 6, 6, 50 | 5, 5, 5, 5, 5, 5, 51 | 4, 4, 4, 4, 4, 4, 52 | 3, 3, 3, 3, 3, 3, 53 | 2, 2, 2, 2, 2, 2, 54 | 1, 1, 1, 1, 1, 1, 55 | 0}; 56 | 57 | /// Generate pixel locations relative to robust shape 58 | _encoder.reset(new NearestEncoding()); 59 | }; 60 | 61 | // ----------------------------------------------------------------------------- 62 | // 63 | // Purpose and Method: 64 | // Inputs: 65 | // Outputs: 66 | // Dependencies: 67 | // Restrictions and Caveats: 68 | // 69 | // ----------------------------------------------------------------------------- 70 | cv::Rect_ 71 | GaussianChannelFeatures::enlargeBbox 72 | ( 73 | const cv::Rect_ &bbox 74 | ) 75 | { 76 | /// Enlarge bounding box 77 | cv::Point2f shift(bbox.width*BBOX_SCALE, bbox.height*BBOX_SCALE); 78 | cv::Rect_ enlarged_bbox = cv::Rect_(bbox.x-shift.x, bbox.y-shift.y, bbox.width+(shift.x*2), bbox.height+(shift.y*2)); 79 | return enlarged_bbox; 80 | }; 81 | 82 | // ----------------------------------------------------------------------------- 83 | // 84 | // Purpose and Method: 85 | // Inputs: 86 | // Outputs: 87 | // Dependencies: 88 | // Restrictions and Caveats: 89 | // 90 | // ----------------------------------------------------------------------------- 91 | std::vector 92 | GaussianChannelFeatures::generateChannels 93 | ( 94 | const cv::Mat &img, 95 | const cv::Rect_ &bbox 96 | ) 97 | { 98 | /// Crop face image 99 | cv::Mat face_scaled, T = (cv::Mat_(2,3) << 1, 0, -bbox.x, 0, 1, -bbox.y); 100 | cv::warpAffine(img, face_scaled, T, bbox.size()); 101 | 102 | /// Several gaussian channel images 103 | const unsigned int num_channels = static_cast(_channel_sigmas.size()); 104 | std::vector img_channels(num_channels); 105 | cv::cvtColor(face_scaled, img_channels[0], cv::COLOR_BGR2GRAY); 106 | for (unsigned int j=1; j < num_channels; j++) 107 | cv::GaussianBlur(img_channels[0], img_channels[j], cv::Size(), _channel_sigmas[j]*bbox.height); 108 | return img_channels; 109 | }; 110 | 111 | // ----------------------------------------------------------------------------- 112 | // 113 | // Purpose and Method: 114 | // Inputs: 115 | // Outputs: 116 | // Dependencies: 117 | // Restrictions and Caveats: 118 | // 119 | // ----------------------------------------------------------------------------- 120 | cv::Mat 121 | GaussianChannelFeatures::extractFeatures 122 | ( 123 | const std::vector &img_channels, 124 | const float face_height, 125 | const cv::Mat &rigid, 126 | const cv::Mat &tform, 127 | const cv::Mat &shape, 128 | float level 129 | ) 130 | { 131 | /// Compute features from a pixel coordinates difference using a pattern 132 | cv::Mat CR = rigid.colRange(0,2).clone(); 133 | float diameter = ((1-level)*_max_diameter) + (level*_min_diameter); 134 | std::vector freak_pattern(_sampling_pattern); 135 | for (cv::Point2f &point : freak_pattern) 136 | point *= diameter; 137 | std::vector pixel_coordinates = _encoder->generatePixelSampling(_robust_shape, freak_pattern); 138 | _encoder->setPixelSamplingEncoding(_robust_shape, _robust_label, pixel_coordinates); 139 | std::vector current_pixel_coordinates = _encoder->getProjectedPixelSampling(CR, tform, shape); 140 | 141 | const unsigned int num_landmarks = static_cast(shape.rows); 142 | const unsigned int num_sampling = static_cast(_channel_per_sampling.size()); 143 | cv::Mat features = cv::Mat(num_landmarks,num_sampling,CV_32FC1); 144 | for (unsigned int i=0; i < num_landmarks; i++) 145 | { 146 | for (unsigned int j=0; j < num_sampling; j++) 147 | { 148 | int x = static_cast(current_pixel_coordinates[(i*num_sampling)+j].x + 0.5f); 149 | int y = static_cast(current_pixel_coordinates[(i*num_sampling)+j].y + 0.5f); 150 | x = x < 0 ? 0 : x; 151 | y = y < 0 ? 0 : y; 152 | unsigned int channel = _channel_per_sampling[j]; 153 | x = x > img_channels[channel].cols-1 ? img_channels[channel].cols-1 : x; 154 | y = y > img_channels[channel].rows-1 ? img_channels[channel].rows-1 : y; 155 | features.at(i,j) = img_channels[channel].at(y,x); 156 | } 157 | } 158 | return features; 159 | }; 160 | 161 | } // namespace upm 162 | -------------------------------------------------------------------------------- /src/HonariChannelFeatures.cpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file HonariChannelFeatures.cpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2017/12 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ----------------------- INCLUDES -------------------------------------------- 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace upm { 17 | 18 | const float BBOX_SCALE = 0.3f; 19 | 20 | // ----------------------------------------------------------------------------- 21 | // 22 | // Purpose and Method: 23 | // Inputs: 24 | // Outputs: 25 | // Dependencies: 26 | // Restrictions and Caveats: 27 | // 28 | // ----------------------------------------------------------------------------- 29 | HonariChannelFeatures::HonariChannelFeatures 30 | ( 31 | const cv::Mat &shape, 32 | const cv::Mat &label, 33 | const std::string &path, 34 | const std::string &database 35 | ) 36 | { 37 | map_scale = 1.0f; 38 | _max_diameter = 0.20f; 39 | _min_diameter = 0.05f; 40 | _robust_shape = shape.clone(); 41 | _robust_label = label.clone(); 42 | _sampling_pattern = { 43 | {0.667f,0.000f}, {0.333f,0.577f}, {-0.333f,0.577f}, {-0.667f,0.000f}, {-0.333f,-0.577f}, {0.333f,-0.577f}, 44 | {0.433f,0.250f}, {0.000f,0.500f}, {-0.433f,0.250f}, {-0.433f,-0.250f}, {0.000f,-0.500f}, {0.433f,-0.250f}, 45 | {0.361f,0.000f}, {0.180f,0.313f}, {-0.180f,0.313f}, {-0.361f,0.000f}, {-0.180f,-0.313f}, {0.180f,-0.313f}, 46 | {0.216f,0.125f}, {0.000f,0.250f}, {-0.216f,0.125f}, {-0.216f,-0.125f}, {0.000f,-0.250f}, {0.216f,-0.125f}, 47 | {0.167f,0.000f}, {0.083f,0.144f}, {-0.083f,0.144f}, {-0.167f,0.000f}, {-0.083f,-0.144f}, {0.083f,-0.144f}, 48 | {0.096f,0.055f}, {0.000f,0.111f}, {-0.096f,0.055f}, {-0.096f,-0.055f}, {0.000f,-0.111f}, {0.096f,-0.055f}, 49 | {0.083f,0.000f}, {0.042f,0.072f}, {-0.042f,0.072f}, {-0.083f,0.000f}, {-0.042f,-0.072f}, {0.042f,-0.072f}, 50 | {0.000f,0.000f}}; 51 | _encoder.reset(new NearestEncoding()); 52 | _path = path; 53 | _database = database; 54 | }; 55 | 56 | // ----------------------------------------------------------------------------- 57 | // 58 | // Purpose and Method: 59 | // Inputs: 60 | // Outputs: 61 | // Dependencies: 62 | // Restrictions and Caveats: 63 | // 64 | // ----------------------------------------------------------------------------- 65 | cv::Rect_ 66 | HonariChannelFeatures::enlargeBbox 67 | ( 68 | const cv::Rect_ &bbox 69 | ) 70 | { 71 | /// Enlarge bounding box 72 | cv::Point2f shift(bbox.width*BBOX_SCALE, bbox.height*BBOX_SCALE); 73 | cv::Rect_ enlarged_bbox = cv::Rect_(bbox.x-shift.x, bbox.y-shift.y, bbox.width+(shift.x*2), bbox.height+(shift.y*2)); 74 | /// Squared bbox required by neural networks 75 | enlarged_bbox.x = enlarged_bbox.x+(enlarged_bbox.width*0.5f)-(enlarged_bbox.height*0.5f); 76 | enlarged_bbox.width = enlarged_bbox.height; 77 | return enlarged_bbox; 78 | }; 79 | 80 | // ----------------------------------------------------------------------------- 81 | // 82 | // Purpose and Method: 83 | // Inputs: 84 | // Outputs: 85 | // Dependencies: 86 | // Restrictions and Caveats: 87 | // 88 | // ----------------------------------------------------------------------------- 89 | tensorflow::Status 90 | HonariChannelFeatures::imageToTensor 91 | ( 92 | const cv::Mat &img, 93 | std::vector* output_tensors 94 | ) 95 | { 96 | /// Copy mat into a tensor 97 | tensorflow::Tensor img_tensor(tensorflow::DT_FLOAT, tensorflow::TensorShape({img.rows,img.cols,img.channels()})); 98 | auto img_tensor_mapped = img_tensor.tensor(); 99 | const uchar *pixel_coordinates = img.ptr(); 100 | for (unsigned int i=0; i < img.rows; i++) 101 | for (unsigned int j=0; j < img.cols; j++) 102 | for (unsigned int k=0; k < img.channels(); k++) 103 | img_tensor_mapped(i,j,k) = pixel_coordinates[i*img.cols*img.channels() + j*img.channels() + k]; 104 | 105 | /// The convention for image ops in TensorFlow is that all images are expected 106 | /// to be in batches, so that they're four-dimensional arrays with indices of 107 | /// [batch, height, width, channel]. Because we only have a single image, we 108 | /// have to add a batch dimension of 1 to the start with ExpandDims() 109 | tensorflow::Scope root = tensorflow::Scope::NewRootScope(); 110 | auto holder = tensorflow::ops::Placeholder(root.WithOpName("input"), tensorflow::DataType::DT_FLOAT); 111 | auto expander = tensorflow::ops::ExpandDims(root.WithOpName("expander"), holder, 0); 112 | auto divider = tensorflow::ops::Div(root.WithOpName("normalized"), expander, {255.0f}); 113 | 114 | /// This runs the GraphDef network definition that we've just constructed 115 | tensorflow::GraphDef graph; 116 | TF_RETURN_IF_ERROR(root.ToGraphDef(&graph)); 117 | std::unique_ptr session(tensorflow::NewSession(tensorflow::SessionOptions())); 118 | TF_RETURN_IF_ERROR(session->Create(graph)); 119 | std::vector> input_tensors = {{"input", img_tensor},}; 120 | TF_RETURN_IF_ERROR(session->Run({input_tensors}, {"normalized"}, {}, output_tensors)); 121 | return tensorflow::Status::OK(); 122 | }; 123 | 124 | // ----------------------------------------------------------------------------- 125 | // 126 | // Purpose and Method: 127 | // Inputs: 128 | // Outputs: 129 | // Dependencies: 130 | // Restrictions and Caveats: 131 | // 132 | // ----------------------------------------------------------------------------- 133 | std::vector 134 | HonariChannelFeatures::tensorToMaps 135 | ( 136 | const tensorflow::Tensor &img_tensor, 137 | const cv::Size &face_size 138 | ) 139 | { 140 | tensorflow::TTypes::ConstFlat data = img_tensor.flat(); 141 | unsigned int num_channels = static_cast(img_tensor.dim_size(0)); 142 | unsigned int channel_size = static_cast(img_tensor.dim_size(1)); 143 | std::vector channels(num_channels); 144 | for (unsigned int i=0; i < num_channels; i++) 145 | { 146 | std::vector vec(channel_size); 147 | for (unsigned int j=0; j < channel_size; j++) 148 | vec[j] = data(i*channel_size+j); 149 | channels[i] = cv::Mat(face_size.height, face_size.width, CV_32FC1); 150 | memcpy(channels[i].data, vec.data(), vec.size()*sizeof(float)); 151 | } 152 | return channels; 153 | }; 154 | 155 | // ----------------------------------------------------------------------------- 156 | // 157 | // Purpose and Method: 158 | // Inputs: 159 | // Outputs: 160 | // Dependencies: 161 | // Restrictions and Caveats: 162 | // 163 | // ----------------------------------------------------------------------------- 164 | void 165 | HonariChannelFeatures::loadChannelsGenerator() 166 | { 167 | /// Loading CNN model 168 | std::string trained_model = "faces_framework/alignment/bobetocalo_eccv18/data/" + _database + ".pb"; 169 | tensorflow::GraphDef graph; 170 | tensorflow::Status load_graph_status = ReadBinaryProto(tensorflow::Env::Default(), trained_model, &graph); 171 | if (not load_graph_status.ok()) 172 | UPM_ERROR("Failed to load graph: " << trained_model); 173 | _session.reset(tensorflow::NewSession(tensorflow::SessionOptions())); 174 | tensorflow::Status session_create_status = _session->Create(graph); 175 | if (not session_create_status.ok()) 176 | UPM_ERROR("Failed to create session"); 177 | if ((_database == "300w_public") or (_database == "300w_private")) 178 | { 179 | _cnn_parts[FacePartLabel::leyebrow] = {1, 119, 2, 121, 3}; 180 | _cnn_parts[FacePartLabel::reyebrow] = {4, 124, 5, 126, 6}; 181 | _cnn_parts[FacePartLabel::leye] = {7, 138, 139, 8, 141, 142}; 182 | _cnn_parts[FacePartLabel::reye] = {11, 144, 145, 12, 147, 148}; 183 | _cnn_parts[FacePartLabel::nose] = {128, 129, 130, 17, 16, 133, 134, 135, 18}; 184 | _cnn_parts[FacePartLabel::tmouth] = {20, 150, 151, 22, 153, 154, 21, 165, 164, 163, 162, 161}; 185 | _cnn_parts[FacePartLabel::bmouth] = {156, 157, 23, 159, 160, 168, 167, 166}; 186 | _cnn_parts[FacePartLabel::lear] = {101, 102, 103, 104, 105, 106}; 187 | _cnn_parts[FacePartLabel::rear] = {112, 113, 114, 115, 116, 117}; 188 | _cnn_parts[FacePartLabel::chin] = {107, 108, 24, 110, 111}; 189 | _cnn_landmarks = {101, 102, 103, 104, 105, 106, 107, 108, 24, 110, 111, 112, 113, 114, 115, 116, 117, 1, 119, 2, 121, 3, 4, 124, 5, 126, 6, 128, 129, 130, 17, 16, 133, 134, 135, 18, 7, 138, 139, 8, 141, 142, 11, 144, 145, 12, 147, 148, 20, 150, 151, 22, 153, 154, 21, 156, 157, 23, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168}; 190 | } 191 | else if (_database == "cofw") 192 | { 193 | _cnn_parts[FacePartLabel::leyebrow] = {1, 101, 3, 102}; 194 | _cnn_parts[FacePartLabel::reyebrow] = {4, 103, 6, 104}; 195 | _cnn_parts[FacePartLabel::leye] = {7, 9, 8, 10, 105}; 196 | _cnn_parts[FacePartLabel::reye] = {11, 13, 12, 14, 106}; 197 | _cnn_parts[FacePartLabel::nose] = {16, 17, 18, 107}; 198 | _cnn_parts[FacePartLabel::tmouth] = {20, 22, 21, 108}; 199 | _cnn_parts[FacePartLabel::bmouth] = {109, 23}; 200 | _cnn_parts[FacePartLabel::chin] = {24}; 201 | _cnn_landmarks = {1, 101, 3, 102, 4, 103, 6, 104, 7, 9, 8, 10, 105, 11, 13, 12, 14, 106, 16, 17, 18, 107, 20, 22, 21, 108, 109, 23, 24}; 202 | } 203 | else if (_database == "aflw") 204 | { 205 | _cnn_parts[FacePartLabel::leyebrow] = {1, 2, 3}; 206 | _cnn_parts[FacePartLabel::reyebrow] = {4, 5, 6}; 207 | _cnn_parts[FacePartLabel::leye] = {7, 101, 8}; 208 | _cnn_parts[FacePartLabel::reye] = {11, 102, 12}; 209 | _cnn_parts[FacePartLabel::nose] = {16, 17, 18}; 210 | _cnn_parts[FacePartLabel::tmouth] = {20, 103, 21}; 211 | _cnn_parts[FacePartLabel::lear] = {15}; 212 | _cnn_parts[FacePartLabel::rear] = {19}; 213 | _cnn_parts[FacePartLabel::chin] = {24}; 214 | _cnn_landmarks = {1, 2, 3, 4, 5, 6, 7, 101, 8, 11, 102, 12, 15, 16, 17, 18, 19, 20, 103, 21, 24}; 215 | } 216 | else if (_database == "wflw") 217 | { 218 | _cnn_parts[upm::FacePartLabel::leyebrow] = {1, 134, 2, 136, 3, 138, 139, 140, 141}; 219 | _cnn_parts[upm::FacePartLabel::reyebrow] = {6, 147, 148, 149, 150, 4, 143, 5, 145}; 220 | _cnn_parts[upm::FacePartLabel::leye] = {7, 161, 9, 163, 8, 165, 10, 167, 196}; 221 | _cnn_parts[upm::FacePartLabel::reye] = {11, 169, 13, 171, 12, 173, 14, 175, 197}; 222 | _cnn_parts[upm::FacePartLabel::nose] = {151, 152, 153, 17, 16, 156, 157, 158, 18}; 223 | _cnn_parts[upm::FacePartLabel::tmouth] = {20, 177, 178, 22, 180, 181, 21, 192, 191, 190, 189, 188}; 224 | _cnn_parts[upm::FacePartLabel::bmouth] = {187, 186, 23, 184, 183, 193, 194, 195}; 225 | _cnn_parts[upm::FacePartLabel::lear] = {100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110}; 226 | _cnn_parts[upm::FacePartLabel::rear] = {122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132}; 227 | _cnn_parts[upm::FacePartLabel::chin] = {111, 112, 113, 114, 115, 24, 117, 118, 119, 120, 121}; 228 | _cnn_landmarks = {100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 24, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 1, 134, 2, 136, 3, 138, 139, 140, 141, 4, 143, 5, 145, 6, 147, 148, 149, 150, 151, 152, 153, 17, 16, 156, 157, 158, 18, 7, 161, 9, 163, 8, 165, 10, 167, 11, 169, 13, 171, 12, 173, 14, 175, 20, 177, 178, 22, 180, 181, 21, 183, 184, 23, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197}; 229 | } 230 | else 231 | { 232 | UPM_ERROR("Database does not exist ...") 233 | } 234 | } 235 | 236 | // ----------------------------------------------------------------------------- 237 | // 238 | // Purpose and Method: 239 | // Inputs: 240 | // Outputs: 241 | // Dependencies: 242 | // Restrictions and Caveats: 243 | // 244 | // ----------------------------------------------------------------------------- 245 | std::vector 246 | HonariChannelFeatures::generateChannels 247 | ( 248 | const cv::Mat &img, 249 | const cv::Rect_ &bbox 250 | ) 251 | { 252 | /// Crop and scale face image 253 | cv::Size_ face_size = cv::Size_(160,160); 254 | cv::Mat face_translated, T = (cv::Mat_(2,3) << 1, 0, -bbox.x, 0, 1, -bbox.y); 255 | cv::warpAffine(img, face_translated, T, bbox.size()); 256 | cv::Mat face_scaled, S = (cv::Mat_(2,3) << face_size.width/bbox.width, 0, 0, 0, face_size.height/bbox.height, 0); 257 | cv::warpAffine(face_translated, face_scaled, S, face_size); 258 | 259 | /// Testing CNN model 260 | std::vector input_tensors; 261 | tensorflow::Status read_tensor_status = imageToTensor(face_scaled, &input_tensors); 262 | if (not read_tensor_status.ok()) 263 | UPM_ERROR(read_tensor_status); 264 | const tensorflow::Tensor &input_tensor = input_tensors[0]; 265 | 266 | std::string input_layer = "input_1:0"; 267 | std::vector output_layers = {"k2tfout_0:0"}; 268 | std::vector output_tensors; 269 | tensorflow::Status run_status = _session->Run({{input_layer, input_tensor}}, output_layers, {}, &output_tensors); 270 | if (not run_status.ok()) 271 | UPM_ERROR("Running model failed: " << run_status); 272 | 273 | /// Convert output tensor to probability maps 274 | std::vector channels = tensorToMaps(output_tensors[0], face_size); 275 | 276 | /// Sort and resize channels to reduce memory required 277 | unsigned int shape_idx = 0; 278 | std::vector img_channels(channels.size()); 279 | for (const auto &cnn_part: _cnn_parts) 280 | for (int feature_idx : cnn_part.second) 281 | { 282 | auto found = std::find(_cnn_landmarks.begin(),_cnn_landmarks.end(),feature_idx); 283 | if (found == _cnn_landmarks.end()) 284 | break; 285 | int pos = static_cast(std::distance(_cnn_landmarks.begin(), found)); 286 | cv::resize(channels[pos], img_channels[shape_idx], cv::Size(), 1.0f/map_scale, 1.0f/map_scale, cv::INTER_LINEAR); 287 | shape_idx++; 288 | } 289 | return img_channels; 290 | }; 291 | 292 | // ----------------------------------------------------------------------------- 293 | // 294 | // Purpose and Method: 295 | // Inputs: 296 | // Outputs: 297 | // Dependencies: 298 | // Restrictions and Caveats: 299 | // 300 | // ----------------------------------------------------------------------------- 301 | cv::Mat 302 | HonariChannelFeatures::extractFeatures 303 | ( 304 | const std::vector &img_channels, 305 | const float face_height, 306 | const cv::Mat &rigid, 307 | const cv::Mat &tform, 308 | const cv::Mat &shape, 309 | float level 310 | ) 311 | { 312 | /// Compute features from a pixel coordinates difference using a pattern 313 | cv::Mat CR = rigid.colRange(0,2).clone(); 314 | float diameter = ((1-level)*_max_diameter) + (level*_min_diameter); 315 | std::vector freak_pattern(_sampling_pattern); 316 | for (cv::Point2f &point : freak_pattern) 317 | point *= diameter; 318 | std::vector pixel_coordinates = _encoder->generatePixelSampling(_robust_shape, freak_pattern); 319 | _encoder->setPixelSamplingEncoding(_robust_shape, _robust_label, pixel_coordinates); 320 | std::vector current_pixel_coordinates = _encoder->getProjectedPixelSampling(CR, tform, shape); 321 | 322 | const unsigned int num_landmarks = static_cast(shape.rows); 323 | const unsigned int num_sampling = static_cast(_sampling_pattern.size()); 324 | cv::Mat features = cv::Mat(num_landmarks,num_sampling,CV_32FC1); 325 | for (unsigned int i=0; i < num_landmarks; i++) 326 | { 327 | cv::Mat channel; 328 | cv::resize(img_channels[i], channel, cv::Size(), map_scale, map_scale, cv::INTER_LINEAR); 329 | for (unsigned int j=0; j < num_sampling; j++) 330 | { 331 | int x = static_cast(current_pixel_coordinates[(i*num_sampling)+j].x + 0.5f); 332 | int y = static_cast(current_pixel_coordinates[(i*num_sampling)+j].y + 0.5f); 333 | x = x < 0 ? 0 : x > channel.cols-1 ? channel.cols-1 : x; 334 | y = y < 0 ? 0 : y > channel.rows-1 ? channel.rows-1 : y; 335 | features.at(i,j) = channel.at(y,x); 336 | } 337 | } 338 | return features; 339 | }; 340 | 341 | } // namespace upm 342 | -------------------------------------------------------------------------------- /test/face_alignment_bobetocalo_eccv18_test.cpp: -------------------------------------------------------------------------------- 1 | /** **************************************************************************** 2 | * @file face_alignment_bobetocalo_eccv18_test.cpp 3 | * @brief Face detection and recognition framework 4 | * @author Roberto Valle Fernandez 5 | * @date 2018/10 6 | * @copyright All rights reserved. 7 | * Software developed by UPM PCR Group: http://www.dia.fi.upm.es/~pcr 8 | ******************************************************************************/ 9 | 10 | // ----------------------- INCLUDES -------------------------------------------- 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | // ----------------------------------------------------------------------------- 21 | // 22 | // Purpose and Method: 23 | // Inputs: 24 | // Outputs: 25 | // Dependencies: 26 | // Restrictions and Caveats: 27 | // 28 | // ----------------------------------------------------------------------------- 29 | int 30 | main 31 | ( 32 | int argc, 33 | char **argv 34 | ) 35 | { 36 | // Read sample annotations 37 | upm::FaceAnnotation ann; 38 | ann.filename = "test/image_070.jpg"; 39 | ann.bbox.pos = cv::Rect2f(195.196984225f,75.148001275f,898.385880775f,760.773923725f); 40 | upm::DB_PARTS[upm::FacePartLabel::leyebrow] = {1, 119, 2, 121, 3}; 41 | upm::DB_PARTS[upm::FacePartLabel::reyebrow] = {4, 124, 5, 126, 6}; 42 | upm::DB_PARTS[upm::FacePartLabel::leye] = {7, 138, 139, 8, 141, 142}; 43 | upm::DB_PARTS[upm::FacePartLabel::reye] = {11, 144, 145, 12, 147, 148}; 44 | upm::DB_PARTS[upm::FacePartLabel::nose] = {128, 129, 130, 17, 16, 133, 134, 135, 18}; 45 | upm::DB_PARTS[upm::FacePartLabel::tmouth] = {20, 150, 151, 22, 153, 154, 21, 165, 164, 163, 162, 161}; 46 | upm::DB_PARTS[upm::FacePartLabel::bmouth] = {156, 157, 23, 159, 160, 168, 167, 166}; 47 | upm::DB_PARTS[upm::FacePartLabel::lear] = {101, 102, 103, 104, 105, 106}; 48 | upm::DB_PARTS[upm::FacePartLabel::rear] = {112, 113, 114, 115, 116, 117}; 49 | upm::DB_PARTS[upm::FacePartLabel::chin] = {107, 108, 24, 110, 111}; 50 | upm::DB_LANDMARKS = {101, 102, 103, 104, 105, 106, 107, 108, 24, 110, 111, 112, 113, 114, 115, 116, 117, 1, 119, 2, 121, 3, 4, 124, 5, 126, 6, 128, 129, 130, 17, 16, 133, 134, 135, 18, 7, 138, 139, 8, 141, 142, 11, 144, 145, 12, 147, 148, 20, 150, 151, 22, 153, 154, 21, 156, 157, 23, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168}; 51 | std::vector coords = {211.939577f,507.508412f,270.328191f,604.681633f,341.748364f,688.39419f,435.651535f,744.294051f,598.333129f,744.449497f,713.827933f,726.477666f,787.85507f,704.49905f,828.583557f,682.753602f,860.909862f,657.546242f,881.643288f,615.222814f,878.531127f,568.503989f,860.753602f,513.459889f,854.140669f,449.001586f,837.218907f,387.812517f,807.382167f,324.638265f,766.264252f,268.660681f,715.655227f,227.855287f,336.108592f,377.038169f,366.878816f,311.452074f,417.25549f,251.700872f,469.381735f,209.183138f,539.401273f,201.482089f,617.939577f,158.925493f,625.953147f,127.338367f,638.28439f,109.405397f,659.874393f,91.472428f,681.231229f,91.978439f,614.204822f,229.217875f,649.214998f,260.376712f,683.330552f,288.851678f,722.425235f,314.837078f,682.201945f,432.431205f,711.766655f,421.851162f,737.674333f,411.037138f,753.779194f,392.67588f,765.254675f,364.472943f,425.967746f,375.404366f,465.06243f,324.250468f,494.276575f,298.49905f,540.178499f,314.875939f,517.460704f,334.559296f,484.784648f,358.249437f,642.368897f,253.686056f,646.531127f,205.488873f,676.48445f,186.544696f,694.572865f,203.038169f,697.256737f,228.206668f,673.760902f,239.682148f,670.221268f,564.108592f,717.32952f,522.679787f,751.872547f,474.171714f,774.745786f,474.210575f,783.809425f,455.266398f,807.227537f,461.801609f,811.584073f,453.204306f,833.135214f,489.965684f,828.232991f,514.27842f,814.578835f,530.110437f,789.254891f,545.787009f,736.079391f,559.246045f,680.996431f,555.161537f,768.561142f,501.323767f,783.653979f,489.848286f,796.802125f,477.984192f,811.350905f,461.995915f,796.802125f,477.984192f,783.653979f,489.848286f,768.561142f,501.323767f}; 52 | for (int cont=0; cont < upm::DB_LANDMARKS.size(); cont++) 53 | { 54 | unsigned int feature_idx = upm::DB_LANDMARKS[cont]; 55 | float x = coords[(2*cont)+0]; 56 | float y = coords[(2*cont)+1]; 57 | float occluded = 0.0f; 58 | if (feature_idx < 1) 59 | continue; 60 | for (const auto &part : upm::DB_PARTS) 61 | if (std::find(part.second.begin(),part.second.end(),feature_idx) != part.second.end()) 62 | { 63 | upm::FaceLandmark landmark; 64 | landmark.feature_idx = feature_idx; 65 | landmark.pos = cv::Point2f(x,y); 66 | landmark.occluded = occluded; 67 | ann.parts[part.first].landmarks.push_back(landmark); 68 | break; 69 | } 70 | } 71 | cv::Mat frame = cv::imread(ann.filename, cv::IMREAD_COLOR); 72 | if (frame.empty()) 73 | return EXIT_FAILURE; 74 | 75 | // Set face detected position 76 | std::vector faces(1); 77 | faces[0].bbox.pos = ann.bbox.pos; 78 | 79 | /// Load face components 80 | boost::shared_ptr composite(new upm::FaceComposite()); 81 | boost::shared_ptr fa(new upm::FaceAlignment3dde("data/")); 82 | composite->addComponent(fa); 83 | 84 | /// Parse face component options 85 | composite->parseOptions(argc, argv); 86 | composite->load(); 87 | 88 | // Process frame 89 | double ticks = processFrame(frame, composite, faces, ann); 90 | UPM_PRINT("FPS = " << cv::getTickFrequency()/ticks); 91 | 92 | // Evaluate results 93 | boost::shared_ptr output(&std::cout, [](std::ostream*){}); 94 | composite->evaluate(output, faces, ann); 95 | 96 | // Draw results 97 | boost::shared_ptr viewer(new upm::Viewer); 98 | viewer->init(0, 0, "face_alignment_bobetocalo_eccv18_test"); 99 | showResults(viewer, ticks, 0, frame, composite, faces, ann); 100 | 101 | UPM_PRINT("End of face_alignment_bobetocalo_eccv18_test"); 102 | return EXIT_SUCCESS; 103 | }; 104 | -------------------------------------------------------------------------------- /test/image_070.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcr-upm/bobetocalo_eccv18/1350856824a7699d3ed658b9f666b37eba5c7505/test/image_070.jpg --------------------------------------------------------------------------------