├── .gitignore ├── configs └── create_data │ ├── dtang │ ├── test_p48.prototxt │ ├── test_p96.prototxt │ ├── train_p48.prototxt │ └── train_p96.prototxt │ └── blender │ ├── pico_p48.prototxt │ ├── pico_p96.prototxt │ ├── creative_p48.prototxt │ └── creative_p96.prototxt ├── README.md ├── include ├── hand_segmentation_result.h ├── inference.h ├── inference_dummy.h ├── preprocess.h ├── hand_segmentation.h ├── inference_regression_net.h ├── make_patch_extraction.h ├── make_preprocess.h ├── hand_segmentation_meanshift.h ├── hand_segmentation_threshold.h ├── data_provider_csv.h ├── data_provider_dtang.h ├── hand_patch.h ├── data_provider_blender.h ├── patch_extraction.h ├── inference_rf_simple.h ├── data_provider_cam.h ├── inference_heatmap_net.h ├── projection.h ├── make_data_provider.h ├── make_hand_segmentation.h ├── create_data.h ├── make_projection.h ├── hand_segmentation_rf.h ├── make_inference.h ├── common.h ├── annotation_viewer.h └── data_provider.h ├── src ├── preprocess.cpp ├── inference_rf_simple.cpp ├── projection.cpp ├── main_nn_train.cpp ├── inference_regression_net.cpp ├── main_create_data.cpp ├── hand_segmentation_threshold.cpp ├── main_rf_train_segmentation.cpp ├── hand_patch.cpp ├── data_provider_cam.cpp ├── data_provider_dtang.cpp ├── patch_extraction.cpp ├── hand_segmentation_meanshift.cpp ├── data_provider_csv.cpp ├── main_rf_train.cpp ├── hand_segmentation_rf.cpp ├── data_provider_blender.cpp ├── common.cpp └── inference_heatmap_net.cpp ├── cmake └── FindQGLVIEWER.cmake ├── CMakeLists.txt └── proto └── pose.proto /.gitignore: -------------------------------------------------------------------------------- 1 | build* 2 | data 3 | .ycm_extra_conf.py* 4 | **/*pyc 5 | **/*npy 6 | matlab/*bin 7 | matlab/*config 8 | matlab/*csv 9 | python/subspace/gt3.csv 10 | -------------------------------------------------------------------------------- /configs/create_data/dtang/test_p48.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: false 2 | rotations: false 3 | mirroring: false 4 | out_prefix: "../data/h5/dtang/48/test" 5 | 6 | data_provider_param { 7 | type: DTANG 8 | bg_value: 1000.0 9 | n_pts: 16 10 | dtang_param { 11 | label_path: "../../datasets/dtang/Testing/test_seq_1.txt" 12 | } 13 | } 14 | 15 | projection_param { 16 | type: ORTHOGRAPHIC 17 | } 18 | 19 | segmentation_param { 20 | type: MEANSHIFT 21 | } 22 | 23 | patch_extraction_param { 24 | patch_width: 48 25 | normalize_depth: true 26 | } 27 | -------------------------------------------------------------------------------- /configs/create_data/dtang/test_p96.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: false 2 | rotations: false 3 | mirroring: false 4 | out_prefix: "../data/h5/dtang/96/test" 5 | 6 | data_provider_param { 7 | type: DTANG 8 | bg_value: 1000.0 9 | n_pts: 16 10 | dtang_param { 11 | label_path: "../../datasets/dtang/Testing/test_seq_1.txt" 12 | } 13 | } 14 | 15 | projection_param { 16 | type: ORTHOGRAPHIC 17 | } 18 | 19 | segmentation_param { 20 | type: MEANSHIFT 21 | } 22 | 23 | patch_extraction_param { 24 | patch_width: 96 25 | normalize_depth: true 26 | } 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Framework for Articulated Hand Pose Estimation 2 | 3 | A framework for articulated hand pose estimation. See the [project site](http://rvlab.icg.tugraz.at/pose) for more details. 4 | 5 | ## Citing 6 | Pleas cite this work if it helps your research 7 | 8 | @inproceedings{riegler2015blenderhandposture, 9 | author = {Riegler, Gernot and Ferstl, David and Rüther, Matthias and Bischof, Horst}, 10 | booktitle = {Scandinavian Conference on Image Analysis}, 11 | title = {A Framework for Articulated Hand Pose Estimation and Evaluation}, 12 | year = {2015} 13 | } 14 | -------------------------------------------------------------------------------- /configs/create_data/dtang/train_p48.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: true 2 | 3 | rotations: true 4 | rotations_from: -90.0 5 | rotations_to: 90.0 6 | rotations_step: 12.5 7 | 8 | mirroring: false 9 | samples_per_hdf5: 128000 10 | out_prefix: "../data/h5/dtang/48/train" 11 | 12 | data_provider_param { 13 | type: DTANG 14 | bg_value: 1000.0 15 | n_pts: 16 16 | dtang_param { 17 | label_path: "../../datasets/dtang/Training/labels.txt" 18 | } 19 | } 20 | 21 | projection_param { 22 | type: ORTHOGRAPHIC 23 | } 24 | 25 | segmentation_param { 26 | type: MEANSHIFT 27 | } 28 | 29 | patch_extraction_param { 30 | patch_width: 48 31 | normalize_depth: true 32 | } 33 | -------------------------------------------------------------------------------- /configs/create_data/dtang/train_p96.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: true 2 | 3 | rotations: true 4 | rotations_from: -90.0 5 | rotations_to: 90.0 6 | rotations_step: 12.5 7 | 8 | mirroring: false 9 | samples_per_hdf5: 128000 10 | out_prefix: "../data/h5/dtang/96/train" 11 | 12 | data_provider_param { 13 | type: DTANG 14 | bg_value: 1000.0 15 | n_pts: 16 16 | dtang_param { 17 | label_path: "../../datasets/dtang/Training/labels.txt" 18 | } 19 | } 20 | 21 | projection_param { 22 | type: ORTHOGRAPHIC 23 | } 24 | 25 | segmentation_param { 26 | type: MEANSHIFT 27 | } 28 | 29 | patch_extraction_param { 30 | patch_width: 96 31 | normalize_depth: true 32 | } 33 | -------------------------------------------------------------------------------- /configs/create_data/blender/pico_p48.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: true 2 | 3 | rotations: true 4 | rotations_from: -90.0 5 | rotations_to: 90.0 6 | rotations_step: 45.0 7 | 8 | mirroring: false 9 | 10 | samples_per_hdf5: 128000 11 | 12 | out_prefix: "/mnt/ssddata/h5/blender/pico_p48" 13 | 14 | data_provider_param { 15 | type: BLENDER 16 | bg_value: 1250.0 17 | n_pts: 20 18 | blender_param { 19 | data_path: "/mnt/ssddata/pico" 20 | recursive: true 21 | 22 | noise_gaussian_sigma: 3 23 | } 24 | } 25 | 26 | projection_param { 27 | type: PROJECTIVE 28 | } 29 | 30 | segmentation_param { 31 | type: MEANSHIFT 32 | } 33 | 34 | patch_extraction_param { 35 | patch_width: 48 36 | normalize_depth: true 37 | } 38 | -------------------------------------------------------------------------------- /configs/create_data/blender/pico_p96.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: true 2 | 3 | rotations: true 4 | rotations_from: -90.0 5 | rotations_to: 90.0 6 | rotations_step: 45.0 7 | 8 | mirroring: false 9 | 10 | samples_per_hdf5: 128000 11 | 12 | out_prefix: "/mnt/ssddata/h5/blender/pico_p96" 13 | 14 | data_provider_param { 15 | type: BLENDER 16 | bg_value: 1250.0 17 | n_pts: 20 18 | blender_param { 19 | data_path: "/mnt/ssddata/pico" 20 | recursive: true 21 | 22 | noise_gaussian_sigma: 3 23 | } 24 | } 25 | 26 | projection_param { 27 | type: PROJECTIVE 28 | } 29 | 30 | segmentation_param { 31 | type: MEANSHIFT 32 | } 33 | 34 | patch_extraction_param { 35 | patch_width: 96 36 | normalize_depth: true 37 | } 38 | -------------------------------------------------------------------------------- /configs/create_data/blender/creative_p48.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: true 2 | 3 | rotations: true 4 | rotations_from: -90.0 5 | rotations_to: 90.0 6 | rotations_step: 45.0 7 | 8 | mirroring: false 9 | 10 | samples_per_hdf5: 128000 11 | 12 | out_prefix: "/mnt/ssddata/h5/blender/creative_p48" 13 | 14 | data_provider_param { 15 | type: BLENDER 16 | bg_value: 1250.0 17 | n_pts: 20 18 | blender_param { 19 | data_path: "/mnt/ssddata/creative" 20 | recursive: true 21 | 22 | noise_gaussian_sigma: 3 23 | } 24 | } 25 | 26 | projection_param { 27 | type: PROJECTIVE 28 | } 29 | 30 | segmentation_param { 31 | type: MEANSHIFT 32 | } 33 | 34 | patch_extraction_param { 35 | patch_width: 48 36 | normalize_depth: true 37 | } 38 | -------------------------------------------------------------------------------- /configs/create_data/blender/creative_p96.prototxt: -------------------------------------------------------------------------------- 1 | shuffle: true 2 | 3 | rotations: true 4 | rotations_from: -90.0 5 | rotations_to: 90.0 6 | rotations_step: 45.0 7 | 8 | mirroring: false 9 | 10 | samples_per_hdf5: 128000 11 | 12 | out_prefix: "/mnt/ssddata/h5/blender/creative_p96" 13 | 14 | data_provider_param { 15 | type: BLENDER 16 | bg_value: 1250.0 17 | n_pts: 20 18 | blender_param { 19 | data_path: "/mnt/ssddata/creative" 20 | recursive: true 21 | 22 | noise_gaussian_sigma: 3 23 | } 24 | } 25 | 26 | projection_param { 27 | type: PROJECTIVE 28 | } 29 | 30 | segmentation_param { 31 | type: MEANSHIFT 32 | } 33 | 34 | patch_extraction_param { 35 | patch_width: 96 36 | normalize_depth: true 37 | } 38 | -------------------------------------------------------------------------------- /include/hand_segmentation_result.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef HAND_SEGMENTATION_RESULT_H 30 | #define HAND_SEGMENTATION_RESULT_H 31 | 32 | #include 33 | 34 | struct HandSegmentationResult { 35 | cv::Mat_ mask_; 36 | cv::Rect roi_; 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/preprocess.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "preprocess.h" 30 | 31 | template 32 | void Preprocess::operator()(std::vector >& hand_patch) const { 33 | if(subtract_mean_) { 34 | hand_patch[0].patch_ -= mean_; 35 | } 36 | } 37 | 38 | 39 | template class Preprocess; 40 | template class Preprocess; 41 | -------------------------------------------------------------------------------- /cmake/FindQGLVIEWER.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find QGLViewer 2 | # Once done this will define 3 | # 4 | # QGLVIEWER_FOUND - system has QGLViewer 5 | # QGLVIEWER_INCLUDE_DIR - the QGLViewer include directory 6 | # QGLVIEWER_LIBRARIES - Link these to use QGLViewer 7 | # QGLVIEWER_DEFINITIONS - Compiler switches required for using QGLViewer 8 | # 9 | 10 | find_path(QGLVIEWER_INCLUDE_DIR 11 | NAMES QGLViewer/qglviewer.h 12 | PATHS /usr/include 13 | /usr/local/include 14 | ENV QGLVIEWERROOT 15 | ) 16 | 17 | find_library(QGLVIEWER_LIBRARY_RELEASE 18 | NAMES qglviewer-qt4 qglviewer QGLViewer QGLViewer2 19 | PATHS /usr/lib 20 | /usr/local/lib 21 | ENV QGLVIEWERROOT 22 | ENV LD_LIBRARY_PATH 23 | ENV LIBRARY_PATH 24 | PATH_SUFFIXES QGLViewer QGLViewer/release 25 | ) 26 | 27 | find_library(QGLVIEWER_LIBRARY_DEBUG 28 | NAMES dqglviewer dQGLViewer dQGLViewer2 29 | PATHS /usr/lib 30 | /usr/local/lib 31 | ENV QGLVIEWERROOT 32 | ENV LD_LIBRARY_PATH 33 | ENV LIBRARY_PATH 34 | PATH_SUFFIXES QGLViewer QGLViewer/debug 35 | ) 36 | 37 | if(QGLVIEWER_LIBRARY_RELEASE) 38 | if(QGLVIEWER_LIBRARY_DEBUG) 39 | set(QGLVIEWER_LIBRARIES_ optimized ${QGLVIEWER_LIBRARY_RELEASE} debug ${QGLVIEWER_LIBRARY_DEBUG}) 40 | else() 41 | set(QGLVIEWER_LIBRARIES_ ${QGLVIEWER_LIBRARY_RELEASE}) 42 | endif() 43 | 44 | set(QGLVIEWER_LIBRARIES ${QGLVIEWER_LIBRARIES_} CACHE FILEPATH "The QGLViewer library") 45 | 46 | endif() 47 | 48 | IF(QGLVIEWER_INCLUDE_DIR AND QGLVIEWER_LIBRARIES) 49 | SET(QGLVIEWER_FOUND TRUE) 50 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DQGLVIEWER_FOUND") 51 | ENDIF(QGLVIEWER_INCLUDE_DIR AND QGLVIEWER_LIBRARIES) 52 | 53 | IF(QGLVIEWER_FOUND) 54 | IF(NOT QGLViewer_FIND_QUIETLY) 55 | MESSAGE(STATUS "Found QGLViewer: ${QGLVIEWER_LIBRARIES}") 56 | ENDIF(NOT QGLViewer_FIND_QUIETLY) 57 | ELSE(QGLVIEWER_FOUND) 58 | IF(QGLViewer_FIND_REQUIRED) 59 | MESSAGE(FATAL_ERROR "Could not find QGLViewer") 60 | ENDIF(QGLViewer_FIND_REQUIRED) 61 | ENDIF(QGLVIEWER_FOUND) 62 | 63 | -------------------------------------------------------------------------------- /include/inference.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef INFERENCE_H 30 | #define INFERENCE_H 31 | 32 | #include "hand_segmentation.h" 33 | #include "hand_patch.h" 34 | 35 | template 36 | class Inference { 37 | public: 38 | Inference() {} 39 | virtual ~Inference() {} 40 | 41 | virtual std::vector operator()(const std::vector >& patches) const = 0; 42 | 43 | protected: 44 | }; 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /include/inference_dummy.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef INFERENCE_DUMMY_H 30 | #define INFERENCE_DUMMY_H 31 | 32 | #include "inference.h" 33 | 34 | template 35 | class InferenceDummy : public Inference { 36 | public: 37 | InferenceDummy(int n_pts) : n_pts_(n_pts) {} 38 | virtual ~InferenceDummy() {} 39 | 40 | virtual std::vector operator()(const std::vector >& patches) const { 41 | std::vector result(n_pts_, cv::Vec3f(0.0f, 0.0f, 0.0f)); 42 | return result; 43 | } 44 | 45 | private: 46 | int n_pts_; 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /include/preprocess.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef PREPROCESS_H 30 | #define PREPROCESS_H 31 | 32 | #include "hand_patch.h" 33 | 34 | #include 35 | 36 | template 37 | class Preprocess { 38 | public: 39 | Preprocess() : subtract_mean_(false) {} 40 | virtual ~Preprocess() {} 41 | 42 | virtual void operator()(std::vector >& hand_patch) const; 43 | 44 | virtual void addSubtractMean(cv::Mat_& mean) { 45 | subtract_mean_ = true; 46 | mean_ = mean; 47 | } 48 | 49 | private: 50 | bool subtract_mean_; 51 | cv::Mat_ mean_; 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /include/hand_segmentation.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef HAND_SEGMENTATION_H 30 | #define HAND_SEGMENTATION_H 31 | 32 | #include "hand_segmentation_result.h" 33 | 34 | #include 35 | 36 | #include "pose.pb.h" 37 | 38 | template 39 | class HandSegmentation { 40 | public: 41 | HandSegmentation(const pose::SegmentationParameter& param) : param_(param) { } 42 | virtual ~HandSegmentation() {} 43 | 44 | virtual std::vector operator()(const cv::Mat_& depth, const cv::Mat_& ir, const cv::Vec3f& hint_2d) const = 0; 45 | 46 | protected: 47 | const pose::SegmentationParameter& param_; 48 | }; 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /include/inference_regression_net.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef INFERENCE_REGRESSION_NET_H 30 | #define INFERENCE_REGRESSION_NET_H 31 | 32 | #include "inference.h" 33 | 34 | #include 35 | #include 36 | 37 | template 38 | class InferenceRegressionNet : public Inference { 39 | public: 40 | InferenceRegressionNet(boost::shared_ptr > net) : 41 | net_(net) {} 42 | virtual ~InferenceRegressionNet() {} 43 | 44 | virtual std::vector operator()(const std::vector >& patches) const; 45 | 46 | private: 47 | boost::shared_ptr > net_; 48 | }; 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /include/make_patch_extraction.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef MAKE_PATCH_EXTRACTION_H 30 | #define MAKE_PATCH_EXTRACTION_H 31 | 32 | #include "patch_extraction.h" 33 | 34 | #include "pose.pb.h" 35 | 36 | #include 37 | 38 | 39 | template 40 | boost::shared_ptr< PatchExtraction > makePatchExtraction(const pose::PatchExtractionParameter& param, Dtype bg_value) { 41 | 42 | std::cout << "[INFO] Creating PatchExtraction" << std::endl; 43 | 44 | std::vector patch_widths(param.patch_width_size()); 45 | for(int idx = 0; idx < param.patch_width_size(); ++idx) { 46 | patch_widths[idx] = param.patch_width(idx); 47 | } 48 | 49 | return boost::make_shared >(patch_widths, bg_value, param.normalize_depth()); 50 | } 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /include/make_preprocess.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef MAKE_PREPROCESS_H 30 | #define MAKE_PREPROCESS_H 31 | 32 | #include "preprocess.h" 33 | 34 | #include "pose.pb.h" 35 | 36 | #include 37 | 38 | #include 39 | 40 | 41 | template 42 | boost::shared_ptr< Preprocess > makePreprocess(const pose::PreprocessParameter& param) { 43 | 44 | std::cout << "[INFO] Creating Preprocess" << std::endl; 45 | 46 | boost::shared_ptr > preprocess = boost::make_shared >(); 47 | 48 | if(param.mean_path() != "") { 49 | std::cout << "[INFO] add mean to preprocess" << std::endl; 50 | cv::Mat_ mean = rv::ocv::ReadCsv(param.mean_path()); 51 | preprocess->addSubtractMean(mean); 52 | } 53 | 54 | return preprocess; 55 | } 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /include/hand_segmentation_meanshift.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef HAND_SEGMENTATION_MEANSHIFT_H 30 | #define HAND_SEGMENTATION_MEANSHIFT_H 31 | 32 | #include "hand_segmentation.h" 33 | #include "projection.h" 34 | 35 | template 36 | class HandSegmentationMeanshift : public HandSegmentation { 37 | public: 38 | HandSegmentationMeanshift(const pose::SegmentationParameter& param, const Projection& projection, Dtype bg_value) : 39 | HandSegmentation(param), projection_(projection), bg_value_(bg_value) {} 40 | virtual ~HandSegmentationMeanshift() {} 41 | 42 | virtual std::vector operator()(const cv::Mat_& depth, const cv::Mat_& ir, const cv::Vec3f& hint_2d) const; 43 | 44 | protected: 45 | const Projection& projection_; 46 | const Dtype bg_value_; 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /include/hand_segmentation_threshold.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef HAND_SEGMENTATION_THRESHOLD_H 30 | #define HAND_SEGMENTATION_THRESHOLD_H 31 | 32 | #include "hand_segmentation.h" 33 | #include "projection.h" 34 | 35 | template 36 | class HandSegmentationThreshold : public HandSegmentation { 37 | public: 38 | HandSegmentationThreshold(const pose::SegmentationParameter& param, const Projection& projection, Dtype bg_value) : 39 | HandSegmentation(param), projection_(projection), bg_value_(bg_value) {} 40 | virtual ~HandSegmentationThreshold() {} 41 | 42 | virtual std::vector operator()(const cv::Mat_& depth, const cv::Mat_& ir, const cv::Vec3f& hint_2d) const; 43 | 44 | protected: 45 | const Projection& projection_; 46 | const Dtype bg_value_; 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /include/data_provider_csv.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | #ifndef DATA_PROVIDER_CSV_H 29 | #define DATA_PROVIDER_CSV_H 30 | 31 | #include 32 | 33 | #include "pose.pb.h" 34 | 35 | template 36 | class DataProviderCsv : public DataProvider { 37 | public: 38 | DataProviderCsv(const pose::DataProviderParameter& param); 39 | virtual ~DataProviderCsv() {} 40 | 41 | virtual boost::filesystem::path depthPath(int idx) const; 42 | virtual void shuffle(); 43 | 44 | protected: 45 | virtual std::vector< cv::Vec3f > gt_internal(int idx) const; 46 | virtual cv::Vec3f hint2d_internal(int idx) const; 47 | virtual cv::Mat_< Dtype > depth_internal(int idx); 48 | virtual cv::Mat_< Dtype > ir_internal(int idx); 49 | 50 | private: 51 | std::vector depth_paths_; 52 | std::vector > annos_; 53 | }; 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /include/data_provider_dtang.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef DATA_PROVIDER_DTANG_H 30 | #define DATA_PROVIDER_DTANG_H 31 | 32 | #include 33 | 34 | #include "pose.pb.h" 35 | 36 | template 37 | class DataProviderDTang : public DataProvider { 38 | public: 39 | DataProviderDTang(const pose::DataProviderParameter& param); 40 | virtual ~DataProviderDTang() {} 41 | 42 | virtual boost::filesystem::path depthPath(int idx) const; 43 | 44 | virtual void shuffle(); 45 | 46 | protected: 47 | virtual std::vector< cv::Vec3f > gt_internal(int idx) const; 48 | virtual cv::Vec3f hint2d_internal(int idx) const; 49 | virtual cv::Mat_< Dtype > depth_internal(int idx); 50 | virtual cv::Mat_< Dtype > ir_internal(int idx); 51 | 52 | private: 53 | std::vector depth_paths_; 54 | std::vector > annos_; 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /include/hand_patch.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef HAND_PATCH_H 30 | #define HAND_PATCH_H 31 | 32 | #include 33 | 34 | template 35 | class HandPatch { 36 | public: 37 | HandPatch() {} 38 | virtual ~HandPatch() {} 39 | 40 | std::vector annoInHandPatch(const std::vector& anno, const cv::Rect& roi) const; 41 | std::vector annoFromHandPatch(const std::vector& anno, const cv::Rect& roi) const; 42 | 43 | std::vector centerAnno(const std::vector& anno) const; 44 | std::vector uncenterAnno(const std::vector& anno) const; 45 | 46 | bool isReasonableCenteredAnno(const std::vector& anno, bool normalized) const; 47 | 48 | public: 49 | cv::Mat_ patch_; 50 | 51 | float scale_; 52 | int row_offset_; 53 | int col_offset_; 54 | 55 | Dtype mean_depth_; 56 | Dtype std_depth_; 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/data_provider_blender.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef DATA_PROVIDER_BLENDER_H 30 | #define DATA_PROVIDER_BLENDER_H 31 | 32 | #include 33 | 34 | #include "pose.pb.h" 35 | 36 | template 37 | class DataProviderBlender : public DataProvider { 38 | public: 39 | DataProviderBlender(const pose::DataProviderParameter& param); 40 | ~DataProviderBlender() {} 41 | 42 | virtual boost::filesystem::path depthPath(int idx) const; 43 | 44 | virtual void shuffle(); 45 | 46 | protected: 47 | virtual std::vector< cv::Vec3f > gt_internal(int idx) const; 48 | virtual cv::Vec3f hint2d_internal(int idx) const; 49 | virtual cv::Mat_< Dtype > depth_internal(int idx); 50 | virtual cv::Mat_< Dtype > ir_internal(int idx); 51 | 52 | boost::filesystem::path annoPath(const boost::filesystem::path& depth_path) const; 53 | 54 | private: 55 | std::vector depth_paths_; 56 | std::vector > annos_; 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/patch_extraction.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef EXTRACT_PATCH_H 30 | #define EXTRACT_PATCH_H 31 | 32 | #include "hand_patch.h" 33 | #include "hand_segmentation_result.h" 34 | 35 | template 36 | class PatchExtraction { 37 | public: 38 | PatchExtraction(const std::vector& patch_widths, Dtype bg_value, bool normalize_depth) : 39 | patch_widths_(patch_widths), bg_value_(bg_value), normalize_depth_(normalize_depth) {} 40 | virtual ~PatchExtraction() {} 41 | 42 | std::vector > operator()(const cv::Mat_& depth, const HandSegmentationResult& segmentation) const; 43 | 44 | bool isReasonableHandPatch(const std::vector >& patches) const; 45 | 46 | const std::vector patchWidths() const { return patch_widths_; } 47 | bool normalizeDepth() const { return normalize_depth_; } 48 | 49 | private: 50 | const std::vector patch_widths_; 51 | Dtype bg_value_; 52 | const bool normalize_depth_; 53 | }; 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /include/inference_rf_simple.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef INFERENCE_RF_SIMPLE_H 30 | #define INFERENCE_RF_SIMPLE_H 31 | 32 | #include "inference.h" 33 | 34 | #include "pose.pb.h" 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | template 42 | class InferenceRfSimple : public Inference { 43 | public: 44 | InferenceRfSimple(const pose::RFParameter& param) { 45 | rv::io::CompressedSerializationIn serialization(param.forest_path()); 46 | forest_ = boost::make_shared(); 47 | forest_->Load(serialization); 48 | 49 | if(param.transform_csv() != "") { 50 | T_ = rv::rf::CreateRfMat(0, 0); 51 | rv::eigen::ReadCsv(param.transform_csv(), *T_); 52 | } 53 | } 54 | 55 | virtual ~InferenceRfSimple() {} 56 | 57 | virtual std::vector operator()(const std::vector >& patches) const; 58 | private: 59 | rv::rf::ForestPtr forest_; 60 | rv::rf::RfMatPtr T_; 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /include/data_provider_cam.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifdef BUILD_DATA_PROVIDER_CAM 30 | 31 | #ifndef DATA_PROVIDER_CAM_H 32 | #define DATA_PROVIDER_CAM_H 33 | 34 | #include "data_provider.h" 35 | #include "pose.pb.h" 36 | 37 | #include 38 | 39 | #include 40 | 41 | template 42 | class DataProviderCam : public DataProvider { 43 | public: 44 | DataProviderCam(const pose::DataProviderParameter& param); 45 | virtual ~DataProviderCam(); 46 | 47 | virtual boost::filesystem::path depthPath(int idx) const; 48 | virtual void shuffle() {} 49 | 50 | protected: 51 | virtual std::vector< cv::Vec3f > gt_internal(int idx) const; 52 | virtual cv::Vec3f hint2d_internal(int idx) const; 53 | virtual cv::Mat_< Dtype > depth_internal(int idx); 54 | virtual cv::Mat_< Dtype > ir_internal(int idx); 55 | 56 | private: 57 | rv::camera::Camera* cam_; 58 | bool delete_cam_; 59 | 60 | std::string im_type_depth_; 61 | std::string im_type_ir_; 62 | 63 | cv::Mat_ ir_; 64 | cv::Mat_ depth_; 65 | }; 66 | 67 | #endif 68 | 69 | #endif //BUILD_DATA_PROVIDER_CAM 70 | -------------------------------------------------------------------------------- /include/inference_heatmap_net.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef INFERENCE_HEATMAP_NET_H 30 | #define INFERENCE_HEATMAP_NET_H 31 | 32 | #include "inference.h" 33 | 34 | #include 35 | #include 36 | 37 | #include "pose.pb.h" 38 | 39 | template 40 | class InferenceHeatmapNet : public Inference { 41 | public: 42 | InferenceHeatmapNet(boost::shared_ptr > net, const pose::NNHeatmapParameter& param, int n_pts) : 43 | net_(net), param_(param), n_pts_(n_pts) {} 44 | virtual ~InferenceHeatmapNet() {} 45 | 46 | virtual std::vector operator()(const std::vector >& patches) const; 47 | 48 | protected: 49 | 50 | virtual std::vector inference2d() const; 51 | 52 | virtual void inferenceD(std::vector& es) const; 53 | virtual void patchFitD(const std::vector >& patches, std::vector& es) const; 54 | 55 | 56 | private: 57 | boost::shared_ptr > net_; 58 | const pose::NNHeatmapParameter& param_; 59 | int n_pts_; 60 | 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/inference_rf_simple.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "inference_rf_simple.h" 30 | 31 | template 32 | std::vector< cv::Vec3f > InferenceRfSimple::operator()(const std::vector< HandPatch< Dtype > >& patches) const { 33 | 34 | cv::Mat_ data = patches[0].patch_; 35 | int data_dim = data.rows * data.cols; 36 | 37 | rv::rf::RfMatPtr patch = rv::rf::CreateRfMat(data_dim, 1); 38 | int idx = 0; 39 | for(int w = 0; w < data.cols; ++w) { 40 | for(int h = 0; h < data.rows; ++h) { 41 | (*patch)(idx, 0) = data(h, w); 42 | idx++; 43 | } 44 | } 45 | 46 | if(T_ != 0) { 47 | *patch = (*T_) * (*patch); 48 | } 49 | 50 | rv::rf::SamplePtr sample = boost::make_shared(patch); 51 | rv::rf::VecPtrTargetPtr estimated_targets = forest_->inferencemt(sample); 52 | rv::rf::RfMatPtr result = (*estimated_targets)[0]->data(); 53 | 54 | int result_blob_data_idx = 0; 55 | std::vector es(result->rows() / 3); 56 | for(int row = 0; row < result->rows() / 3; ++row, result_blob_data_idx += 3) { 57 | Dtype x = (*result)(result_blob_data_idx, 0); 58 | Dtype y = (*result)(result_blob_data_idx + 1, 0); 59 | Dtype z = (*result)(result_blob_data_idx + 2, 0); 60 | 61 | es[row](0) = x; 62 | es[row](1) = y; 63 | es[row](2) = z; 64 | 65 | // std::cout << es[row] << std::endl; 66 | } 67 | 68 | return es; 69 | } 70 | 71 | 72 | template class InferenceRfSimple; 73 | template class InferenceRfSimple; 74 | -------------------------------------------------------------------------------- /include/projection.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef PROJECTION_H 30 | #define PROJECTION_H 31 | 32 | #include 33 | 34 | namespace cv { 35 | typedef Matx Matx18d; 36 | } 37 | 38 | 39 | class Projection { 40 | public: 41 | Projection(const cv::Matx33d& K, const cv::Matx18d d) : K_(K), d_(d) {}; 42 | virtual ~Projection() {} 43 | 44 | virtual cv::Vec3f to2D(const cv::Vec3f& v3) const = 0; 45 | virtual cv::Vec3f to3D(const cv::Vec3f& v2) const = 0; 46 | 47 | virtual void to2D(const std::vector& v3, std::vector& v2) const; 48 | virtual void to3D(const std::vector& v2, std::vector& v3) const; 49 | 50 | virtual cv::Matx33d K() const { return K_; } 51 | virtual cv::Matx18d d() const { return d_; } 52 | 53 | protected: 54 | const cv::Matx33d K_; 55 | const cv::Matx18d d_; 56 | }; 57 | 58 | 59 | class OrthographicProjection : public Projection { 60 | public: 61 | OrthographicProjection(const cv::Matx33d& K, const cv::Matx18d d) : 62 | Projection(K, d) {}; 63 | virtual ~OrthographicProjection() {} 64 | 65 | virtual cv::Vec3f to2D(const cv::Vec3f& v3) const; 66 | virtual cv::Vec3f to3D(const cv::Vec3f& v2) const; 67 | }; 68 | 69 | 70 | class ProjectiveProjection : public Projection { 71 | public: 72 | ProjectiveProjection(const cv::Matx33d& K, const cv::Matx18d d) : 73 | Projection(K, d) {}; 74 | virtual ~ProjectiveProjection() {} 75 | 76 | virtual cv::Vec3f to2D(const cv::Vec3f& v3) const; 77 | virtual cv::Vec3f to3D(const cv::Vec3f& v2) const; 78 | }; 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /include/make_data_provider.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef MAKE_DATA_PROVIDER_H 30 | #define MAKE_DATA_PROVIDER_H 31 | 32 | #include 33 | 34 | #include "data_provider.h" 35 | #include "data_provider_blender.h" 36 | #include "data_provider_csv.h" 37 | #include "data_provider_dtang.h" 38 | #include "data_provider_cam.h" 39 | 40 | #include "pose.pb.h" 41 | 42 | #include 43 | 44 | 45 | template 46 | boost::shared_ptr< DataProvider > makeDataProvider(const pose::DataProviderParameter& param) { 47 | boost::shared_ptr > data_provider; 48 | 49 | if(param.type() == pose::DataProviderParameter_DataProviderType_DTANG) { 50 | std::cout << "[INFO] Creating DataProviderDTang" << std::endl; 51 | data_provider = boost::make_shared >(param); 52 | } 53 | else if(param.type() == pose::DataProviderParameter_DataProviderType_CSV) { 54 | std::cout << "[INFO] Creating DataProviderCsv" << std::endl; 55 | data_provider = boost::make_shared >(param); 56 | } 57 | else if(param.type() == pose::DataProviderParameter_DataProviderType_BLENDER) { 58 | std::cout << "[INFO] Creating DataProviderBlender" << std::endl; 59 | data_provider = boost::make_shared >(param); 60 | } 61 | #ifdef BUILD_DATA_PROVIDER_CAM 62 | else if(param.type() == pose::DataProviderParameter_DataProviderType_CAM) { 63 | std::cout << "[INFO] Creating DataProviderCam" << std::endl; 64 | data_provider = boost::make_shared >(param); 65 | } 66 | #endif 67 | 68 | if(param.shuffle()) { 69 | data_provider->shuffle(); 70 | } 71 | 72 | return data_provider; 73 | } 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /include/make_hand_segmentation.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef MAKE_HAND_SEGMENTATION_H 30 | #define MAKE_HAND_SEGMENTATION_H 31 | 32 | #include "hand_segmentation.h" 33 | 34 | #include "hand_segmentation_meanshift.h" 35 | #include "hand_segmentation_rf.h" 36 | #include "hand_segmentation_threshold.h" 37 | 38 | #include "pose.pb.h" 39 | 40 | #include 41 | 42 | 43 | template 44 | boost::shared_ptr > makeHandSegmentation(const pose::SegmentationParameter& param, 45 | const Projection& projection, 46 | Dtype bg_value) { 47 | boost::shared_ptr > hand_segmentation; 48 | 49 | if(param.type() == pose::SegmentationParameter_SegmentationType_MEANSHIFT) { 50 | std::cout << "[INFO] Creating HandSegmentationMeanshift" << std::endl; 51 | hand_segmentation = boost::make_shared >(param, projection, bg_value); 52 | } 53 | else if(param.type() == pose::SegmentationParameter_SegmentationType_THRESHOLD) { 54 | std::cout << "[INFO] Creating HandSegmentationThreshold" << std::endl; 55 | hand_segmentation = boost::make_shared >(param, projection, bg_value); 56 | } 57 | else if(param.type() == pose::SegmentationParameter_SegmentationType_RF) { 58 | std::cout << "[INFO] Creating HandSegmentationRf" << std::endl; 59 | hand_segmentation = boost::make_shared >(param, projection, bg_value); 60 | } 61 | else { 62 | throw std::runtime_error("[ERROR] unknown hand segmentation"); 63 | } 64 | 65 | return hand_segmentation; 66 | } 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /src/projection.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "projection.h" 30 | 31 | 32 | void Projection::to2D(const std::vector< cv::Vec3f >& v3, std::vector< cv::Vec3f >& v2) const { 33 | v2.resize(v3.size()); 34 | 35 | for(size_t pts_idx = 0; pts_idx < v3.size(); ++pts_idx) { 36 | v2[pts_idx] = to2D(v3[pts_idx]); 37 | } 38 | } 39 | 40 | void Projection::to3D(const std::vector< cv::Vec3f >& v2, std::vector< cv::Vec3f >& v3) const { 41 | v3.resize(v2.size()); 42 | 43 | for(size_t pts_idx = 0; pts_idx < v2.size(); ++pts_idx) { 44 | v3[pts_idx] = to3D(v2[pts_idx]); 45 | } 46 | } 47 | 48 | 49 | //------------------------------------------------------------------------------ 50 | cv::Vec3f OrthographicProjection::to2D(const cv::Vec3f& v3) const { 51 | cv::Vec3f v2; 52 | 53 | v2[0] = K_(0, 0) * v3(0) / v3(2) + K_(0, 2); 54 | v2[1] = K_(1, 1) * v3(1) / v3(2) + K_(1, 2); 55 | v2[2] = v3(2); 56 | 57 | return v2; 58 | } 59 | 60 | cv::Vec3f OrthographicProjection::to3D(const cv::Vec3f& v2) const { 61 | cv::Vec3f v3; 62 | 63 | v3[0] = v2(2) * (v2(0) - K_(0, 2)) / K_(0, 0); 64 | v3[1] = v2(2) * (v2(1) - K_(1, 2)) / K_(1, 1); 65 | v3[2] = v2(2); 66 | 67 | return v3; 68 | } 69 | 70 | 71 | //------------------------------------------------------------------------------ 72 | cv::Vec3f ProjectiveProjection::to2D(const cv::Vec3f& v3) const { 73 | //TODO 74 | cv::Vec3f v2; 75 | 76 | v2[0] = K_(0, 0) * v3(0) / v3(2) + K_(0, 2); 77 | v2[1] = K_(1, 1) * v3(1) / v3(2) + K_(1, 2); 78 | v2[2] = v3(2); 79 | 80 | return v2; 81 | } 82 | 83 | cv::Vec3f ProjectiveProjection::to3D(const cv::Vec3f& v2) const { 84 | //TODO 85 | cv::Vec3f v3; 86 | 87 | v3[0] = v2(2) * (v2(0) - K_(0, 2)) / K_(0, 0); 88 | v3[1] = v2(2) * (v2(1) - K_(1, 2)) / K_(1, 1); 89 | v3[2] = v2(2); 90 | 91 | return v3; 92 | } 93 | -------------------------------------------------------------------------------- /include/create_data.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef CREATE_DATA_H 30 | #define CREATE_DATA_H 31 | 32 | #include "data_provider.h" 33 | #include "hand_segmentation.h" 34 | #include "hand_patch.h" 35 | #include "patch_extraction.h" 36 | #include "pose.pb.h" 37 | 38 | #include 39 | #include 40 | 41 | #include 42 | #include 43 | 44 | #include 45 | 46 | template 47 | class CreateData { 48 | public: 49 | CreateData(const pose::CreateDataParameter& param, 50 | DataProvider& data_provider, 51 | const HandSegmentation& hand_segmentation, 52 | const PatchExtraction& patch_extraction) : 53 | param_(param), data_provider_(data_provider), 54 | hand_segmentation_(hand_segmentation), 55 | patch_extraction_(patch_extraction) { 56 | 57 | createRotations(); 58 | } 59 | 60 | virtual ~CreateData() {}; 61 | 62 | virtual void data2Hdf5() const; 63 | 64 | 65 | virtual void mirror(const cv::Mat_& depth, 66 | const cv::Mat_& ir, const std::vector& anno, 67 | cv::Mat_& mirrored_depth, cv::Mat_& mirrored_ir, 68 | std::vector& mirrored_anno) const; 69 | virtual void rotate(const cv::Mat_& depth, 70 | const cv::Mat_& ir, const std::vector& anno, 71 | cv::Mat_& rotated_depth, cv::Mat_& rotated_ir, 72 | std::vector& rotated_anno, float rot_deg) const; 73 | 74 | protected: 75 | virtual void createRotations(); 76 | 77 | protected: 78 | const pose::CreateDataParameter& param_; 79 | DataProvider& data_provider_; 80 | const HandSegmentation& hand_segmentation_; 81 | const PatchExtraction& patch_extraction_; 82 | 83 | std::vector rotations_; 84 | }; 85 | 86 | 87 | 88 | 89 | #endif 90 | -------------------------------------------------------------------------------- /src/main_nn_train.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | namespace po = boost::program_options; 38 | 39 | 40 | typedef float Dtype; 41 | 42 | 43 | int main(int argc, char** argv) { 44 | 45 | boost::filesystem::path solver_path; 46 | boost::filesystem::path weights_path; 47 | int cuda_device; 48 | 49 | po::options_description desc; 50 | desc.add_options() 51 | ("solver_path", po::value()->required(), "") 52 | ("weights_path", po::value()->default_value(""), "") 53 | ("cuda_device", po::value()->default_value(0), "") 54 | ; 55 | 56 | po::variables_map vm; 57 | try { 58 | po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); 59 | po::notify(vm); 60 | 61 | solver_path = vm["solver_path"].as(); 62 | weights_path = vm["weights_path"].as(); 63 | 64 | cuda_device = vm["cuda_device"].as(); 65 | if(cuda_device >= 0) { 66 | caffe::Caffe::SetDevice(cuda_device); 67 | caffe::Caffe::set_mode(caffe::Caffe::GPU); 68 | } 69 | else { 70 | caffe::Caffe::set_mode(caffe::Caffe::CPU); 71 | } 72 | } catch(std::exception& e) { 73 | std::cout << "ERROR: " << e.what() << std::endl << std::endl; 74 | std::cout << desc << std::endl; 75 | return -1; 76 | } 77 | 78 | //read solver config 79 | caffe::SolverParameter solver_param; 80 | caffe::ReadProtoFromTextFileOrDie(solver_path.string(), &solver_param); 81 | 82 | //get solver 83 | boost::shared_ptr > solver(caffe::GetSolver(solver_param)); 84 | 85 | //copy weights if given 86 | if(weights_path != "") { 87 | if(!boost::filesystem::exists(weights_path)) { 88 | std::cout << "[ERROR] weights path does not exist" << std::endl; 89 | return -1; 90 | } 91 | solver->net()->CopyTrainedLayersFrom(weights_path.string()); 92 | } 93 | 94 | //solve network 95 | solver->Solve(); 96 | 97 | return 0; 98 | } 99 | -------------------------------------------------------------------------------- /src/inference_regression_net.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "inference_regression_net.h" 30 | #include "create_data.h" 31 | 32 | #include 33 | #include 34 | 35 | #include 36 | 37 | template 38 | std::vector< cv::Vec3f > InferenceRegressionNet::operator()(const std::vector >& patches) const { 39 | std::vector > > blobs(patches.size()); //have to store shared_ptr, otherwise => bad time 40 | std::vector* > bottom(patches.size()); 41 | for(size_t hp_idx = 0; hp_idx < patches.size(); ++hp_idx) { 42 | cv::Mat_ patch = patches[hp_idx].patch_; 43 | blobs[hp_idx] = rv::ocv::mat2Blob(patch); 44 | 45 | bottom[hp_idx] = blobs[hp_idx].get(); 46 | } 47 | 48 | Dtype loss; 49 | net_->Forward(bottom, &loss); 50 | 51 | boost::shared_ptr > blob_ptr; 52 | if(net_->has_blob("ip_regression")) { 53 | blob_ptr = net_->blob_by_name("ip_regression"); 54 | } 55 | else if(net_->has_blob("ip_regression_16")) { 56 | blob_ptr = net_->blob_by_name("ip_regression_16"); 57 | } 58 | else if(net_->has_blob("ip_regression_20")) { 59 | blob_ptr = net_->blob_by_name("ip_regression_20"); 60 | } 61 | else { 62 | LOG(ERROR) << "no blob for regression found"; 63 | } 64 | const Dtype* result_blob_data = blob_ptr->cpu_data(); 65 | 66 | int result_blob_data_idx = 0; 67 | std::vector es(blob_ptr->channels() / 3); 68 | for(int row = 0; row < blob_ptr->channels() / 3; ++row, result_blob_data_idx += 3) { 69 | Dtype x = result_blob_data[result_blob_data_idx]; 70 | Dtype y = result_blob_data[result_blob_data_idx + 1]; 71 | Dtype z = result_blob_data[result_blob_data_idx + 2]; 72 | 73 | es[row](0) = x; 74 | es[row](1) = y; 75 | es[row](2) = z; 76 | } 77 | 78 | // for(size_t idx = 0; idx < es.size(); ++idx) { 79 | // std::cout << es[idx] << std::endl; 80 | // } 81 | 82 | return es; 83 | } 84 | 85 | template class InferenceRegressionNet; 86 | template class InferenceRegressionNet; 87 | -------------------------------------------------------------------------------- /include/make_projection.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef MAKE_PROJECTION_H 30 | #define MAKE_PROJECTION_H 31 | 32 | #include 33 | #include 34 | 35 | #include "projection.h" 36 | 37 | #include "pose.pb.h" 38 | 39 | #include 40 | #include 41 | 42 | 43 | inline boost::shared_ptr makeProjection(const pose::ProjectionParameter& param, 44 | const boost::filesystem::path& hint) { 45 | 46 | boost::shared_ptr projection; 47 | 48 | //automatic find calib if possible from data_provider depth path 49 | std::string calib_path = param.calib_path(); 50 | if(calib_path == "") { 51 | boost::filesystem::path parent = hint; 52 | 53 | while(parent.parent_path() != parent && calib_path == "") { 54 | if(boost::filesystem::exists(parent / "calibration_depth.txt")) { 55 | calib_path = (parent / "calibration_depth.txt").string(); 56 | } 57 | 58 | if(boost::filesystem::exists(parent / "calib.txt")) { 59 | calib_path = (parent / "calib.txt").string(); 60 | } 61 | 62 | parent = parent.parent_path(); 63 | } 64 | } 65 | 66 | if(calib_path != "") { 67 | cv::Matx33d K = 0; 68 | cv::Matx18d d = 0; 69 | 70 | std::ifstream fin(calib_path.c_str()); 71 | 72 | fin >> K(0, 0); 73 | fin >> K(1, 1); 74 | fin >> K(0, 2); 75 | fin >> K(1, 2); 76 | K(2, 2) = 1; 77 | 78 | for(int d_idx = 0; d_idx < d.cols; ++d_idx) { 79 | if(fin.good()) { 80 | fin >> d(0, d_idx); 81 | } 82 | } 83 | 84 | fin.close(); 85 | 86 | if(param.type() == pose::ProjectionParameter_ProjectionType_ORTHOGRAPHIC) { 87 | std::cout << "[INFO] Creating OrthographicProjection" << std::endl; 88 | projection = boost::make_shared(K, d); 89 | } 90 | else if(param.type() == pose::ProjectionParameter_ProjectionType_PROJECTIVE) { 91 | std::cout << "[INFO] Creating ProjectiveProjection" << std::endl; 92 | projection = boost::make_shared(K, d); 93 | } 94 | } 95 | else { 96 | std::cout << "[ERROR] no calibration file found" << std::endl; 97 | } 98 | 99 | if(projection == 0) { 100 | std::cout << "[ERROR] No Projection created" << std::endl; 101 | } 102 | 103 | return projection; 104 | } 105 | 106 | #endif 107 | -------------------------------------------------------------------------------- /include/hand_segmentation_rf.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef HAND_SEGMENTATION_RF_H 30 | #define HAND_SEGMENTATION_RF_H 31 | 32 | #include "hand_segmentation.h" 33 | #include "data_provider.h" 34 | #include "projection.h" 35 | 36 | #include 37 | 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | 48 | template 49 | class HandSegmentationRf : public HandSegmentation { 50 | public: 51 | HandSegmentationRf(const pose::SegmentationParameter& param, const Projection& projection, Dtype bg_value) : 52 | HandSegmentation(param), bg_value_(bg_value) { 53 | //create and load forest 54 | forest_ = boost::make_shared(); 55 | 56 | std::string forest_path = param.rf_param().forest_path(); 57 | if(boost::filesystem::exists(forest_path)) { 58 | rv::io::CompressedSerializationIn serialization(param.rf_param().forest_path()); 59 | forest_->Load(serialization); 60 | } 61 | else { 62 | throw std::runtime_error("[ERROR] forest path does not exist in HandSegmentationRf"); 63 | } 64 | 65 | //create morph structure element 66 | int struc_elem_size = param.rf_param().struc_elem_size(); 67 | struc_elem_ = cv::getStructuringElement(cv::MORPH_CROSS, cv::Size(struc_elem_size, struc_elem_size)); 68 | std::cout << "struc_elem_: " << std::endl << struc_elem_ << std::endl; 69 | } 70 | 71 | virtual ~HandSegmentationRf() {} 72 | 73 | virtual std::vector operator()(const cv::Mat_& depth, const cv::Mat_& ir, const cv::Vec3f& hint_2d) const; 74 | 75 | static rv::rf::ForestPtr train( 76 | boost::shared_ptr > data_provider, 77 | boost::shared_ptr > hand_segmentation, 78 | const rv::rf::ForestParameter& params); 79 | 80 | protected: 81 | rv::rf::ForestPtr forest_; 82 | cv::Mat struc_elem_; 83 | 84 | Dtype bg_value_; 85 | }; 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /src/main_create_data.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include 30 | 31 | #include 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include 41 | 42 | #include 43 | #include 44 | #include 45 | #include 46 | namespace po = boost::program_options; 47 | 48 | typedef float Dtype; 49 | 50 | 51 | int main(int argc, char** argv) { 52 | 53 | boost::filesystem::path config_path; 54 | int n_threads; 55 | 56 | po::options_description desc; 57 | desc.add_options() 58 | ("config_path", po::value()->required(), "") 59 | ("n_threads", po::value()->default_value(0), "<=0 as many as possible") 60 | ; 61 | 62 | po::variables_map vm; 63 | try { 64 | po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); 65 | po::notify(vm); 66 | config_path = vm["config_path"].as(); 67 | n_threads = vm["n_threads"].as(); 68 | 69 | if(n_threads > 0) { 70 | omp_set_num_threads(n_threads); 71 | } 72 | } catch(std::exception& e) { 73 | std::cout << "ERROR: " << e.what() << std::endl << std::endl; 74 | std::cout << desc << std::endl; 75 | return -1; 76 | } 77 | 78 | //read config 79 | pose::CreateDataParameter param; 80 | if(!readProtoFromTextFile(config_path, ¶m)) { 81 | std::cout << "ERROR: parsing config_path: " << config_path << std::endl; 82 | return -1; 83 | } 84 | 85 | //Init DataProvider 86 | boost::shared_ptr > data_provider = makeDataProvider(param.data_provider_param()); 87 | 88 | //Init HandSegmentation 89 | boost::shared_ptr > hand_segmentation = makeHandSegmentation(param.segmentation_param(), *(data_provider->projection()), data_provider->bgValue()); 90 | 91 | //Init ExtractPatch 92 | boost::shared_ptr > patch_extraction = makePatchExtraction(param.patch_extraction_param(), data_provider->bgValue()); 93 | 94 | //Init CreateData 95 | CreateData create_data(param, *data_provider, *hand_segmentation, *patch_extraction); 96 | // create_data.addHeatmaps(18, 3); 97 | 98 | create_data.data2Hdf5(); 99 | 100 | return 0; 101 | } 102 | -------------------------------------------------------------------------------- /include/make_inference.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef CREATE_TEST_PATCH_H 30 | #define CREATE_TEST_PATCH_H 31 | 32 | #include "inference_heatmap_net.h" 33 | #include "inference_regression_net.h" 34 | #include "inference_rf_simple.h" 35 | #include "inference_dummy.h" 36 | 37 | #include "pose.pb.h" 38 | 39 | #include 40 | 41 | template 42 | boost::shared_ptr< Inference< Dtype > > makeInference(const pose::InferenceParameter& param, int n_pts) { 43 | boost::shared_ptr > inference; 44 | 45 | if(param.type() == pose::InferenceParameter_InferenceType_RF_SIMPLE_REGRESSION) { 46 | std::cout << "[INFO] Creating InferenceRfSimple" << std::endl; 47 | inference = boost::make_shared >(param.rf_param()); 48 | } 49 | else if(param.type() == pose::InferenceParameter_InferenceType_NN_REGRESSION) { 50 | std::cout << "[INFO] Creating InferenceRegressionNet" << std::endl; 51 | caffe::Caffe::set_mode(caffe::Caffe::GPU); 52 | // caffe::Caffe::set_mode(caffe::Caffe::CPU); 53 | 54 | std::string net_path = param.nn_regression_param().net_path(); 55 | std::string weights_path = param.nn_regression_param().weights_path(); 56 | boost::shared_ptr > net = boost::make_shared >(net_path, caffe::TEST); 57 | net->CopyTrainedLayersFrom(weights_path); 58 | std::cout << "caffe::mode: " << caffe::Caffe::mode() << " CPU: " << caffe::Caffe::CPU << "/GPU: " << caffe::Caffe::GPU << std::endl; 59 | 60 | inference = boost::make_shared >(net); 61 | } 62 | else if(param.type() == pose::InferenceParameter_InferenceType_NN_HEATMAP) { 63 | std::cout << "[INFO] Creating InferenceHeatmapNet" << std::endl; 64 | caffe::Caffe::set_mode(caffe::Caffe::GPU); 65 | // caffe::Caffe::set_mode(caffe::Caffe::CPU); 66 | 67 | std::string net_path = param.nn_heatmap_param().net_path(); 68 | std::string weights_path = param.nn_heatmap_param().weights_path(); 69 | boost::shared_ptr > net = boost::make_shared >(net_path, caffe::TEST); 70 | net->CopyTrainedLayersFrom(weights_path); 71 | std::cout << "caffe::mode: " << caffe::Caffe::mode() << " CPU: " << caffe::Caffe::CPU << "/GPU: " << caffe::Caffe::GPU << std::endl; 72 | 73 | inference = boost::make_shared >(net, param.nn_heatmap_param(), n_pts); 74 | } 75 | else { 76 | std::cout << "[INFO] Make InferenceDummy" << std::endl; 77 | inference = boost::make_shared >(n_pts); 78 | } 79 | 80 | return inference; 81 | } 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /src/hand_segmentation_threshold.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "hand_segmentation_threshold.h" 30 | 31 | #include 32 | 33 | #include 34 | 35 | 36 | template 37 | std::vector< HandSegmentationResult > HandSegmentationThreshold::operator()(const cv::Mat_< Dtype >& depth_in, const cv::Mat_& ir, const cv::Vec3f& hint_2d) const { 38 | const pose::SegmentationThresholdParameter& threshold_param = this->param_.threshold_param(); 39 | float min_ir = threshold_param.min_ir(); 40 | float max_ir = threshold_param.max_ir(); 41 | float min_d = threshold_param.min_d(); 42 | float max_d = threshold_param.max_d(); 43 | float t = threshold_param.t(); 44 | 45 | 46 | cv::Mat_ depth; 47 | if(threshold_param.type() == pose::SegmentationThresholdParameter_Type_NEAREST) { 48 | cv::medianBlur(depth_in, depth, 5); 49 | } 50 | else { 51 | depth = depth_in; 52 | } 53 | 54 | 55 | float min_depth = 1e9; 56 | if(threshold_param.type() == pose::SegmentationThresholdParameter_Type_NEAREST) { 57 | for(int row = 0; row < depth.rows; ++row) { 58 | for(int col = 0; col < depth.cols; ++col) { 59 | float d = depth(row, col); 60 | if(d < min_depth) min_depth = d; 61 | } 62 | } 63 | } 64 | 65 | 66 | cv::Mat_ mask = cv::Mat_::zeros(depth.rows, depth.cols); 67 | int min_y = mask.rows; 68 | int max_y = 0; 69 | int min_x = mask.cols; 70 | int max_x = 0; 71 | for(int row = 0; row < mask.rows; ++row) { 72 | for(int col = 0; col < mask.cols; ++col) { 73 | Dtype d = depth(row, col); 74 | Dtype i = ir(row, col); 75 | 76 | if(threshold_param.type() == pose::SegmentationThresholdParameter_Type_RANGE) { 77 | if(d >= min_d && d <= max_d && i >= min_ir && i <= max_ir) { 78 | mask(row, col ) = 1; 79 | 80 | if(row < min_y) min_y = row; 81 | if(row > max_y) max_y = row; 82 | if(col < min_x) min_x = col; 83 | if(col > max_x) max_x = col; 84 | } 85 | } 86 | else if(threshold_param.type() == pose::SegmentationThresholdParameter_Type_NEAREST) { 87 | if(d >= min_depth && d <= (min_depth + t)) { 88 | mask(row, col ) = 1; 89 | 90 | if(row < min_y) min_y = row; 91 | if(row > max_y) max_y = row; 92 | if(col < min_x) min_x = col; 93 | if(col > max_x) max_x = col; 94 | } 95 | } 96 | 97 | } 98 | } 99 | 100 | cv::Rect roi(min_x, min_y, max_x - min_x + 1, max_y - min_y + 1); 101 | 102 | std::vector result(1); 103 | result[0].roi_ = roi; 104 | result[0].mask_ = mask; 105 | 106 | return result; 107 | } 108 | 109 | 110 | template class HandSegmentationThreshold; 111 | template class HandSegmentationThreshold; 112 | -------------------------------------------------------------------------------- /src/main_rf_train_segmentation.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include 30 | 31 | #include "make_hand_segmentation.h" 32 | #include "make_data_provider.h" 33 | #include "make_projection.h" 34 | #include "common.h" 35 | #include "hand_segmentation_rf.h" 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | 45 | #include 46 | 47 | #include 48 | #include 49 | namespace po = boost::program_options; 50 | 51 | 52 | typedef float Dtype; 53 | 54 | 55 | int main(int argc, char** argv) { 56 | std::string forest_config_path; 57 | boost::filesystem::path config_path; 58 | boost::filesystem::path forest_path; 59 | 60 | po::options_description desc; 61 | desc.add_options() 62 | ("config_path", po::value()->required(), "") 63 | ("forest_config_path", po::value()->required(), "") 64 | ("forest_out_path", po::value()->required(), "") 65 | ; 66 | 67 | po::variables_map vm; 68 | try { 69 | po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); 70 | po::notify(vm); 71 | config_path = vm["config_path"].as(); 72 | forest_config_path = vm["forest_config_path"].as(); 73 | forest_path = vm["forest_out_path"].as(); 74 | } catch(std::exception& e) { 75 | std::cout << "ERROR: " << e.what() << std::endl << std::endl; 76 | std::cout << desc << std::endl; 77 | return -1; 78 | } 79 | 80 | //read forest config 81 | rv::rf::ForestParameter forest_param; 82 | if(!readProtoFromTextFile(forest_config_path, &forest_param)) { 83 | std::cout << "ERROR: parsing config_path: " << forest_config_path << std::endl; 84 | return -1; 85 | } 86 | 87 | //read config 88 | pose::CreateDataParameter param; 89 | if(!readProtoFromTextFile(config_path, ¶m)) { 90 | std::cout << "ERROR: parsing config_path: " << config_path << std::endl; 91 | return -1; 92 | } 93 | 94 | //Init DataProvider 95 | boost::shared_ptr > data_provider = makeDataProvider(param.data_provider_param()); 96 | boost::shared_ptr projection = data_provider->projection(); 97 | 98 | //Init HandSegmentation 99 | boost::shared_ptr > hand_segmentation = makeHandSegmentation(param.segmentation_param(), *projection, data_provider->bgValue()); 100 | 101 | //train forest 102 | rv::rf::ForestPtr forest = HandSegmentationRf::train(data_provider, hand_segmentation, forest_param); 103 | 104 | //save forest 105 | if(!boost::filesystem::exists(forest_path.parent_path())) { 106 | boost::filesystem::create_directories(forest_path.parent_path()); 107 | } 108 | rv::io::CompressedSerializationOut serialization(forest_path.string()); 109 | forest->Save(serialization); 110 | std::cout << "[INFO] stored forest to " << forest_path << std::endl; 111 | } 112 | -------------------------------------------------------------------------------- /include/common.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef COMMON_H 30 | #define COMMON_H 31 | 32 | #include 33 | #include 34 | 35 | #include 36 | #include 37 | 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | #include 45 | #include 46 | 47 | #include "hand_patch.h" 48 | 49 | #include "pose.pb.h" 50 | 51 | 52 | void depthmap2pts(const cv::Mat_& dm, const cv::Mat_& ir, 53 | const cv::Mat_& img, const float& max_d, const float& min_ir, 54 | std::vector& pts, std::vector& colors); 55 | 56 | std::vector readAnno(boost::filesystem::path anno_path, 57 | int n_joints); 58 | 59 | void writePoints(std::ostream& out, const std::vector& pts); 60 | 61 | cv::Mat_ readExrDepth(const boost::filesystem::path& p); 62 | 63 | bool readProtoFromTextFile(const boost::filesystem::path& path, 64 | google::protobuf::Message* proto); 65 | 66 | //----------------------------------------------------------------------------- 67 | void getIdTs(boost::filesystem::path depth_map_path, std::string& id, 68 | std::string& ts); 69 | std::string getAnnotator(boost::filesystem::path anno_path); 70 | std::vector lsAnnoPaths( 71 | boost::filesystem::path depth_map_path); 72 | boost::filesystem::path infraredPath(boost::filesystem::path depth_map_path); 73 | boost::filesystem::path blenderRgbPath(boost::filesystem::path depth_map_path); 74 | 75 | 76 | cv::Rect enclosingRect(const std::vector& pts); 77 | 78 | 79 | template 80 | cv::Mat_ annoShow(const cv::Mat_& im, 81 | const std::vector& anno, Dtype bg_value) { 82 | std::vector colors = rv::ocv::hsv(anno.size()); 83 | static rv::ocv::ColorMap& cmap = rv::ocv::ColorMapCoolWarm::i(); 84 | 85 | std::vector pts(anno.size()); 86 | for(int idx = 0; idx < pts.size(); ++idx) { 87 | pts[idx].x = anno[idx](0); 88 | pts[idx].y = anno[idx](1); 89 | } 90 | cv::Mat_ im3 = cmap.Map(im, 0, bg_value); 91 | if(pts.size() > 0) { 92 | rv::ocv::drawPoints(im3, pts, colors); 93 | } 94 | 95 | return im3; 96 | } 97 | 98 | template 99 | cv::Mat_ handPatchShow(const HandPatch& hp, 100 | const std::vector& anno) { 101 | std::vector colors = rv::ocv::hsv(anno.size()); 102 | static rv::ocv::ColorMap& cmap = rv::ocv::ColorMapCoolWarm::i(); 103 | 104 | std::vector adjusted_anno; 105 | if(anno.size() > 0) { 106 | adjusted_anno = hp.uncenterAnno(anno); 107 | } 108 | 109 | std::vector pts(anno.size()); 110 | for(int idx = 0; idx < pts.size(); ++idx) { 111 | pts[idx].x = adjusted_anno[idx](0); 112 | pts[idx].y = adjusted_anno[idx](1); 113 | } 114 | 115 | cv::Mat_ im3 = cmap.Map(hp.patch_, -3, 3); 116 | if(pts.size() > 0) { 117 | rv::ocv::drawPoints(im3, pts, colors, 2); 118 | } 119 | 120 | return im3; 121 | } 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /src/hand_patch.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "hand_patch.h" 30 | 31 | template 32 | std::vector< cv::Vec3f > HandPatch::annoInHandPatch(const std::vector< cv::Vec3f >& anno, const cv::Rect& roi) const { 33 | std::vector adjusted_anno(anno.size()); 34 | for(int row = 0; row < adjusted_anno.size(); ++row) { 35 | Dtype x = (anno[row](0) - roi.x) * scale_ + col_offset_; 36 | Dtype y = (anno[row](1) - roi.y) * scale_ + row_offset_; 37 | Dtype z = (anno[row](2) - mean_depth_) / std_depth_; 38 | 39 | adjusted_anno[row](0) = x; 40 | adjusted_anno[row](1) = y; 41 | adjusted_anno[row](2) = z; 42 | } 43 | 44 | return centerAnno(adjusted_anno); 45 | } 46 | 47 | template 48 | std::vector< cv::Vec3f > HandPatch::annoFromHandPatch(const std::vector< cv::Vec3f >& anno, const cv::Rect& roi) const { 49 | std::vector adjusted_anno = uncenterAnno(anno); 50 | for(int row = 0; row < anno.size(); ++row) { 51 | Dtype x = adjusted_anno[row](0); 52 | Dtype y = adjusted_anno[row](1); 53 | Dtype z = adjusted_anno[row](2); 54 | 55 | x = ((x - col_offset_) / scale_) + roi.x; 56 | y = ((y - row_offset_) / scale_) + roi.y; 57 | z = z * std_depth_ + mean_depth_; 58 | 59 | adjusted_anno[row](0) = x; 60 | adjusted_anno[row](1) = y; 61 | adjusted_anno[row](2) = z; 62 | } 63 | 64 | return adjusted_anno; 65 | } 66 | 67 | template 68 | std::vector< cv::Vec3f > HandPatch::centerAnno(const std::vector< cv::Vec3f >& anno) const { 69 | std::vector adjusted_anno(anno.size()); 70 | for(int row = 0; row < anno.size(); ++row) { 71 | Dtype x = anno[row](0); 72 | Dtype y = anno[row](1); 73 | Dtype z = anno[row](2); 74 | 75 | x = (x - patch_.cols / 2.0) / (patch_.cols / 2.0); 76 | y = (y - patch_.rows / 2.0) / (patch_.rows / 2.0); 77 | 78 | adjusted_anno[row](0) = x; 79 | adjusted_anno[row](1) = y; 80 | adjusted_anno[row](2) = z; 81 | } 82 | return adjusted_anno; 83 | } 84 | 85 | template 86 | std::vector< cv::Vec3f > HandPatch::uncenterAnno(const std::vector< cv::Vec3f >& anno) const { 87 | std::vector adjusted_anno(anno.size()); 88 | for(int row = 0; row < anno.size(); ++row) { 89 | Dtype x = anno[row](0); 90 | Dtype y = anno[row](1); 91 | Dtype z = anno[row](2); 92 | 93 | x = x * (patch_.cols / 2.0) + patch_.cols / 2.0; 94 | y = y * (patch_.rows / 2.0) + patch_.rows / 2.0; 95 | 96 | adjusted_anno[row](0) = x; 97 | adjusted_anno[row](1) = y; 98 | adjusted_anno[row](2) = z; 99 | } 100 | return adjusted_anno; 101 | } 102 | 103 | 104 | template 105 | bool HandPatch::isReasonableCenteredAnno(const std::vector< cv::Vec3f >& anno, bool normalized) const { 106 | for(size_t anno_idx = 0; anno_idx < anno.size(); ++anno_idx) { 107 | cv::Vec3f a = anno[anno_idx]; 108 | Dtype x = a(0); 109 | Dtype y = a(1); 110 | Dtype z = a(2); 111 | 112 | if(x < -1.2 || x > 1.2 || y < -1.2 || y > 1.2 || (normalized && z < -5) || (normalized && z > 3.2)) { 113 | return false; 114 | } 115 | } 116 | 117 | return true; 118 | } 119 | 120 | 121 | template class HandPatch; 122 | template class HandPatch; 123 | -------------------------------------------------------------------------------- /src/data_provider_cam.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifdef BUILD_DATA_PROVIDER_CAM 30 | 31 | #include "data_provider_cam.h" 32 | #include "common.h" 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #include 40 | 41 | #include 42 | 43 | 44 | template 45 | DataProviderCam::DataProviderCam(const pose::DataProviderParameter& param) : DataProvider(param) { 46 | 47 | if(param.cam_param().type() == pose::CamParameter::NONE) { 48 | this->max_idx_ = 0; 49 | } 50 | else if(param.cam_param().type() == pose::CamParameter::CREATIVE) { 51 | rv::camera::Creative325Camera& cam = rv::camera::Creative325Camera::i(); 52 | cam_ = &cam; 53 | delete_cam_ = false; 54 | this->max_idx_ = std::numeric_limits::max(); 55 | im_type_depth_ = rv::camera::Creative325Camera::typeDepth(); 56 | im_type_ir_ = rv::camera::Creative325Camera::typeIr(); 57 | } 58 | else if(param.cam_param().type() == pose::CamParameter::PMD_PICO) { 59 | cam_ = new rv::camera::PmdPicoCamera(); 60 | delete_cam_ = true; 61 | this->max_idx_ = std::numeric_limits::max(); 62 | im_type_depth_ = rv::camera::PmdPicoCamera::typeDepth(); 63 | im_type_ir_ = rv::camera::PmdPicoCamera::typeIr(); 64 | } 65 | 66 | if(cam_ != 0) { 67 | cam_->registerType(im_type_depth_); 68 | cam_->registerType(im_type_ir_); 69 | 70 | cam_->start(); 71 | } 72 | } 73 | 74 | template 75 | DataProviderCam::~DataProviderCam() { 76 | if(cam_ != 0) { 77 | cam_->stop(); 78 | if(delete_cam_) { 79 | delete cam_; 80 | } 81 | } 82 | } 83 | 84 | template 85 | boost::filesystem::path DataProviderCam::depthPath(int idx) const { 86 | return ""; 87 | } 88 | 89 | template 90 | cv::Mat_< Dtype > DataProviderCam::depth_internal(int idx) { 91 | rv::camera::CameraImage img_depth; 92 | cam_->read(im_type_depth_, img_depth); 93 | depth_ = img_depth.im_; 94 | 95 | rv::camera::CameraImage img_ir; 96 | cam_->read(im_type_ir_, img_ir); 97 | ir_ = img_ir.im_; 98 | 99 | 100 | Dtype bg_value = this->param_.bg_value(); 101 | int median_ksize = 5; 102 | float min_ir = 150; 103 | 104 | for(int row = 0; row < depth_.rows; ++row) { 105 | for(int col = 0; col < depth_.cols; ++col) { 106 | Dtype d = depth_(row, col); 107 | Dtype i = ir_(row, col); 108 | 109 | if(d < 0 || d > bg_value || i < min_ir) { 110 | depth_(row, col) = bg_value; 111 | } 112 | } 113 | } 114 | 115 | if(median_ksize > 0) { 116 | cv::medianBlur(depth_, depth_, median_ksize); 117 | } 118 | 119 | return depth_; 120 | } 121 | 122 | template 123 | cv::Mat_< Dtype > DataProviderCam::ir_internal(int idx) { 124 | return ir_; 125 | } 126 | 127 | template 128 | std::vector< cv::Vec3f > DataProviderCam::gt_internal(int idx) const { 129 | return std::vector(this->param_.n_pts(), cv::Vec3f(0, 0, 0)); 130 | } 131 | 132 | template 133 | cv::Vec3f DataProviderCam::hint2d_internal(int idx) const { 134 | return cv::Vec3f(0, 0, 0); 135 | } 136 | 137 | template class DataProviderCam; 138 | template class DataProviderCam; 139 | 140 | 141 | #endif //BUILD_DATA_PROVIDER_CAM 142 | -------------------------------------------------------------------------------- /src/data_provider_dtang.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "data_provider_dtang.h" 30 | #include "common.h" 31 | 32 | #include 33 | #include 34 | 35 | 36 | template 37 | DataProviderDTang::DataProviderDTang(const pose::DataProviderParameter& param) : DataProvider(param) { 38 | std::cout << "[INFO] read label file " << param.dtang_param().label_path() << std::endl; 39 | 40 | boost::filesystem::path label_path = param.dtang_param().label_path(); 41 | 42 | std::ifstream label_file(label_path.c_str()); 43 | std::string line; 44 | 45 | int idx = 0; 46 | 47 | while(std::getline(label_file, line)) { 48 | std::istringstream iss(line); 49 | 50 | std::string relative_depth_path; 51 | iss >> relative_depth_path; 52 | 53 | boost::filesystem::path depth_path = label_path.parent_path() / "Depth" / 54 | boost::filesystem::path(relative_depth_path); 55 | 56 | std::vector anno(param.n_pts()); 57 | for(int pt_idx = 0; pt_idx < anno.size(); ++pt_idx) { 58 | iss >> anno[pt_idx](0); 59 | iss >> anno[pt_idx](1); 60 | iss >> anno[pt_idx](2); 61 | } 62 | 63 | //only load original data - no rotated 64 | if(depth_path.parent_path().parent_path().filename().string() == "Depth" && (idx % this->param_.inc() == 0)) { 65 | depth_paths_.push_back(depth_path); 66 | annos_.push_back(anno); 67 | } 68 | 69 | idx++; 70 | } 71 | 72 | this->max_idx_ = depth_paths_.size(); 73 | 74 | label_file.close(); 75 | 76 | std::cout << "[INFO] label file " << label_path << " contained " << depth_paths_.size() << " annotated depth images" << std::endl; 77 | } 78 | 79 | template 80 | boost::filesystem::path DataProviderDTang::depthPath(int idx) const { 81 | return depth_paths_[idx]; 82 | } 83 | 84 | template 85 | cv::Mat_< Dtype > DataProviderDTang::depth_internal(int idx) { 86 | std::string depth_path_str = depth_paths_[idx].string(); 87 | cv::Mat depthIn = cv::imread(depth_path_str, CV_LOAD_IMAGE_ANYDEPTH); 88 | cv::Mat_ depth; 89 | 90 | depthIn.convertTo(depth, depth.type()); 91 | 92 | Dtype bg_value = this->param_.bg_value(); 93 | 94 | for(int row = 0; row < depth.rows; ++row) { 95 | for(int col = 0; col < depthIn.cols; ++col) { 96 | if(depth(row, col) > bg_value) { 97 | depth(row, col) = bg_value; 98 | } 99 | } 100 | } 101 | 102 | return depth; 103 | } 104 | 105 | template 106 | cv::Mat_< Dtype > DataProviderDTang::ir_internal(int idx) { 107 | return depth_internal(idx); 108 | } 109 | 110 | template 111 | std::vector< cv::Vec3f > DataProviderDTang::gt_internal(int idx) const { 112 | return annos_[idx]; 113 | } 114 | 115 | template 116 | cv::Vec3f DataProviderDTang::hint2d_internal(int idx) const { 117 | return annos_[idx][0]; 118 | } 119 | 120 | 121 | template 122 | void DataProviderDTang::shuffle() { 123 | static cv::RNG rng(time(0)); 124 | 125 | std::vector shuffled_depth_paths(depth_paths_.size()); 126 | std::vector > shuffled_annos(annos_.size()); 127 | 128 | for(size_t idx = 0; idx < depth_paths_.size(); ++idx) { 129 | int rand_idx = rng.uniform(0, depth_paths_.size()); 130 | shuffled_depth_paths[idx] = depth_paths_[rand_idx]; 131 | shuffled_annos[idx] = annos_[rand_idx]; 132 | } 133 | 134 | depth_paths_ = shuffled_depth_paths; 135 | annos_ = shuffled_annos; 136 | } 137 | 138 | 139 | template class DataProviderDTang; 140 | template class DataProviderDTang; 141 | -------------------------------------------------------------------------------- /src/patch_extraction.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include 30 | 31 | #include "patch_extraction.h" 32 | 33 | #include 34 | 35 | #include 36 | 37 | template 38 | std::vector< HandPatch > PatchExtraction::operator()(const cv::Mat_< Dtype >& depth, const HandSegmentationResult& segmentation) const { 39 | 40 | cv::Rect roi = segmentation.roi_; 41 | cv::Mat_ mask = segmentation.mask_; 42 | 43 | //extract hand pixels in roi where mask(row, col) == 1 44 | cv::Mat_ hand_depth(roi.height, roi.width); 45 | for(int y = 0; y < roi.height; ++y) { 46 | for(int x = 0; x < roi.width; ++x) { 47 | int col = x + roi.x; 48 | int row = y + roi.y; 49 | hand_depth(y, x) = mask(row, col) == 0 ? bg_value_ : depth(row, col); 50 | } 51 | } 52 | 53 | //cut out patches 54 | std::vector > hand_patches(patch_widths_.size()); 55 | for(size_t hp_idx = 0; hp_idx < hand_patches.size(); ++hp_idx) { 56 | int patch_width = patch_widths_[hp_idx]; 57 | 58 | int max_width = roi.height > roi.width ? roi.height : roi.width; 59 | hand_patches[hp_idx].scale_ = float(patch_width) / float(max_width); 60 | 61 | cv::Mat_ hand_depth_scaled; 62 | cv::resize(hand_depth, hand_depth_scaled, cv::Size(), hand_patches[hp_idx].scale_, hand_patches[hp_idx].scale_, cv::INTER_NEAREST); 63 | 64 | hand_patches[hp_idx].patch_ = cv::Mat_(patch_width, patch_width, bg_value_); 65 | hand_patches[hp_idx].row_offset_ = (patch_width - hand_depth_scaled.rows) / 2; 66 | hand_patches[hp_idx].col_offset_ = (patch_width - hand_depth_scaled.cols) / 2; 67 | 68 | for(int row = 0; row < hand_depth_scaled.rows; ++row) { 69 | for(int col = 0; col < hand_depth_scaled.cols; ++col) { 70 | hand_patches[hp_idx].patch_(hand_patches[hp_idx].row_offset_ + row, hand_patches[hp_idx].col_offset_ + col) = hand_depth_scaled(row, col); 71 | } 72 | } 73 | 74 | //normalize 75 | if(normalize_depth_) { 76 | std::vector values; 77 | for(int row = 0; row < hand_patches[hp_idx].patch_.rows; ++row) { 78 | for(int col = 0; col < hand_patches[hp_idx].patch_.cols; ++col) { 79 | Dtype val = hand_patches[hp_idx].patch_(row, col); 80 | if(val < bg_value_) { 81 | values.push_back(val); 82 | } 83 | } 84 | } 85 | 86 | hand_patches[hp_idx].mean_depth_ = rv::stat::Mean(values); 87 | hand_patches[hp_idx].std_depth_ = rv::stat::StandardDeviation(values, hand_patches[hp_idx].mean_depth_); 88 | } 89 | else { 90 | hand_patches[hp_idx].mean_depth_ = 0; 91 | hand_patches[hp_idx].std_depth_ = 1; 92 | } 93 | 94 | 95 | for(int row = 0; row < hand_patches[hp_idx].patch_.rows; ++row) { 96 | for(int col = 0; col < hand_patches[hp_idx].patch_.cols; ++col) { 97 | Dtype val = (hand_patches[hp_idx].patch_(row, col) - hand_patches[hp_idx].mean_depth_) 98 | / hand_patches[hp_idx].std_depth_; 99 | hand_patches[hp_idx].patch_(row, col) = val < 3.0 ? val : 3.0; 100 | } 101 | } 102 | } 103 | 104 | return hand_patches; 105 | } 106 | 107 | 108 | template 109 | bool PatchExtraction::isReasonableHandPatch(const std::vector< HandPatch< Dtype > >& patches) const { 110 | for(size_t idx = 0; idx < patches.size(); ++idx) { 111 | for(int row = 0; row < patches[idx].patch_.rows; ++row) { 112 | for(int col = 0; col < patches[idx].patch_.cols; ++col) { 113 | Dtype d = patches[idx].patch_(row, col); 114 | if(d < -6 || d > 3.2) { 115 | std::cout << "mu = " << patches[idx].mean_depth_ << " sigma = " << patches[idx].std_depth_ << std::endl; 116 | std::cout << "d = " << d << std::endl; 117 | return false; 118 | } 119 | } 120 | } 121 | } 122 | 123 | return true; 124 | } 125 | 126 | 127 | 128 | template class PatchExtraction; 129 | template class PatchExtraction; 130 | -------------------------------------------------------------------------------- /src/hand_segmentation_meanshift.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "hand_segmentation_meanshift.h" 30 | #include "common.h" 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include 37 | 38 | 39 | 40 | template 41 | std::vector HandSegmentationMeanshift::operator()(const cv::Mat_& depth, const cv::Mat_& ir, const cv::Vec3f& hint_2d_c) const { 42 | 43 | const pose::SegmentationMeanshiftParameter param = this->param_.meanshift_param(); 44 | bool use_hint = param.use_hint(); 45 | float bandwidth = param.bandwidth(); 46 | float neighbourhood = param.neighbourhood(); 47 | float max_iters = param.max_iters(); 48 | float eps = param.eps(); 49 | 50 | // smooth 51 | cv::Mat_ smoothed; 52 | cv::medianBlur(depth, smoothed, 5); 53 | 54 | //get starting point 55 | cv::Vec3f hint_2d = hint_2d_c; 56 | if(!use_hint || smoothed(hint_2d(1), hint_2d(0)) >= bg_value_) { 57 | hint_2d(2) = bg_value_; 58 | for(int row = 0; row < smoothed.rows; ++row) { 59 | for(int col = 0; col < smoothed.cols; ++col) { 60 | Dtype d = smoothed(row, col); 61 | if(d < hint_2d(2)) { 62 | hint_2d(0) = col; 63 | hint_2d(1) = row; 64 | hint_2d(2) = d; 65 | } 66 | } 67 | } 68 | } 69 | 70 | 71 | //----------------------------------- 72 | //meanshift clustering 73 | 74 | // project points 75 | std::vector pts3; 76 | for(int y = 0; y < smoothed.rows; ++y) { 77 | for(int x = 0; x < smoothed.cols; ++x) { 78 | Dtype d = smoothed(y, x); 79 | if(d < bg_value_) { 80 | cv::Vec3f pt3 = projection_.to3D(cv::Vec3f(x, y, d)); 81 | pts3.push_back(pt3); 82 | } 83 | } 84 | } 85 | 86 | //cluster hand with mean shift 87 | std::vector indices; 88 | cv::Vec3f hint_3d = projection_.to3D(hint_2d); 89 | // std::vector centers = rv::ocv::meanshift(pts3, hint_3d, 0.075, 1e-5, 100, 0.1, indices); 90 | std::vector centers = rv::ocv::meanshift(pts3, hint_3d, bandwidth, neighbourhood, max_iters, eps, indices); 91 | 92 | 93 | //find bb 94 | int min_x = depth.cols; 95 | int min_y = depth.rows; 96 | int max_x = 0; 97 | int max_y = 0; 98 | Dtype max_d = 0; 99 | for(size_t idx = 0; idx < indices.size(); ++idx) { 100 | cv::Vec3f pt2 = projection_.to2D(pts3[indices[idx]]); 101 | 102 | float x = pt2(0); 103 | float y = pt2(1); 104 | float d = pt2(2); 105 | 106 | if(min_x > x) min_x = x; 107 | if(max_x < x) max_x = x; 108 | if(min_y > y) min_y = y; 109 | if(max_y < y) max_y = y; 110 | if(max_d < d) max_d = d; 111 | } 112 | 113 | if(min_x == depth.cols || min_y == depth.rows || max_x == 0 || max_y == 0) { 114 | return std::vector(); 115 | } 116 | 117 | cv::Rect roi = cv::Rect(min_x, min_y, max_x - min_x, max_y - min_y); //TODO: add +1 to width/height 118 | 119 | //create hand mask 120 | cv::Mat_ mask = cv::Mat_::zeros(smoothed.rows, smoothed.cols); 121 | size_t idx = 0; 122 | size_t pt_idx = 0; 123 | for(int y = 0; y < smoothed.rows; ++y) { 124 | for(int x = 0; x < smoothed.cols; ++x) { 125 | Dtype d = smoothed(y, x); 126 | if(d < bg_value_) { 127 | bool contains = false; 128 | if(pt_idx < indices.size() && idx == indices[pt_idx]) { 129 | contains = true; 130 | pt_idx++; 131 | } 132 | 133 | if(d < max_d) { 134 | contains = true; 135 | } 136 | 137 | if(contains) { 138 | mask(y, x) = 1; 139 | } 140 | 141 | idx++; 142 | } 143 | } 144 | } 145 | 146 | std::vector result(1); 147 | result[0].roi_ = roi; 148 | result[0].mask_ = mask; 149 | 150 | return result; 151 | } 152 | 153 | 154 | 155 | template class HandSegmentationMeanshift; 156 | template class HandSegmentationMeanshift; 157 | -------------------------------------------------------------------------------- /src/data_provider_csv.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "data_provider_csv.h" 30 | #include "common.h" 31 | 32 | #include 33 | #include 34 | 35 | 36 | template 37 | DataProviderCsv::DataProviderCsv(const pose::DataProviderParameter& param) : 38 | DataProvider(param) { 39 | 40 | std::vector all_depth_paths; 41 | if(param.csv_param().recursive()) { 42 | rv::io::ListFilesRecursive(param.csv_param().data_path(), ".*_depth.csv", all_depth_paths); 43 | } 44 | else { 45 | rv::io::ListFiles(param.csv_param().data_path(), ".*_depth.csv", all_depth_paths); 46 | } 47 | std::cout << "[INFO] found " << all_depth_paths.size() << " depth images" << std::endl; 48 | all_depth_paths = this->sortReducePaths(all_depth_paths); 49 | std::cout << "[INFO] sorted and reduced to " << all_depth_paths.size() << " depth images" << std::endl; 50 | 51 | 52 | for(int depth_idx = 0; depth_idx < all_depth_paths.size(); ++depth_idx) { 53 | boost::filesystem::path depth_path = all_depth_paths[depth_idx]; 54 | std::vector anno_paths = lsAnnoPaths(depth_path); 55 | 56 | if(param.csv_param().anno_type() == pose::CsvParameter_AnnoType_ALL) { 57 | depth_paths_.push_back(depth_path); 58 | annos_.push_back(std::vector(param.n_pts(), cv::Vec3f(0.0, 0.0, 0.0))); 59 | } 60 | else { 61 | for(int anno_idx = 0; anno_idx < anno_paths.size(); ++anno_idx) { 62 | boost::filesystem::path anno_path = anno_paths[anno_idx]; 63 | std::string annotator = getAnnotator(anno_path); 64 | 65 | if((param.csv_param().anno_type() == pose::CsvParameter_AnnoType_ANNO) || 66 | (param.csv_param().anno_type() == pose::CsvParameter_AnnoType_VALID_ANNO && 67 | annotator != "extrapolated" && annotator != "init")) { 68 | std::vector anno = readAnno(anno_path, param.n_pts()); 69 | 70 | depth_paths_.push_back(depth_path); 71 | annos_.push_back(anno); 72 | } 73 | } 74 | } 75 | } 76 | 77 | this->max_idx_ = depth_paths_.size(); 78 | 79 | std::cout << "[INFO] label file " << param.csv_param().data_path() << " contained " << depth_paths_.size() << " annotated depth images" << std::endl; 80 | 81 | cv::Vec3f mean, mean_0, std, std_0; 82 | this->stat(annos_, mean, std, mean_0, std_0); 83 | std::cout << "[INFO] anno[0] = " << mean_0 << " +- " << std_0 << " | anno[all] = " << mean << " +- " << std << std::endl; 84 | } 85 | 86 | template 87 | boost::filesystem::path DataProviderCsv::depthPath(int idx) const { 88 | return depth_paths_[idx]; 89 | } 90 | 91 | template 92 | cv::Mat_< Dtype > DataProviderCsv::depth_internal(int idx) { 93 | boost::filesystem::path depth_path = depth_paths_[idx]; 94 | cv::Mat_ dm = rv::ocv::ReadCsv(depth_path); 95 | 96 | cv::Mat_ ir_img = ir_internal(idx); 97 | Dtype bg_value = this->param_.bg_value(); 98 | this->param_.csv_param(); 99 | int median_ksize = this->param_.csv_param().median_ksize(); 100 | float min_ir = this->param_.csv_param().min_ir(); 101 | 102 | for(int row = 0; row < dm.rows; ++row) { 103 | for(int col = 0; col < dm.cols; ++col) { 104 | Dtype d = dm(row, col); 105 | Dtype i = ir_img(row, col); 106 | 107 | if(d < 0 || d > bg_value || i < min_ir) { 108 | dm(row, col) = bg_value; 109 | } 110 | } 111 | } 112 | 113 | if(median_ksize > 0) { 114 | cv::medianBlur(dm, dm, median_ksize); 115 | } 116 | 117 | return dm; 118 | } 119 | 120 | template 121 | cv::Mat_< Dtype > DataProviderCsv::ir_internal(int idx) { 122 | boost::filesystem::path depth_path = depth_paths_[idx]; 123 | boost::filesystem::path ir_path = infraredPath(depth_path); 124 | cv::Mat_ ir_img = rv::ocv::ReadCsv(ir_path); 125 | 126 | return ir_img; 127 | } 128 | 129 | template 130 | std::vector< cv::Vec3f > DataProviderCsv::gt_internal(int idx) const { 131 | return annos_[idx]; 132 | } 133 | 134 | template 135 | cv::Vec3f DataProviderCsv::hint2d_internal(int idx) const { 136 | return annos_[idx][0]; 137 | } 138 | 139 | 140 | 141 | template 142 | void DataProviderCsv::shuffle() { 143 | static cv::RNG rng(time(0)); 144 | 145 | std::vector shuffled_depth_paths(depth_paths_.size()); 146 | std::vector > shuffled_annos(annos_.size()); 147 | 148 | for(size_t idx = 0; idx < depth_paths_.size(); ++idx) { 149 | int rand_idx = rng.uniform(0, depth_paths_.size()); 150 | shuffled_depth_paths[idx] = depth_paths_[rand_idx]; 151 | shuffled_annos[idx] = annos_[rand_idx]; 152 | } 153 | 154 | depth_paths_ = shuffled_depth_paths; 155 | annos_ = shuffled_annos; 156 | } 157 | 158 | 159 | 160 | template class DataProviderCsv; 161 | template class DataProviderCsv; 162 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | # Graz University of Technology (TU GRAZ) 3 | 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 1. Redistributions of source code must retain the above copyright 7 | # notice, this list of conditions and the following disclaimer. 8 | # 2. Redistributions in binary form must reproduce the above copyright 9 | # notice, this list of conditions and the following disclaimer in the 10 | # documentation and/or other materials provided with the distribution. 11 | # 3. All advertising materials mentioning features or use of this software 12 | # must display the following acknowledgement: 13 | # This product includes software developed by the ICG, TU GRAZ. 14 | # 4. Neither the name of the ICG, TU GRAZ nor the 15 | # names of its contributors may be used to endorse or promote products 16 | # derived from this software without specific prior written permission. 17 | 18 | # THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 30 | 31 | PROJECT(handpose) 32 | 33 | 34 | option(BUILD_POSE "build create_data/eval/train apps" ON) 35 | option(BUILD_LIVE "link creative and pmd camera" OFF) 36 | option(BUILD_ANNOTATION_TOOL "build annotation tool" ON) 37 | 38 | 39 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) 40 | 41 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 42 | 43 | set(CMAKE_BUILD_TYPE Release) 44 | # set(CMAKE_BUILD_TYPE Debug) 45 | message("CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) 46 | 47 | 48 | 49 | #packages 50 | # find_package(OpenMP) 51 | # if (OPENMP_FOUND) 52 | # message(STATUS "OpenMP was found.") 53 | # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 54 | # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 55 | # endif() 56 | 57 | find_package(OpenCV REQUIRED core imgproc highgui objdetect flann) 58 | message(STATUS "OpenCV ${OpenCV_VERSION} was found.") 59 | include_directories(${OpenCV_INCLUDE_DIR}) 60 | 61 | find_package(Boost COMPONENTS system date_time chrono regex thread filesystem program_options timer REQUIRED) 62 | message(STATUS "Boost was found.") 63 | include_directories(${Boost_INCLUDE_DIR}) 64 | 65 | if(BUILD_LIVE) 66 | find_package(rv_cpp_utils REQUIRED CAMERA_PMD CAMERA_CREATIVE_325) 67 | add_definitions(-DBUILD_DATA_PROVIDER_CAM) 68 | else() 69 | find_package(rv_cpp_utils REQUIRED) 70 | endif() 71 | message(STATUS "RV_CPP_UTILS_INCLUDE_DIRS: ${RV_CPP_UTILS_INCLUDE_DIRS}") 72 | message(STATUS "RV_CPP_UTILS_LIBRARIES: ${RV_CPP_UTILS_LIBRARIES}") 73 | include_directories(${RV_CPP_UTILS_INCLUDE_DIRS}) 74 | 75 | find_package(Caffe REQUIRED) 76 | message(STATUS "CAFFE_INCLUDE_DIRS: ${CAFFE_INCLUDE_DIRS}") 77 | message(STATUS "CAFFE_LIBRARIES: ${CAFFE_LIBRARIES}") 78 | include_directories(${CAFFE_INCLUDE_DIRS}) 79 | include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include") 80 | 81 | find_package(Protobuf REQUIRED) 82 | message(STATUS "PROTOBUF_LIBRARIES: ${PROTOBUF_LIBRARIES}") 83 | PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS proto/pose.proto) 84 | message(STATUS "ProtoSources: ${PROTO_SRCS}") 85 | message(STATUS "ProtoHeaders: ${PROTO_HDRS}") 86 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 87 | 88 | 89 | message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}/include") 90 | 91 | 92 | if(BUILD_POSE) 93 | add_library(pose 94 | src/common.cpp 95 | src/create_data.cpp 96 | src/data_provider_blender.cpp 97 | src/data_provider_csv.cpp 98 | src/data_provider_dtang.cpp 99 | src/data_provider_cam.cpp 100 | src/hand_patch.cpp 101 | src/hand_segmentation_meanshift.cpp 102 | src/hand_segmentation_rf.cpp 103 | src/hand_segmentation_threshold.cpp 104 | src/patch_extraction.cpp 105 | src/preprocess.cpp 106 | src/projection.cpp 107 | src/inference_regression_net.cpp 108 | src/inference_heatmap_net.cpp 109 | src/inference_rf_simple.cpp 110 | ${PROTO_SRCS} 111 | ) 112 | 113 | set(POSE_LIBRARIES 114 | pose 115 | ${OpenCV_LIBS} 116 | ${Boost_LIBRARIES} 117 | ${CAFFE_LIBRARIES} 118 | ${RV_CPP_UTILS_LIBRARIES} 119 | ${PROTOBUF_LIBRARIES} 120 | ) 121 | 122 | message(STATUS "POSE_LIBRARIES: ${POSE_LIBRARIES}") 123 | 124 | add_executable(create_data src/main_create_data.cpp) 125 | add_dependencies(create_data pose) 126 | target_link_libraries(create_data ${POSE_LIBRARIES}) 127 | 128 | add_executable(eval src/main_eval.cpp) 129 | add_dependencies(eval pose) 130 | target_link_libraries(eval ${POSE_LIBRARIES}) 131 | 132 | add_executable(nn_train src/main_nn_train.cpp) 133 | add_dependencies(nn_train pose) 134 | target_link_libraries(nn_train ${POSE_LIBRARIES}) 135 | 136 | add_executable(rf_train src/main_rf_train.cpp) 137 | add_dependencies(rf_train pose) 138 | target_link_libraries(rf_train ${POSE_LIBRARIES}) 139 | 140 | add_executable(seg_rf_train src/main_rf_train_segmentation.cpp) 141 | add_dependencies(seg_rf_train pose) 142 | target_link_libraries(seg_rf_train ${POSE_LIBRARIES}) 143 | 144 | endif() 145 | 146 | 147 | if(BUILD_ANNOTATION_TOOL) 148 | 149 | find_package(OpenGL REQUIRED) 150 | include_directories(${OPENGL_INCLUDE_DIRS}) 151 | 152 | find_package(Qt5Core REQUIRED) 153 | find_package(Qt5Xml REQUIRED) 154 | find_package(Qt5OpenGL REQUIRED) 155 | find_package(Qt5Widgets REQUIRED) 156 | 157 | find_package(QGLVIEWER REQUIRED) 158 | if(QGLVIEWER_FOUND) 159 | message(STATUS "QGLVIEWER was found: ${QGLVIEWER_INCLUDE_DIR}") 160 | message(STATUS "QGLVIEWER library: ${QGLVIEWER_LIBRARIES}") 161 | include_directories(${QGLVIEWER_INCLUDE_DIR}) 162 | #add_definitions(${QGLVIEWER_DEFINITIONS}) 163 | endif(QGLVIEWER_FOUND) 164 | 165 | 166 | add_executable(annotation_tool 167 | src/main_annotation.cpp 168 | src/annotation_viewer.cpp 169 | src/common.cpp 170 | src/projection.cpp 171 | ${PROTO_SRCS} 172 | ) 173 | target_link_libraries(annotation_tool 174 | Qt5::Widgets Qt5::OpenGL Qt5::Core Qt5::Xml 175 | ${QGLVIEWER_LIBRARIES} 176 | ${OPENGL_LIBRARIES} 177 | ${OpenCV_LIBS} 178 | ${Boost_LIBRARIES} 179 | ${PROTOBUF_LIBRARIES} 180 | ) 181 | 182 | endif() 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /include/annotation_viewer.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef ANNOTATION_VIEWER_H 30 | #define ANNOTATION_VIEWER_H 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | #include 39 | 40 | #include "projection.h" 41 | 42 | 43 | class AnnotationViewer : public QGLViewer { 44 | public: 45 | AnnotationViewer(const std::vector pts3, 46 | const std::vector pts3_colors, 47 | const cv::Mat_& img, 48 | const std::vector >& anno_proposals, 49 | const std::vector& constraint_anno, 50 | const Projection& projection) : 51 | pts3_(pts3), pts3_colors_(pts3_colors), img_(img), 52 | anno_proposals_(anno_proposals), constraint_anno_(constraint_anno), 53 | projection_(projection), active_anno_proposal_(0), 54 | active_joint_idx_(0), draw_3d_(false), free_edit_mode_(true), 55 | project_constrained_depth_mode_(0), deviation_tolerance_(2.0), 56 | changed_(false), size_3d_pts_(2.0), size_3d_annotation_(3.0) { 57 | 58 | anno_ = anno_proposals_[active_anno_proposal_]; 59 | 60 | color_pts_ = cv::Vec3f(119.0, 125.0, 210.0) / 255.0; 61 | color_joints_ = cv::Vec3f(221.0, 144.0, 144.0) / 255.0; 62 | color_joint_active_ = cv::Vec3f( 63.0, 135.0, 68.0) / 255.0; 63 | color_skelet_ = cv::Vec3f( 77.0, 77.0, 77.0) / 255.0; 64 | color_background_ = cv::Vec3f(255.0, 255.0, 255.0) / 255.0; 65 | } 66 | 67 | virtual ~AnnotationViewer() {} 68 | 69 | protected : 70 | virtual int parentJointIdx(int joint_idx) = 0; 71 | 72 | virtual void glColorVec3f(const cv::Vec3f& v); 73 | 74 | virtual float dist(const cv::Vec3f& v1, const cv::Vec3f& v2); 75 | 76 | virtual cv::Vec3f cartesian2spherical(const cv::Vec3f& c_cart, 77 | const cv::Vec3f& p_cart); 78 | virtual cv::Vec3f spherical2cartesian(const cv::Vec3f& c_cart, 79 | const cv::Vec3f& p_spher); 80 | 81 | virtual void moveAnnoX(int joint_idx, float t); 82 | virtual void moveAnnoY(int joint_idx, float t); 83 | virtual void moveAnnoZ(int joint_idx, float t); 84 | virtual void moveAnnoD(int joint_idx, float t); 85 | virtual void moveAnnoR(int joint_idx, float t); 86 | virtual void moveAnnoTheta(int joint_idx, float t); 87 | virtual void moveAnnoPhi(int joint_idx, float t); 88 | 89 | virtual void moveAnnoAllDependent(int joint_idx, const cv::Vec3f& t3); 90 | virtual void projectAnnoToConstraints(); 91 | 92 | virtual void rotateAnno(float x, float y, float z); 93 | 94 | 95 | virtual void generateSphericalConstraints(); 96 | 97 | virtual void init(); 98 | 99 | 100 | virtual void drawHandSkeleton(const std::vector& anno3); 101 | virtual void drawPointWithCoords(const cv::Vec3f& pos, float radius, 102 | const cv::Vec3f& color, bool coord); 103 | virtual void drawSphericalConstraint(const std::vector& anno3); 104 | virtual void drawIn3d(); 105 | virtual void drawIn2d(); 106 | 107 | virtual void draw(); 108 | 109 | virtual void keyPressEvent(QKeyEvent* e); 110 | virtual void mousePressEvent(QMouseEvent* e); 111 | 112 | 113 | virtual QString helpString() const { 114 | return QString("..."); 115 | } 116 | 117 | 118 | public: 119 | virtual std::vector getAnno() const { 120 | return anno_; 121 | } 122 | 123 | virtual bool getChanged() const { 124 | return changed_; 125 | } 126 | 127 | virtual int getReturnDirection() const { 128 | return return_direction_; 129 | } 130 | 131 | protected: 132 | std::vector pts3_; 133 | std::vector pts3_colors_; 134 | const cv::Mat_ img_; 135 | const std::vector > anno_proposals_; 136 | const std::vector constraint_anno_; 137 | const Projection& projection_; 138 | 139 | int active_anno_proposal_; 140 | 141 | std::vector anno_; 142 | std::vector spherical_constraints_; 143 | int active_joint_idx_; 144 | 145 | bool draw_3d_; 146 | bool free_edit_mode_; 147 | int project_constrained_depth_mode_; 148 | float deviation_tolerance_; 149 | 150 | bool changed_; 151 | int return_direction_; 152 | 153 | float size_3d_pts_; 154 | float size_3d_annotation_; 155 | 156 | cv::Vec3f color_pts_; 157 | cv::Vec3f color_joints_; 158 | cv::Vec3f color_joint_active_; 159 | cv::Vec3f color_skelet_; 160 | cv::Vec3f color_background_; 161 | }; 162 | 163 | 164 | class Mine20AnnotationViewer : public AnnotationViewer { 165 | public: 166 | Mine20AnnotationViewer(const std::vector pts3, 167 | const std::vector pts3_colors, 168 | const cv::Mat_& img, 169 | const std::vector >& anno_proposals, 170 | const std::vector& constraint_anno, 171 | const Projection& projection) : 172 | AnnotationViewer(pts3, pts3_colors, img, anno_proposals, 173 | constraint_anno, projection) {} 174 | 175 | protected: 176 | virtual int parentJointIdx(int joint_idx) { 177 | if(joint_idx == 4 || joint_idx == 8 || joint_idx == 12 178 | || joint_idx == 16) { 179 | return 0; 180 | } 181 | else { 182 | return joint_idx - 1; 183 | } 184 | } 185 | }; 186 | 187 | #endif 188 | -------------------------------------------------------------------------------- /proto/pose.proto: -------------------------------------------------------------------------------- 1 | package pose; 2 | 3 | //------------------------------------------------------------------------------ 4 | message ProjectionParameter { 5 | enum ProjectionType { 6 | NONE = 0; 7 | ORTHOGRAPHIC = 1; 8 | PROJECTIVE = 2; 9 | } 10 | 11 | optional ProjectionType type = 1 [default = NONE]; 12 | 13 | //if empty try to find in data dir 14 | optional string calib_path = 2 [default = ""]; 15 | } 16 | 17 | 18 | //------------------------------------------------------------------------------ 19 | message DataProviderParameter { 20 | enum DataProviderType { 21 | NONE = 0; 22 | DTANG = 1; 23 | CSV = 2; 24 | BLENDER = 3; 25 | CAM = 4; 26 | } 27 | 28 | optional DataProviderType type = 1 [default = NONE]; 29 | 30 | optional ProjectionParameter projection_param = 2; 31 | 32 | optional float bg_value = 3 [default = 1000.0]; 33 | optional uint32 n_pts = 4 [default = 20]; 34 | optional bool shuffle = 5 [default = false]; 35 | optional uint32 inc = 6 [default = 1]; 36 | optional bool undistort = 7 [default = false]; 37 | optional bool flip = 8 [default = false]; 38 | 39 | optional DTangParameter dtang_param = 9; 40 | optional CsvParameter csv_param = 10; 41 | optional BlenderParameter blender_param = 11; 42 | optional CamParameter cam_param = 12; 43 | } 44 | 45 | message DTangParameter { 46 | optional string label_path = 1; 47 | } 48 | 49 | message CsvParameter { 50 | optional string data_path = 1; 51 | optional bool recursive = 3 [default = true]; 52 | 53 | enum AnnoType { 54 | ALL = 0; //load also depth maps without anno 55 | ANNO = 1; //load only depth maps with anno (also if only anno is init, ...) 56 | VALID_ANNO = 2; 57 | } 58 | optional AnnoType anno_type = 4 [default = VALID_ANNO]; 59 | 60 | optional float min_ir = 5 [default = 250.0]; 61 | optional uint32 median_ksize = 6 [default = 3]; 62 | } 63 | 64 | message BlenderParameter { 65 | optional string data_path = 1; 66 | optional bool recursive = 3 [default = true]; 67 | 68 | optional float noise_gaussian_sigma = 4 [default = 0.0]; 69 | } 70 | 71 | message CamParameter { 72 | enum CameraType { 73 | NONE = 0; 74 | CREATIVE = 1; 75 | PMD_PICO = 2; 76 | } 77 | 78 | optional CameraType type = 1 [default = NONE]; 79 | } 80 | 81 | 82 | //------------------------------------------------------------------------------ 83 | message SegmentationParameter { 84 | enum SegmentationType { 85 | NONE = 0; 86 | MEANSHIFT = 1; 87 | RF = 2; 88 | THRESHOLD = 3; 89 | } 90 | 91 | optional SegmentationType type = 1 [default = NONE]; 92 | 93 | optional SegmentationMeanshiftParameter meanshift_param = 4; 94 | optional SegmentationRfParameter rf_param = 2; 95 | optional SegmentationThresholdParameter threshold_param = 3; 96 | } 97 | 98 | message SegmentationMeanshiftParameter { 99 | optional bool use_hint = 1 [default = true]; 100 | optional double bandwidth = 2 [default = 0.075]; 101 | optional double neighbourhood = 3 [default = 1e-5]; 102 | optional uint32 max_iters = 4 [default = 100]; 103 | optional float eps = 5 [default = 0.1]; 104 | } 105 | 106 | message SegmentationRfParameter { 107 | optional string forest_path = 1; 108 | optional float min_prob = 2 [default = 0.8]; 109 | optional uint32 struc_elem_size = 3 [default = 5]; 110 | } 111 | 112 | message SegmentationThresholdParameter { 113 | enum Type { 114 | RANGE = 0; 115 | NEAREST = 1; 116 | } 117 | optional Type type = 1 [default = RANGE]; 118 | 119 | //range 120 | optional float min_ir = 2 [default = 0]; 121 | optional float max_ir = 3 [default = 1e9]; 122 | optional float min_d = 4 [default = 0]; 123 | optional float max_d = 5 [default = 1e9]; 124 | 125 | //nearest 126 | optional float t = 6 [default = 500]; 127 | } 128 | 129 | 130 | //------------------------------------------------------------------------------ 131 | message PatchExtractionParameter { 132 | repeated uint32 patch_width = 1 [packed = true]; 133 | optional bool normalize_depth = 2 [default = true]; 134 | } 135 | 136 | 137 | //------------------------------------------------------------------------------ 138 | message PreprocessParameter { 139 | optional string mean_path = 1 [default = ""]; 140 | } 141 | 142 | //------------------------------------------------------------------------------ 143 | message InferenceParameter { 144 | enum InferenceType { 145 | NONE = 0; 146 | NN_REGRESSION = 1; 147 | NN_HEATMAP = 2; 148 | RF_SIMPLE_REGRESSION = 3; 149 | } 150 | 151 | optional InferenceType type = 1 [default = NONE]; 152 | 153 | optional NNRegressionParameter nn_regression_param = 2; 154 | optional NNHeatmapParameter nn_heatmap_param = 4; 155 | optional RFParameter rf_param = 3; 156 | } 157 | 158 | message NNRegressionParameter { 159 | optional string net_path = 1; 160 | optional string weights_path = 2; 161 | } 162 | 163 | message NNHeatmapParameter { 164 | optional string net_path = 1; 165 | optional string weights_path = 2; 166 | 167 | enum HeatmapType { 168 | NONE = 0; 169 | TWO_D = 1; //2D 170 | TWO_POINT_FIVE_D = 2; //2D + 1D depth 171 | THREE_D = 3; //3D 172 | } 173 | 174 | optional HeatmapType type = 3 [default = NONE]; 175 | optional string layer_name_2d = 4 [default = ""]; 176 | optional string layer_name_depth = 5 [default = ""]; 177 | optional string layer_name_3d = 6 [default = ""]; 178 | 179 | optional float min_heatmap_val = 16 [default = 0.001]; 180 | 181 | optional uint32 dim_x = 8 [default = 18]; 182 | optional uint32 dim_y = 7 [default = 18]; 183 | optional uint32 dim_z = 9 [default = 25]; 184 | 185 | optional int32 x_from = 10 [default = -1]; 186 | optional int32 x_to = 11 [default = 1]; 187 | optional int32 y_from = 12 [default = -1]; 188 | optional int32 y_to = 13 [default = 1]; 189 | optional int32 z_from = 14 [default = -1]; 190 | optional int32 z_to = 15 [default = 1]; 191 | } 192 | 193 | message RFParameter { 194 | optional string forest_path = 1; 195 | optional string transform_csv = 2; 196 | } 197 | 198 | 199 | 200 | 201 | 202 | //------------------------------------------------------------------------------ 203 | message CreateDataParameter { 204 | optional DataProviderParameter data_provider_param =1; 205 | optional SegmentationParameter segmentation_param = 2; 206 | optional PatchExtractionParameter patch_extraction_param = 3; 207 | 208 | optional bool shuffle = 4 [default = true]; 209 | 210 | optional bool rotations = 5 [default = true]; 211 | optional float rotations_from = 6 [default = -90.0]; 212 | optional float rotations_to = 7 [default = 90.0]; 213 | optional float rotations_step = 8 [default = 15.0]; 214 | 215 | optional bool mirroring = 9 [default = false]; 216 | 217 | optional uint32 samples_per_hdf5 = 10 [default = 1000000]; 218 | 219 | optional string out_prefix = 11; 220 | } 221 | 222 | 223 | //------------------------------------------------------------------------------ 224 | message EvaluationParameter { 225 | optional DataProviderParameter data_provider_param =1; 226 | optional SegmentationParameter segmentation_param = 2; 227 | optional PatchExtractionParameter patch_extraction_param = 3; 228 | optional PreprocessParameter preprocess_param = 4; 229 | optional InferenceParameter inference_param = 5; 230 | 231 | optional string result_path = 6; 232 | //empty => no names txt file 233 | optional string names_path = 7 [default = ""]; 234 | 235 | optional string segmentation_result_path = 8 [default = ""]; 236 | } 237 | -------------------------------------------------------------------------------- /src/main_rf_train.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include 30 | 31 | #include "common.h" 32 | #include "make_data_provider.h" 33 | #include "make_hand_segmentation.h" 34 | #include "make_inference.h" 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | #include 45 | 46 | #include 47 | #include 48 | namespace po = boost::program_options; 49 | 50 | 51 | typedef float Dtype; 52 | 53 | void hdf2samples(const boost::filesystem::path& hdf_path, 54 | const std::string& data_name, 55 | int inc, 56 | std::vector& samples, 57 | rv::rf::VecPtrTargetPtr& targets) { 58 | rv::io::H5BlockReader data_reader(hdf_path, data_name); 59 | rv::io::H5BlockReader regression_reader(hdf_path, "regression"); 60 | 61 | int data_channels = data_reader.channels(); 62 | int data_height = data_reader.height(); 63 | int data_width = data_reader.width(); 64 | int data_dim = data_channels * data_height * data_width; 65 | 66 | int regression_channels = regression_reader.channels(); 67 | int regression_height = regression_reader.height(); 68 | int regression_width = regression_reader.width(); 69 | int regression_dim = regression_channels * regression_height * regression_width; 70 | 71 | while(data_reader.hasNext() && regression_reader.hasNext()) { 72 | rv::rf::RfMatPtr patch = rv::rf::CreateRfMat(data_dim, 1); 73 | int patch_idx = 0; 74 | for(int c = 0; c < data_channels; ++c) { 75 | for(int w = 0; w < data_width; ++w) { 76 | for(int h = 0; h < data_height; ++h) { 77 | (*patch)(patch_idx, 0) = data_reader.data(c, h, w); 78 | patch_idx++; 79 | } 80 | } 81 | } 82 | 83 | rv::rf::RfMatPtr regression = rv::rf::CreateRfMat(regression_dim, 1); 84 | int regression_idx = 0; 85 | for(int c = 0; c < regression_channels; ++c) { 86 | for(int h = 0; h < regression_height; ++h) { 87 | for(int w = 0; w < regression_width; ++w) { 88 | (*regression)(regression_idx, 0) = regression_reader.data(c, h, w); 89 | regression_idx++; 90 | } 91 | } 92 | } 93 | 94 | // cv::imshow("sample", rv::ocv::clamp(patches[0])); 95 | // std::cout << "regression_vec: " << regression << std::endl; 96 | // cv::waitKey(0); 97 | 98 | samples.push_back(boost::make_shared(patch)); 99 | targets->push_back(boost::make_shared(regression)); 100 | 101 | for(int i = 0; i < inc && data_reader.hasNext() && regression_reader.hasNext(); ++i) { 102 | data_reader.next(); 103 | regression_reader.next(); 104 | } 105 | } 106 | } 107 | 108 | 109 | int main(int argc, char** argv) { 110 | std::string forest_config_path; 111 | boost::filesystem::path out_forest_path; 112 | boost::filesystem::path train_hdf_path; 113 | boost::filesystem::path test_hdf_path; 114 | std::string data_name; 115 | int inc; 116 | 117 | po::options_description desc; 118 | desc.add_options() 119 | ("forest_config_path", po::value()->required(), "") 120 | ("out_forest_path", po::value()->required(), "") 121 | ("train_hdf_path", po::value()->required(), "") 122 | ("test_hdf_path", po::value()->default_value(""), "") 123 | ("data_name", po::value()->required(), "") 124 | ("inc", po::value()->required(), "") 125 | ; 126 | 127 | po::variables_map vm; 128 | try { 129 | po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); 130 | po::notify(vm); 131 | 132 | forest_config_path = vm["forest_config_path"].as(); 133 | out_forest_path = vm["out_forest_path"].as(); 134 | train_hdf_path = vm["train_hdf_path"].as(); 135 | test_hdf_path = vm["test_hdf_path"].as(); 136 | data_name = vm["data_name"].as(); 137 | inc = vm["inc"].as(); 138 | } catch(std::exception& e) { 139 | std::cout << "ERROR: " << e.what() << std::endl << std::endl; 140 | std::cout << desc << std::endl; 141 | return -1; 142 | } 143 | 144 | rv::rf::ForestParameter forest_param; 145 | if(!readProtoFromTextFile(forest_config_path, &forest_param)) { 146 | std::cout << "ERROR: parsing config_path: " << forest_config_path << std::endl; 147 | return -1; 148 | } 149 | 150 | 151 | //read hdf data to samples and targets 152 | std::vector train_samples; 153 | rv::rf::VecPtrTargetPtr train_targets = boost::make_shared(); 154 | hdf2samples(train_hdf_path, data_name, inc, train_samples, train_targets); 155 | 156 | std::vector vec_train_targets; 157 | vec_train_targets.push_back(train_targets); 158 | 159 | //sample stat 160 | int data_width = train_samples[0]->width(); 161 | int data_height = train_samples[0]->height(); 162 | std::cout << "[INFO] loaded " << train_samples.size() << " samples and targets for training" << std::endl; 163 | 164 | //train forest 165 | rv::rf::TrainForest rf_train(forest_param, true); 166 | rv::rf::ForestPtr forest; 167 | forest = rf_train.Train(train_samples, vec_train_targets, vec_train_targets, rv::rf::TRAIN, forest); 168 | if(!boost::filesystem::exists(out_forest_path.parent_path())) { 169 | boost::filesystem::create_directories(out_forest_path.parent_path()); 170 | } 171 | 172 | rv::io::CompressedSerializationOut serialization_bin(out_forest_path.string()); 173 | forest->Save(serialization_bin); 174 | 175 | return 0; 176 | } 177 | -------------------------------------------------------------------------------- /src/hand_segmentation_rf.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "hand_segmentation_rf.h" 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | #include 37 | #include 38 | 39 | template 40 | std::vector HandSegmentationRf::operator()(const cv::Mat_& depth, const cv::Mat_& ir, const cv::Vec3f& hint_2d) const { 41 | cv::Mat_ fg_prob = cv::Mat_::zeros(depth.rows, depth.cols); 42 | cv::Mat_ mask = cv::Mat_::zeros(depth.rows, depth.cols); 43 | 44 | int n_found = 0; 45 | 46 | for(int row = 0; row < depth.rows; ++row) { 47 | for(int col = 0; col < depth.cols; ++col) { 48 | Dtype d = depth(row, col); 49 | if(d < bg_value_ && d > 0) { 50 | rv::rf::SamplePtr sample = boost::make_shared(depth, col, row); 51 | rv::rf::VecPtrTargetPtr estimated_targets = forest_->inferencest(sample); 52 | rv::rf::RfMatPtr es_mat = (*estimated_targets)[0]->data(); 53 | float p = (*es_mat)(0, 1); 54 | 55 | fg_prob(row, col) = p; 56 | 57 | if(p > this->param_.rf_param().min_prob()) { 58 | mask(row, col) = 1; 59 | n_found++; 60 | } 61 | } 62 | } 63 | } 64 | 65 | if(n_found < 10) { 66 | std::cout << "[INFO] only 10 pixels classified as hand" << std::endl; 67 | return std::vector(); 68 | } 69 | 70 | static rv::ocv::ColorMap& cmap = rv::ocv::ColorMapCoolWarm::i(); 71 | // cv::imshow("depth", rv::ocv::clamp(depth)); 72 | cv::imshow("bg_prob", cmap.Map(1 - fg_prob)); 73 | cv::imshow("fg_prob", cmap.Map(fg_prob)); 74 | cv::imshow("mask", mask * 255); 75 | 76 | //clean up mask 77 | cv::morphologyEx(mask, mask, cv::MORPH_CLOSE, struc_elem_); 78 | cv::morphologyEx(mask, mask, cv::MORPH_OPEN, struc_elem_); 79 | 80 | cv::imshow("mask2", mask * 255); 81 | // cv::waitKey(0); 82 | 83 | 84 | //find roi 85 | int min_y = mask.rows; 86 | int max_y = 0; 87 | int min_x = mask.cols; 88 | int max_x = 0; 89 | 90 | for(int row = 0; row < mask.rows; ++row) { 91 | for(int col = 0; col < mask.cols; ++col) { 92 | if(mask(row, col) > 0) { 93 | if(row < min_y) min_y = row; 94 | if(row > max_y) max_y = row; 95 | if(col < min_x) min_x = col; 96 | if(col > max_x) max_x = col; 97 | } 98 | } 99 | } 100 | 101 | cv::Rect roi(min_x, min_y, max_x - min_x + 1, max_y - min_y + 1); 102 | 103 | 104 | //return result 105 | std::vector result(1); 106 | result[0].roi_ = roi; 107 | result[0].mask_ = mask; 108 | 109 | return result; 110 | } 111 | 112 | 113 | 114 | 115 | template 116 | rv::rf::ForestPtr HandSegmentationRf::train( 117 | boost::shared_ptr > data_provider, 118 | boost::shared_ptr > hand_segmentation, 119 | const rv::rf::ForestParameter& params) { 120 | 121 | data_provider->shuffle(); 122 | 123 | const int pos_samples_p_image = 120; 124 | const int neg_samples_p_image = 60; 125 | 126 | const float pos_samples_weight = float(neg_samples_p_image) / float(pos_samples_p_image + neg_samples_p_image); 127 | const float neg_samples_weight = float(pos_samples_p_image) / float(pos_samples_p_image + neg_samples_p_image); 128 | 129 | //create samples 130 | std::vector samples; 131 | rv::rf::VecPtrTargetPtr targets = boost::make_shared(); 132 | 133 | for(; data_provider->hasNext(); data_provider->next()) { 134 | std::cout << "extract template from: " << data_provider->depthPath() << std::endl; 135 | 136 | cv::Mat_ depth = data_provider->depth(); 137 | cv::Mat_ ir = data_provider->ir(); 138 | cv::Vec3f hint_2d = data_provider->hint2d(); 139 | 140 | std::vector segmentations = (*hand_segmentation)(depth, ir, hint_2d); 141 | 142 | for(int segmentation_idx = 0; segmentation_idx < segmentations.size(); ++segmentation_idx) { 143 | cv::Mat_ mask = segmentations[segmentation_idx].mask_; 144 | 145 | int n_neg_samples = 0; 146 | int n_pos_samples = 0; 147 | while(n_neg_samples < neg_samples_p_image) { 148 | int x = rv::rand::Rand::Uniform(0, depth.cols); 149 | int y = rv::rand::Rand::Uniform(0, depth.rows); 150 | 151 | if(mask(y, x) == 0 && depth(y, x) < data_provider->bgValue()) { 152 | samples.push_back(boost::make_shared(depth, x, y)); 153 | targets->push_back(boost::make_shared(0, 2, neg_samples_weight)); 154 | n_neg_samples++; 155 | } 156 | } 157 | 158 | while(n_pos_samples < pos_samples_p_image) { 159 | int x = rv::rand::Rand::Uniform(0, depth.cols); 160 | int y = rv::rand::Rand::Uniform(0, depth.rows); 161 | 162 | if(mask(y, x) > 0) { 163 | samples.push_back(boost::make_shared(depth, x, y)); 164 | targets->push_back(boost::make_shared(1, 2, pos_samples_weight)); 165 | n_pos_samples++; 166 | } 167 | } 168 | } 169 | } 170 | 171 | std::vector vec_targets; 172 | vec_targets.push_back(targets); 173 | 174 | std::cout << "n samples: " << samples.size() << std::endl; 175 | std::cout << "n targets: " << targets->size() << std::endl; 176 | 177 | rv::rf::TrainForest rf_train(params, true); 178 | rv::rf::ForestPtr forest; 179 | forest = rf_train.Train(samples, vec_targets, vec_targets, rv::rf::TRAIN, forest); 180 | 181 | return forest; 182 | } 183 | 184 | 185 | template class HandSegmentationRf; 186 | -------------------------------------------------------------------------------- /src/data_provider_blender.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "data_provider_blender.h" 30 | #include "common.h" 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include 37 | 38 | #include 39 | 40 | template 41 | DataProviderBlender::DataProviderBlender(const pose::DataProviderParameter& param) : 42 | DataProvider(param) { 43 | 44 | std::vector all_depth_paths; 45 | if(param.blender_param().recursive()) { 46 | rv::io::ListFilesRecursive(param.blender_param().data_path(), ".*_depth_0002.exr", all_depth_paths); 47 | } 48 | else { 49 | rv::io::ListFiles(param.blender_param().data_path(), ".*_depth_0002.exr", all_depth_paths); 50 | } 51 | std::cout << "[INFO] found " << all_depth_paths.size() << " depth images" << std::endl; 52 | all_depth_paths = this->sortReducePaths(all_depth_paths); 53 | std::cout << "[INFO] sorted and reduced to " << all_depth_paths.size() << " depth images" << std::endl; 54 | 55 | rv::timer::CpuBatchedTimer& timer = rv::timer::CpuBatchedTimer::i(); 56 | 57 | unsigned int valid_anno_idx = 0; 58 | depth_paths_.resize(all_depth_paths.size()); 59 | annos_.resize(all_depth_paths.size()); 60 | 61 | timer.start("overall"); 62 | // #pragma omp parallel for 63 | for(unsigned int depth_idx = 0; depth_idx < all_depth_paths.size(); ++depth_idx) { 64 | timer.start("all_depth_paths"); 65 | boost::filesystem::path depth_path = all_depth_paths[depth_idx]; 66 | timer.stop("all_depth_paths"); 67 | 68 | timer.start("annoPath"); 69 | boost::filesystem::path anno_path = annoPath(depth_path); 70 | timer.stop("annoPath"); 71 | 72 | if(boost::filesystem::exists(anno_path)) { 73 | timer.start("readAnno"); 74 | std::vector anno = readAnno(anno_path, param.n_pts()); 75 | timer.stop("readAnno"); 76 | 77 | #pragma omp critical (DataProviderBlender_constructor_append) 78 | { 79 | timer.start("push_back"); 80 | depth_paths_[valid_anno_idx] = depth_path; 81 | annos_[valid_anno_idx] = anno; 82 | valid_anno_idx++; 83 | if(valid_anno_idx % 10000 == 0) { 84 | std::cout << "read " << valid_anno_idx << " annotations" << std::endl; 85 | } 86 | timer.stop("push_back"); 87 | } 88 | } 89 | } 90 | timer.stop("overall"); 91 | 92 | timer.print(std::cout); 93 | 94 | depth_paths_.resize(valid_anno_idx); 95 | annos_.resize(valid_anno_idx); 96 | this->max_idx_ = depth_paths_.size(); 97 | 98 | std::cout << "[INFO] label file " << param.blender_param().data_path() << " contained " << depth_paths_.size() << " annotated depth images" << std::endl; 99 | 100 | cv::Vec3f mean, mean_0, std, std_0; 101 | this->stat(annos_, mean, std, mean_0, std_0); 102 | std::cout << "[INFO] anno[0] = " << mean_0 << " +- " << std_0 << " | anno[all] = " << mean << " +- " << std << std::endl; 103 | } 104 | 105 | 106 | template 107 | boost::filesystem::path DataProviderBlender::annoPath(const boost::filesystem::path& depth_path) const { 108 | std::string id, ts; 109 | getIdTs(depth_path, id, ts); 110 | 111 | boost::format fmt("%s_%s_anno_blender.txt"); 112 | return depth_path.parent_path() / (fmt % id % ts).str(); 113 | } 114 | 115 | 116 | template 117 | boost::filesystem::path DataProviderBlender::depthPath(int idx) const { 118 | return depth_paths_[idx]; 119 | } 120 | 121 | template 122 | cv::Mat_< Dtype > DataProviderBlender::depth_internal(int idx) { 123 | cv::Mat_ dm = readExrDepth(depth_paths_[idx]) * 1000; //to mm 124 | 125 | Dtype bg_value = this->param_.bg_value(); 126 | float noise_gaussian_sigma = this->param_.blender_param().noise_gaussian_sigma(); 127 | 128 | for(int row = 0; row < dm.rows; ++row) { 129 | for(int col = 0; col < dm.cols; ++col) { 130 | Dtype d = dm(row, col); 131 | if(d < 0 || d > bg_value) { 132 | d = bg_value; 133 | } 134 | else { 135 | if(noise_gaussian_sigma > 0) { 136 | d = d + rv::rand::Rand::i().Gaussian(0, noise_gaussian_sigma); 137 | } 138 | } 139 | 140 | dm(row, col) = d; 141 | } 142 | } 143 | 144 | return dm; 145 | } 146 | 147 | template 148 | cv::Mat_< Dtype > DataProviderBlender::ir_internal(int idx) { 149 | std::string ir_path = depth_paths_[idx].string(); 150 | boost::replace_last(ir_path, "depth", "rgb"); 151 | boost::replace_last(ir_path, "exr", "png"); 152 | 153 | cv::Mat_ ir; 154 | if(boost::filesystem::exists(ir_path)) { 155 | cv::Mat_ bgr = cv::imread(ir_path); 156 | 157 | ir = cv::Mat_::zeros(bgr.rows, bgr.cols); 158 | for(int row = 0; row < ir.rows; ++row) { 159 | for(int col = 0; col < ir.cols; ++col) { 160 | cv::Vec3b v = bgr(row, col); 161 | if(v(2) >= 128 && v(1) <= 32 && v(0) <= 32) { 162 | ir(row, col) = 750; 163 | } 164 | } 165 | } 166 | } 167 | else { 168 | ir = 1.0 / readExrDepth(depth_paths_[idx]); 169 | } 170 | 171 | return ir; 172 | } 173 | 174 | template 175 | std::vector< cv::Vec3f > DataProviderBlender::gt_internal(int idx) const { 176 | return annos_[idx]; 177 | } 178 | 179 | template 180 | cv::Vec3f DataProviderBlender::hint2d_internal(int idx) const { 181 | return annos_[idx][0]; 182 | } 183 | 184 | 185 | 186 | template 187 | void DataProviderBlender::shuffle() { 188 | static cv::RNG rng(time(0)); 189 | 190 | std::vector shuffled_depth_paths(depth_paths_.size()); 191 | std::vector > shuffled_annos(annos_.size()); 192 | 193 | for(size_t idx = 0; idx < depth_paths_.size(); ++idx) { 194 | int rand_idx = rng.uniform(0, depth_paths_.size()); 195 | shuffled_depth_paths[idx] = depth_paths_[rand_idx]; 196 | shuffled_annos[idx] = annos_[rand_idx]; 197 | } 198 | 199 | depth_paths_ = shuffled_depth_paths; 200 | annos_ = shuffled_annos; 201 | } 202 | 203 | 204 | 205 | template class DataProviderBlender; 206 | template class DataProviderBlender; 207 | -------------------------------------------------------------------------------- /src/common.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "common.h" 30 | 31 | #include 32 | 33 | #include 34 | 35 | #include 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #include 43 | #include 44 | #include 45 | 46 | 47 | //------------------------------------------------------------------------------ 48 | void depthmap2pts(const cv::Mat_& dm, const cv::Mat_& ir, const cv::Mat_& img, const float& max_d, const float& min_ir, 49 | std::vector& pts, std::vector& colors) { 50 | 51 | for(int row = 0; row < dm.rows; ++row) { 52 | for(int col = 0; col < dm.cols; ++col) { 53 | float d = dm(row, col); 54 | if(d < max_d && d > 0 && ir(row, col) >= min_ir) { 55 | pts.push_back(cv::Vec3f(col, row, d)); 56 | colors.push_back(cv::Vec3f(img(row, col)[0] / 255.0, img(row, col)[1] / 255.0, img(row, col)[2] / 255.0)); 57 | } 58 | } 59 | } 60 | } 61 | 62 | 63 | 64 | 65 | //------------------------------------------------------------------------------ 66 | std::vector readAnno(boost::filesystem::path anno_path, int n_joints) { 67 | std::vector joints(n_joints); 68 | 69 | std::ifstream fin(anno_path.c_str()); 70 | 71 | for(size_t joint_idx = 0; joint_idx < joints.size(); ++joint_idx) { 72 | fin >> joints[joint_idx][0]; 73 | fin >> joints[joint_idx][1]; 74 | fin >> joints[joint_idx][2]; 75 | } 76 | 77 | fin.close(); 78 | 79 | return joints; 80 | } 81 | 82 | 83 | //------------------------------------------------------------------------------ 84 | void writePoints(std::ostream& out, const std::vector& pts) { 85 | for(size_t pts_idx = 0; pts_idx < pts.size(); ++pts_idx) { 86 | out << pts[pts_idx][0] << " " << pts[pts_idx][1] << " " << pts[pts_idx][2]; 87 | if(pts_idx < pts.size() - 1) { 88 | out << " "; 89 | } 90 | else { 91 | out << std::endl; 92 | } 93 | } 94 | } 95 | 96 | 97 | //------------------------------------------------------------------------------ 98 | cv::Mat_ readExrDepth(const boost::filesystem::path& p) { 99 | cv::Mat_ exrC3 = cv::imread(p.string(), -1); 100 | std::vector > exrChannels; 101 | cv::split(exrC3, exrChannels); 102 | cv::Mat_ depth = exrChannels[0]; 103 | 104 | return depth; 105 | } 106 | 107 | 108 | bool readProtoFromTextFile(const boost::filesystem::path& path, google::protobuf::Message* proto) { 109 | int fd = open(path.c_str(), O_RDONLY); 110 | if(fd < 0) return false; 111 | 112 | google::protobuf::io::FileInputStream* input = new google::protobuf::io::FileInputStream(fd); 113 | bool success = google::protobuf::TextFormat::Parse(input, proto); 114 | delete input; 115 | close(fd); 116 | 117 | return success; 118 | } 119 | 120 | 121 | 122 | //------------------------------------------------------------------------------ 123 | void getIdTs(boost::filesystem::path depth_map_path, std::string& id, std::string& ts) { 124 | std::string depth_map_filename = depth_map_path.filename().string(); 125 | 126 | boost::regex expression("(\\d+)_(\\d+)_depth.*"); 127 | boost::smatch what; 128 | if(boost::regex_match(depth_map_filename, what, expression, boost::match_extra)) { 129 | id = what[1].str(); 130 | ts = what[2].str(); 131 | } 132 | } 133 | 134 | //------------------------------------------------------------------------------ 135 | std::string getAnnotator(boost::filesystem::path anno_path) { 136 | std::string anno_filename = anno_path.filename().string(); 137 | 138 | boost::regex expression("\\d+_\\d+_anno_(.+).txt"); 139 | boost::smatch what; 140 | std::string annotator = "unknown"; 141 | if(boost::regex_match(anno_filename, what, expression, boost::match_extra)) { 142 | annotator = what[1].str(); 143 | } 144 | 145 | return annotator; 146 | } 147 | 148 | //------------------------------------------------------------------------------ 149 | std::vector lsAnnoPaths(boost::filesystem::path depth_map_path) { 150 | 151 | std::string id, ts; 152 | getIdTs(depth_map_path, id, ts); 153 | 154 | boost::format fmt("%s_%s_.*.txt"); 155 | std::vector anno_paths; 156 | rv::io::ListFiles(depth_map_path.parent_path(), (fmt % id % ts).str(), anno_paths); 157 | 158 | return anno_paths; 159 | } 160 | 161 | //------------------------------------------------------------------------------ 162 | boost::filesystem::path infraredPath(boost::filesystem::path depth_map_path) { 163 | std::string id, ts; 164 | getIdTs(depth_map_path, id, ts); 165 | 166 | boost::format fmt("%s_%s_ir.csv"); 167 | return depth_map_path.parent_path() / (fmt % id % ts).str(); 168 | } 169 | 170 | //------------------------------------------------------------------------------ 171 | boost::filesystem::path blenderRgbPath(boost::filesystem::path depth_map_path) { 172 | std::string id, ts, blender_id; 173 | 174 | std::string depth_map_filename = depth_map_path.filename().string(); 175 | boost::regex expression("(\\d+)_(\\d+)_depth_(\\d+).*"); 176 | boost::smatch what; 177 | if(boost::regex_match(depth_map_filename, what, expression, boost::match_extra)) { 178 | id = what[1].str(); 179 | ts = what[2].str(); 180 | blender_id = what[3].str(); 181 | } 182 | 183 | boost::format fmt("%s_%s_rgb_%s.png"); 184 | return depth_map_path.parent_path() / (fmt % id % ts % blender_id).str(); 185 | } 186 | 187 | 188 | 189 | cv::Rect enclosingRect(const std::vector< cv::Vec3f >& pts) { 190 | float min_x = 1e9; 191 | float min_y = 1e9; 192 | float max_x = -1e9; 193 | float max_y = -1e9; 194 | 195 | for(size_t idx = 0; idx < pts.size(); ++idx) { 196 | float x = pts[idx](0); 197 | float y = pts[idx](1); 198 | 199 | if(x < min_x) min_x = x; 200 | if(y < min_y) min_y = y; 201 | if(x > max_x) max_x = x; 202 | if(y > max_y) max_y = y; 203 | } 204 | 205 | cv::Rect rect = cv::Rect(min_x, min_y, max_x - min_x + 1, max_y - min_y + 1); 206 | return rect; 207 | } 208 | 209 | 210 | -------------------------------------------------------------------------------- /src/inference_heatmap_net.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #include "inference_heatmap_net.h" 30 | #include "create_data.h" 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include 38 | 39 | template 40 | std::vector< cv::Vec3f > InferenceHeatmapNet::operator()(const std::vector >& patches) const { 41 | std::vector > > blobs(patches.size()); //have to store shared_ptr, otherwise => bad time 42 | std::vector* > bottom(patches.size()); 43 | for(size_t hp_idx = 0; hp_idx < patches.size(); ++hp_idx) { 44 | cv::Mat_ patch = patches[hp_idx].patch_; 45 | blobs[hp_idx] = rv::ocv::mat2Blob(patch); 46 | 47 | bottom[hp_idx] = blobs[hp_idx].get(); 48 | } 49 | 50 | Dtype loss; 51 | net_->Forward(bottom, &loss); 52 | 53 | if(param_.type() == pose::NNHeatmapParameter_HeatmapType_TWO_D) { 54 | std::vector es = inference2d(); 55 | patchFitD(patches, es); 56 | return es; 57 | } 58 | else if(param_.type() == pose::NNHeatmapParameter_HeatmapType_TWO_POINT_FIVE_D) { 59 | std::vector es = inference2d(); 60 | inferenceD(es); 61 | return es; 62 | } 63 | else { 64 | return std::vector(n_pts_, cv::Vec3f(0.0, 0.0, 0.0)); 65 | } 66 | } 67 | 68 | 69 | template 70 | std::vector< cv::Vec3f > InferenceHeatmapNet::inference2d() const { 71 | int heatmap_width = param_.dim_x(); 72 | int heatmap_height = param_.dim_y(); 73 | float min_heatmap_val = param_.min_heatmap_val(); 74 | 75 | boost::shared_ptr > blob_ptr; 76 | if(net_->has_blob(param_.layer_name_2d())) { 77 | blob_ptr = net_->blob_by_name(param_.layer_name_2d()); 78 | } 79 | else { 80 | LOG(ERROR) << "no blob for 2d heatmap found"; 81 | } 82 | CHECK_EQ(blob_ptr->count(), n_pts_ * heatmap_height * heatmap_width); 83 | 84 | const Dtype* result_blob_data = blob_ptr->cpu_data(); 85 | 86 | int result_blob_data_idx = 0; 87 | std::vector estimates; 88 | for(int heatmap_idx = 0; heatmap_idx < n_pts_; ++heatmap_idx) { 89 | // cv::Mat_ heatmap = cv::Mat_::zeros(heatmap_height, heatmap_width); 90 | 91 | //collect points 92 | std::vector pts; 93 | std::vector w; 94 | cv::Vec2f pt_0; 95 | float w_0 = 0; 96 | for(int x = 0; x < heatmap_width; ++x) { 97 | for(int y = 0; y < heatmap_height; ++y) { 98 | Dtype val = result_blob_data[result_blob_data_idx]; 99 | result_blob_data_idx++; 100 | if(val > min_heatmap_val) { 101 | pts.push_back(cv::Vec2f(x, y)); 102 | w.push_back(val); 103 | } 104 | 105 | if(val > w_0) { 106 | w_0 = val; 107 | pt_0 = cv::Vec2f(x, y); 108 | } 109 | } 110 | } 111 | 112 | //mean shift 113 | float bandwidth = 0.75; 114 | int max_iters = 10; 115 | for(int iter = 0; iter < max_iters; ++iter) { 116 | cv::Vec2f pt_1(0.0, 0.0); 117 | float sum_k = 0; 118 | for(size_t pts_idx = 0; pts_idx < pts.size(); ++pts_idx) { 119 | float diff_x = (pt_0(0) - pts[pts_idx](0)) / bandwidth; 120 | float diff_y = (pt_0(1) - pts[pts_idx](1)) / bandwidth; 121 | float dist = std::sqrt(diff_x * diff_x + diff_y * diff_y); 122 | float k = w[pts_idx] * std::exp(-dist); 123 | 124 | sum_k += k; 125 | pt_1 = pt_1 + k * pts[pts_idx]; 126 | } 127 | pt_1 /= sum_k; 128 | 129 | // std::cout << "from: " << pt_0 << " to " << pt_1 << std::endl; 130 | pt_0 = pt_1; 131 | } 132 | 133 | 134 | // static rv::ocv::ColorMap& cmap = rv::ocv::CoolWarmColorMap::i(); 135 | // cv::imshow((boost::format("hm_%d") % heatmap_idx).str(), cmap.map(heatmap)); 136 | 137 | cv::Vec3f es(pt_0(0), pt_0(1), 0); 138 | //to [-1, 1] 139 | es(0) = 2 * es(0) / heatmap_width - 1; 140 | es(1) = 2 * es(1) / heatmap_height - 1; 141 | estimates.push_back(es); 142 | 143 | // std::cout << "es_" << heatmap_idx << " : " << es << std::endl; 144 | } 145 | 146 | return estimates; 147 | } 148 | 149 | 150 | template 151 | void InferenceHeatmapNet::patchFitD(const std::vector< HandPatch< Dtype > >& patches, std::vector< cv::Vec3f >& es) const { 152 | int heatmap_width = param_.dim_x(); 153 | int heatmap_height = param_.dim_y(); 154 | 155 | //estimate depth 156 | std::vector valid_depth_values; 157 | std::vector invalid_depth_indices; 158 | for(int idx = 0; idx < es.size(); ++idx) { 159 | // to patch coord 160 | int col = (es[idx](0) + 1) / 2.0 * patches[0].patch_.cols; 161 | int row = (es[idx](1) + 1) / 2.0 * patches[0].patch_.rows; 162 | Dtype depth = patches[0].patch_(row, col); 163 | if(depth >= 3.0 || depth <= -3.0) { //TODO: not always true 164 | invalid_depth_indices.push_back(idx); 165 | } 166 | else { 167 | valid_depth_values.push_back(depth); 168 | es[idx](2) = depth; 169 | } 170 | } 171 | 172 | Dtype median_depth = 0; 173 | if(valid_depth_values.size() > 0) { 174 | median_depth = rv::stat::Median(valid_depth_values); 175 | } 176 | 177 | for(int idx = 0; idx < invalid_depth_indices.size(); ++idx) { 178 | es[invalid_depth_indices[idx]](2) = median_depth; 179 | } 180 | } 181 | 182 | 183 | 184 | template 185 | void InferenceHeatmapNet::inferenceD(std::vector< cv::Vec3f >& es) const { 186 | 187 | boost::shared_ptr > blob_ptr; 188 | if(net_->has_blob(param_.layer_name_depth())) { 189 | blob_ptr = net_->blob_by_name(param_.layer_name_depth()); 190 | LOG(INFO) << "infer d"; 191 | } 192 | else { 193 | LOG(ERROR) << "no blob for depth heatmap found"; 194 | } 195 | CHECK_EQ(blob_ptr->count(), n_pts_ * param_.dim_z()); 196 | 197 | const Dtype* result_blob_data = blob_ptr->cpu_data(); 198 | 199 | int result_blob_data_idx = 0; 200 | for(int pt_idx = 0; pt_idx < n_pts_; ++pt_idx) { 201 | Dtype mean = 0; 202 | Dtype normalizer = 0; 203 | for(int z = 0; z < param_.dim_z(); ++z) { 204 | Dtype val = result_blob_data[result_blob_data_idx]; 205 | result_blob_data_idx++; 206 | if(val > param_.min_heatmap_val()) { 207 | mean = mean + val * z; 208 | normalizer += val; 209 | } 210 | } 211 | 212 | mean /= normalizer; 213 | 214 | //to [-1, 1] 215 | es[pt_idx](2) = 2 * mean / param_.dim_z() - 1; 216 | } 217 | } 218 | 219 | 220 | 221 | template class InferenceHeatmapNet; 222 | template class InferenceHeatmapNet; 223 | -------------------------------------------------------------------------------- /include/data_provider.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 Institute for Computer Graphics and Vision (ICG), 2 | // Graz University of Technology (TU GRAZ) 3 | 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 1. Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // 2. Redistributions in binary form must reproduce the above copyright 9 | // notice, this list of conditions and the following disclaimer in the 10 | // documentation and/or other materials provided with the distribution. 11 | // 3. All advertising materials mentioning features or use of this software 12 | // must display the following acknowledgement: 13 | // This product includes software developed by the ICG, TU GRAZ. 14 | // 4. Neither the name of the ICG, TU GRAZ nor the 15 | // names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | 18 | // THIS SOFTWARE IS PROVIDED BY ICG, TU GRAZ ''AS IS'' AND ANY 19 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | // DISCLAIMED. IN NO EVENT SHALL ICG, TU GRAZ BE LIABLE FOR ANY 22 | // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #ifndef DATA_PROVIDER_H 30 | #define DATA_PROVIDER_H 31 | 32 | #include 33 | #include 34 | 35 | #include 36 | 37 | #include 38 | #include 39 | #include "make_projection.h" 40 | 41 | template 42 | class DataProvider { 43 | public: 44 | DataProvider(const pose::DataProviderParameter& param) : 45 | idx_(0), max_idx_(0), param_(param) { 46 | } 47 | 48 | virtual ~DataProvider() {} 49 | 50 | virtual bool hasNext() { return idx_ < max_idx_; } 51 | virtual void next() { idx_++; } 52 | virtual void reset() { idx_ = 0; } 53 | 54 | virtual boost::filesystem::path depthPath(int idx) const = 0; 55 | 56 | virtual boost::filesystem::path depthPath() const { return depthPath(idx_); } 57 | virtual cv::Mat_ depth() { return depth(idx_); } 58 | virtual cv::Mat_ ir() { return ir(idx_); } 59 | virtual std::vector gt() { return gt(idx_); } 60 | virtual cv::Vec3f hint2d() { return hint2d(idx_); } 61 | 62 | virtual cv::Mat_ depth(int idx) { 63 | cv::Mat_ img = depth_internal(idx); 64 | cv::Mat_ img_und; 65 | if(param_.undistort()) { 66 | createProjection(); 67 | cv::remap(img, img_und, undist_map1_, undist_map2_, cv::INTER_NEAREST, cv::BORDER_CONSTANT, bgValue()); 68 | 69 | // static rv::ocv::ColorMap& cmap = rv::ocv::ColorMapCoolWarm::i(); 70 | // cv::imshow("input", cmap.Map(img)); 71 | // cv::imshow("undistort", cmap.Map(img_und)); 72 | // cv::waitKey(0); 73 | } 74 | else { 75 | img_und = img; 76 | } 77 | 78 | if(this->param_.flip()) { 79 | cv::flip(img_und, img_und, 1); 80 | } 81 | 82 | return img_und; 83 | } 84 | 85 | virtual cv::Mat_ ir(int idx) { 86 | cv::Mat_ img = ir_internal(idx); 87 | cv::Mat_ img_und; 88 | if(param_.undistort()) { 89 | createProjection(); 90 | cv::remap(img, img_und, undist_map1_, undist_map2_, cv::INTER_LINEAR, cv::BORDER_CONSTANT, 0); 91 | } 92 | else { 93 | img_und = img; 94 | } 95 | 96 | if(this->param_.flip()) { 97 | cv::flip(img_und, img_und, 1); 98 | } 99 | 100 | return img_und; 101 | } 102 | 103 | virtual std::vector gt(int idx) { 104 | std::vector pts = gt_internal(idx); 105 | if(param_.undistort()) { 106 | createProjection(); 107 | cv::Mat_ pts2(pts.size(), 1); 108 | for(int pts_idx = 0; pts_idx < pts.size(); ++pts_idx) { 109 | pts2(pts_idx, 0)(0) = pts[pts_idx][0]; 110 | pts2(pts_idx, 0)(1) = pts[pts_idx][1]; 111 | } 112 | 113 | cv::Mat_ pts2_und(pts.size(), 1); 114 | cv::Matx33d K = projection_->K(); 115 | cv::undistortPoints(pts2, pts2_und, K, projection_->d(), cv::noArray(), K); 116 | 117 | for(int pts_idx = 0; pts_idx < pts.size(); ++pts_idx) { 118 | pts[pts_idx](0) = pts2_und(pts_idx, 0)(0); 119 | pts[pts_idx](1) = pts2_und(pts_idx, 0)(1); 120 | } 121 | } 122 | 123 | return pts; 124 | } 125 | 126 | virtual cv::Vec3f hint2d(int idx) { 127 | cv::Vec3f pt = hint2d_internal(idx); 128 | if(param_.undistort()) { 129 | createProjection(); 130 | 131 | cv::Mat_ pts2(1, 1); 132 | pts2(0, 0)(0) = pt[0]; 133 | pts2(0, 0)(1) = pt[1]; 134 | 135 | cv::Mat_ pts2_und(1, 1); 136 | cv::Matx33d K = projection_->K(); 137 | cv::undistortPoints(pts2, pts2_und, K, projection_->d(), cv::noArray(), K); 138 | 139 | pt(0) = pts2_und(0, 0)(0); 140 | pt(1) = pts2_und(0, 0)(1); 141 | } 142 | 143 | return pt; 144 | } 145 | 146 | virtual int nPts() const { return param_.n_pts(); } 147 | virtual Dtype bgValue() const { return param_.bg_value(); } 148 | 149 | virtual void shuffle() = 0; 150 | 151 | virtual int maxIdx() const { return max_idx_; } 152 | 153 | virtual boost::shared_ptr projection() { 154 | createProjection(); 155 | return projection_; 156 | } 157 | 158 | protected: 159 | virtual cv::Mat_ depth_internal(int idx) = 0; 160 | virtual cv::Mat_ ir_internal(int idx) = 0; 161 | virtual std::vector gt_internal(int idx) const = 0; 162 | virtual cv::Vec3f hint2d_internal(int idx) const = 0; 163 | 164 | 165 | virtual void createProjection() { 166 | if(projection_ == 0) { 167 | cv::Mat_ img = depth_internal(0); 168 | projection_ = makeProjection(param_.projection_param(), depthPath()); 169 | 170 | cv::Matx33d K = projection_->K(); 171 | cv::Matx18d d = projection_->d(); 172 | 173 | std::cout << "K = " << std::endl << K << std::endl; 174 | std::cout << "d = " << std::endl << d << std::endl; 175 | 176 | cv::initUndistortRectifyMap(K, d, cv::Mat_::eye(3, 3), K, img.size(), CV_16SC2, undist_map1_, undist_map2_); 177 | } 178 | } 179 | 180 | virtual std::vector sortReducePaths(std::vector& paths) { 181 | std::sort(paths.begin(), paths.end()); 182 | 183 | //mind inc parameter 184 | int inc = param_.inc(); 185 | std::vector reduced_paths; 186 | for(int idx = 0; idx < paths.size(); idx = idx + inc) { 187 | reduced_paths.push_back(paths[idx]); 188 | } 189 | 190 | return reduced_paths; 191 | } 192 | 193 | virtual void stat(const std::vector >& anno, 194 | cv::Vec3f& mean, cv::Vec3f& std, 195 | cv::Vec3f& mean_0, cv::Vec3f& std_0) { 196 | mean = 0; 197 | std = 0; 198 | size_t n = 0; 199 | 200 | mean_0 = 0; 201 | std_0 = 0; 202 | 203 | //mean 204 | for(size_t anno_idx = 0; anno_idx < anno.size(); ++anno_idx) { 205 | mean_0 += anno[anno_idx][0]; 206 | 207 | for(size_t joint_idx = 0; joint_idx < anno[anno_idx].size(); ++joint_idx) { 208 | mean += anno[anno_idx][joint_idx]; 209 | n++; 210 | } 211 | } 212 | mean_0 = mean_0 / double(anno.size()); 213 | mean = mean / double(n); 214 | 215 | //std 216 | for(size_t anno_idx = 0; anno_idx < anno.size(); ++anno_idx) { 217 | cv::Vec3f d_0 = anno[anno_idx][0] - mean_0; 218 | std_0 += cv::Vec3f(d_0(0) * d_0(0), d_0(1) * d_0(1), d_0(2) * d_0(2)); 219 | 220 | for(size_t joint_idx = 0; joint_idx < anno[anno_idx].size(); ++joint_idx) { 221 | cv::Vec3f d = anno[anno_idx][0] - mean_0; 222 | std += cv::Vec3f(d(0) * d(0), d(1) * d(1), d(2) * d(2)); 223 | } 224 | } 225 | std_0 = std_0 / double(anno.size() - 1); 226 | std = std / double(n - 1); 227 | 228 | for(int i = 0; i < 3; ++i) { 229 | std_0(i) = std::sqrt(std_0(i)); 230 | std(i) = std::sqrt(std(i)); 231 | } 232 | } 233 | 234 | protected: 235 | boost::shared_ptr projection_; 236 | cv::Mat undist_map1_; 237 | cv::Mat undist_map2_; 238 | 239 | int idx_; 240 | int max_idx_; 241 | 242 | const pose::DataProviderParameter& param_; 243 | }; 244 | 245 | #endif // DATAPROVIDER_H 246 | --------------------------------------------------------------------------------