├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── build.sh ├── config ├── Levi_ground_truth_tracking_feature_images_test.yml ├── Yiwan_loading.yml ├── Yiwan_loading_fast.yml └── Yiwan_tracking.yml ├── examples └── Yiwan.sh ├── include ├── gui_app │ ├── BasicGLPane.h │ ├── CameraControl.h │ ├── ImagePanel.h │ ├── MainFrame.h │ ├── Matrices.h │ ├── PangaeaTracking.h │ ├── Vectors.h │ └── controlPanel.h ├── main_engine │ ├── MainEngine.h │ ├── image_source │ │ └── ImageSourceEngine.h │ ├── rendering │ │ ├── CCamera.h │ │ ├── DepthBuffer.h │ │ └── NcvGlXContext.h │ ├── tracker │ │ ├── DeformNRSFMTracker.h │ │ ├── FeaturePyramid.h │ │ ├── FeatureReader.h │ │ ├── ImagePyramid.h │ │ ├── Mesh.h │ │ ├── MeshData.h │ │ ├── MeshIO.h │ │ ├── MeshPyramid.h │ │ ├── OptimizationStrategy.h │ │ ├── ProblemWrapper.h │ │ ├── TrackingEngine.h │ │ └── residual.h │ └── utils │ │ ├── global.h │ │ ├── qx_basic.h │ │ ├── qx_ctbf_ss.h │ │ ├── qx_highlight_removal_bf.h │ │ └── settings.h └── third_party │ ├── KDTreeAdaptor.hpp │ ├── Stopwatch.h │ ├── jet_extras.h │ ├── msvc │ └── Stopwatch.h │ ├── nanoflann.hpp │ ├── ply.h │ └── sample.h ├── msvc ├── ConsoleApp │ ├── ConsoleApp.vcxproj │ └── ConsoleApp.vcxproj.filters ├── GUIApp │ ├── GUIApp.vcxproj │ └── GUIApp.vcxproj.filters ├── MainEngine │ ├── MainEngine.vcxproj │ └── MainEngine.vcxproj.filters ├── PangaeaTracking.sln └── README.txt └── src ├── console_app └── PangaeaTracking_console.cpp ├── gui_app ├── BasicGLPane.cpp ├── CameraControl.cpp ├── ImagePanel.cpp ├── MainFrame.cpp ├── Matrices.cpp ├── PangaeaTracking.cpp └── controlPanel.cpp ├── main_engine ├── MainEngine.cpp ├── image_source │ ├── ImageSequenceReader.cpp │ └── ImagesBufferReader.cpp ├── rendering │ ├── CCamera.cpp │ ├── DepthBuffer.cpp │ └── NcvGlXContext.cpp ├── tracker │ ├── DeformNRSFMTracker.cpp │ ├── FeaturePyramid.cpp │ ├── FeatureReader.cpp │ ├── ImagePyramid.cpp │ ├── Mesh.cpp │ ├── MeshBufferReader.cpp │ ├── MeshPyramidReader.cpp │ ├── MeshSequenceReader.cpp │ ├── OptimizationStrategy.cpp │ ├── ProblemWrapper.cpp │ ├── ShapeSequenceReader.cpp │ └── ShapesBufferReader.cpp └── utils │ ├── global.cpp │ ├── qx_basic.cpp │ ├── qx_ctbf_ss.cpp │ ├── qx_highlight_removal_bf.cpp │ └── settings.cpp └── mesh_rescale └── MeshRescale.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | #files 2 | 3 | GPATH 4 | GRTAGS 5 | GSYMS 6 | GTAGS 7 | 8 | *.mex 9 | ~* 10 | *.mat 11 | *.zip 12 | *.mexa64 13 | *.raw 14 | *~ 15 | *.ppm 16 | *.png 17 | *.jpg 18 | tmp.* 19 | */generate_scores.o* 20 | pangaea 21 | 22 | 23 | #folder 24 | /build_* 25 | /old_stuff 26 | 27 | # MSVC files 28 | *.opensdf 29 | *.sdf 30 | *.suo 31 | *.user 32 | 33 | # MSVC folders 34 | msvc/.localhistory/ 35 | msvc/x64 36 | msvc/MainEngine/x64 37 | msvc/ConsoleApp/x64 38 | msvc/GUIApp/x64 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Rui Yu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # enable pretty build (comment to see full commands) 2 | Q ?= @ 3 | 4 | # Set DEBUG to 1 for debugging 5 | DEBUG := 0 6 | 7 | # if only want to visualize saving results and don't do tracking, 8 | # you could set VIS_ONLY to 1 and compile without ceres solver, 9 | # but eigen3 is still needed. 10 | # VIS_ONLY := 1 11 | 12 | ifeq ($(DEBUG), 1) 13 | BUILD_DIR := build_debug 14 | else 15 | BUILD_DIR := build 16 | endif 17 | 18 | VIS_ONLY ?= 0 19 | ifeq ($(VIS_ONLY), 1) 20 | BUILD_DIR := $(BUILD_DIR)/PangaeaVis 21 | else 22 | BUILD_DIR := $(BUILD_DIR)/PangaeaTracking 23 | endif 24 | 25 | LIB_BUILD_DIR := $(BUILD_DIR)/lib 26 | BIN_BUILD_DIR := $(BUILD_DIR)/bin 27 | 28 | # All the directories containing code 29 | # SRC_DIRS := src/main_engine 30 | MAIN_DIRS := $(shell find src/main_engine -type d -exec bash -c "find {} -maxdepth 1 \ 31 | \( -name '*.cpp' \) | grep -q ." \; -print) 32 | GUI_DIRS := $(shell find src/gui_app -type d -exec bash -c "find {} -maxdepth 1 \ 33 | \( -name '*.cpp' \) | grep -q ." \; -print) 34 | SRC_DIRS := $(MAIN_DIRS) $(GUI_DIRS) 35 | 36 | ALL_BUILD_DIRS := $(addprefix $(BUILD_DIR)/, $(SRC_DIRS)) 37 | ALL_BUILD_DIRS += $(LIB_BUILD_DIR) 38 | 39 | EIGEN_INCLUDE := -I/usr/include/eigen3/unsupported -I/usr/include/eigen3 40 | 41 | WX_INCLUDE := `wx-config --cxxflags` 42 | 43 | HDF5_INCLUDE := -I/usr/include/hdf5/serial 44 | 45 | FLAGS_INCLUDE := $(EIGEN_INCLUDE) $(WX_INCLUDE) $(HDF5_INCLUDE) -I./include -I./include/third_party 46 | 47 | # Library dependencies 48 | GL_LIB := -lGL -lGLU -lX11 -lGLEW 49 | 50 | WX_LIB := `wx-config --libs --gl-libs` 51 | 52 | BOOST_LIB := -lboost_filesystem -lboost_system -lboost_thread 53 | 54 | OPENCV_LIB := -lopencv_core -lopencv_highgui -lopencv_imgproc -lopencv_imgcodecs 55 | 56 | CERES_LIB := -lceres -lglog -ltbb -ltbbmalloc -lcholmod -lccolamd \ 57 | -lcamd -lcolamd -lamd -lsuitesparseconfig -llapack -lf77blas -latlas 58 | 59 | LMDB_LIB := -llmdb 60 | 61 | HDF5_LIB := -lhdf5_hl -lhdf5 62 | 63 | LIBRARY_DIRS += $(LIB_BUILD_DIR) 64 | LDFLAGS := $(WX_LIB) $(BOOST_LIB) $(OPENCV_LIB) $(CERES_LIB) $(GL_LIB) $(LMDB_LIB) $(HDF5_LIB) 65 | LDFLAGS += $(foreach library_dir, $(LIBRARY_DIRS), -L$(library_dir)) 66 | 67 | # Setting compiler and building flags 68 | CXX := g++ 69 | #CXXFLAGS += -std=c++11 -fopenmp -fPIC $(FLAGS_INCLUDE) 70 | CXXFLAGS += -std=c++11 -fopenmp $(FLAGS_INCLUDE) -w 71 | 72 | # Debugging 73 | ifeq ($(DEBUG), 1) 74 | CXXFLAGS += -DDEBUG -g -O0 75 | else 76 | CXXFLAGS += -DNDEBUG -O2 -ffast-math -Wno-unused-result 77 | endif 78 | 79 | # Automatic dependency generation 80 | CXXFLAGS += -MMD -MP 81 | 82 | # Disable offsetof macro warnings 83 | CXXFLAGS += -Wno-invalid-offsetof 84 | 85 | # Get all source files 86 | MAIN_ENGINE_SRCS := $(shell find src/main_engine -name "*.cpp" ! -name "Deform*.cpp") 87 | #MAIN_ENGINE_SRCS := $(shell find src/main_engine -name "*.cpp" ) 88 | GUI_SRCS := $(shell find src/gui_app -name "*.cpp" ! -name "PangaeaTracking.cpp") 89 | GUI_APP_SRCS := $(shell find src/gui_app -name "PangaeaTracking.cpp") 90 | CONSOLE_APP_SRCS := $(shell find src/console_app -name "PangaeaTracking_console.cpp") 91 | 92 | CONSOLE_BIN := $(BIN_BUILD_DIR)/PangaeaTracking_console 93 | 94 | ifeq ($(VIS_ONLY), 0) 95 | MAIN_ENGINE_SRCS += src/main_engine/tracker/DeformNRSFMTracker.cpp 96 | MAIN_LIB := $(LIB_BUILD_DIR)/libmain_engine.a 97 | GUI_BIN := $(BIN_BUILD_DIR)/PangaeaTracking 98 | LDFLAGS := -lmain_engine $(LDFLAGS) 99 | else 100 | CXXFLAGS += -DVIS_ONLY 101 | MAIN_LIB := $(LIB_BUILD_DIR)/libmain_engine_vis.a 102 | GUI_BIN := $(BIN_BUILD_DIR)/PangaeaTracking_vis 103 | LDFLAGS := -lmain_engine_vis $(LDFLAGS) 104 | endif 105 | 106 | # The objects corresponding to the source files 107 | MAIN_OBJS := $(addprefix $(BUILD_DIR)/, ${MAIN_ENGINE_SRCS:.cpp=.o}) 108 | GUI_OBJS := $(addprefix $(BUILD_DIR)/, ${GUI_SRCS:.cpp=.o}) 109 | 110 | # Output files for automatic dependency generation 111 | DEPS := $(MAIN_OBJS:.o=.d) $(GUI_OBJS:.o=.d) 112 | 113 | ifeq ($(VIS_ONLY), 1) 114 | APP := $(GUI_BIN) 115 | else 116 | APP := $(CONSOLE_BIN) $(GUI_BIN) 117 | endif 118 | 119 | #ORIGIN := \$$ORIGIN 120 | #.PHONY: 121 | 122 | .PHONY: all 123 | 124 | all: $(APP) 125 | 126 | $(CONSOLE_BIN): $(MAIN_LIB) | $(BIN_BUILD_DIR) 127 | @ echo CXX/LD -o $@ 128 | $(Q) $(CXX) $(CONSOLE_APP_SRCS) -o $@ $(CXXFLAGS) $(LINKFLAGS) $(LDFLAGS) 129 | 130 | $(GUI_BIN): $(GUI_OBJS) $(MAIN_LIB) | $(BIN_BUILD_DIR) 131 | # @ echo $(MAIN_ENGINE_SRCS) 132 | # @ echo $(MAIN_OBJS) 133 | @ echo CXX/LD -o $@ 134 | $(Q) $(CXX) $(GUI_APP_SRCS) -o $@ $(GUI_OBJS) $(CXXFLAGS) $(LINKFLAGS) $(LDFLAGS) 135 | 136 | # $(CXX) $(GUI_APP_SRCS) $(CXXFLAGS) -o $@ $(GUI_OBJS) $(LINKFLAGS) $(WX_PATH) $(LIBS_LINKER) -L./build/lib -Wl,-rpath,$(ORIGIN)/../lib -lmain_engine_vis 137 | # $(CXX) $(GUI_APP_SRCS) $(CXXFLAGS) -o $@ $(GUI_OBJS) -lmain_engine_vis $(LINKFLAGS) $(LDFLAGS) 138 | # $(Q)$(CXX) $< -o $@ $(LINKFLAGS) $(LDFLAGS) \ 139 | # -Wl,-rpath, -lmain_engine_vis 140 | 141 | #@ echo AR -o $@ 142 | #$(Q)ar rcs $@ $(MAIN_OBJS) 143 | #@ echo $(MAIN_OBJS) 144 | 145 | $(MAIN_LIB): $(MAIN_OBJS) | $(LIB_BUILD_DIR) 146 | @ echo AR -o $@ 147 | $(Q)ar rcs $@ $(MAIN_OBJS) 148 | # @ echo $(MAIN_OBJS) 149 | # @ echo LD -o $@ 150 | # $(Q)$(CXX) -shared -o $@ $(MAIN_OBJS) $(LINKFLAGS) $(LDFLAGS) 151 | # $(Q)$(CXX) -o $@ $(MAIN_OBJS) $(LINKFLAGS) $(LDFLAGS) 152 | 153 | # Define build targets 154 | $(BUILD_DIR)/%.o: %.cpp | $(ALL_BUILD_DIRS) 155 | @ echo CXX $< 156 | $(Q)$(CXX) $< $(CXXFLAGS) -c -o $@ 157 | 158 | $(ALL_BUILD_DIRS): 159 | @ mkdir -p $@ 160 | 161 | $(BIN_BUILD_DIR): 162 | @ mkdir -p $@ 163 | 164 | -include $(DEPS) 165 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #PangaeaTracking 2 | 3 | This is the implementation of the following BMVC2016 paper: 4 | 5 | '*Better Together: Joint Reasoning for Non-rigid 3D Reconstruction with Specularities and Shading*'. Qi Liu-Yin, Rui Yu, Lourdes Agapito, Andrew Fitzgibbon, Chris Russell 6 | 7 | For more information about this work, please visit the [project website](http://www0.cs.ucl.ac.uk/staff/Qi.Liu/bmvc16/better_together.html). 8 | 9 | This github repository is maintained by Qi Liu (Qi.Liu@cs.ucl.ac.uk). 10 | Contact me if you have any questions. 11 | 12 | #1. Building the System 13 | 14 | ###1.1 Requirements 15 | 16 | PangaeaTracking has been tested in Ubuntu 14.04 only. Several 3rd party libraries are needed for compiling PangaeaTracking. 17 | 18 | - OpenGL / GLU / GLEW / X11 / TBB / LMDB / HDF5 19 | ``` 20 | sudo apt-get install libgl1-mesa-dev 21 | sudo apt-get install libglu1-mesa-dev 22 | sudo apt-get install libglew1.8 libglew-dev 23 | sudo apt-get install libx11-dev 24 | sudo apt-get install libtbb-dev 25 | sudo apt-get install liblmdb-dev 26 | sudo apt-get install libhdf5-serial-dev 27 | ``` 28 | - OPENCV (e.g. version 2.4.8 or later) 29 | available at http://opencv.org/ 30 | 31 | - Ceres Solver 32 | available at http://ceres-solver.org/ 33 | 34 | - wxWidgets 35 | available at https://www.wxwidgets.org/ 36 | 37 | - Boost 38 | available at http://www.boost.org/ 39 | 40 | ###1.2 Build Process 41 | 42 | To compile the system, do the following: 43 | 44 | ``` 45 | ./build.sh 46 | ``` 47 | 48 | #2. Data 49 | 50 | One example sequence is available at [google drive](https://drive.google.com/drive/folders/0B8-9V4y1N7pxZExaMlE3bnc3Mzg). 51 | 52 | #3. Examples 53 | 54 | After building PangaeaTracking and preparing the data, you are ready to run the scripts in examples folder. 55 | Check examples/Yiwan.sh for usage. 56 | 57 | #4. GUI Usage 58 | 59 | For rotating the 3d model in 2d image plane, use the middle mouse button. 60 | 61 | ------ 62 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #make VIS_ONLY=1 4 | make VIS_ONLY=0 -------------------------------------------------------------------------------- /config/Levi_ground_truth_tracking_feature_images_test.yml: -------------------------------------------------------------------------------- 1 | %YAML:1.0 2 | ImageSourceType: 'ImageSequence' 3 | TrackingType: 'DeformNRSFMTracker' 4 | ImageSourceSettings: 5 | dataPath: '/cvfish/home/Work/data/Qi_synthetic/images/' 6 | numFrames: 379 7 | width: 1920 8 | height: 1088 9 | startFrame: 180 10 | imageFormat: 'rgb%04d.png' 11 | useMultiImages: 1 12 | dataPathLevelRoot: "/cvfish/home/Work/data/Qi_synthetic/remeshed_0180/images_render/" 13 | dataPathLevelFormat: "%02dk/" 14 | imageLevelFormat: "render%04d.png" 15 | # dataPathLevelList: [25, 10, 5, 5, 5] 16 | dataPathLevelList: [25, 10, 5] 17 | TrackerSettings: 18 | savePath: '/home/cvfish/Work/code/github/PangaeaTracking/results/Levi/feature_images/feature_term_1000/' 19 | # error_type: 'gray' 20 | # error_type: 'color' 21 | # error_type: 'ncc' 22 | error_type: 'color_ncc' 23 | ba_type: 'motstr' 24 | photometric_huber_width: 0.1 25 | tv_huber_width: 1 26 | tv_tukey_width: 0 27 | # photometric_weight: 1000 28 | photometric_weight: 1 29 | grad_weight: 0 30 | tv_weight: 0.05 31 | deform_weight: 0.2 32 | do_alternation: 1 33 | arap_weight: 1 34 | inextent_weight: 0 35 | trans_weight: 100000 36 | linear_solver: CG 37 | save_results: 1 38 | save_mesh: 1 39 | # save_mesh_pyramid: 1 40 | # save_prop_pyramid: 1 41 | save_mesh_pyramid: 0 42 | save_prop_pyramid: 0 43 | use_depth_pyramid: 0 44 | use_prev_for_template_in_tv: 0 45 | load_mesh: 1 46 | use_visibility_mask: 1 47 | use_opengl_mask: 0 48 | mesh_pyramid_file: '/cvfish/home/Work/data/Qi_synthetic/remeshed_0180/cropped_remeshed_color_obj/cropped_remeshed_color_0180_%02dk.obj' 49 | show_window: 0 50 | blurFilterSize (at each level): [0,0,0,0,0] 51 | imageGradientsScalingFactor (at each level): [0.03125,0.03125,0.03125,0.03125,0.03125] 52 | # max_num_iterations (at each level): [5,5,10,10,30] 53 | max_num_iterations (at each level): [10,20,30,30,30] 54 | function_tolerance (at each level): [1e-4, 1e-4, 1e-4, 1e-4, 1e-4] 55 | gradient_tolerance (at each level): [1e-3, 1e-3, 1e-3, 1e-3, 1e-3] 56 | parameter_tolerance (at each level): [1e-6, 1e-6, 1e-6, 1e-6, 1e-6] 57 | initial_trust_region_radius (at each level): [1e4, 1e4, 1e4, 1e4, 1e4] 58 | max_trust_region_radius (at each level): [1e8, 1e8, 1e8, 1e8, 1e8] 59 | min_trust_region_radius (at each level): [1e-32,1e-32,1e-32,1e-32,1e-32] 60 | min_relative_decrease (at each level): [1e-3,1e-3,1e-3,1e-3,1e-3] 61 | # mesh_pyramid_vertex_num: [25, 10, 5, 5, 5] 62 | mesh_pyramid_vertex_num: [25, 10, 5] 63 | # imagePyramidSamplingFactors (at each level): [1,2,4,8,16] 64 | imagePyramidSamplingFactors (at each level): [1,2,8] 65 | mesh_pyramid_neighbor_num: [5, 5, 5, 5, 5] 66 | mesh_pyramid_neighbor_radius: [500, 200, 100, 50, 10] 67 | clockwise: 0 68 | hasGT: 1 69 | meshPathGT: "/cvfish/home/Work/data/Qi_synthetic/remeshed_0180/cropped_remeshed_color_obj/" 70 | meshLevelFormatGT: "cropped_remeshed_color_%04d_%02dk.obj" 71 | # meshLevelListGT: [25,10,5,5,5] 72 | meshLevelListGT: [25,10,5] 73 | print_energy: 1 74 | meshPyramidFormat: 'mesh%04d_level%02d.ply' 75 | propPyramidFormat: 'prop_mesh%04d_level%02d.ply' 76 | save_ply: 1 77 | # use_feature_images: 0 78 | # use_rgb_images: 1 79 | neighbor_patch_radius: 1 80 | use_feature_images: 1 81 | use_rgb_images: 0 82 | data_term_pair: 83 | - { first:[0], second:[1] } 84 | - { first:[1], second:[2] } 85 | reg_term_pair: 86 | - { first:[2], second:[2] } 87 | - { first:[2], second:[2] } 88 | FeatureSettings: 89 | use_ncc: 1 90 | channels: 3 91 | scaling_factor: 1.0 92 | db_path: '/cvfish/home/Work/data/Qi_synthetic/remeshed_0180/images_render/25k/feature_images/sift/binSize2_lmdb_double/features.h5' 93 | # feature_term_weight: 1000 94 | feature_term_weight: 1 95 | feature_huber_width: 0.1 96 | key_name_format: '/sift%04d' 97 | blurFeatureFilterSize (at each level): [5,7,9,11,13] 98 | # blurFeatureFilterSize (at each level): [0,0,0,0,0] 99 | featurePyramidSamplingFactors (at each level): [1,2,4,8,16] 100 | featureGradientScalingFactor (at each level): [0.03125,0.03125,0.03125,0.03125,0.03125] 101 | # MeshLoadingSettings: 102 | # meshPath: "/cvfish/home/Work/data/Qi_synthetic/remeshed_0180/remeshed_color_obj/" 103 | # meshFormat: "remeshed_color_%04d_05k.obj" 104 | # meshLevelFormat: "remeshed_color_%04d_%02dk.obj" 105 | # # meshLevelList: [5,10,25,30,40,50] 106 | # meshLevelList: [25,10,5] 107 | # visibilityMask: 1 108 | # loadProp: 0 109 | # fastLoading: 0 110 | # clockwise: 0 111 | -------------------------------------------------------------------------------- /config/Yiwan_loading.yml: -------------------------------------------------------------------------------- 1 | %YAML:1.0 2 | ImageSourceType: 'ImageSequence' 3 | TrackingType: 'MeshPyramid' 4 | #TrackingType: 'MeshBuffer' 5 | ImageSourceSettings: 6 | dataPath: '../data/Yiwan/images/' 7 | numFrames: 500 8 | width: 1920 9 | height: 1080 10 | startFrame: 1 11 | MeshLoadingSettings: 12 | meshPath: "../data/Yiwan/results/arap1_tv0.05_deform0.2_trans100000/mesh_pyramid/" 13 | meshFormat: "mesh%04d.obj" 14 | meshPyramidFormat: "mesh%04d_level%02d.obj" 15 | propPyramidFormat: "prop_mesh%04d_level%02d.obj" 16 | meshLevelList: [0,1,2] 17 | # meshLevelList: [0] 18 | visibilityMask: 0 19 | loadProp: 0 20 | -------------------------------------------------------------------------------- /config/Yiwan_loading_fast.yml: -------------------------------------------------------------------------------- 1 | %YAML:1.0 2 | ImageSourceType: 'ImageSequence' 3 | TrackingType: 'MeshBuffer' 4 | ImageSourceSettings: 5 | dataPath: '../data/Yiwan/images/' 6 | numFrames: 500 7 | width: 1920 8 | height: 1080 9 | startFrame: 1 10 | MeshLoadingSettings: 11 | meshPath: "../data/Yiwan/results/arap1_tv0.05_deform0.2_trans100000/mesh_pyramid/" 12 | meshFormat: "mesh%04d.obj" 13 | meshPyramidFormat: "mesh%04d_level%02d.obj" 14 | propPyramidFormat: "prop_mesh%04d_level%02d.obj" 15 | meshLevelList: [0,1,2] 16 | # meshLevelList: [0] 17 | visibilityMask: 0 18 | loadProp: 0 19 | -------------------------------------------------------------------------------- /config/Yiwan_tracking.yml: -------------------------------------------------------------------------------- 1 | %YAML:1.0 2 | ImageSourceType: 'ImageSequence' 3 | TrackingType: 'DeformNRSFMTracker' 4 | ImageSourceSettings: 5 | dataPath: '../data/Yiwan/images/' 6 | numFrames: 500 7 | width: 1920 8 | height: 1080 9 | isOrthoCamera: 0 10 | TrackerSettings: 11 | # savePath: '../data/Yiwan/results/arap1_tv0.05_deform0.2_trans100000/' 12 | savePath: '../results/' 13 | error_type: 'gray' 14 | ba_type: 'motstr' 15 | photometric_huber_width: 0.1 16 | tv_huber_width: 1 17 | tv_tukey_width: 0 18 | photometric_weight: 1000 19 | grad_weight: 0 20 | tv_weight: 0.05 21 | deform_weight: 0.2 22 | do_alternation: 1 23 | arap_weight: 1 24 | inextent_weight: 0 25 | trans_weight: 100000 26 | linear_solver: CG 27 | save_results: 1 28 | save_mesh: 1 29 | save_mesh_pyramid: 1 30 | save_prop_pyramid: 1 31 | use_depth_pyramid: 0 32 | use_prev_for_template_in_tv: 0 33 | load_mesh: 1 34 | use_visibility_mask: 1 35 | use_opengl_mask: 0 36 | mesh_file: '../data/Yiwan/mesh_pyramid_crop/mesh_vertex5000.obj' 37 | mesh_pyramid_file: '../data/Yiwan/mesh_pyramid_crop/mesh_vertex%d.obj' 38 | show_window: 1 39 | blurFilterSize (at each level): [5,7,9,11,13] 40 | imageGradientsScalingFactor (at each level): [0.03125,0.03125,0.03125,0.03125,0.03125] 41 | max_num_iterations (at each level): [5,5,30,30,30] 42 | function_tolerance (at each level): [1e-4, 1e-4, 1e-4, 1e-4, 1e-4] 43 | gradient_tolerance (at each level): [1e-3, 1e-3, 1e-3, 1e-3, 1e-3] 44 | parameter_tolerance (at each level): [1e-6, 1e-6, 1e-6, 1e-6, 1e-6] 45 | initial_trust_region_radius (at each level): [1e4, 1e4, 1e4, 1e4, 1e4] 46 | max_trust_region_radius (at each level): [1e8, 1e8, 1e8, 1e8, 1e8] 47 | min_trust_region_radius (at each level): [1e-32,1e-32,1e-32,1e-32,1e-32] 48 | min_relative_decrease (at each level): [1e-3,1e-3,1e-3,1e-3,1e-3] 49 | mesh_pyramid_vertex_num: [25000,10000,5000] 50 | imagePyramidSamplingFactors (at each level): [2,4,8] 51 | mesh_pyramid_neighbor_num: [5, 5, 5] 52 | mesh_pyramid_neighbor_radius: [500, 200, 100] 53 | -------------------------------------------------------------------------------- /examples/Yiwan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # loading results 4 | ../build/PangaeaTracking/bin/PangaeaTracking ../config/Yiwan_loading.yml 5 | #../build_debug/PangaeaTracking/bin/PangaeaTracking ../config/Yiwan_loading.yml 6 | 7 | # loading results into a buffer first 8 | #../build_debug/PangaeaTracking/bin/PangaeaTracking ../config/Yiwan_loading_fast.yml 9 | 10 | # do real tracking 11 | #../build/PangaeaTracking/bin/PangaeaTracking ../config/Yiwan_tracking.yml 12 | #../build_debug/PangaeaTracking/bin/PangaeaTracking ../config/Yiwan_tracking.yml 13 | 14 | -------------------------------------------------------------------------------- /include/gui_app/BasicGLPane.h: -------------------------------------------------------------------------------- 1 | #ifndef _BASIC_GLPANE_ 2 | #define _BASIC_GLPANE_ 3 | 4 | #include "gui_app/CameraControl.h" 5 | #include "gui_app/MainFrame.h" 6 | 7 | class controlPanel; 8 | 9 | class BasicGLPane : public wxGLCanvas 10 | { 11 | public: 12 | 13 | BasicGLPane(wxWindow* parent, int* args); 14 | ~BasicGLPane(); 15 | 16 | void OnPaint(wxPaintEvent &event); 17 | void resized(wxSizeEvent& evt); 18 | 19 | int getWidth(); 20 | int getHeight(); 21 | void setImageHeight(int height); 22 | void setImageWidth(int width); 23 | void setIntrinsicMatrix(double K[3][3]); 24 | void setProjectionMatrix(double zNear=0.1, double zFar=20000); 25 | 26 | void render(wxPaintEvent& evt); 27 | 28 | // events 29 | void mouseMoved(wxMouseEvent& evt); 30 | void mouseWheelMoved(wxMouseEvent& evt); 31 | 32 | void initLighting(bool spot,GLfloat ambient,GLfloat spot1,GLfloat spot2,GLfloat spot3); 33 | 34 | void pangaeaMeshRender(TrackerOutputInfo& outputInfo, bool showTexture, bool 35 | showGT, bool showEST, bool threshOn, GLfloat thresh); 36 | void meshRender(PangaeaMeshData& mesh, bool showTexture, bool isGT, bool threshOn, GLfloat thresh); 37 | 38 | void pangaeaPointRender(TrackerOutputInfo& outputInfo, bool showTexture, bool 39 | showGT, bool showEST, bool threshOn, GLfloat thresh); 40 | void pointRender(PangaeaMeshData& mesh, bool showTexture, bool isGT, bool threshOn, GLfloat thresh); 41 | 42 | void renderHelper(PangaeaMeshData& mesh, int faceID, bool showTexture, bool isGT); 43 | 44 | wxGLContext* m_pGLContext; 45 | CCameraControl* m_pCameraControl; 46 | double KK[3][3]; //camera intrinsics 47 | float PMatrix[4][4]; //projection matrix for opengl, column major order 48 | 49 | // controlPanel* m_pControlPanel; 50 | controlPanel* m_pControlPanel; 51 | MainFrame* m_pMainFrame; 52 | 53 | float m_nX; 54 | float m_nY; 55 | 56 | bool m_bLeftDragging; 57 | bool m_bRightDragging; 58 | bool m_bMidDragging; 59 | 60 | bool m_bIsOrthoCamera; 61 | bool m_IsColMajor; 62 | 63 | int m_nImageHeight; 64 | int m_nImageWidth; 65 | 66 | private: 67 | 68 | DECLARE_EVENT_TABLE(); 69 | 70 | }; 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /include/gui_app/CameraControl.h: -------------------------------------------------------------------------------- 1 | #ifndef _CAMERACONTROL_H 2 | #define _CAMERACONTROL_H 3 | 4 | #include "main_engine/utils/global.h" 5 | #include "gui_app/Matrices.h" 6 | 7 | //include // Need to include it here because the GL* types are required 8 | const float PI=3.1415926535897932384626433832795; 9 | const float PIdiv180=(PI/180.0); 10 | 11 | class CCameraControl 12 | { 13 | public: 14 | 15 | float camAngleX; 16 | float camAngleY; 17 | float camAngleZ; 18 | float camTransX; 19 | float camTransY; 20 | float camTransZ; 21 | 22 | Matrix4 matrixView; // for camera 23 | Matrix4 matrixModel; // for object 24 | Matrix4 matrixModelView; 25 | 26 | Vector3 center; 27 | Vector4 cameraLoc; 28 | 29 | public: 30 | 31 | CCameraControl(); //inits the values (Position: (0|0|0) Target: (0|0|-1) ) 32 | void Reset(); // resets values to init 33 | void setObjectCenter(double objectCenter[3]); 34 | // anyway, we need to computed the reprojections of 3d points 35 | // void setTrackingPose(CoordinateType trackingRot[9], 36 | // trackingTrans[3]); 37 | const float* getModelViewMatrix(); 38 | 39 | 40 | }; 41 | 42 | #endif -------------------------------------------------------------------------------- /include/gui_app/ImagePanel.h: -------------------------------------------------------------------------------- 1 | #ifndef _WX_IMAGE_PANEL 2 | #define _WX_IMAGE_PANEL 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | class MainFrame; 10 | 11 | class wxImagePanel : public wxPanel 12 | { 13 | wxImage image; 14 | wxBitmap resized; 15 | int w, h; 16 | bool doOverlay; 17 | std::vector overlayPnts; // x,y coordinate 18 | 19 | MainFrame* pMainFrame; 20 | int numOverlayPnts; 21 | 22 | bool updated; 23 | 24 | public: 25 | wxImagePanel(wxFrame* parent, wxString file, wxBitmapType format); 26 | wxImagePanel(wxPanel* parent, unsigned char* pData, int width, int height, bool overlay); 27 | void updateImage(unsigned char* pData, int width, int height); 28 | 29 | void setMainFrame(MainFrame* pMainFrm); 30 | void setNumPnts(int numPnts); 31 | 32 | void paintEvent(wxPaintEvent & evt); 33 | void paintNow(); 34 | void OnSize(wxSizeEvent& event); 35 | void render(wxDC& dc); 36 | 37 | DECLARE_EVENT_TABLE() 38 | 39 | }; 40 | 41 | #endif -------------------------------------------------------------------------------- /include/gui_app/MainFrame.h: -------------------------------------------------------------------------------- 1 | #ifndef _MAINFRAME_H 2 | #define _MAINFRAME_H 3 | 4 | #include "main_engine/MainEngine.h" 5 | #include "gui_app/PangaeaTracking.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class controlPanel; 13 | class BasicGLPane; 14 | class wxImagePanel; 15 | 16 | class MainFrame: public wxFrame, public MainEngine 17 | { 18 | 19 | public: 20 | 21 | // MainFrame(const wxString& title, int argc, wxChar* argv[]); 22 | //MainFrame(const wxString& title, int argc, char** argv); 23 | MainFrame(const wxString& title, int argc, char* argv[]); 24 | 25 | 26 | ~MainFrame(); 27 | 28 | void OnTimer(wxTimerEvent& event); 29 | void OnIdle(wxIdleEvent& event); 30 | 31 | bool ProcessOneFrame(int nFrame); 32 | 33 | void UpdateVisibilityMask(double toleranceRatio); 34 | void UpdateRenderingLevel(int nRenderLevel, bool renderType = false); 35 | 36 | void SaveImage(); 37 | void SaveOverlay(); 38 | 39 | // void ReadModelInfo(); 40 | 41 | controlPanel* m_pControlPanel; 42 | BasicGLPane* m_pGLPane; 43 | 44 | wxImagePanel* m_pOverlayPane; 45 | wxImagePanel* m_pImagePane; 46 | 47 | // Image sequence, timer callback 48 | wxTimer m_nTimer; 49 | 50 | boost::thread threadTracking; 51 | bool isTrackingFinished; 52 | 53 | 54 | private: 55 | 56 | DECLARE_EVENT_TABLE() 57 | 58 | }; 59 | 60 | enum{ 61 | ID_TIMER = 100 62 | }; 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /include/gui_app/PangaeaTracking.h: -------------------------------------------------------------------------------- 1 | #ifndef _PANGAEA_TRACKING_ 2 | #define _PANGAEA_TRACKING_ 3 | 4 | #include 5 | 6 | // #include "gui_app/MainFrame.h" 7 | class MainFrame; 8 | 9 | class PangaeaTracking: public wxApp 10 | { 11 | 12 | public: 13 | 14 | MainFrame* m_pMainFrame; 15 | char **argv_char; 16 | int arg_num; 17 | 18 | public: 19 | 20 | virtual bool OnInit(); 21 | virtual int OnExit(); 22 | 23 | }; 24 | 25 | DECLARE_APP(PangaeaTracking) 26 | 27 | #endif -------------------------------------------------------------------------------- /include/gui_app/controlPanel.h: -------------------------------------------------------------------------------- 1 | #ifndef _CONTROL_PANEL_H 2 | #define _CONTROL_PANEL_H 3 | 4 | #include "gui_app/MainFrame.h" 5 | class BasicGLPane; 6 | 7 | class controlPanel : public wxPanel 8 | { 9 | 10 | public: 11 | 12 | controlPanel(wxWindow *parent); 13 | virtual ~controlPanel(); 14 | 15 | void Update(bool fromTracker=false); 16 | void OnEvent(wxMouseEvent & event); 17 | void ScrollEvent(wxScrollEvent & event); 18 | void CommandEvent(wxCommandEvent & event); 19 | void Reset(wxCommandEvent& event); 20 | void Spotlight( wxCommandEvent& event); 21 | // void FlipNorms( wxCommandEvent& event); 22 | void SaveImage( wxCommandEvent& event); 23 | void SaveOverlay( wxCommandEvent& event); 24 | 25 | void SaveImageSequence( wxCommandEvent& event); 26 | void SaveOverlaySequence( wxCommandEvent& event); 27 | 28 | void ShowSequence( wxCommandEvent& event); 29 | 30 | void ProcessOneFrame( wxCommandEvent& event); 31 | void ProcessWholeSequence( wxCommandEvent& event); 32 | void StopProcessing( wxCommandEvent& event); 33 | 34 | wxButton *m_pResetButton; 35 | wxButton *m_pSpotlightButton; 36 | wxCheckBox *m_pRenderWndCheckBox; 37 | wxCheckBox *m_pShowGTCheckBox; 38 | 39 | wxCheckBox *m_pShowESTCheckBox; 40 | wxCheckBox *m_pShowTextureCheckBox; 41 | wxCheckBox *m_pShowSurfaceCheckBox; 42 | wxCheckBox *m_pShowEdgesCheckBox; 43 | wxCheckBox *m_pShowPointsCheckBox; 44 | wxCheckBox *m_pUseSpotlightCheckBox; 45 | wxCheckBox *m_pBreakBoundaryCheckBox; 46 | wxCheckBox *m_pDynamicThreshCheckBox; 47 | wxCheckBox *m_pDeletePointsCheckBox; 48 | wxCheckBox *m_pWhiteBackground; 49 | wxCheckBox *m_pHideOcclusion; 50 | wxCheckBox *m_pOcclusionMode; 51 | wxCheckBox *m_pColorDiffMode; 52 | wxCheckBox *m_pFlipNormsCheckBox; 53 | wxCheckBox *m_pRenderPropCheckBox; 54 | wxCheckBox *m_pShowNormalsBox; 55 | wxCheckBox *m_pShowErrorHeatMap; 56 | wxCheckBox *m_pTurnOffLighting; 57 | 58 | wxStaticText *m_pFrameNumText; 59 | wxSlider *m_pFrameSlider; 60 | wxStaticText *m_pMaxPolyScaleText; 61 | wxSlider *m_pMaxPolyScaleSlider; 62 | wxStaticText *m_pAmbientLightText; 63 | wxSlider *m_pAmbientLightSlider; 64 | wxStaticText *m_pPntSizeText; 65 | wxSlider *m_pPntSizeSlider; 66 | wxStaticText *m_pOverlaySizeText; 67 | wxSlider *m_pOverlaySizeSlider; 68 | wxStaticText *m_pVisTolRatioText; 69 | wxSlider *m_pVisTolRatioSlider; 70 | wxStaticText *m_pPyramidLevelText; 71 | wxSlider *m_pPyramidLevelSlider; 72 | 73 | int m_nRenderLevel; // pyramid level rendering 74 | bool m_nRenderProp; // render type, whether rendering propagation result 75 | 76 | wxButton *m_pSaveImageButton; 77 | wxButton *m_pSaveOverlayButton; 78 | wxButton *m_pSaveSequenceButton; 79 | wxButton *m_pSaveOverlaySequenceButton; 80 | wxButton *m_pShowSequenceButton; 81 | 82 | wxButton *m_pRunOneFrameButton; 83 | wxButton *m_pRunSequenceButton; 84 | wxButton *m_pStopButton; 85 | 86 | bool m_bStop; 87 | 88 | bool m_bRenderWndOn; 89 | bool m_bShowGT; 90 | bool m_bShowEST; 91 | bool m_bShowTexture; 92 | bool m_bShowSurface; 93 | bool m_bShowPoints; 94 | bool m_bShowEdges; 95 | bool m_bSpotlightOn; 96 | bool m_bBreakBoundaryOn; 97 | bool m_bDynamicThreshOn; 98 | bool m_bWhiteBackground; 99 | bool m_bHideOcclusion; 100 | bool m_bOcclusionMode; 101 | bool m_bColorDiffMode; 102 | bool m_bFlipNorm; 103 | bool m_bRenderProp; 104 | bool m_bShowNormals; 105 | bool m_bShowErrorHeatMap; 106 | bool m_bTurnOffLighting; 107 | 108 | BasicGLPane* m_pGLPane; // pointer to the rendering pane 109 | //wxGLCanvas* m_pGLPane; 110 | wxPanel* m_pOverlayPane; 111 | wxPanel* m_pImagePane; 112 | 113 | bool m_bDeletePoints; // if we delete points belong to huge triangles 114 | bool m_bIsCentered; // if we have translated the camera and object 115 | int m_nStartFrame; 116 | int m_nCurrentFrame; 117 | int m_nFrameStep; 118 | 119 | int m_nFrames; 120 | float m_nMaxPolyScale; 121 | float m_nAmbientLight; 122 | 123 | double m_nToleranceRatio; 124 | 125 | int m_nPntSize; 126 | int m_nOverlaySize; 127 | 128 | MainFrame* m_pMainFrame; 129 | 130 | private: 131 | 132 | DECLARE_EVENT_TABLE(); 133 | 134 | }; 135 | 136 | enum{ 137 | ID_RESET = 1, 138 | // ID_SPOTLIGHT, 139 | // ID_FLIP_NORMALS, 140 | ID_SAVE_IMAGE, 141 | ID_SAVE_OVERLAY, 142 | ID_SAVE_SEQUENCE, 143 | ID_SAVE_OVERLAY_SEQUENCE, 144 | ID_SHOW_SEQUENCE, 145 | ID_RUN_FRAME, 146 | ID_RUN_SEQUENCE, 147 | ID_STOP 148 | }; 149 | 150 | #endif 151 | -------------------------------------------------------------------------------- /include/main_engine/MainEngine.h: -------------------------------------------------------------------------------- 1 | #ifndef _MAINEngine_H 2 | #define _MAINEngine_H 3 | 4 | #include "./utils/settings.h" 5 | 6 | #include "./image_source/ImageSourceEngine.h" 7 | 8 | #include "./tracker/Mesh.h" 9 | #include "./tracker/ImagePyramid.h" 10 | 11 | #include "./tracker/TrackingEngine.h" 12 | 13 | #ifndef VIS_ONLY 14 | #include "./tracker/DeformNRSFMTracker.h" 15 | #endif 16 | 17 | class MainEngine 18 | { 19 | 20 | public: 21 | 22 | MainEngine(); 23 | virtual ~MainEngine(); 24 | 25 | void LoadInitialMeshUVD(); 26 | void LoadInitialMeshFromFile(PangaeaMeshPyramid &templateMeshPyramid, 27 | const string &meshLevelFormat); 28 | void SetIntrinsicMatrix(double K[3][3]); 29 | void SetupInputAndTracker(); 30 | 31 | // void ReadConfigurationFile(int argc, wxChar* argv[]); 32 | void ReadConfigurationFile(int argc, char* argv[]); 33 | 34 | void GetInput(int nFrame); 35 | 36 | void Run(); 37 | bool ProcessNextFrame(); 38 | virtual bool ProcessOneFrame(int nFrame); 39 | 40 | ImageSourceEngine* m_pImageSourceEngine; 41 | TrackingEngine* m_pTrackingEngine; 42 | 43 | unsigned char* m_pColorImageRGB; 44 | unsigned char* m_pColorImageRGBBuffer; 45 | 46 | int m_nWidth; 47 | int m_nHeight; 48 | int m_nStartFrame; 49 | int m_nCurrentFrame; 50 | int m_NumTrackingFrames; 51 | 52 | int m_nFrameStep; 53 | 54 | double center[3]; 55 | double KK[3][3]; 56 | 57 | TrackerOutputInfo outputInfo; 58 | TrackerOutputInfo* pOutputInfo; 59 | PangaeaMeshData templateMesh; 60 | PangaeaMeshPyramid templateMeshPyramid; 61 | 62 | PangaeaMeshPyramid templateMeshIntensityPyramid; 63 | 64 | int m_nNumMeshLevels; 65 | 66 | boost::mutex mutex; 67 | //boost::thread_group inputThreadGroup; 68 | boost::thread* pInputThread; 69 | bool inputFlag; 70 | 71 | }; 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/main_engine/image_source/ImageSourceEngine.h: -------------------------------------------------------------------------------- 1 | #ifndef _IMAGE_SOURCE_ENGINE_H 2 | #define _IMAGE_SOURCE_ENGINE_H 3 | 4 | #include "../utils/global.h" 5 | #include "../utils/settings.h" 6 | 7 | class ImageSourceEngine 8 | { 9 | 10 | public: 11 | 12 | ImageSourceEngine(){}; 13 | virtual ~ImageSourceEngine(){}; 14 | 15 | virtual void setCurrentFrame(int curFrame) = 0; 16 | virtual unsigned char* getColorImage() = 0; 17 | virtual unsigned char* getLevelColorImage(int nLevel) = 0; 18 | virtual void readUVDImage(DepthImageType& uImage, DepthImageType& vImage, 19 | DepthImageType& dImage, InternalIntensityImageType& maskImage) = 0; 20 | 21 | }; 22 | 23 | class ImagesBufferReader : public ImageSourceEngine 24 | { 25 | 26 | public: 27 | 28 | ImagesBufferReader(ImageSourceSettings& settings); 29 | 30 | ~ImagesBufferReader(); 31 | 32 | void setCurrentFrame(int curFrame); 33 | unsigned char* getColorImage(); 34 | unsigned char* getLevelColorImage(int nLevel) { return getColorImage(); }; 35 | void readUVDImage(DepthImageType& uImage, DepthImageType& vImage, 36 | DepthImageType& dImage, InternalIntensityImageType& maskImage); 37 | void ReadRawDepth(std::stringstream& data_path, std::string filename, 38 | int width, int height, DepthImageType& resImage); 39 | 40 | int startFrameNo; 41 | int currentFrameNo; 42 | int totalFrameNo; 43 | 44 | int m_nHeight; 45 | int m_nWidth; 46 | double KK[3][3]; 47 | 48 | double m_ShapeScale; 49 | 50 | std::string inputPath; 51 | std::string imgFormat; 52 | 53 | ColorImageContainerType m_vImages; // all images 54 | 55 | // uvd and mask image gives the initial shape to start with 56 | DepthImageType uImage; 57 | DepthImageType vImage; 58 | DepthImageType dImage; 59 | InternalIntensityImageType maskImage; 60 | 61 | int nFrameStep; 62 | 63 | }; 64 | 65 | class ImageSequenceReader : public ImageSourceEngine 66 | { 67 | 68 | public: 69 | 70 | ImageSequenceReader(ImageSourceSettings& settings); 71 | 72 | ImageSequenceReader(std::string& inputPath, std::string& imgFormat, 73 | int nHeight, int nWidth, int startFrame, int numTrackingFrames, double shapeScale = 1.0); 74 | 75 | ~ImageSequenceReader(); 76 | 77 | void setCurrentFrame(int curFrame); 78 | unsigned char* getColorImage(); 79 | unsigned char* getLevelColorImage(int nLevel); 80 | void readUVDImage(DepthImageType& uImage, DepthImageType& vImage, 81 | DepthImageType& dImage, InternalIntensityImageType& maskImage); 82 | void ReadRawDepth(std::stringstream& data_path, std::string filename, 83 | int width, int height, DepthImageType& resImage); 84 | 85 | int startFrameNo; 86 | int currentFrameNo; 87 | int totalFrameNo; 88 | 89 | int m_nHeight; 90 | int m_nWidth; 91 | double KK[3][3]; 92 | 93 | double m_ShapeScale; 94 | 95 | std::string inputPath; 96 | std::string imgFormat; 97 | 98 | ColorImageType m_curImage; // current image 99 | ColorImageContainerType m_curImages; // all current images 100 | 101 | // uvd and mask image gives the initial shape to start with 102 | DepthImageType uImage; 103 | DepthImageType vImage; 104 | DepthImageType dImage; 105 | InternalIntensityImageType maskImage; 106 | 107 | }; 108 | 109 | #endif 110 | -------------------------------------------------------------------------------- /include/main_engine/rendering/CCamera.h: -------------------------------------------------------------------------------- 1 | #ifndef _CCAMERA_H 2 | #define _CCAMERA_H 3 | 4 | #include "../utils/global.h" 5 | 6 | class CCamera 7 | { 8 | 9 | // a camera class based on eigen3 10 | 11 | // By convention with this class: 12 | // u = 2D coord in image plane 13 | // c = 3D coord in camera centred coords 14 | // X = 3D coord in world coords 15 | 16 | protected: 17 | 18 | Matrix3d _K; 19 | Matrix3d _R; 20 | Vector3d _t; 21 | 22 | // derived parameters 23 | Matrix3d _Kinv; 24 | Matrix3d _KR; 25 | Vector3d _Kt; 26 | 27 | Matrix3d _Rtrans; 28 | 29 | // Image dimensions 30 | 31 | unsigned int _W; 32 | unsigned int _H; 33 | 34 | // Orthographic camera ? 35 | bool _IsOrtho; 36 | 37 | void setDerivedParams(); 38 | 39 | public: 40 | 41 | CCamera(double K[3][3], double camPose[6], unsigned int W, 42 | unsigned int H); 43 | 44 | CCamera(double K[3][3], double R[3][3], double t[3], unsigned int W, 45 | unsigned int H); 46 | 47 | ~CCamera(); 48 | 49 | Vector3d worldToCamera(vector& X) const 50 | { 51 | return _R * Vector3d::Map(&X[0]) + _t; 52 | } 53 | 54 | Vector2d worldToPixel(vector& X) const 55 | { 56 | Vector3d temp = _KR * Vector3d::Map(&X[0]) + _Kt; 57 | return Vector2d(temp[0]/temp[2], temp[1]/temp[2]); 58 | } 59 | 60 | const Matrix3d& K() const 61 | { 62 | return _K; 63 | } 64 | 65 | const Matrix3d& R() const 66 | { 67 | return _R; 68 | } 69 | 70 | const Vector3d& t() const 71 | { 72 | return _t; 73 | } 74 | 75 | unsigned int W() const 76 | { 77 | return _W; 78 | } 79 | 80 | unsigned int H() const 81 | { 82 | return _H; 83 | } 84 | 85 | bool IsOrtho() const 86 | { 87 | return _IsOrtho; 88 | } 89 | 90 | const Matrix3d& Kinv() const 91 | { 92 | return _Kinv; 93 | } 94 | 95 | const Matrix3d& Rtrans() const 96 | { 97 | return _Rtrans; 98 | } 99 | 100 | const Matrix3d& KR() const 101 | { 102 | return _KR; 103 | } 104 | 105 | const Vector3d& Kt() const 106 | { 107 | return _Kt; 108 | } 109 | 110 | inline Vector3d OC() const 111 | { 112 | return (- _Rtrans * _t ); 113 | } 114 | 115 | inline Vector3d OA() const 116 | { 117 | Vector3d zDir; 118 | zDir << 0, 0, 1; 119 | return _Rtrans * zDir; 120 | } 121 | 122 | Matrix4d getProjectionMatrix(double zNear = 0.1, double zFar = 1000.0) const; 123 | 124 | Matrix4d getModelViewMatrix() const; 125 | 126 | }; 127 | 128 | #endif -------------------------------------------------------------------------------- /include/main_engine/rendering/DepthBuffer.h: -------------------------------------------------------------------------------- 1 | #ifndef _DEPTH_BUFFER_H 2 | #define _DEPTH_BUFFER_H 3 | 4 | #include "../utils/global.h" 5 | #include "../tracker/Mesh.h" 6 | 7 | #include "./CCamera.h" 8 | 9 | #include 10 | 11 | class DepthBuffer 12 | { 13 | 14 | public: 15 | 16 | DepthBuffer(const CCamera* pCamera, PangaeaMeshData* pMesh); 17 | ~DepthBuffer() {}; 18 | 19 | // MatrixXCordRow _depthBufferMin; 20 | // MatrixXCordRow _depthBufferMax; 21 | MatrixXfRow _depthBufferMin; 22 | MatrixXfRow _depthBufferMax; 23 | 24 | float _zMin; 25 | float _zMax; 26 | 27 | protected: 28 | 29 | void getTotalDepthRange(float& zMin, float& zMax) const; 30 | void renderMeshGL(); 31 | void drawMeshGL(); 32 | 33 | const CCamera* _pCamera; 34 | PangaeaMeshData* _pMesh; 35 | 36 | int _width; 37 | int _height; 38 | 39 | }; 40 | 41 | #endif -------------------------------------------------------------------------------- /include/main_engine/rendering/NcvGlXContext.h: -------------------------------------------------------------------------------- 1 | #ifndef NCVGLXCONTEXT_H 2 | #define NCVGLXCONTEXT_H 3 | 4 | #include "../utils/global.h" 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace ncv 14 | { 15 | class GlXOffscreenContext 16 | { 17 | public: 18 | 19 | GlXOffscreenContext(uint width, uint height); 20 | 21 | ~GlXOffscreenContext(); 22 | 23 | inline bool isReady() const 24 | { 25 | return _ready; 26 | } 27 | 28 | bool makeActive(); 29 | 30 | void clearActive(); 31 | 32 | inline bool isExtensionSupported(const string extensionStr) 33 | { 34 | ContextRestorer restorer(!isActive()); 35 | makeActive(); 36 | return glewIsSupported(extensionStr.c_str()); 37 | } 38 | 39 | int getIntProperty(GLenum intProperty); 40 | 41 | inline bool isActive() 42 | { 43 | if (!isReady()) 44 | return false; 45 | 46 | GLXContext currContext = glXGetCurrentContext(); 47 | 48 | return (currContext == _context); 49 | } 50 | 51 | inline static bool someContextIsActive() 52 | { 53 | return (glXGetCurrentContext() != NULL); 54 | } 55 | 56 | class ContextRestorer 57 | { 58 | public: 59 | explicit ContextRestorer(bool shouldRestore = true) 60 | { 61 | reset(); 62 | 63 | _displayToRestore = glXGetCurrentDisplay(); 64 | _drawableToRestore = glXGetCurrentDrawable(); 65 | _contextToRestore = glXGetCurrentContext(); 66 | } 67 | 68 | ~ContextRestorer() 69 | { 70 | restore(); 71 | } 72 | 73 | void restore() 74 | { 75 | if (_shouldRestore) 76 | { 77 | if ((_displayToRestore) && (_contextToRestore) && 78 | (_drawableToRestore != None)) 79 | { 80 | if (!glXMakeCurrent(_displayToRestore, _drawableToRestore, 81 | _contextToRestore)) 82 | { 83 | cerr << "ERROR: GlXContext: ContextRestorer: " 84 | "Unable to make context active." << endl; 85 | } 86 | 87 | reset(); 88 | } 89 | } 90 | } 91 | 92 | private: 93 | 94 | void reset() 95 | { 96 | _shouldRestore = false; 97 | _displayToRestore = NULL; 98 | _drawableToRestore = None; 99 | _contextToRestore = NULL; 100 | } 101 | 102 | bool _shouldRestore; 103 | Display* _displayToRestore; 104 | GLXDrawable _drawableToRestore; 105 | GLXContext _contextToRestore; 106 | }; 107 | 108 | private: 109 | 110 | void createContext(); 111 | 112 | inline void localError(const char* msg) 113 | { 114 | string errorString = string("ERROR: GlXContext: ") + string(msg); 115 | throw runtime_error(errorString.c_str()); 116 | } 117 | 118 | private: 119 | Display* _display; 120 | GLXFBConfig _fbConfig; 121 | GLXPbuffer _pBuffer; 122 | GLXContext _context; 123 | 124 | uint _width; 125 | uint _height; 126 | 127 | bool _ready; 128 | }; 129 | 130 | typedef boost::shared_ptr GlXOffscreenContextPtr; 131 | } 132 | 133 | #endif // NCVGLXCONTEXT_H 134 | -------------------------------------------------------------------------------- /include/main_engine/tracker/FeaturePyramid.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "FeatureReader.h" 4 | #include "ImagePyramid.h" 5 | 6 | struct FeatureLevel: public Level 7 | { 8 | FeatureImageContainerType featureImageVec; 9 | FeatureImageContainerType featureImageGradXVec; 10 | FeatureImageContainerType featureImageGradYVec; 11 | }; 12 | 13 | class FeaturePyramid 14 | { 15 | public: 16 | 17 | // has to given the path to the feature image database 18 | FeaturePyramid(); 19 | ~FeaturePyramid(); 20 | 21 | void create(int nW, int nH, int nChannels, int numLevels); 22 | void setupCameraPyramid(int numLevels, CameraInfo& camInfo); 23 | void setupPyramid(std::string key); 24 | 25 | void updateData(); 26 | void updatePrev(); 27 | 28 | CameraInfo& getCameraInfo(int nLevel); 29 | FeatureLevel& getPrevFeatureLevel(int nLevel); 30 | FeatureLevel& getCurrFeatureLevel(int nLevel); 31 | 32 | private: 33 | 34 | // whether previous frame has been initialized 35 | bool m_bInitialized; 36 | 37 | FeatureLevel* prevLevels; 38 | FeatureLevel* currLevels; 39 | FeatureLevel* currLevelsBuffer; 40 | 41 | FeatureImageType featureBufferImage; 42 | FeatureImageType blurBufferImage; 43 | 44 | vector camInfoLevels; 45 | 46 | int m_nWidth; 47 | int m_nHeight; 48 | int m_nNumChannels; 49 | 50 | int m_nNumLevels; 51 | 52 | FeatureReader* pFeatureReader; 53 | 54 | }; 55 | -------------------------------------------------------------------------------- /include/main_engine/tracker/FeatureReader.h: -------------------------------------------------------------------------------- 1 | #ifndef _FEATURE_READER_H 2 | #define _FEATURE_READER_H 3 | 4 | #include "../utils/global.h" 5 | 6 | class FeatureReader 7 | { 8 | 9 | public: 10 | 11 | FeatureReader(){}; 12 | FeatureReader(string featureFile){}; 13 | virtual ~FeatureReader(){}; 14 | 15 | virtual void InitializeDB(int height, int width, int numChannels) = 0; 16 | virtual void ShutDownDB() = 0; 17 | 18 | virtual void getFeatureLevel(string key, int channel, 19 | FeatureImageType& featureBufferImage) = 0; 20 | 21 | int m_nWidth; 22 | int m_nHeight; 23 | int m_nNumChannels; 24 | 25 | }; 26 | 27 | class LMDBReader : public FeatureReader 28 | { 29 | 30 | public: 31 | 32 | // for lmdb, we only need the folder of the lock files 33 | LMDBReader(string folder); 34 | ~LMDBReader(); 35 | 36 | void InitializeDB(int height, int width, int numChannels); 37 | void ShutDownDB(); 38 | 39 | void getFeatureLevel(string key, int channel, 40 | FeatureImageType& featureBufferImage); 41 | 42 | private: 43 | 44 | string db_path; 45 | 46 | // lmdb 47 | MDB_env *mdb_env; 48 | MDB_dbi mdb_dbi; 49 | MDB_val mdb_key, mdb_data; 50 | MDB_txn *mdb_txn; 51 | 52 | //MDB_cursor* mdb_cursor; 53 | 54 | }; 55 | 56 | class HDF5Reader : public FeatureReader 57 | { 58 | 59 | public: 60 | 61 | HDF5Reader(string file); 62 | ~HDF5Reader(); 63 | 64 | void InitializeDB(int height, int width, int numChannels); 65 | void ShutDownDB(); 66 | 67 | void getFeatureLevel(string key, int channel, 68 | FeatureImageType& featureBufferImage); 69 | 70 | private: 71 | 72 | string hdf5file; 73 | 74 | hid_t file_id, dataset_id; /* identifiers */ 75 | hid_t attr; 76 | herr_t ret; /* Return value */ 77 | 78 | double* pFeatureImages; 79 | string currentKey; 80 | 81 | }; 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /include/main_engine/tracker/ImagePyramid.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../utils/global.h" 4 | 5 | struct CameraInfo 6 | { 7 | bool isOrthoCamera; 8 | double KK[3][3]; 9 | //double pos[6]; 10 | int width; 11 | int height; 12 | 13 | // 14 | double invKK[3][3]; 15 | }; 16 | 17 | struct Level 18 | { 19 | }; 20 | 21 | struct ImageLevel: public Level 22 | { 23 | InternalIntensityImageType grayImage; 24 | InternalIntensityImageType gradXImage; 25 | InternalIntensityImageType gradYImage; 26 | 27 | InternalColorImageType colorImage; 28 | InternalIntensityImageType colorImageSplit[3]; 29 | InternalIntensityImageType colorImageGradXSplit[3]; 30 | InternalIntensityImageType colorImageGradYSplit[3]; 31 | 32 | // support for depth image to be added 33 | // depth image, just ignore if not used 34 | InternalIntensityImageType depthImage; 35 | InternalIntensityImageType depthGradXImage; 36 | InternalIntensityImageType depthGradYImage; 37 | 38 | // depth normal image 39 | InternalIntensityImageType depthNormalImageSplit[3]; 40 | InternalIntensityImageType depthNormalImageGradXSplit[3]; 41 | InternalIntensityImageType depthNormalImageGradYSplit[3]; 42 | 43 | }; 44 | 45 | class ImagePyramid 46 | { 47 | public: 48 | ImagePyramid(); 49 | ~ImagePyramid(); 50 | 51 | ImagePyramid& operator=(const ImagePyramid&); 52 | 53 | void create(int width, int height); 54 | 55 | void allocateMemory(int width, int height); // allocate memory based on the size 56 | void deallocateMemory(); 57 | 58 | void setupPyramid(unsigned char* pColorImageRGB, int numLevels); 59 | void setupCameraPyramid(int numLevels, CameraInfo& camInfo); 60 | 61 | void updateData(); 62 | 63 | CameraInfo& getCameraInfo(int nLevel); 64 | ImageLevel& getImageLevel(int nLevel); 65 | 66 | InternalIntensityImageType* getColorImageSplit(int nLevel); 67 | 68 | private: 69 | 70 | unsigned char* pCurrentColorImageRGB; 71 | unsigned char* pCurrentColorImageBGR; 72 | unsigned char* pCurrentGrayImage; 73 | 74 | vector levels; 75 | vector levelsBuffer; 76 | 77 | vector camInfoLevels; 78 | 79 | int m_nWidth; 80 | int m_nHeight; 81 | 82 | InternalIntensityImageType grayBufferImage; 83 | InternalIntensityImageType blurGrayBufferImage; 84 | 85 | InternalColorImageType colorBufferImage; 86 | InternalColorImageType blurColorBufferImage; 87 | 88 | IntensityImageType grayImageBYTE; 89 | 90 | }; 91 | -------------------------------------------------------------------------------- /include/main_engine/tracker/MeshPyramid.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "./MeshData.h" 4 | #include "./MeshIO.h" 5 | 6 | #include "third_party/KDTreeAdaptor.hpp" 7 | 8 | typedef pair MeshPair; 9 | typedef vector > MeshWeights; 10 | typedef vector > MeshNeighbors; 11 | typedef vector > MeshNeighborsNano; 12 | 13 | class MeshPropagation 14 | { 15 | 16 | public: 17 | 18 | typedef map, int > NeighborsMap; 19 | 20 | MeshNeighbors& getPatchNeighbors(int level) 21 | { 22 | return patchNeighborsVec[ level ]; 23 | } 24 | 25 | MeshWeights& getPatchWeights(int level) 26 | { 27 | return patchWeightsVec[ level ]; 28 | } 29 | 30 | MeshNeighbors& getPatchRadii(int level) 31 | { 32 | return patchRadiiVec[ level ]; 33 | } 34 | 35 | MeshNeighbors& getNeighbors(pair& meshPair) 36 | { 37 | return neighborsVec[ neighborMap[ meshPair ] ]; 38 | } 39 | 40 | MeshWeights& getWeights(pair& meshPair) 41 | { 42 | return weightsVec[ neighborMap[ meshPair ] ]; 43 | } 44 | 45 | bool isAdded(pair meshPair) 46 | { 47 | map,int>::iterator it = neighborMap.find( meshPair ); 48 | if( it == neighborMap.end() ) 49 | return false; 50 | else 51 | return true; 52 | } 53 | 54 | void addNeighborsAndWeights(pair meshPair, MeshNeighbors& meshNeighbors, MeshWeights& meshWeights) 55 | { 56 | // std::pair ret; 57 | // ret = neighborMap.insert( pair(meshPair, neighborMap.size() ) ); 58 | 59 | // if(ret.second == false){ 60 | // cout << "already inserted:" << endl; 61 | // } 62 | // else{ 63 | // neighborsVec.push_back( move( meshNeighbors ) ); 64 | // weightsVec.push_back( move( meshWeights ) ); 65 | // } 66 | if(!isAdded( meshPair )) 67 | { 68 | neighborsVec.push_back( move( meshNeighbors ) ); 69 | weightsVec.push_back( move( meshWeights ) ); 70 | neighborMap.insert( pair(meshPair, neighborMap.size() ) ); 71 | } 72 | } 73 | 74 | // private: 75 | public: 76 | 77 | // support propagation between arbitary two different levels, 78 | // every time we need to get the weights and neighbors, we give 79 | // a pair of levels as input, and get the index/corresponding weights or 80 | // neighbors back 81 | map, int > neighborMap; 82 | 83 | vector< MeshWeights > weightsVec; 84 | vector< MeshNeighbors > neighborsVec; 85 | 86 | // patch support on mesh 87 | // note the difference with weights/neighors for data term interpolation 88 | // or regularization terms 89 | vector< MeshWeights > patchWeightsVec; 90 | vector< MeshNeighbors > patchNeighborsVec; 91 | vector< MeshNeighbors > patchRadiiVec; 92 | 93 | }; 94 | 95 | // mesh pyramid 96 | template 97 | class MeshPyramid 98 | { 99 | 100 | public: 101 | 102 | typedef MeshData Mesh; 103 | 104 | MeshPyramid(){}; 105 | MeshPyramid(Mesh& mesh); 106 | MeshPyramid(string meshLevelFormat, 107 | IntegerContainerType& meshVertexNum, bool clockwise = true); 108 | MeshPyramid(string meshPath, string meshLevelFormat, int frame, 109 | IntegerContainerType& meshLevelList, bool clockwise = true); 110 | 111 | MeshPyramid& operator=(const MeshPyramid& d) = default; 112 | MeshPyramid& operator=(MeshPyramid&& d) 113 | { 114 | levels = std::move(d.levels); 115 | numLevels = d.numLevels; 116 | meshPyramidVertexNum = std::move(d.meshPyramidVertexNum); 117 | 118 | return *this; 119 | } 120 | 121 | void swapFeatures(); 122 | void updatePyramid(string meshPath, string meshLevelFormat, int frame, 123 | IntegerContainerType& meshLevelList); 124 | 125 | // pyramid data, the finest level is level 0 126 | // and level number increases as we go up the pyramid 127 | int numLevels; 128 | vector levels; 129 | IntegerContainerType meshPyramidVertexNum; 130 | 131 | // put this outside levels as we would need multiple copies of meshes, 132 | // but only need one copy of neighbors and weights 133 | // // these correspond to the weights and neighbors 134 | // // from fine mesh to coarse mesh, for each vertex 135 | // // in the fine mesh, we search for its neighbors 136 | // // in the coarse mesh and calculate corresponding weights. 137 | // // vector kNN; //number of nearest neighbors 138 | // // vector radius; //the radius range for searching neighbors 139 | 140 | // IntegerContainerType kNN; 141 | // CoordinateContainerType radius; 142 | 143 | // vector neighborsPyramid; 144 | // vector weightsPyramid; 145 | // vector distancesPyramid; 146 | // vector sigmasPyramid; 147 | // bool useRadius; 148 | 149 | // should we keep the neighbors from coarse level to fine level, 150 | // not at the moment 151 | 152 | // member methods 153 | // setup the neighbors and weights for all the levels, 154 | // we will have n-1 levels of neighbors and weights for n level pyramid 155 | // void prepareMeshPyramid( 156 | // IntegerContainerType& meshNeighborNum, 157 | // CoordinateContainerType& meshNeighborRadius, 158 | // bool meshPyramidUseRadius, 159 | // MeshPropagation& meshPropagation); 160 | 161 | }; 162 | 163 | template 164 | MeshPyramid::MeshPyramid(MeshData& mesh) 165 | { 166 | // creare a MeshPyramid from a single mesh 167 | // in this case, we just got one level, 168 | // mesh subsampling to be added if necessary 169 | 170 | numLevels = 1; 171 | levels.resize(1); 172 | meshPyramidVertexNum.resize(1); 173 | meshPyramidVertexNum[0] = mesh.numVertices; 174 | levels[0] = std::move(mesh); 175 | 176 | } 177 | 178 | template 179 | MeshPyramid::MeshPyramid(string meshLevelFormat, 180 | IntegerContainerType& meshVertexNum, bool clockwise) 181 | { 182 | numLevels = meshVertexNum.size(); 183 | levels.resize(numLevels); 184 | meshPyramidVertexNum.resize(numLevels); 185 | 186 | // load meshes 187 | char buffer[BUFFER_SIZE]; 188 | Mesh tempMesh; 189 | for(int i = 0; i < numLevels; ++i) 190 | { 191 | std::stringstream meshFile; 192 | sprintf(buffer, meshLevelFormat.c_str(), meshVertexNum[i]); 193 | meshFile << buffer; 194 | PangaeaMeshIO::loadfromFile(meshFile.str(), tempMesh, clockwise); 195 | 196 | // // Set orientation of the faces of the mesh 197 | // tempMesh.clockwise = trackerSettings.clockwise; 198 | 199 | levels[i] = std::move(tempMesh); 200 | meshPyramidVertexNum[i] = levels[i].numVertices; 201 | 202 | } 203 | 204 | } 205 | 206 | template 207 | MeshPyramid::MeshPyramid(string meshPath, string meshLevelFormat, 208 | int frame, IntegerContainerType& meshLevelList, bool clockwise) 209 | { 210 | numLevels = meshLevelList.size(); 211 | levels.resize(numLevels); 212 | meshPyramidVertexNum.resize(numLevels); 213 | 214 | // load meshes 215 | char buffer[BUFFER_SIZE]; 216 | Mesh tempMesh; 217 | for(int i = 0; i < numLevels; ++i) 218 | { 219 | std::stringstream meshFile; 220 | sprintf(buffer, meshLevelFormat.c_str(), frame, meshLevelList[i]); 221 | meshFile << meshPath << buffer; 222 | cout << "loading meshes" << endl; 223 | cout << meshFile.str() << endl; 224 | PangaeaMeshIO::loadfromFile(meshFile.str(), tempMesh, clockwise); 225 | levels[i] = std::move(tempMesh); 226 | 227 | meshPyramidVertexNum[i] = levels[i].numVertices; 228 | } 229 | } 230 | 231 | template 232 | void MeshPyramid::swapFeatures() 233 | { 234 | 235 | for(int i = 0; i < numLevels; ++i) 236 | swap(levels[i].features, levels[i].featuresBuffer); 237 | 238 | } 239 | 240 | template 241 | void MeshPyramid::updatePyramid(string meshPath, 242 | string meshLevelFormat, int frame, IntegerContainerType& meshLevelList) 243 | { 244 | // just do update, no memory allocation and much faster 245 | // load meshes 246 | char buffer[BUFFER_SIZE]; 247 | for(int i = 0; i < numLevels; ++i) 248 | { 249 | std::stringstream meshFile; 250 | sprintf(buffer, meshLevelFormat.c_str(), frame, meshLevelList[i]); 251 | meshFile << meshPath << buffer; 252 | PangaeaMeshIO::updateFromFile(meshFile.str(), levels[i]); 253 | } 254 | } 255 | 256 | typedef MeshPyramid PangaeaMeshPyramid; 257 | -------------------------------------------------------------------------------- /include/main_engine/tracker/OptimizationStrategy.h: -------------------------------------------------------------------------------- 1 | // 2 | // OptimizationStrategy file 3 | 4 | #pragma once 5 | 6 | #include "../utils/global.h" 7 | #include "../utils/settings.h" 8 | 9 | struct OptimizationLevel 10 | { 11 | // data term mesh levels for each level 12 | // the first one gives the projection point 13 | // the second one gives its neighbors, 14 | // if these two are the same level, do not need to do any interpolation 15 | // otherwise we interpolate the position based on its neighbors 16 | vector > dataTermPairs; 17 | vector > regTermPairs; 18 | 19 | // propagation to be done after optimization 20 | vector > propPairs; 21 | 22 | // 23 | vector deformTermLevelIDVec; 24 | }; 25 | 26 | struct WeightPara 27 | { 28 | // weighting parameters of each energy term 29 | double dataTermWeight; // photometric term 30 | double dataIntensityTermWeight; // photometric term 31 | double tvTermWeight; 32 | double tvRotTermWeight; 33 | 34 | double arapTermWeight; 35 | double inextentTermWeight; 36 | double deformWeight; 37 | 38 | // Laplacian smoothing weight 39 | double smoothingTermWeight; 40 | 41 | // rigid transformation weight 42 | double rotWeight; 43 | double transWeight; 44 | 45 | // Huber width 46 | double dataHuberWidth; 47 | double dataIntensityHuberWidth; 48 | double tvHuberWidth; 49 | double tvRotHuberWidth; 50 | double smoothingHuberWidth; 51 | 52 | // feature weighting parameters 53 | double featureTermWeight; 54 | double featureHuberWidth; 55 | 56 | }; 57 | 58 | struct WeightScale 59 | { 60 | vector dataTermScale; 61 | vector tvTermScale; 62 | vector rotTVTermScale; 63 | vector arapTermScale; 64 | vector inextentTermScale; 65 | vector deformTermScale; 66 | vector smoothingTermScale; 67 | 68 | // rigid transformation weighting scale 69 | // change over different levels of the pyramid 70 | vector rotScale; 71 | vector transScale; 72 | 73 | vector featureTermScale; 74 | }; 75 | 76 | class OptimizationStrategy 77 | { 78 | public: 79 | 80 | 81 | vector optimizationSettings; 82 | int numOptimizationLevels; 83 | int numMeshLevels; 84 | 85 | OptimizationStrategy(){}; 86 | OptimizationStrategy(int numMeshLevels); 87 | ~OptimizationStrategy(){}; 88 | 89 | virtual void Initialize(){}; 90 | 91 | virtual void setWeightParameters(WeightPara& inputWeightPara); 92 | virtual void setWeightScale(IntegerContainerType& meshVertexNum); 93 | virtual void setWeightScale(WeightScale& inputWeightScale); 94 | virtual void setWeightParametersVec(); 95 | 96 | WeightPara weightPara; 97 | WeightScale weightScale; 98 | 99 | // weightPara 100 | vector< WeightPara > weightParaVec; 101 | 102 | vector< pair > propPairsFinal; 103 | }; 104 | 105 | // arbitary neighbors support for all terms goes here 106 | class FreeNeighborStrategy:public OptimizationStrategy 107 | { 108 | public: 109 | 110 | FreeNeighborStrategy(int numMeshLevels); 111 | ~FreeNeighborStrategy(){}; 112 | 113 | void Initialize(); 114 | 115 | void AddPropPairs(vector >& propPairs, 116 | vector >& nextPairs, 117 | vector& updatedLevels); 118 | 119 | }; 120 | -------------------------------------------------------------------------------- /include/main_engine/tracker/ProblemWrapper.h: -------------------------------------------------------------------------------- 1 | #include "../utils/global.h" 2 | #include "ceres/ceres.h" 3 | 4 | #define NUM_PRINT_COSTS 15 5 | 6 | class ProblemWrapper 7 | { 8 | public: 9 | ProblemWrapper(); 10 | ProblemWrapper(int numLevels); 11 | ~ProblemWrapper(); 12 | 13 | void Initialize(int numLevels); 14 | int getLevelsNum(); 15 | 16 | ceres::Problem& getProblem(int nLevel); 17 | bool getLevelFlag(int nLevel); 18 | void setLevelFlag(int nLevel); 19 | 20 | void addDataTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 21 | void addFeatureTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 22 | void addTVTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 23 | void addRotTVTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 24 | void addARAPTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 25 | void addINEXTENTTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 26 | void addDeformTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 27 | void addTemporalTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 28 | void addSmoothingTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 29 | void addTemporalSHCoeffTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 30 | void addSpecularSmoothnessTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 31 | void addSpecularMagnitudeTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 32 | void addTemporalSpecularTerm(int nLevel, ceres::ResidualBlockId& residualBlockId); 33 | 34 | void addDataTermCost(int nLevel, ceres::CostFunction* pCostFunction); 35 | void addFeatureTermCost(int nLevel, ceres::CostFunction* pCostFunction); 36 | void addRegTermCost(int nLevel, ceres::CostFunction* pCostFunction); 37 | 38 | // void addTVTermCost(int nLevel, ceres::CostFunction* pCostFunction); 39 | // void addRotTVTermCost(int nLevel, ceres::CostFunction* pCostFunction); 40 | // void addARAPTermCost(int nLevel, ceres::CostFunction* pCostFunction); 41 | // void addINEXTENTTermCost(int nLevel, ceres::CostFunction* pCostFunction); 42 | // void addDeformTermCost(int nLevel, ceres::CostFunction* pCostFunction); 43 | // void addTemporalTermCost(int nLevel, ceres::CostFunction* pCostFunction); 44 | 45 | 46 | void addDataTermLoss(int nLevel, ceres::LossFunction* pLossFunction); 47 | void addFeatureTermLoss(int nLevel, ceres::LossFunction* pLossFunction); 48 | void addRegTermLoss(int nLevel, ceres::LossFunction* pLossFunction); 49 | // void addLossFunction(int nLevel, ceres::LossFunction* pLossFunction); 50 | 51 | 52 | void clearDataTerm(int nLevel); 53 | void clearFeatureTerm(int nLevel); 54 | int getDataTermNum(int nLevel); 55 | int getFeatureTermNum(int nLevel); 56 | 57 | void getTotalEnergy(int nLevel, double* cost); 58 | void getDataTermCost(int nLevel, double* cost); 59 | void getFeatureTermCost(int nLevel, double* cost); 60 | void getTVTermCost(int nLevel, double* cost); 61 | void getRotTVTermCost(int nLevel, double* cost); 62 | void getARAPTermCost(int nLevel, double* cost); 63 | void getINEXTENTTermCost(int nLevel, double* cost); 64 | void getDeformTermCost(int nLevel, double* cost); 65 | void getTemporalTermCost(int nLevel, double* cost); 66 | void getSmoothingTermCost(int nLevel, double* cost); 67 | void getTemporalSHCoeffTermCost(int nLevel, double* cost); 68 | void getSpecularSmoothnessTermCost(int nLevel, double* cost); 69 | void getSpecularMagnitudeTermCost(int nLevel, double* cost); 70 | void getTemporalSpecularTermCost(int nLevel, double* cost); 71 | 72 | void getAllCost(int nLevel, double *cost, double* total_cost, double* sum_cost); 73 | 74 | 75 | void clearDataTermCost(int nLevel); 76 | void clearFeatureTermCost(int nLevel); 77 | void clearCostFunctions(int nLevel); 78 | void clearCostFunctionsHelper(vector& costFunctions); 79 | 80 | void clearDataTermLoss(int nLevel); 81 | void clearFeatureTermLoss(int nLevel); 82 | void clearLossFunctions(int nLevel); 83 | void clearLossFunctionsHelper(vector& lossFunctions); 84 | 85 | // double* getRigidTransformation(); 86 | // PangaeaMeshData& getMeshData(int nLevel); 87 | // MeshDeformation& getMeshRotation(int nLevel); 88 | // MeshDeformation& getMeshTranslation(int nLevel); 89 | // MeshDeformation& getPrevMeshRotation(int nLevel); 90 | // MeshDeformation& getPrevMeshTranslation(int nLevel); 91 | 92 | // void setRigidTransformation(double* pRigidTransform); 93 | // void setMeshData(PangaeaMeshPyramid* pMeshPyr); 94 | // void setMeshRotation(vector* pMeshRotPyr); 95 | // void setMeshTranslation(vector* pMeshTransPyr); 96 | 97 | // void setOptimizationVariables(double* pRigidTransform, 98 | // PangaeaMeshPyramid* pMeshPyr, 99 | // vector* pMeshRotPyr, 100 | // vector* pMeshTransPyr, 101 | // vector* pPrevMeshRotPyr, 102 | // vector* pPrevMeshTransPyr); 103 | 104 | private: 105 | 106 | vector setupFlag; 107 | vector problems; 108 | 109 | // keep the pointer to all possible optimization variables 110 | // double* pRigidTransform_; 111 | // PangaeaMeshPyramid* pMeshPyr_; 112 | // vector* pMeshRotPyr_; 113 | // vector* pMeshTransPyr_; 114 | // vector* pPrevMeshRotPyr_; 115 | // vector* pPrevMeshTransPyr_; 116 | 117 | 118 | vector > dataTermResidualBlocks; 119 | vector > featureTermResidualBlocks; 120 | vector > tvTermResidualBlocks; 121 | vector > rotTVTermResidualBlocks; 122 | vector > arapTermResidualBlocks; 123 | vector > inextentTermResidualBlocks; 124 | vector > deformTermResidualBlocks; 125 | vector > temporalTermResidualBlocks; 126 | vector > smoothingTermResidualBlocks; 127 | vector > temporalSHCoeffTermResidualBlocks; 128 | vector > specularSmoothnessTermResidualBlocks; 129 | vector > specularMagnitudeTermResidualBlocks; 130 | vector > temporalSpecularTermResidualBlocks; 131 | 132 | 133 | vector > dataTermCostFunctions; 134 | vector > featureTermCostFunctions; 135 | vector > regTermCostFunctions; 136 | // vector > tvTermCostFunctions; 137 | // vector > rotTVTermCostFunctions; 138 | // vector > arapTermCostFunctions; 139 | // vector > inextentTermCostFunctions; 140 | // vector > deformTermCostFunctions; 141 | // vector > temporalTermCostFunctions; 142 | 143 | vector > dataTermLossFunctions; 144 | vector > featureTermLossFunctions; 145 | vector > regTermLossFunctions; 146 | 147 | int numLevels; 148 | }; 149 | -------------------------------------------------------------------------------- /include/main_engine/utils/qx_basic.h: -------------------------------------------------------------------------------- 1 | /******************************************************* 2 | \Author: Qingxiong Yang 3 | \Function: Basic functions. 4 | ********************************************************/ 5 | #ifndef QX_CVPR09_CTBF_BASIC_H 6 | #define QX_CVPR09_CTBF_BASIC_H 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | //#include 13 | // #include 14 | // #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include // For greater() 20 | #include 21 | // #if _MSC_VER > 1020 // if VC++ version is > 4.2 22 | using namespace std; // std c++ libs implemented in std 23 | // #endif 24 | #define QX_DEF_PADDING 10 25 | #define QX_DEF_THRESHOLD_ZERO 1e-6 26 | class qx_timer {public: void start(); float stop(); void time_display(char *disp=""); void fps_display(char *disp=""); private: clock_t m_begin; clock_t m_end;}; 27 | 28 | inline float qx_max_f3(float*a){return(max(max(a[0],a[1]),a[2]));} 29 | inline float qx_min_f3(float*a){return(min(min(a[0],a[1]),a[2]));} 30 | /*Box filter*/ 31 | void boxcar_sliding_window(double **out,double **in,double **temp,int h,int w,int radius); 32 | void boxcar_sliding_window_x(double *out,double *in,int h,int w,int radius); 33 | void boxcar_sliding_window_y(double *out,double *in,int h,int w,int radius); 34 | /*Gaussian filter*/ 35 | int gaussian_recursive(double **image,double **temp,double sigma,int order,int h,int w); 36 | void gaussian_recursive_x(double **od,double **id,int w,int h,double a0,double a1,double a2,double a3,double b1,double b2,double coefp,double coefn); 37 | void gaussian_recursive_y(double **od,double **id,int w,int h,double a0,double a1,double a2,double a3,double b1,double b2,double coefp,double coefn); 38 | /*basic functions*/ 39 | 40 | //inline float min(float a,float b){if(ab) return(a); else return(b);} 42 | inline int qx_sum_u3(unsigned char *a) {return(a[0]+a[1]+a[2]);} 43 | inline double qx_sum_d3(double*a){return(a[0]+a[1]+a[2]);} 44 | inline unsigned char qx_min_u3(unsigned char *a){return(min(min(a[0],a[1]),a[2]));} 45 | inline unsigned char qx_max_u3(unsigned char *a){return(max(max(a[0],a[1]),a[2]));} 46 | inline void image_zero(float **in,int h,int w,float zero=0){memset(in[0],zero,sizeof(float)*h*w);} 47 | inline void image_zero(double **in,int h,int w,double zero=0){memset(in[0],zero,sizeof(double)*h*w);} 48 | inline void image_zero(unsigned char**in,int h,int w,unsigned char zero=0){memset(in[0],zero,sizeof(unsigned char)*h*w);} 49 | inline void image_zero(double ***in,int h,int w,int d,double zero=0){memset(in[0][0],zero,sizeof(double)*h*w*d);} 50 | inline unsigned char rgb_2_gray(unsigned char*in) 51 | { 52 | return (unsigned char)(0.299*in[0]+0.587*in[1]+0.114*in[2]+0.5); 53 | } 54 | inline int qx_square_difference_u3(unsigned char *a,unsigned char *b){int d1,d2,d3; d1=(*a++)-(*b++); d2=(*a++)-(*b++); d3=(*a++)-(*b++); return(int(d1*d1+d2*d2+d3*d3));} 55 | void qx_specular_free_image(unsigned char ***image_specular_free,unsigned char ***image_normalized,float **diffuse_chromaticity_max,int h,int w); 56 | inline double *get_color_weighted_table(double sigma_range,int len) 57 | { 58 | double *table_color,*color_table_x; int y; 59 | table_color=new double [len]; 60 | color_table_x=&table_color[0]; 61 | for(y=0;ymax_val) max_val=in[i]; 85 | } 86 | inline void vec_max_val(unsigned char &max_val,unsigned char *in,int len) 87 | { 88 | max_val=in[0]; 89 | for(int i=1;imax_val) max_val=in[i]; 90 | } 91 | inline void down_sample_1(unsigned char **out,unsigned char **in,int h,int w,int scale_exp) 92 | { 93 | int y,x; int ho,wo; unsigned char *out_y,*in_x; 94 | ho=(h>>scale_exp); wo=(w>>scale_exp); 95 | for(y=0;y>scale_exp); wo=(w>>scale_exp); 105 | for(y=0;y 2 | #include 3 | 4 | #include "main_engine/utils/global.h" 5 | 6 | using namespace nanoflann; 7 | 8 | // ===================================================================================== 9 | 10 | /** A simple vector-of-vectors adaptor for nanoflann, without duplicating the storage. 11 | * The i'th vector represents a point in the state space. 12 | * 13 | * \tparam DIM If set to >0, it specifies a compile-time fixed dimensionality for the points in the data set, allowing more compiler optimizations. 14 | * \tparam num_t The type of the point coordinates (typically, double or float). 15 | * \tparam Distance The distance metric to use: nanoflann::metric_L1, nanoflann::metric_L2, nanoflann::metric_L2_Simple, etc. 16 | * \tparam IndexType The type for indices in the KD-tree index (typically, size_t of int) 17 | */ 18 | template 19 | struct KDTreeVectorOfVectorsAdaptor 20 | { 21 | typedef KDTreeVectorOfVectorsAdaptor self_t; 22 | typedef typename Distance::template traits::distance_t metric_t; 23 | typedef KDTreeSingleIndexAdaptor< metric_t,self_t,DIM,IndexType> index_t; 24 | 25 | index_t* index; //! The kd-tree index for the user to call its methods as usual with any other FLANN index. 26 | 27 | /// Constructor: takes a const ref to the vector of vectors object with the data points 28 | KDTreeVectorOfVectorsAdaptor(const VectorOfVectorsType &mat, const int leaf_max_size = 10) : m_data(mat) 29 | { 30 | assert(mat.size()!=0 && mat[0].size()!=0); 31 | const size_t dims = mat[0].size(); 32 | if (DIM>0 && static_cast(dims)!=DIM) 33 | throw std::runtime_error("Data set dimensionality does not match the 'DIM' template argument"); 34 | index = new index_t( dims, *this /* adaptor */, nanoflann::KDTreeSingleIndexAdaptorParams(leaf_max_size ) ); 35 | index->buildIndex(); 36 | } 37 | 38 | ~KDTreeVectorOfVectorsAdaptor() { 39 | delete index; 40 | } 41 | 42 | const VectorOfVectorsType &m_data; 43 | 44 | /** Query for the \a num_closest closest points to a given point (entered as query_point[0:dim-1]). 45 | * Note that this is a short-cut method for index->findNeighbors(). 46 | * The user can also call index->... methods as desired. 47 | * \note nChecks_IGNORED is ignored but kept for compatibility with the original FLANN interface. 48 | */ 49 | inline void query(const num_t *query_point, const size_t num_closest, IndexType *out_indices, num_t *out_distances_sq, const int nChecks_IGNORED = 10) const 50 | { 51 | nanoflann::KNNResultSet resultSet(num_closest); 52 | resultSet.init(out_indices, out_distances_sq); 53 | index->findNeighbors(resultSet, query_point, nanoflann::SearchParams()); 54 | } 55 | 56 | /** @name Interface expected by KDTreeSingleIndexAdaptor 57 | * @{ */ 58 | 59 | const self_t & derived() const { 60 | return *this; 61 | } 62 | self_t & derived() { 63 | return *this; 64 | } 65 | 66 | // Must return the number of data points 67 | inline size_t kdtree_get_point_count() const { 68 | return m_data.size(); 69 | } 70 | 71 | // Returns the distance between the vector "p1[0:size-1]" and the data point with index "idx_p2" stored in the class: 72 | inline num_t kdtree_distance(const num_t *p1, const size_t idx_p2,size_t size) const 73 | { 74 | num_t s=0; 75 | for (size_t i=0; i 91 | bool kdtree_get_bbox(BBOX & /*bb*/) const { 92 | return false; 93 | } 94 | 95 | /** @} */ 96 | 97 | }; // end of KDTreeVectorOfVectorsAdaptor 98 | 99 | typedef std::vector > my_vector_of_vectors_t; 100 | typedef KDTreeVectorOfVectorsAdaptor my_kd_tree_t; -------------------------------------------------------------------------------- /include/third_party/Stopwatch.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of ElasticFusion. 3 | * 4 | * Copyright (C) 2015 Imperial College London 5 | * 6 | * The use of the code within this file and all code within files that 7 | * make up the software that is ElasticFusion is permitted for 8 | * non-commercial purposes only. The full terms and conditions that 9 | * apply to the code within this file are detailed within the LICENSE.txt 10 | * file and at 11 | * unless explicitly stated. By downloading this file you agree to 12 | * comply with these terms. 13 | * 14 | * If you wish to use any of this code for commercial purposes then 15 | * please email researchcontracts.engineering@imperial.ac.uk. 16 | * 17 | */ 18 | 19 | #ifndef STOPWATCH_H_ 20 | #define STOPWATCH_H_ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | #define SEND_INTERVAL_MS 10000 37 | 38 | #ifndef DISABLE_STOPWATCH 39 | #define STOPWATCH(name, expression) \ 40 | do \ 41 | { \ 42 | const unsigned long long int startTime = Stopwatch::getInstance().getCurrentSystemTime(); \ 43 | expression \ 44 | const unsigned long long int endTime = Stopwatch::getInstance().getCurrentSystemTime(); \ 45 | Stopwatch::getInstance().addStopwatchTiming(name, endTime - startTime); \ 46 | } \ 47 | while(false) 48 | 49 | #define TICK(name) \ 50 | do \ 51 | { \ 52 | Stopwatch::getInstance().tick(name, Stopwatch::getInstance().getCurrentSystemTime()); \ 53 | } \ 54 | while(false) 55 | 56 | #define TOCK(name) \ 57 | do \ 58 | { \ 59 | Stopwatch::getInstance().tock(name, Stopwatch::getInstance().getCurrentSystemTime()); \ 60 | } \ 61 | while(false) 62 | #else 63 | #define STOPWATCH(name, expression) \ 64 | expression 65 | 66 | #define TOCK(name) ((void)0) 67 | 68 | #define TICK(name) ((void)0) 69 | 70 | #endif 71 | 72 | class Stopwatch 73 | { 74 | public: 75 | static Stopwatch & getInstance() 76 | { 77 | static Stopwatch instance; 78 | return instance; 79 | } 80 | 81 | void addStopwatchTiming(std::string name, unsigned long long int duration) 82 | { 83 | if(duration > 0) 84 | { 85 | timings[name] = (float)(duration) / 1000.0f; 86 | } 87 | } 88 | 89 | void setCustomSignature(unsigned long long int newSignature) 90 | { 91 | signature = newSignature; 92 | } 93 | 94 | const std::map & getTimings() 95 | { 96 | return timings; 97 | } 98 | 99 | void printAll() 100 | { 101 | for(std::map::const_iterator it = timings.begin(); it != timings.end(); it++) 102 | { 103 | std::cout << it->first << ": " << it->second << "ms" << std::endl; 104 | } 105 | 106 | std::cout << std::endl; 107 | } 108 | 109 | void pulse(std::string name) 110 | { 111 | timings[name] = 1; 112 | } 113 | 114 | void sendAll() 115 | { 116 | gettimeofday(&clock, 0); 117 | 118 | if((currentSend = (clock.tv_sec * 1000000 + clock.tv_usec)) - lastSend > SEND_INTERVAL_MS) 119 | { 120 | int size = 0; 121 | unsigned char * data = serialiseTimings(size); 122 | 123 | sendto(sockfd, data, size, 0, (struct sockaddr *) &servaddr, sizeof(servaddr)); 124 | 125 | free(data); 126 | 127 | lastSend = currentSend; 128 | } 129 | } 130 | 131 | static unsigned long long int getCurrentSystemTime() 132 | { 133 | timeval tv; 134 | gettimeofday(&tv, 0); 135 | unsigned long long int time = (unsigned long long int)(tv.tv_sec * 1000000 + tv.tv_usec); 136 | return time; 137 | } 138 | 139 | void tick(std::string name, unsigned long long int start) 140 | { 141 | tickTimings[name] = start; 142 | } 143 | 144 | void tock(std::string name, unsigned long long int end) 145 | { 146 | float duration = (float)(end - tickTimings[name]) / 1000.0f; 147 | 148 | if(duration > 0) 149 | { 150 | timings[name] = duration; 151 | } 152 | } 153 | 154 | private: 155 | Stopwatch() 156 | { 157 | memset(&servaddr, 0, sizeof(servaddr)); 158 | servaddr.sin_family = AF_INET; 159 | servaddr.sin_addr.s_addr = inet_addr("127.0.0.1"); 160 | servaddr.sin_port = htons(45454); 161 | sockfd = socket(AF_INET, SOCK_DGRAM, 0); 162 | 163 | gettimeofday(&clock, 0); 164 | 165 | signature = clock.tv_sec * 1000000 + clock.tv_usec; 166 | 167 | currentSend = lastSend = clock.tv_sec * 1000000 + clock.tv_usec; 168 | } 169 | 170 | virtual ~Stopwatch() 171 | { 172 | close(sockfd); 173 | } 174 | 175 | unsigned char * serialiseTimings(int & packetSize) 176 | { 177 | packetSize = sizeof(int) + sizeof(unsigned long long int); 178 | 179 | for(std::map::const_iterator it = timings.begin(); it != timings.end(); it++) 180 | { 181 | packetSize += it->first.length() + 1 + sizeof(float); 182 | } 183 | 184 | int * dataPacket = (int *)calloc(packetSize, sizeof(unsigned char)); 185 | 186 | dataPacket[0] = packetSize * sizeof(unsigned char); 187 | 188 | *((unsigned long long int *)&dataPacket[1]) = signature; 189 | 190 | float * valuePointer = (float *)&((unsigned long long int *)&dataPacket[1])[1]; 191 | 192 | for(std::map::const_iterator it = timings.begin(); it != timings.end(); it++) 193 | { 194 | valuePointer = (float *)mempcpy(valuePointer, it->first.c_str(), it->first.length() + 1); 195 | *valuePointer++ = it->second; 196 | } 197 | 198 | return (unsigned char *)dataPacket; 199 | } 200 | 201 | timeval clock; 202 | long long int currentSend, lastSend; 203 | unsigned long long int signature; 204 | int sockfd; 205 | struct sockaddr_in servaddr; 206 | std::map timings; 207 | std::map tickTimings; 208 | }; 209 | 210 | #endif /* STOPWATCH_H_ */ 211 | -------------------------------------------------------------------------------- /include/third_party/jet_extras.h: -------------------------------------------------------------------------------- 1 | // Ceres Solver - A fast non-linear least squares minimizer 2 | // Copyright 2010, 2011, 2012 Google Inc. All rights reserved. 3 | // http://code.google.com/p/ceres-solver/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are met: 7 | // 8 | // * Redistributions of source code must retain the above copyright notice, 9 | // this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above copyright notice, 11 | // this list of conditions and the following disclaimer in the documentation 12 | // and/or other materials provided with the distribution. 13 | // * Neither the name of Google Inc. nor the names of its contributors may be 14 | // used to endorse or promote products derived from this software without 15 | // specific prior written permission. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | // POSSIBILITY OF SUCH DAMAGE. 28 | // 29 | // Author: keir@google.com (Keir Mierle) 30 | // 31 | 32 | #ifndef CERES_PUBLIC_JET_EXTRAS_H_ 33 | #define CERES_PUBLIC_JET_EXTRAS_H_ 34 | 35 | #include "ceres/jet.h" 36 | #include "Eigen/Core" 37 | 38 | namespace ceres { 39 | 40 | // A jet traits class to make it easier to work with mixed auto / numeric diff. 41 | template 42 | struct JetOps { 43 | static bool IsScalar() { 44 | return true; 45 | } 46 | static T GetScalar(const T& t) { 47 | return t; 48 | } 49 | static void SetScalar(const T& scalar, T* t) { 50 | *t = scalar; 51 | } 52 | static void ScaleDerivative(double scale_by, T *value) { 53 | // For double, there is no derivative to scale. 54 | } 55 | }; 56 | 57 | template 58 | struct JetOps > { 59 | static bool IsScalar() { 60 | return false; 61 | } 62 | static T GetScalar(const Jet& t) { 63 | return t.a; 64 | } 65 | static void SetScalar(const T& scalar, Jet* t) { 66 | t->a = scalar; 67 | } 68 | static void ScaleDerivative(double scale_by, Jet *value) { 69 | value->v *= scale_by; 70 | } 71 | }; 72 | 73 | template 74 | struct Chain { 75 | static ArgumentType Rule(const FunctionType &f, 76 | const FunctionType dfdx[kNumArgs], 77 | const ArgumentType x[kNumArgs]) { 78 | // In the default case of scalars, there's nothing to do since there are no 79 | // derivatives to propagate. 80 | return f; 81 | } 82 | }; 83 | 84 | // XXX Add documentation here! 85 | template 86 | struct Chain > { 87 | static Jet Rule(const FunctionType &f, 88 | const FunctionType dfdx[kNumArgs], 89 | const Jet x[kNumArgs]) { 90 | // x is itself a function of another variable ("z"); what this function 91 | // needs to return is "f", but with the derivative with respect to z 92 | // attached to the jet. So combine the derivative part of x's jets to form 93 | // a Jacobian matrix between x and z (i.e. dx/dz). 94 | Eigen::Matrix dxdz; 95 | for (int i = 0; i < kNumArgs; ++i) { 96 | dxdz.row(i) = x[i].v.transpose(); 97 | } 98 | 99 | // Map the input gradient dfdx into an Eigen row vector. 100 | Eigen::Map > 101 | vector_dfdx(dfdx, 1, kNumArgs); 102 | 103 | // Now apply the chain rule to obtain df/dz. Combine the derivative with 104 | // the scalar part to obtain f with full derivative information. 105 | Jet jet_f; 106 | jet_f.a = f; 107 | jet_f.v = vector_dfdx.template cast() * dxdz; // Also known as dfdz. 108 | return jet_f; 109 | } 110 | }; 111 | 112 | } // namespace ceres 113 | 114 | #endif // CERES_PUBLIC_JET_EXTRAS_H_ 115 | -------------------------------------------------------------------------------- /include/third_party/msvc/Stopwatch.h: -------------------------------------------------------------------------------- 1 | /* 2 | * MODIFIED VERSION FOR WINDOWS 3 | * 4 | * This file is part of ElasticFusion. 5 | * 6 | * Copyright (C) 2015 Imperial College London 7 | * 8 | * The use of the code within this file and all code within files that 9 | * make up the software that is ElasticFusion is permitted for 10 | * non-commercial purposes only. The full terms and conditions that 11 | * apply to the code within this file are detailed within the LICENSE.txt 12 | * file and at 13 | * unless explicitly stated. By downloading this file you agree to 14 | * comply with these terms. 15 | * 16 | * If you wish to use any of this code for commercial purposes then 17 | * please email researchcontracts.engineering@imperial.ac.uk. 18 | * 19 | */ 20 | 21 | #ifndef STOPWATCH_H_ 22 | #define STOPWATCH_H_ 23 | 24 | 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | 38 | #define SEND_INTERVAL_MS 10000 39 | 40 | #ifndef DISABLE_STOPWATCH 41 | #define STOPWATCH(name, expression) \ 42 | do \ 43 | { \ 44 | const unsigned long long int startTime = Stopwatch::getInstance().getCurrentSystemTime(); \ 45 | expression \ 46 | const unsigned long long int endTime = Stopwatch::getInstance().getCurrentSystemTime(); \ 47 | Stopwatch::getInstance().addStopwatchTiming(name, endTime - startTime); \ 48 | } \ 49 | while(false) 50 | 51 | #define TICK(name) \ 52 | do \ 53 | { \ 54 | Stopwatch::getInstance().tick(name, Stopwatch::getInstance().getCurrentSystemTime()); \ 55 | } \ 56 | while(false) 57 | 58 | #define TOCK(name) \ 59 | do \ 60 | { \ 61 | Stopwatch::getInstance().tock(name, Stopwatch::getInstance().getCurrentSystemTime()); \ 62 | } \ 63 | while(false) 64 | #else 65 | #define STOPWATCH(name, expression) \ 66 | expression 67 | 68 | #define TOCK(name) ((void)0) 69 | 70 | #define TICK(name) ((void)0) 71 | 72 | #endif 73 | 74 | class Stopwatch 75 | { 76 | public: 77 | static Stopwatch & getInstance() 78 | { 79 | static Stopwatch instance; 80 | return instance; 81 | } 82 | 83 | void addStopwatchTiming(std::string name, unsigned long long int duration) 84 | { 85 | if(duration > 0) 86 | { 87 | timings[name] = (float)(duration) / 1000.0f; 88 | } 89 | } 90 | 91 | void setCustomSignature(unsigned long long int newSignature) 92 | { 93 | signature = newSignature; 94 | } 95 | 96 | const std::map & getTimings() 97 | { 98 | return timings; 99 | } 100 | 101 | void printAll() 102 | { 103 | for(std::map::const_iterator it = timings.begin(); it != timings.end(); it++) 104 | { 105 | std::cout << it->first << ": " << it->second << "ms" << std::endl; 106 | } 107 | 108 | std::cout << std::endl; 109 | } 110 | 111 | void pulse(std::string name) 112 | { 113 | timings[name] = 1; 114 | } 115 | 116 | void sendAll() 117 | { 118 | gettimeofday(&clock, 0); 119 | 120 | if((currentSend = (clock.tv_sec * 1000000 + clock.tv_usec)) - lastSend > SEND_INTERVAL_MS) 121 | { 122 | int size = 0; 123 | unsigned char * data = serialiseTimings(size); 124 | 125 | sendto(sockfd, (const char*) data, size, 0, (struct sockaddr *) &servaddr, sizeof(servaddr)); 126 | 127 | free(data); 128 | 129 | lastSend = currentSend; 130 | } 131 | } 132 | 133 | static unsigned long long int getCurrentSystemTime() 134 | { 135 | timeval tv; 136 | gettimeofday(&tv, 0); 137 | unsigned long long int time = (unsigned long long int)(tv.tv_sec * 1000000 + tv.tv_usec); 138 | return time; 139 | } 140 | 141 | void tick(std::string name, unsigned long long int start) 142 | { 143 | tickTimings[name] = start; 144 | } 145 | 146 | void tock(std::string name, unsigned long long int end) 147 | { 148 | float duration = (float)(end - tickTimings[name]) / 1000.0f; 149 | 150 | if(duration > 0) 151 | { 152 | timings[name] = duration; 153 | } 154 | } 155 | 156 | private: 157 | Stopwatch() 158 | { 159 | memset(&servaddr, 0, sizeof(servaddr)); 160 | servaddr.sin_family = AF_INET; 161 | servaddr.sin_addr.s_addr = inet_addr("127.0.0.1"); 162 | servaddr.sin_port = htons(45454); 163 | sockfd = socket(AF_INET, SOCK_DGRAM, 0); 164 | 165 | gettimeofday(&clock, 0); 166 | 167 | signature = clock.tv_sec * 1000000 + clock.tv_usec; 168 | 169 | currentSend = lastSend = clock.tv_sec * 1000000 + clock.tv_usec; 170 | } 171 | 172 | virtual ~Stopwatch() 173 | { 174 | closesocket(sockfd); 175 | } 176 | 177 | unsigned char * serialiseTimings(int & packetSize) 178 | { 179 | packetSize = sizeof(int) + sizeof(unsigned long long int); 180 | 181 | for(std::map::const_iterator it = timings.begin(); it != timings.end(); it++) 182 | { 183 | packetSize += it->first.length() + 1 + sizeof(float); 184 | } 185 | 186 | int * dataPacket = (int *)calloc(packetSize, sizeof(unsigned char)); 187 | 188 | dataPacket[0] = packetSize * sizeof(unsigned char); 189 | 190 | *((unsigned long long int *)&dataPacket[1]) = signature; 191 | 192 | float * valuePointer = (float *)&((unsigned long long int *)&dataPacket[1])[1]; 193 | 194 | for(std::map::const_iterator it = timings.begin(); it != timings.end(); it++) 195 | { 196 | valuePointer = (float *)memcpy(valuePointer, it->first.c_str(), it->first.length() + 1); 197 | *valuePointer++ = it->second; 198 | } 199 | 200 | return (unsigned char *)dataPacket; 201 | } 202 | 203 | static int gettimeofday(struct timeval* tp, void* tzp) 204 | { 205 | DWORD t; 206 | t = timeGetTime(); 207 | tp->tv_sec = t / 1000; 208 | tp->tv_usec = t % 1000; 209 | /* 0 indicates that the call succeeded. */ 210 | return 0; 211 | } 212 | 213 | timeval clock; 214 | long long int currentSend, lastSend; 215 | unsigned long long int signature; 216 | int sockfd; 217 | struct sockaddr_in servaddr; 218 | std::map timings; 219 | std::map tickTimings; 220 | }; 221 | 222 | #endif /* STOPWATCH_H_ */ 223 | -------------------------------------------------------------------------------- /include/third_party/sample.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 libmv authors. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | // 21 | // Author: mierle@google.com (Keir Mierle) 22 | // 23 | 24 | #ifndef SAMPLE_H_ 25 | #define SAMPLE_H_ 26 | 27 | #include "main_engine/utils/global.h" 28 | 29 | #include "opencv2/imgproc/imgproc.hpp" 30 | #include "ceres/ceres.h" 31 | #include "jet_extras.h" 32 | 33 | 34 | // Sample the image at position (x, y) but use the gradient to 35 | // propagate derivatives from x and y. This is needed to integrate the numeric 36 | // image gradients with Ceres's autodiff framework. 37 | template< typename T, class TImage > 38 | T SampleWithDerivative(const TImage & intensityImage, 39 | const TImage & intensityGradientX, 40 | const TImage & intensityGradientY, 41 | const T & x, 42 | const T & y) 43 | { 44 | typedef TImage ImageType; 45 | typedef typename ImageType::value_type PixelType; 46 | 47 | PixelType scalar_x = ceres::JetOps::GetScalar(x); 48 | PixelType scalar_y = ceres::JetOps::GetScalar(y); 49 | 50 | PixelType sample[3]; 51 | // Sample intensity image and gradients 52 | SampleLinear( intensityImage, intensityGradientX, intensityGradientY, 53 | scalar_y, scalar_x, sample ); 54 | T xy[2] = { x, y }; 55 | return ceres::Chain< PixelType, 2, T >::Rule( sample[0], sample + 1, xy ); 56 | } 57 | 58 | #endif // SAMPLE_H_ 59 | -------------------------------------------------------------------------------- /msvc/ConsoleApp/ConsoleApp.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | -------------------------------------------------------------------------------- /msvc/GUIApp/GUIApp.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Header Files 20 | 21 | 22 | Header Files 23 | 24 | 25 | Header Files 26 | 27 | 28 | Header Files 29 | 30 | 31 | Header Files 32 | 33 | 34 | Header Files 35 | 36 | 37 | Header Files 38 | 39 | 40 | Header Files 41 | 42 | 43 | 44 | 45 | Source Files 46 | 47 | 48 | Source Files 49 | 50 | 51 | Source Files 52 | 53 | 54 | Source Files 55 | 56 | 57 | Source Files 58 | 59 | 60 | Source Files 61 | 62 | 63 | Source Files 64 | 65 | 66 | -------------------------------------------------------------------------------- /msvc/MainEngine/MainEngine.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Header Files 23 | 24 | 25 | Header Files 26 | 27 | 28 | Header Files 29 | 30 | 31 | Header Files 32 | 33 | 34 | Header Files 35 | 36 | 37 | Header Files 38 | 39 | 40 | Header Files 41 | 42 | 43 | Header Files 44 | 45 | 46 | Header Files 47 | 48 | 49 | Header Files 50 | 51 | 52 | Header Files 53 | 54 | 55 | Header Files 56 | 57 | 58 | Header Files 59 | 60 | 61 | Header Files 62 | 63 | 64 | Header Files 65 | 66 | 67 | Header Files 68 | 69 | 70 | Header Files 71 | 72 | 73 | Header Files 74 | 75 | 76 | Header Files 77 | 78 | 79 | Header Files 80 | 81 | 82 | Header Files 83 | 84 | 85 | 86 | 87 | Source Files 88 | 89 | 90 | Source Files 91 | 92 | 93 | Source Files 94 | 95 | 96 | Source Files 97 | 98 | 99 | Source Files 100 | 101 | 102 | Source Files 103 | 104 | 105 | Source Files 106 | 107 | 108 | Source Files 109 | 110 | 111 | Source Files 112 | 113 | 114 | Source Files 115 | 116 | 117 | Source Files 118 | 119 | 120 | Source Files 121 | 122 | 123 | Source Files 124 | 125 | 126 | Source Files 127 | 128 | 129 | Source Files 130 | 131 | 132 | Source Files 133 | 134 | 135 | Source Files 136 | 137 | 138 | Source Files 139 | 140 | 141 | Source Files 142 | 143 | 144 | Source Files 145 | 146 | 147 | Source Files 148 | 149 | 150 | Source Files 151 | 152 | 153 | -------------------------------------------------------------------------------- /msvc/PangaeaTracking.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.21005.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MainEngine", "MainEngine\MainEngine.vcxproj", "{4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}" 7 | EndProject 8 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ConsoleApp", "ConsoleApp\ConsoleApp.vcxproj", "{9C90F969-3724-4336-B3E2-180BC465AABB}" 9 | ProjectSection(ProjectDependencies) = postProject 10 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF} = {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF} 11 | EndProjectSection 12 | EndProject 13 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GUIApp", "GUIApp\GUIApp.vcxproj", "{EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}" 14 | ProjectSection(ProjectDependencies) = postProject 15 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF} = {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF} 16 | EndProjectSection 17 | EndProject 18 | Global 19 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 20 | Debug|Win32 = Debug|Win32 21 | Debug|x64 = Debug|x64 22 | Release|Win32 = Release|Win32 23 | Release|x64 = Release|x64 24 | EndGlobalSection 25 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 26 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Debug|Win32.ActiveCfg = Debug|Win32 27 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Debug|Win32.Build.0 = Debug|Win32 28 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Debug|x64.ActiveCfg = Debug|x64 29 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Debug|x64.Build.0 = Debug|x64 30 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Release|Win32.ActiveCfg = Release|Win32 31 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Release|Win32.Build.0 = Release|Win32 32 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Release|x64.ActiveCfg = Release|x64 33 | {4D8EC26E-FF70-43E6-B3FC-895F332E3BEF}.Release|x64.Build.0 = Release|x64 34 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Debug|Win32.ActiveCfg = Debug|Win32 35 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Debug|Win32.Build.0 = Debug|Win32 36 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Debug|x64.ActiveCfg = Debug|x64 37 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Debug|x64.Build.0 = Debug|x64 38 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Release|Win32.ActiveCfg = Release|Win32 39 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Release|Win32.Build.0 = Release|Win32 40 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Release|x64.ActiveCfg = Release|x64 41 | {9C90F969-3724-4336-B3E2-180BC465AABB}.Release|x64.Build.0 = Release|x64 42 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Debug|Win32.ActiveCfg = Debug|Win32 43 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Debug|Win32.Build.0 = Debug|Win32 44 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Debug|x64.ActiveCfg = Debug|x64 45 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Debug|x64.Build.0 = Debug|x64 46 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Release|Win32.ActiveCfg = Release|Win32 47 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Release|Win32.Build.0 = Release|Win32 48 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Release|x64.ActiveCfg = Release|x64 49 | {EC357A5A-B13C-4DFA-8A2B-D0E64235C73D}.Release|x64.Build.0 = Release|x64 50 | EndGlobalSection 51 | GlobalSection(SolutionProperties) = preSolution 52 | HideSolutionNode = FALSE 53 | EndGlobalSection 54 | EndGlobal 55 | -------------------------------------------------------------------------------- /msvc/README.txt: -------------------------------------------------------------------------------- 1 | #PangaeaTracking for MSVC 2 | 3 | This is Microsoft Visual Studio project to compile PangaeaTracking on Windows. 4 | 5 | 1. The following libraries will be required to be compiled in your machine: 6 | 7 | - OpenCV 8 | available at https://github.com/Itseez/opencv 9 | - Ceres Solver 10 | available at https://ceres-solver.googlesource.com/ceres-solver/ 11 | - SuiteSparse 12 | compiled from Jose Luis Blanco-Claraco's msvc project: https://github.com/jlblancoc/suitesparse-metis-for-windows 13 | - glog 14 | available at https://github.com/google/glog 15 | - Eigen 16 | available at http://eigen.tuxfamily.org/ 17 | - Boost 18 | available at http://www.boost.org/ 19 | - freeGLUT 20 | available at http://freeglut.sourceforge.net/ 21 | - GLEW 22 | available at http://glew.sourceforge.net/ 23 | - wxWidgets 24 | available at https://www.wxwidgets.org 25 | 26 | 2. Open the msvc project solution 'PangaeaTracking.sln' and make sure the following header directories are correct in each project. Change environmental variables and/or directories if required: 27 | 28 | $(OPENCV_PATH)\include 29 | $(EIGEN_PATH) 30 | $(EIGEN_PATH)\unsupported 31 | $(BOOST_ROOT) 32 | $(SUITESPARSE_PATH)\include 33 | $(CERES_PATH)\include 34 | $(GLOG_PATH)\src\windows 35 | $(GLUT_ROOT)\include 36 | $(GLEW_ROOT)\include 37 | 38 | These two are only required for the GUI app: 39 | $(WXWIN)\include\msvc 40 | $(WXWIN)\include 41 | 42 | 3. Build 'MainEngine' 43 | 44 | 4. Check for the 'ConsoleApp' and 'GUIApp' projects that the following library directories are correct. Change the environmental variables and/or directories if required. This example was made for x64 so change it to x86 if that is your case. Also, this is for Release mode so if you want to compile it in Debug Mode make the required modifications. 45 | 46 | $(SolutionDir)\x64\Release 47 | $(GLUT_ROOT)\lib 48 | $(GLEW_ROOT)\lib 49 | $(BOOST_ROOT)\stage\lib 50 | $(SUITESPARSE_PATH)\lib64\lapack_blas_windows 51 | $(SUITESPARSE_PATH)\lib64 52 | $(OPENCV_PATH)\x64\vc12\lib 53 | $(CERES_PATH)\lib 54 | $(GLOG_PATH)\x64\Release 55 | 56 | This one is only required for the GUI app: 57 | $(WXWIN)\lib\vc_x64_lib 58 | 59 | 5. This are the list of libraries that have to be linked to the project. Here are shown the ones for the Release mode. Change the name of the libraries to your case when required: 60 | 61 | - Main Engine (this is generated after compiling the MainEngine project) 62 | MainEngine.lib 63 | 64 | - OpenCV 65 | opencv_core300.lib 66 | opencv_highgui300.lib 67 | opencv_imgcodecs300.lib 68 | opencv_imgproc300.lib 69 | 70 | - SuiteSparse 71 | libblas.libliblapack.lib 72 | libamdd.lib 73 | libbtfd.lib 74 | libcamd.lib 75 | libccolamd.lib 76 | libcholmod.lib 77 | libcolamd.lib 78 | libcxsparse.lib 79 | libklu.lib 80 | libldl.lib 81 | libspqr.lib 82 | libumfpack.lib 83 | suitesparseconfig.lib 84 | 85 | - glog 86 | libglog.lib 87 | 88 | - Ceres Solver 89 | ceres.lib 90 | 91 | - freeGLUT 92 | freeglut.lib 93 | GLU32.lib 94 | OPENGL32.lib 95 | 96 | - GLEW 97 | glew32.lib 98 | 99 | - Windows socket (only required for 'StopWatch.h' but not really used) 100 | Ws2_32.lib 101 | 102 | - wxWidgets 103 | wxmsw30u_core.lib 104 | wxbase30u.lib 105 | wxmsw30u_gl.lib 106 | wxexpat.lib 107 | wxjpeg.lib 108 | wxpng.lib 109 | wxregexu.lib 110 | wxtiff.lib 111 | wxzlib.lib 112 | winmm.lib 113 | comctl32.lib 114 | rpcrt4.lib 115 | wsock32.lib 116 | odbc32.lib 117 | 118 | - Boost 119 | libboost_filesystem-vc120-mt-1_59.lib 120 | 121 | 6. Run ConsoleApp and GUIApp with a modified configuration file to your data and result directories. -------------------------------------------------------------------------------- /src/console_app/PangaeaTracking_console.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/MainEngine.h" 2 | #if defined(_DEBUG) && defined(_MSC_VER) 3 | //#include "vld.h" 4 | #endif 5 | int main(int argc, char* argv[]) 6 | { 7 | MainEngine mainEngine; 8 | mainEngine.ReadConfigurationFile(argc, argv); 9 | mainEngine.SetupInputAndTracker(); 10 | mainEngine.Run(); 11 | 12 | return 0; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/gui_app/CameraControl.cpp: -------------------------------------------------------------------------------- 1 | #include "math.h" 2 | #include 3 | 4 | #include "gui_app/CameraControl.h" 5 | 6 | CCameraControl::CCameraControl() 7 | { 8 | Reset(); 9 | } 10 | 11 | void CCameraControl::Reset() 12 | { 13 | 14 | camAngleX = 0; 15 | camAngleY = 0; 16 | camAngleZ = 0; 17 | camTransX = 0; 18 | camTransY = 0; 19 | camTransZ = 0; 20 | 21 | matrixView.identity(); 22 | matrixModel.identity(); 23 | 24 | cameraLoc.x = 0; 25 | cameraLoc.y = 0; 26 | cameraLoc.z = 0; 27 | 28 | } 29 | 30 | void CCameraControl::setObjectCenter(double objectCenter[3]) 31 | { 32 | center[0] = objectCenter[0]; 33 | center[1] = objectCenter[1]; 34 | center[2] = objectCenter[2]; 35 | } 36 | 37 | const float* CCameraControl::getModelViewMatrix() 38 | { 39 | // transformation in normal order, different from 40 | // opengl operations 41 | matrixView.identity(); 42 | // matrixView.translate(-(camTransX-center[0]), 43 | // -camTransY+center[1],-camTransZ+center[2]); 44 | matrixView.rotateX(180); 45 | matrixView.translate(-(camTransX-center[0]), 46 | camTransY-center[1],camTransZ-center[2]); 47 | 48 | matrixModel.identity(); 49 | matrixModel.translate(-center); 50 | matrixModel.rotateZ(camAngleZ); 51 | matrixModel.rotateX(camAngleX); 52 | matrixModel.rotateY(camAngleY); 53 | matrixModelView = matrixView*matrixModel; 54 | 55 | Matrix4 matrixModelViewInv; 56 | matrixModelViewInv = matrixModelView; 57 | matrixModelViewInv.invertAffine(); 58 | Vector4 temp(0,0,0,1); 59 | cameraLoc = matrixModelViewInv * temp; 60 | 61 | return matrixModelView.getTranspose(); 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/gui_app/ImagePanel.cpp: -------------------------------------------------------------------------------- 1 | #include "gui_app/ImagePanel.h" 2 | 3 | #include "gui_app/MainFrame.h" 4 | #include "gui_app/controlPanel.h" 5 | 6 | BEGIN_EVENT_TABLE(wxImagePanel, wxPanel) 7 | EVT_PAINT(wxImagePanel::paintEvent) 8 | EVT_SIZE(wxImagePanel::OnSize) 9 | END_EVENT_TABLE() 10 | 11 | wxImagePanel::wxImagePanel(wxFrame* parent, wxString file, wxBitmapType format): 12 | wxPanel(parent), updated(false) 13 | { 14 | // load the file... ideally add a check to see if loading was successful 15 | image.LoadFile(file, format); 16 | w = -1; 17 | h = -1; 18 | } 19 | 20 | // Initialize from memory 21 | wxImagePanel::wxImagePanel(wxPanel* parent, unsigned char* pData, int width, int height, bool overlay): 22 | wxPanel(parent), updated(false) 23 | { 24 | image = wxImage(width, height, pData, true); 25 | w = width; 26 | h = height; 27 | doOverlay = overlay; 28 | } 29 | 30 | void wxImagePanel::updateImage(unsigned char* pData, int width, int height) 31 | { 32 | image = wxImage(width, height, pData, true); 33 | updated = true; 34 | Refresh(false); 35 | } 36 | 37 | void wxImagePanel::setMainFrame(MainFrame* pMainFrm) 38 | { 39 | pMainFrame = pMainFrm; 40 | } 41 | 42 | void wxImagePanel::setNumPnts(int numPnts) 43 | { 44 | numOverlayPnts = numPnts; 45 | } 46 | 47 | void wxImagePanel::paintEvent(wxPaintEvent & evt) 48 | { 49 | // depending on your system you may need to look at double-buffered dcs 50 | wxPaintDC dc(this); 51 | render(dc); 52 | } 53 | 54 | 55 | void wxImagePanel::paintNow() 56 | { 57 | // depending on your system you may need to look at double-buffered dcs 58 | wxClientDC dc(this); 59 | render(dc); 60 | } 61 | 62 | 63 | void wxImagePanel::render(wxDC& dc) 64 | { 65 | int neww, newh; 66 | dc.GetSize( &neww, &newh ); 67 | 68 | if(( neww != w || newh != h ) || updated) 69 | { 70 | resized = wxBitmap( image.Scale( neww, newh /*, wxIMAGE_QUALITY_HIGH*/ ) ); 71 | w = neww; 72 | h = newh; 73 | dc.DrawBitmap( resized, 0, 0, false ); 74 | updated = false; 75 | } 76 | else 77 | { 78 | dc.DrawBitmap( resized, 0, 0, false ); 79 | } 80 | 81 | if(doOverlay) 82 | { 83 | // draw 2d projection points 84 | wxPen mypen = wxPen(wxColor(wxT("Red")), 1); 85 | dc.SetPen(mypen); 86 | int numPnts = (*(pMainFrame->pOutputInfo)).meshProj.size(); 87 | int size = pMainFrame->m_pControlPanel->m_nOverlaySize; 88 | cout << "overlay size: " << size << endl; 89 | for(int k = 0; k < numPnts; ++k) 90 | { 91 | if((*(pMainFrame->pOutputInfo)).visibilityMask[k]) 92 | { 93 | for(int i = 0; i < size; ++i) 94 | { 95 | for(int j = 0; j < size; ++j) 96 | { 97 | dc.DrawPoint(((*(pMainFrame->pOutputInfo)).meshProj[k][0]+i) * w / pMainFrame->m_nWidth, 98 | ((*(pMainFrame->pOutputInfo)).meshProj[k][1]+j) * h / pMainFrame->m_nHeight); 99 | } 100 | } 101 | } 102 | } 103 | 104 | wxPen occluder_pen = wxPen(wxColor(wxT("Blue")), 1); 105 | dc.SetPen(occluder_pen); 106 | for(int k = 0; k < numPnts; ++k) 107 | { 108 | if(!(*(pMainFrame->pOutputInfo)).visibilityMask[k]) 109 | { 110 | for(int i = 0; i < size; ++i) 111 | { 112 | for(int j = 0; j < size; ++j) 113 | { 114 | dc.DrawPoint(((*(pMainFrame->pOutputInfo)).meshProj[k][0]+i) * w / pMainFrame->m_nWidth, 115 | ((*(pMainFrame->pOutputInfo)).meshProj[k][1]+j) * h / pMainFrame->m_nHeight); 116 | } 117 | } 118 | } 119 | } 120 | 121 | } 122 | } 123 | 124 | void wxImagePanel::OnSize(wxSizeEvent& event){ 125 | Refresh(); 126 | //skip the event. 127 | event.Skip(); 128 | } 129 | -------------------------------------------------------------------------------- /src/gui_app/PangaeaTracking.cpp: -------------------------------------------------------------------------------- 1 | #include "gui_app/PangaeaTracking.h" 2 | #include "gui_app/MainFrame.h" 3 | 4 | IMPLEMENT_APP(PangaeaTracking) 5 | 6 | bool PangaeaTracking::OnInit() 7 | { 8 | // m_pMainFrame = new MainFrame(wxT("PangaeaTracking Render"), argc, argv); 9 | // new MainFrame(wxT("PangaeaTracking Render"), argc, argv); 10 | // argv_char = new char*[argc]; 11 | arg_num = argc; 12 | SafeAllocArrayType(argv_char, argc, char*); 13 | for(int i = 0; i < argc; ++i) 14 | { 15 | argv_char[i] = new char[1024]; 16 | SafeAllocArrayType(argv_char[i], 1024, char); 17 | strncpy( argv_char[i], 18 | (const char*)( (wxString( argv[i] ) ).mb_str(wxConvUTF8) ), 19 | 1023 ); 20 | } 21 | 22 | new MainFrame("PangaeaTracking Render", argc, argv_char); 23 | 24 | return true; 25 | } 26 | 27 | int PangaeaTracking::OnExit() 28 | { 29 | for(int i = 0; i < arg_num; ++i) 30 | SafeDeleteArray(argv_char[i]); 31 | 32 | SafeDeleteArray(argv_char); 33 | 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /src/main_engine/image_source/ImageSequenceReader.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/image_source/ImageSourceEngine.h" 2 | 3 | ImageSequenceReader::ImageSequenceReader(ImageSourceSettings& settings) 4 | { 5 | startFrameNo = settings.startFrame; 6 | totalFrameNo = settings.numFrames; 7 | m_nWidth = settings.width; 8 | m_nHeight = settings.height; 9 | inputPath = settings.dataPath; 10 | imgFormat = settings.imageFormat; 11 | 12 | currentFrameNo = startFrameNo; 13 | 14 | char buffer[BUFFER_SIZE]; 15 | std::stringstream imagePath; 16 | sprintf(buffer, imgFormat.c_str(), currentFrameNo); 17 | imagePath << inputPath << buffer; 18 | //memset(&buffer[0], 0, sizeof(buffer)); 19 | 20 | m_curImage = cv::imread(imagePath.str().c_str(),1); // read in color image 21 | 22 | cout << imagePath.str() << endl; 23 | 24 | // read the calibration information 25 | std::stringstream intrinsicsFileName; 26 | intrinsicsFileName << inputPath << settings.intrinsicsFile; 27 | ifstream intrinsicsFile(intrinsicsFileName.str().c_str()); 28 | for(int i = 0; i < 3; ++i) 29 | { 30 | for(int j = 0; j < 3; ++j) 31 | { 32 | intrinsicsFile >> KK[i][j]; 33 | std::cout << KK[i][j] << " "; 34 | } 35 | std::cout << std::endl; 36 | } 37 | 38 | // read different level images if necessary 39 | if(imageSourceSettings.useMultiImages) 40 | { 41 | 42 | char buffer1[BUFFER_SIZE]; 43 | char buffer2[BUFFER_SIZE]; 44 | 45 | int level_num = imageSourceSettings.dataPathLevelList.size(); 46 | m_curImages.resize(level_num); 47 | 48 | for(int i = 0; i < level_num; ++i) 49 | { 50 | std::stringstream imagePath; 51 | sprintf(buffer1, imageSourceSettings.dataPathLevelFormat.c_str(), 52 | imageSourceSettings.dataPathLevelList[i]); 53 | sprintf(buffer2, imageSourceSettings.imageLevelFormat.c_str(), 54 | currentFrameNo); 55 | imagePath << imageSourceSettings.dataPathLevelRoot << buffer1 << buffer2; 56 | 57 | cout << imagePath.str() << endl; 58 | 59 | m_curImages[i] = cv::imread(imagePath.str().c_str()); 60 | } 61 | } 62 | 63 | } 64 | 65 | ImageSequenceReader::ImageSequenceReader(std::string& inputPath, std::string& imgFormat, 66 | int nHeight, int nWidth, int startFrame, int numTrackingFrames, double shapeScale): 67 | startFrameNo(startFrame), 68 | totalFrameNo(numTrackingFrames), 69 | currentFrameNo(startFrame), 70 | m_nHeight(nHeight), 71 | m_nWidth(nWidth), 72 | m_ShapeScale(shapeScale), 73 | inputPath(inputPath), 74 | imgFormat(imgFormat) 75 | { 76 | char buffer[BUFFER_SIZE]; 77 | std::stringstream imagePath; 78 | sprintf(buffer, imgFormat.c_str(), currentFrameNo); 79 | imagePath << inputPath << buffer; 80 | //memset(&buffer[0], 0, sizeof(buffer)); 81 | 82 | m_curImage = cv::imread(imagePath.str().c_str(),1); // read in color image 83 | 84 | // read different level images if necessary 85 | if(imageSourceSettings.useMultiImages) 86 | { 87 | 88 | char buffer1[BUFFER_SIZE]; 89 | char buffer2[BUFFER_SIZE]; 90 | 91 | int level_num = imageSourceSettings.dataPathLevelList.size(); 92 | m_curImages.resize(level_num); 93 | 94 | for(int i = 0; i < level_num; ++i) 95 | { 96 | std::stringstream imagePath; 97 | sprintf(buffer1, imageSourceSettings.dataPathLevelFormat.c_str(), 98 | imageSourceSettings.dataPathLevelList[i]); 99 | sprintf(buffer2, imageSourceSettings.imageLevelFormat.c_str(), 100 | currentFrameNo); 101 | imagePath << imageSourceSettings.dataPathLevelRoot << buffer1 << buffer2; 102 | 103 | m_curImages[i] = cv::imread(imagePath.str().c_str()); 104 | } 105 | } 106 | } 107 | 108 | ImageSequenceReader::~ImageSequenceReader() 109 | {} 110 | 111 | void ImageSequenceReader::setCurrentFrame(int curFrame) 112 | { 113 | if(currentFrameNo != curFrame) 114 | { 115 | currentFrameNo = curFrame; // update current frame number 116 | 117 | char buffer[BUFFER_SIZE]; 118 | std::stringstream imagePath; 119 | sprintf(buffer, imgFormat.c_str(), currentFrameNo); 120 | imagePath << inputPath << buffer; 121 | //memset(&buffer[0], 0, sizeof(buffer)); 122 | 123 | std::cout << imagePath.str() << std::endl; 124 | 125 | try 126 | { 127 | m_curImage = cv::imread(imagePath.str().c_str(),1); // read in color image 128 | } 129 | catch(exception& e) 130 | { 131 | cerr << "ERROR: cv::imread failed." << endl; 132 | cerr << "(Exception = " << e.what() << ")" << endl; 133 | } 134 | 135 | // read different level images if necessary 136 | if(imageSourceSettings.useMultiImages) 137 | { 138 | 139 | char buffer1[BUFFER_SIZE]; 140 | char buffer2[BUFFER_SIZE]; 141 | 142 | int level_num = imageSourceSettings.dataPathLevelList.size(); 143 | m_curImages.resize(level_num); 144 | 145 | for(int i = 0; i < level_num; ++i) 146 | { 147 | std::stringstream imagePath; 148 | sprintf(buffer1, imageSourceSettings.dataPathLevelFormat.c_str(), 149 | imageSourceSettings.dataPathLevelList[i]); 150 | sprintf(buffer2, imageSourceSettings.imageLevelFormat.c_str(), 151 | currentFrameNo); 152 | imagePath << imageSourceSettings.dataPathLevelRoot << buffer1 << buffer2; 153 | 154 | cout << imagePath.str() << endl; 155 | 156 | m_curImages[i] = cv::imread(imagePath.str().c_str()); 157 | } 158 | } 159 | 160 | } 161 | } 162 | 163 | unsigned char* ImageSequenceReader::getColorImage() 164 | { 165 | return m_curImage.data; 166 | } 167 | 168 | unsigned char* ImageSequenceReader::getLevelColorImage(int nLevel) 169 | { 170 | 171 | return m_curImages[nLevel].data; 172 | 173 | } 174 | 175 | void ImageSequenceReader::readUVDImage(DepthImageType& uImage, DepthImageType& vImage, 176 | DepthImageType& dImage, InternalIntensityImageType& maskImageAux) 177 | { 178 | // read in uImage, vImage and dImage 179 | std::stringstream data_path; 180 | data_path << inputPath; 181 | if(trackerSettings.useXYZ) 182 | { 183 | // for perspective camera, in this case 184 | ReadRawDepth(data_path,"refX.raw",m_nWidth,m_nHeight,uImage); 185 | ReadRawDepth(data_path,"refY.raw",m_nWidth,m_nHeight,vImage); 186 | } 187 | else 188 | { 189 | // mainly for orthographic camera 190 | for(int i=0; i::type); 206 | 207 | } 208 | 209 | void ImageSequenceReader::ReadRawDepth(std::stringstream& data_path, std::string filename, 210 | int width, int height, DepthImageType& resImage) 211 | { 212 | std::ifstream inputFileStream; 213 | std::stringstream imagePath; 214 | CoordinateType* ref_buffer; 215 | imagePath << data_path.str() << filename; 216 | if(bfs::exists(imagePath.str())) 217 | { 218 | ref_buffer = new CoordinateType[width*height]; 219 | // std::cout << data_path.str() << filename << std::endl; 220 | inputFileStream.open(imagePath.str().c_str(),std::ios::binary); 221 | inputFileStream.read((char*)ref_buffer,sizeof(CoordinateType)*width*height); 222 | inputFileStream.close(); 223 | DepthImageType imgTemp(width,height,ref_buffer); 224 | imgTemp = imgTemp.t(); 225 | imgTemp.copyTo(resImage); 226 | delete[] ref_buffer; 227 | } 228 | else 229 | { 230 | cerr << imagePath.str() << " does not exist! " << endl; 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /src/main_engine/image_source/ImagesBufferReader.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/image_source/ImageSourceEngine.h" 2 | 3 | ImagesBufferReader::ImagesBufferReader(ImageSourceSettings& settings) 4 | { 5 | startFrameNo = settings.startFrame; 6 | totalFrameNo = settings.numFrames; 7 | m_nWidth = settings.width; 8 | m_nHeight = settings.height; 9 | inputPath = settings.dataPath; 10 | imgFormat = settings.imageFormat; 11 | 12 | currentFrameNo = startFrameNo; 13 | 14 | // for loop load all the images 15 | nFrameStep = settings.frameStep; 16 | int bufferSize = (totalFrameNo - startFrameNo)/nFrameStep + 1; 17 | m_vImages.resize(bufferSize); 18 | 19 | nFrameStep = settings.frameStep; 20 | char buffer[BUFFER_SIZE]; 21 | for(int i = startFrameNo; i <= totalFrameNo; i = i + nFrameStep) 22 | { 23 | std::stringstream imagePath; 24 | sprintf(buffer, imgFormat.c_str(), i); 25 | imagePath << inputPath << buffer; 26 | //memset(&buffer[0], 0, sizeof(buffer)); 27 | 28 | std::cout << imagePath.str() << std::endl; 29 | 30 | // ColorImageType tempImage; 31 | // tempImage = cv::imread(imagePath.str().c_str(),1); // read in color image 32 | // // cv::resize(tempImage, m_vImages[i-startFrame], cv::Size(), m_ShapeScale, m_ShapeScale); 33 | // m_vImages[i-startFrame] = tempImage; 34 | int bufferPos = (i-startFrameNo)/nFrameStep; 35 | m_vImages[ bufferPos ] = cv::imread(imagePath.str().c_str(),1); // read in color image 36 | } 37 | 38 | // read the calibration information 39 | std::stringstream intrinsicsFileName; 40 | intrinsicsFileName << inputPath << settings.intrinsicsFile; 41 | ifstream intrinsicsFile(intrinsicsFileName.str().c_str()); 42 | for(int i = 0; i < 3; ++i) 43 | { 44 | for(int j = 0; j < 3; ++j) 45 | { 46 | intrinsicsFile >> KK[i][j]; 47 | std::cout << KK[i][j] << " "; 48 | } 49 | std::cout << std::endl; 50 | } 51 | 52 | } 53 | 54 | ImagesBufferReader::~ImagesBufferReader() 55 | { 56 | } 57 | 58 | void ImagesBufferReader::setCurrentFrame(int curFrame) 59 | { 60 | currentFrameNo = curFrame; // update current frame number 61 | } 62 | 63 | // void ImagesBufferReader::getColorImage(unsigned char* pColorImage) 64 | // { 65 | // pColorImage = m_vImages[currentFrameNo - startFrameNo].data; 66 | // } 67 | 68 | unsigned char* ImagesBufferReader::getColorImage() 69 | { 70 | int bufferPos = (currentFrameNo - startFrameNo) / nFrameStep; 71 | return m_vImages[bufferPos].data; 72 | } 73 | 74 | void ImagesBufferReader::readUVDImage(DepthImageType& uImage, DepthImageType& vImage, 75 | DepthImageType& dImage, InternalIntensityImageType& maskImageAux) 76 | { 77 | // // read in uImage, vImage and dImage 78 | // ReadRawDepth(inputPath,"refX.raw",m_nWidth,m_nHeight,uImage); 79 | // ReadRawDepth(inputPath,"refY.raw",m_nWidth,m_nHeight,vImage); 80 | // ReadRawDepth(inputPath,"depth0001.raw",m_nWidth,m_nHeight,dImage); 81 | 82 | // // read in maskImage 83 | // std::stringstream imagePath; 84 | // imagePath << inputPath << "mask.png"; 85 | // IntensityImageType maskImage = cv::imread(imagePath.str().c_str(),0); 86 | // maskImage.convertTo( maskImageAux, cv::DataType::type); 87 | 88 | // read in uImage, vImage and dImage 89 | std::stringstream data_path; 90 | data_path << inputPath; 91 | // DepthImageType uImage(m_nHeight,m_nWidth); 92 | // DepthImageType vImage(m_nHeight,m_nWidth); 93 | // DepthImageType dImage(m_nHeight,m_nWidth); 94 | if(trackerSettings.useXYZ) 95 | { 96 | ReadRawDepth(data_path,"refX.raw",m_nWidth,m_nHeight,uImage); 97 | ReadRawDepth(data_path,"refY.raw",m_nWidth,m_nHeight,vImage); 98 | } 99 | else 100 | { 101 | for(int i=0; i::type); 119 | 120 | } 121 | 122 | void ImagesBufferReader::ReadRawDepth(std::stringstream& data_path, std::string filename, 123 | int width, int height, DepthImageType& resImage) 124 | { 125 | std::ifstream inputFileStream; 126 | std::stringstream imagePath; 127 | CoordinateType* ref_buffer; 128 | ref_buffer = new CoordinateType[width*height]; 129 | imagePath << data_path.str() << filename; 130 | // std::cout << data_path.str() << filename << std::endl; 131 | inputFileStream.open(imagePath.str().c_str(),std::ios::binary); 132 | inputFileStream.read((char*)ref_buffer,sizeof(CoordinateType)*width*height); 133 | inputFileStream.close(); 134 | DepthImageType imgTemp(width,height,ref_buffer); 135 | imgTemp = imgTemp.t(); 136 | imgTemp.copyTo(resImage); 137 | delete[] ref_buffer; 138 | } -------------------------------------------------------------------------------- /src/main_engine/rendering/CCamera.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/rendering/CCamera.h" 2 | 3 | CCamera::CCamera(double K[3][3], double camPose[6], unsigned int W, 4 | unsigned int H) 5 | { 6 | for(int i = 0; i < 3; ++i) 7 | { 8 | for(int j = 0; j < 3; ++j) 9 | { 10 | _K(i,j) = K[i][j]; 11 | cout << _K(i,j) << endl; 12 | } 13 | _t(i) = camPose[i+3]; 14 | } 15 | 16 | _W = W; 17 | _H = H; 18 | 19 | // get the rotation matrix from axis-angle representation 20 | // double angle = Vector3d::Map(camPose).norm(); 21 | // Vector3d axis = Vector3d::Map(camPose)/angle; 22 | Vector3d axis; 23 | double angle = Map(camPose).norm(); 24 | if(angle != 0) 25 | axis = Map(camPose)/angle; 26 | else 27 | axis << 1, 0, 0; 28 | _R = AngleAxisd(angle, axis).toRotationMatrix(); 29 | 30 | setDerivedParams(); 31 | } 32 | 33 | CCamera::CCamera(double K[3][3], double R[3][3], double t[3], 34 | unsigned int W, unsigned int H) 35 | { 36 | for(int i = 0; i < 3; ++i) 37 | { 38 | for(int j = 0; j < 3; ++j) 39 | { 40 | _K(i,j) = K[i][j]; 41 | _R(i,j) = R[i][j]; 42 | } 43 | _t(i) = t[i]; 44 | } 45 | 46 | _W = W; 47 | _H = H; 48 | 49 | setDerivedParams(); 50 | 51 | } 52 | 53 | CCamera::~CCamera() 54 | { 55 | } 56 | 57 | void CCamera::setDerivedParams() 58 | { 59 | _K(0,1) = 0; 60 | _K(1,0) = 0; 61 | _K(2,0) = 0; 62 | _K(2,1) = 0; 63 | 64 | _Kinv.fill(0); 65 | 66 | _Kinv(0,0) = 1 / _K(0,0); 67 | _Kinv(1,1) = 1 / _K(1,1); 68 | _Kinv(0,2) = - _K(0,2) / _K(0,0); 69 | _Kinv(1,2) = - _K(1,2) / _K(1,1); 70 | _Kinv(2,2) = 1; 71 | 72 | _KR = _K * _R; 73 | _Kt = _K * _t; 74 | 75 | _Rtrans = _R.transpose(); 76 | 77 | if(_K(0,2) == 0) 78 | _IsOrtho = true; 79 | else 80 | _IsOrtho = false; 81 | 82 | } 83 | 84 | Matrix4d CCamera::getProjectionMatrix(double zNear, double zFar) const 85 | { 86 | double au = _K(0,0); 87 | double u0 = _K(0,2); 88 | double av = _K(1,1); 89 | double v0 = _K(1,2); 90 | 91 | double factor = 1.0; 92 | 93 | u0 = (_W / 2.0) + factor * (u0 - (_W / 2.0)); 94 | v0 = (_H / 2.0) + factor * (v0 - (_H / 2.0)); 95 | 96 | Matrix4d P = MatrixXd::Zero(4,4); 97 | 98 | if(!_IsOrtho) 99 | { 100 | P(0,0) = 2 * factor * au / _W; 101 | P(1,1) = 2 * factor * av / _H; 102 | 103 | P(0,2) = (1.0 - (2 * u0 / _W)); 104 | P(1,2) = -(1.0 - (2 * v0 / _H)); 105 | 106 | P(2,2) = (zNear + zFar) / (zNear - zFar); 107 | P(2,3) = (2 * zNear * zFar) / (zNear - zFar); 108 | 109 | P(3,2) = -1.0; 110 | } 111 | else 112 | { 113 | P(0,0) = 2/_W; 114 | P(1,1) = 2/_H; 115 | 116 | P(0,3) = -1; 117 | P(1,3) = 1; 118 | 119 | zNear = -1000; 120 | 121 | P(2,2) = 2 / (zNear - zFar); 122 | P(2,3) = (zNear + zFar) / (zNear - zFar); 123 | 124 | P(3,3) = 1; 125 | } 126 | 127 | return P; 128 | 129 | } 130 | 131 | Matrix4d CCamera::getModelViewMatrix() const 132 | { 133 | Matrix3d Flip = MatrixXd::Identity(3,3); 134 | 135 | Flip(1,1) = -1.0; 136 | Flip(2,2) = -1.0; 137 | 138 | Matrix4d MV = MatrixXd::Identity(4,4); 139 | MV.block(0,0,3,3) = Flip * _R; 140 | MV.block(0,3,3,1) = Flip * _t; 141 | 142 | return MV; 143 | } -------------------------------------------------------------------------------- /src/main_engine/rendering/DepthBuffer.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/rendering/DepthBuffer.h" 2 | 3 | #ifndef _MSC_VER 4 | #include "main_engine/rendering/NcvGlXContext.h" 5 | #endif 6 | 7 | class DepthConverter 8 | { 9 | public: 10 | DepthConverter(const float a, const float b) : _a(a), _b(b) 11 | {} 12 | 13 | void operator() (float& v) 14 | { 15 | // opengl uses fixed-point representation inside, range [0 1] 16 | v = _b / (_a - 2.0*v + 1.0); 17 | // v = _b / (_a - v); // if the range is [-1 1] 18 | } 19 | 20 | private: 21 | const float _a; 22 | const float _b; 23 | }; 24 | 25 | DepthBuffer::DepthBuffer(const CCamera* pCamera, PangaeaMeshData* pMesh): 26 | _pCamera(pCamera), _pMesh(pMesh) 27 | { 28 | _width = _pCamera->W(); 29 | _height = _pCamera->H(); 30 | 31 | _depthBufferMin.setZero(_width, _height); 32 | _depthBufferMax.setZero(_width, _height); 33 | 34 | // cout << "number of rows and cols" << endl; 35 | // cout << _depthbuffermin.rows() << endl; 36 | // cout << _depthbuffermin.cols() << endl; 37 | // cout << _depthbuffermin(0,0) << endl; 38 | 39 | float zMin, zMax; 40 | getTotalDepthRange(zMin, zMax); 41 | 42 | _zMin = zMin * 0.95f; 43 | _zMax = zMax * 1.05f; 44 | 45 | _depthBufferMin.fill(_zMin); 46 | _depthBufferMax.fill(_zMax); 47 | 48 | renderMeshGL(); 49 | } 50 | 51 | void DepthBuffer::getTotalDepthRange(float& zMin, float& zMax) const 52 | { 53 | zMax = - numeric_limits::max(); 54 | zMin = numeric_limits::max(); 55 | 56 | for(int I = _pMesh->numVertices, i = 0; i < I; ++i) 57 | { 58 | Vector3d Xc = _pCamera->worldToCamera(_pMesh->vertices[i]); 59 | float z = Xc[2]; 60 | 61 | if(z > zMax) 62 | zMax = z; 63 | 64 | if(z < zMin) 65 | zMin = z; 66 | } 67 | 68 | } 69 | 70 | void DepthBuffer::renderMeshGL() 71 | { 72 | uint W = _pCamera->W(); 73 | uint H = _pCamera->H(); 74 | 75 | #ifndef _MSC_VER 76 | ncv::GlXOffscreenContextPtr context(new ncv::GlXOffscreenContext(W, H)); 77 | 78 | context->makeActive(); 79 | #endif 80 | 81 | // Should be able to use openGL here.. 82 | 83 | glViewport(0,0,W,H); 84 | glDisable(GL_LIGHTING); 85 | 86 | glClearColor(0.0f, 0.0f, 0.0f, 1.0f); 87 | 88 | glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST); 89 | 90 | glMatrixMode(GL_PROJECTION); 91 | glLoadIdentity(); 92 | glLoadMatrix(_pCamera->getProjectionMatrix(_zMin, _zMax)); 93 | 94 | glMatrixMode(GL_MODELVIEW); 95 | glLoadIdentity(); 96 | glLoadMatrix(_pCamera->getModelViewMatrix()); 97 | 98 | glEnable(GL_DEPTH_TEST); 99 | 100 | glClearDepth(1.0f); 101 | glDepthFunc(GL_LEQUAL); 102 | glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); 103 | 104 | // Draw mesh.. 105 | drawMeshGL(); 106 | 107 | glFlush(); 108 | 109 | double P[16]; 110 | glGetDoublev(GL_PROJECTION_MATRIX, P); 111 | 112 | float a = - Map(P)(2,2); 113 | float b = - Map(P)(2,3); 114 | 115 | MatrixXfRow depthBuffer(H,W); 116 | MatrixXfRow depthBufferReverse(H,W); 117 | // MatrixXfRow depthBufferReverseTest(H,W); 118 | glReadPixels(0, 0, W, H, GL_DEPTH_COMPONENT, GL_FLOAT, depthBuffer.data()); 119 | depthBufferReverse = depthBuffer.colwise().reverse(); 120 | 121 | for_each(depthBufferReverse.data(), depthBufferReverse.data() + W*H, DepthConverter(a,b)); 122 | _depthBufferMin = depthBufferReverse; 123 | 124 | // output depth data to txt file 125 | // std::ofstream depthBufferFile; 126 | // depthBufferFile.open("/cs/research/vision/humanis3/Rui/data/newsequence_3_19/photo_metric/depthBuffer.txt", 127 | // std::ofstream::trunc); 128 | // depthBufferFile << _depthBufferMin << endl; 129 | 130 | // // render again to get the maximum depth 131 | // glClearDepth(0.0f); 132 | // glDepthFunc(GL_GEQUAL); 133 | // glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); 134 | 135 | // drawMeshGL(); 136 | 137 | // glFlush(); 138 | 139 | // glReadPixels(0, 0, W, H, GL_DEPTH_COMPONENT, GL_FLOAT, depthBuffer.data()); 140 | // depthBufferReverse = depthBuffer.colwise().reverse(); 141 | // for_each(depthBufferReverse.data(), depthBufferReverse.data() + W*H, DepthConverter(a,b)); 142 | // _depthBufferMax = depthBufferReverse; 143 | 144 | } 145 | 146 | void DepthBuffer::drawMeshGL() 147 | { 148 | int numFaces = _pMesh -> numFaces; 149 | glBegin(GL_TRIANGLES); 150 | for(int i = 0; i < numFaces; ++i) 151 | { 152 | for(int k = 0; k < 3; ++k) 153 | { 154 | int offset = _pMesh->facesVerticesInd[i][k]; 155 | glVertex3f(_pMesh->vertices[offset][0], 156 | _pMesh->vertices[offset][1], 157 | _pMesh->vertices[offset][2]); 158 | } 159 | } 160 | glEnd(); 161 | } -------------------------------------------------------------------------------- /src/main_engine/rendering/NcvGlXContext.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "main_engine/rendering/NcvGlXContext.h" 4 | 5 | namespace ncv 6 | { 7 | 8 | // Return true if GL error occured, print out error with message msg 9 | bool checkGlError( string msg ) 10 | { 11 | GLenum error = GL_NO_ERROR; 12 | 13 | error = glGetError(); 14 | 15 | if (error != GL_NO_ERROR) 16 | { 17 | cerr << msg << endl; 18 | cerr << "\t(" << gluErrorString(error) << ")" << endl; 19 | 20 | return true; 21 | } 22 | 23 | return false; 24 | } 25 | 26 | GlXOffscreenContext::GlXOffscreenContext(uint width, uint height) : 27 | _display(NULL), 28 | _fbConfig(None), 29 | _pBuffer(0), 30 | _context(NULL), 31 | _width(width), 32 | _height(height), 33 | _ready(false) 34 | { 35 | if ((_width == 0) || (_height == 0)) 36 | localError("Both width and height must be greater than zero."); 37 | 38 | createContext(); 39 | } 40 | 41 | GlXOffscreenContext::~GlXOffscreenContext() 42 | { 43 | clearActive(); 44 | 45 | _ready = false; 46 | 47 | glXDestroyContext(_display, _context); 48 | glXDestroyPbuffer(_display, _pBuffer); 49 | 50 | XCloseDisplay(_display); 51 | } 52 | 53 | bool GlXOffscreenContext::makeActive() 54 | { 55 | if (!isReady()) 56 | return false; 57 | 58 | if (isActive()) 59 | return true; 60 | 61 | if (!glXMakeContextCurrent(_display, _pBuffer, _pBuffer, _context)) 62 | { 63 | localError("Unable to make context active."); 64 | return false; 65 | } 66 | else 67 | return true; 68 | } 69 | 70 | void GlXOffscreenContext::clearActive() 71 | { 72 | if (!isReady()) 73 | return; 74 | 75 | if (!glXMakeCurrent(_display, None, NULL)) 76 | localError("Unable to release context."); 77 | } 78 | 79 | int GlXOffscreenContext::getIntProperty(GLenum intProperty) 80 | { 81 | ContextRestorer restorer(!isActive()); 82 | makeActive(); 83 | 84 | int v = 0; 85 | glGetIntegerv(intProperty, &v); 86 | checkGlError("After GlXOffscreenContext::getIntProperty.."); 87 | return v; 88 | } 89 | 90 | void GlXOffscreenContext::createContext() 91 | { 92 | _ready = false; 93 | 94 | _display = XOpenDisplay(NULL); 95 | 96 | if (!_display) 97 | localError("Unable to open XDisplay."); 98 | 99 | int errorBase, eventBase; 100 | if (!glXQueryExtension(_display, &errorBase, &eventBase)) 101 | localError("OpenGL GLX Extension not supported."); 102 | 103 | int screen = DefaultScreen(_display); 104 | 105 | int glxVersionMajor, glxVersionMinor; 106 | glXQueryVersion(_display, &glxVersionMajor, &glxVersionMinor); 107 | 108 | dbg << "GLX Version: " << glxVersionMajor << "." << glxVersionMinor << endl; 109 | 110 | int fbConfigCount = 0; 111 | 112 | int attribList[] = 113 | { 114 | //GLX_X_RENDERABLE, True, // Implied 115 | GLX_DRAWABLE_TYPE, GLX_PBUFFER_BIT,// | GLX_WINDOW_BIT, 116 | GLX_RENDER_TYPE, GLX_RGBA_BIT, 117 | GLX_RED_SIZE, 8, 118 | GLX_BLUE_SIZE, 8, 119 | GLX_GREEN_SIZE, 8, 120 | //GLX_ALPHA_SIZE, 8, 121 | //GLX_MAX_PBUFFER_WIDTH, _width, 122 | //GLX_MAX_PBUFFER_HEIGHT, _height, 123 | GLX_DEPTH_SIZE, 24, 124 | GLX_DOUBLEBUFFER, False, 125 | None 126 | }; 127 | 128 | GLXFBConfig* fbConfigs = glXChooseFBConfig(_display, screen, attribList, &fbConfigCount); 129 | 130 | if ((fbConfigs == NULL) || (fbConfigCount <= 0)) 131 | localError("Unable to choose fb config."); 132 | 133 | int pBufferAttribList[] = 134 | { 135 | GLX_PBUFFER_WIDTH, (int)_width, 136 | GLX_PBUFFER_HEIGHT, (int)_height, 137 | GLX_LARGEST_PBUFFER, False, 138 | None 139 | }; 140 | 141 | for (int i = 0; i < fbConfigCount; ++i) 142 | { 143 | _pBuffer = glXCreatePbuffer(_display, fbConfigs[i], pBufferAttribList); 144 | 145 | if (_pBuffer != None) 146 | { 147 | dbg << "Found successful frame buffer on attempt " << (i+1) << ".." << endl; 148 | _fbConfig = fbConfigs[i]; 149 | break; 150 | } 151 | } 152 | 153 | XSync(_display, False); 154 | 155 | XFree(fbConfigs); 156 | 157 | if (_pBuffer == None) 158 | localError("Unable to create pbuffer."); 159 | 160 | // Get visual.. 161 | XVisualInfo* visInfo = glXGetVisualFromFBConfig(_display, _fbConfig); 162 | if (!visInfo) 163 | { 164 | localError("Unable to get X visual."); 165 | } 166 | 167 | // Create context.. 168 | _context = glXCreateContext(_display, visInfo, NULL, True); 169 | 170 | if (!_context) 171 | { 172 | dbg << "Unable to create a direct context, attempting indirect one.." << endl; 173 | _context = glXCreateContext(_display, visInfo, NULL, False); 174 | } 175 | else 176 | { 177 | dbg << "Created a direct context.." << endl; 178 | } 179 | 180 | XFree(visInfo); 181 | 182 | if (!_context) 183 | localError("Unable to create GL context."); 184 | 185 | if (!glXMakeContextCurrent(_display, _pBuffer, _pBuffer, _context)) 186 | localError("Unable to make context active."); 187 | 188 | // Should be able to use openGL here.. 189 | 190 | GLenum glewErr = glewInit(); 191 | if (glewErr != GLEW_OK) 192 | { 193 | cerr << "GLEW ERROR: " << glewGetErrorString(glewErr) << endl; 194 | } 195 | 196 | int majorVersion, minorVersion; 197 | glGetIntegerv(GL_MAJOR_VERSION, &majorVersion); 198 | glGetIntegerv(GL_MINOR_VERSION, &minorVersion); 199 | 200 | dbg << "GL Version: " << majorVersion << "." << minorVersion << endl; 201 | 202 | checkGlError("After context creation.."); 203 | 204 | // Yay 205 | _ready = true; 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /src/main_engine/tracker/FeaturePyramid.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/tracker/FeaturePyramid.h" 2 | 3 | #include "main_engine/utils/settings.h" 4 | 5 | FeaturePyramid::FeaturePyramid() 6 | :prevLevels(NULL), 7 | currLevels(NULL), 8 | currLevelsBuffer(NULL) 9 | { 10 | pFeatureReader = new HDF5Reader(featureSettings.dbPath); 11 | } 12 | 13 | FeaturePyramid::~FeaturePyramid() 14 | { 15 | SafeDeleteArray(prevLevels); 16 | SafeDeleteArray(currLevels); 17 | SafeDeleteArray(currLevelsBuffer); 18 | 19 | delete pFeatureReader; 20 | } 21 | 22 | void FeaturePyramid::create(int nW, int nH, int nChannels, int numLevels) 23 | { 24 | m_nWidth = nW; 25 | m_nHeight = nH; 26 | m_nNumChannels = nChannels; 27 | m_nNumLevels = numLevels; 28 | m_bInitialized = false; 29 | 30 | prevLevels = new FeatureLevel[numLevels]; 31 | currLevels = new FeatureLevel[numLevels]; 32 | currLevelsBuffer = new FeatureLevel[numLevels]; 33 | 34 | pFeatureReader->InitializeDB(nH, nW, nChannels); 35 | } 36 | 37 | void FeaturePyramid::setupCameraPyramid(int numLevels, CameraInfo& camInfo) 38 | { 39 | double factor = featureSettings.scalingFactor; 40 | 41 | for(int i = 0; i < numLevels; ++i) 42 | { 43 | double factor; 44 | if(featureSettings.featurePyramidSamplingFactors.size() > 0) 45 | { 46 | factor = featureSettings.scalingFactor * 47 | featureSettings.featurePyramidSamplingFactors[i]; 48 | } 49 | else 50 | { 51 | factor = featureSettings.scalingFactor; 52 | } 53 | 54 | CameraInfo camInfoLevel = camInfo; 55 | // update the parameters 56 | camInfoLevel.width = camInfo.width / factor; 57 | camInfoLevel.height = camInfo.height / factor; 58 | camInfoLevel.KK[0][0] = camInfo.KK[0][0] / factor; 59 | camInfoLevel.KK[0][2] = camInfo.KK[0][2] / factor; 60 | camInfoLevel.KK[1][1] = camInfo.KK[1][1] / factor; 61 | camInfoLevel.KK[1][2] = camInfo.KK[1][2] / factor; 62 | camInfoLevel.invKK[0][0] = camInfo.invKK[0][0] * factor; 63 | camInfoLevel.invKK[1][1] = camInfo.invKK[1][1] * factor; 64 | // the invKK[0][2] and invKK[1][2] remains the same 65 | camInfoLevels.push_back(camInfoLevel); 66 | } 67 | } 68 | 69 | void FeaturePyramid::setupPyramid(string key) 70 | { 71 | 72 | for(int i = 0; i < m_nNumLevels; ++i) 73 | { 74 | currLevelsBuffer[i].featureImageVec.resize(m_nNumChannels); 75 | currLevelsBuffer[i].featureImageGradXVec.resize(m_nNumChannels); 76 | currLevelsBuffer[i].featureImageGradYVec.resize(m_nNumChannels); 77 | } 78 | 79 | int factor = 1; 80 | double blurSigma; 81 | 82 | for(int j = 0; j < m_nNumChannels; ++j) 83 | { 84 | for(int i = 0; i < m_nNumLevels; ++i) 85 | { 86 | pFeatureReader->getFeatureLevel(key, j, featureBufferImage); 87 | // features reading test 88 | // if(j == 1) 89 | // { 90 | // for(int mm = 881; mm < 891; ++mm) 91 | // cout << featureBufferImage.at(544, mm) << endl; 92 | // } 93 | 94 | int blurSize = featureSettings.blurFeatureFilterSizes[i]; 95 | if(featureSettings.blurFeatureSigmaSizes.size() > 0) 96 | blurSigma = featureSettings.blurFeatureSigmaSizes[i]; 97 | else 98 | blurSigma = -1; 99 | 100 | cout << "channel " << j << " " << "level " << i << endl; 101 | cout << "feature buffer: " << featureBufferImage.rows << " " << featureBufferImage.cols << endl; 102 | cout << "blur size: " << blurSize << endl; 103 | cout << "blur sigma: " << blurSigma << endl; 104 | 105 | if(blurSize > 0){ 106 | cv::GaussianBlur(featureBufferImage, 107 | blurBufferImage, 108 | cv::Size(blurSize, blurSize), 109 | blurSigma); 110 | }else 111 | featureBufferImage.copyTo(blurBufferImage); 112 | 113 | cout << "blur buffer: " << blurBufferImage.rows << " " << blurBufferImage.cols << endl; 114 | 115 | // do some proper downsampling at this point 116 | if(featureSettings.featurePyramidSamplingFactors.size() > 0){ 117 | factor = featureSettings.featurePyramidSamplingFactors[i]; 118 | resize(blurBufferImage, 119 | currLevelsBuffer[i].featureImageVec[j], 120 | cv::Size(), 121 | 1.0/factor, 122 | 1.0/factor); 123 | } 124 | else{ 125 | blurBufferImage.copyTo(currLevelsBuffer[i].featureImageVec[j]); 126 | } 127 | 128 | cout << "currLevelsBuffer: " << currLevelsBuffer[i].featureImageVec[j].rows << " " 129 | << currLevelsBuffer[i].featureImageVec[j].cols << endl; 130 | 131 | // setup gradient 132 | double gradScale = featureSettings.featureGradientScalingFactors[i]; 133 | 134 | cv::Scharr(currLevelsBuffer[i].featureImageVec[j], 135 | currLevelsBuffer[i].featureImageGradXVec[j], 136 | featureSettings.gradTypeINT, 137 | 1,0, 138 | gradScale); 139 | 140 | cv::Scharr(currLevelsBuffer[i].featureImageVec[j], 141 | currLevelsBuffer[i].featureImageGradYVec[j], 142 | featureSettings.gradTypeINT, 143 | 0,1, 144 | gradScale); 145 | 146 | } 147 | } 148 | 149 | // setup feature pyramid for previous frame 150 | if(!m_bInitialized) 151 | { 152 | // copy over the data from currLevelsBuffer 153 | for(int i = 0; i < m_nNumLevels; ++i){ 154 | prevLevels[i].featureImageVec.resize(m_nNumChannels); 155 | prevLevels[i].featureImageGradXVec.resize(m_nNumChannels); 156 | prevLevels[i].featureImageGradYVec.resize(m_nNumChannels); 157 | for(int j = 0; j < m_nNumChannels; ++j){ 158 | currLevelsBuffer[i].featureImageVec[j].copyTo( prevLevels[i].featureImageVec[j] ); 159 | currLevelsBuffer[i].featureImageGradXVec[j].copyTo( prevLevels[i].featureImageGradXVec[j] ); 160 | currLevelsBuffer[i].featureImageGradYVec[j].copyTo( prevLevels[i].featureImageGradYVec[j] ); 161 | } 162 | } 163 | 164 | m_bInitialized = true; 165 | } 166 | 167 | } 168 | 169 | void FeaturePyramid::updateData() 170 | { 171 | FeatureLevel* tempPointer; 172 | tempPointer = currLevelsBuffer; 173 | currLevelsBuffer = currLevels; 174 | currLevels = tempPointer; 175 | } 176 | 177 | void FeaturePyramid::updatePrev() 178 | { 179 | // make sure prevLevels has already been initialized 180 | 181 | // exchange prevLevels and currLevels 182 | FeatureLevel* tempPointer; 183 | tempPointer = prevLevels; 184 | prevLevels = currLevels; 185 | currLevels = tempPointer; 186 | 187 | } 188 | 189 | CameraInfo& FeaturePyramid::getCameraInfo(int nLevel) 190 | { 191 | return camInfoLevels[nLevel]; 192 | } 193 | 194 | FeatureLevel& FeaturePyramid::getPrevFeatureLevel(int nLevel) 195 | { 196 | return prevLevels[nLevel]; 197 | } 198 | 199 | FeatureLevel& FeaturePyramid::getCurrFeatureLevel(int nLevel) 200 | { 201 | return currLevels[nLevel]; 202 | } 203 | -------------------------------------------------------------------------------- /src/main_engine/tracker/FeatureReader.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/tracker/FeatureReader.h" 2 | 3 | #include "main_engine/utils/settings.h" 4 | 5 | //// LMDB DataBase 6 | 7 | LMDBReader::LMDBReader(string folder) 8 | { 9 | db_path = folder; 10 | } 11 | 12 | LMDBReader::~LMDBReader( ) 13 | { 14 | ShutDownDB(); 15 | } 16 | 17 | void LMDBReader::InitializeDB(int height, int width, int numChannels) 18 | { 19 | 20 | m_nWidth = width; 21 | m_nHeight = height; 22 | m_nNumChannels = numChannels; 23 | 24 | // check the status of the database 25 | std::cout << "Openning lmdb " << db_path << endl; 26 | // check the folder already exists 27 | 28 | // bool valid; 29 | 30 | // valid = (mdb_env_create(&mdb_env) == MDB_SUCCESS) && 31 | // (mdb_env_set_mapsize(mdb_env, 10485760000000) == MDB_SUCCESS) && 32 | // (mdb_env_open(mdb_env, db_path, 0, 0664) == MDB_SUCCESS) && 33 | // (mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn) == MDB_SUCCESS) && 34 | // (mdb_open(mdb_txn, NULL, 0, &mdb_dbi) == MDB_SUCCESS); 35 | 36 | // assert(mdb_cursor_open(mdb_txn, mdb_dbi, &mdb_cursor) == MDB_SUCCESS); 37 | // int mdb_status = mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_data, MDB_NEXT); 38 | // if(mdb_status == MDB_NOTFOUND) 39 | // valid = false; 40 | // else{ 41 | // MDB_CHECK(mdb_status); 42 | // valid = true; 43 | // } 44 | 45 | int test1 = mdb_env_create(&mdb_env); 46 | std::cout << "return value of environment creation " << test1 << endl; 47 | 48 | // int test2 = mdb_env_set_mapsize(mdb_env, 10485760000000); 49 | int test2 = mdb_env_set_mapsize(mdb_env, 10485760000); 50 | std::cout << "return value of setting environment mapsize " << test2 << endl; 51 | 52 | // int test3 = mdb_env_open(mdb_env, db_path.c_str(), 0, 0664); 53 | int test3 = mdb_env_open(mdb_env, db_path.c_str(), MDB_NOLOCK, 0664); 54 | std::cout << "return value of environment openning " << test3 << endl; 55 | 56 | int test4 = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn); 57 | std::cout << "return value of context beginning " << test4 << endl; 58 | 59 | int test5 = mdb_open(mdb_txn, NULL, 0, &mdb_dbi); 60 | std::cout << "return value of mdb openning " << test5 << endl; 61 | 62 | // if(!valid) 63 | // { 64 | // std::cout << "Open lmdb error" << std::endl; 65 | // throw 20; 66 | // }; 67 | 68 | } 69 | 70 | void LMDBReader::ShutDownDB() 71 | { 72 | mdb_close(mdb_env, mdb_dbi); 73 | mdb_env_close(mdb_env); 74 | } 75 | 76 | void LMDBReader::getFeatureLevel(string key, int channel, 77 | FeatureImageType& featureBufferImage) 78 | { 79 | 80 | // setup current pyramid 81 | mdb_key.mv_size = key.length(); 82 | mdb_key.mv_data = reinterpret_cast(&key[0]); 83 | 84 | mdb_get(mdb_txn, mdb_dbi, &mdb_key, &mdb_data); 85 | 86 | // setup the 0th level of current feature image pyramid 87 | 88 | int data_size = mdb_data.mv_size; 89 | int channel_num = m_nHeight * m_nWidth; 90 | int channel_size = channel_num * featureSettings.dataElemSize; 91 | int real_size = m_nNumChannels * channel_size; 92 | int shift = (data_size - real_size) / featureSettings.dataElemSize; 93 | 94 | switch(featureSettings.dataElemSize) 95 | { 96 | case 1: 97 | { 98 | unsigned char* data_pointer = reinterpret_cast(mdb_data.mv_data); 99 | featureBufferImage = cv::Mat(m_nHeight, 100 | m_nWidth, 101 | featureSettings.dataTypeINT, 102 | data_pointer + shift + channel * channel_num); 103 | } 104 | break; 105 | case 4: 106 | { 107 | float* data_pointer = reinterpret_cast(mdb_data.mv_data); 108 | featureBufferImage = cv::Mat(m_nHeight, 109 | m_nWidth, 110 | featureSettings.dataTypeINT, 111 | data_pointer + shift + channel * channel_num); 112 | } 113 | break; 114 | case 8: 115 | { 116 | double* data_pointer = reinterpret_cast(mdb_data.mv_data); 117 | featureBufferImage = cv::Mat(m_nHeight, 118 | m_nWidth, 119 | featureSettings.dataTypeINT, 120 | data_pointer + shift + channel * channel_num); 121 | // cv::namedWindow("featureBufferImage", cv::WINDOW_AUTOSIZE); 122 | // cv::imshow("featureBufferImage", featureBufferImage); 123 | // cv::waitKey(0); 124 | 125 | } 126 | break; 127 | } 128 | 129 | } 130 | 131 | //// HDF5 DataBase 132 | 133 | HDF5Reader::HDF5Reader(string file) 134 | :pFeatureImages(NULL) 135 | { 136 | hdf5file = file; 137 | } 138 | 139 | HDF5Reader::~HDF5Reader() 140 | { 141 | if(pFeatureImages) 142 | delete[] pFeatureImages; 143 | } 144 | 145 | void HDF5Reader::InitializeDB(int height, int width, int numChannels) 146 | { 147 | m_nWidth = width; 148 | m_nHeight = height; 149 | m_nNumChannels = numChannels; 150 | 151 | pFeatureImages = new double[height*width*numChannels]; 152 | 153 | } 154 | 155 | void HDF5Reader::ShutDownDB(){} 156 | 157 | void HDF5Reader::getFeatureLevel(string key, int channel, 158 | FeatureImageType& featureBufferImage) 159 | { 160 | 161 | cout << "feature " << key << endl 162 | << "channel " << channel << endl; 163 | 164 | featureBufferImage = cv::Mat(m_nHeight, m_nWidth, featureSettings.dataTypeINT); 165 | 166 | if(currentKey == key) 167 | { 168 | memcpy(featureBufferImage.data, (double*)pFeatureImages + channel*m_nHeight*m_nWidth, 8*m_nHeight*m_nWidth); 169 | return; 170 | } 171 | 172 | file_id = H5Fopen(hdf5file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); 173 | 174 | dataset_id = H5Dopen2(file_id, key.c_str(), H5P_DEFAULT); 175 | 176 | H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, pFeatureImages); 177 | memcpy(featureBufferImage.data, (double*)pFeatureImages + channel*m_nHeight*m_nWidth, 8*m_nHeight*m_nWidth); 178 | 179 | currentKey = key; 180 | 181 | H5Dclose(dataset_id); 182 | 183 | H5Fclose(file_id); 184 | 185 | } 186 | -------------------------------------------------------------------------------- /src/main_engine/tracker/MeshBufferReader.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/tracker/TrackingEngine.h" 2 | 3 | MeshBufferReader::MeshBufferReader(MeshLoadingSettings& settings, int width, 4 | int height, double K[3][3], int startFrame, int numTrackingFrames): trackerInitialized(false) 5 | { 6 | m_nWidth = width; 7 | m_nHeight = height; 8 | startFrameNo = startFrame; 9 | currentFrameNo = startFrame; 10 | 11 | pCurrentColorImageRGB = new unsigned char[3*width*height]; 12 | // in this case camPose will always be zero 13 | for(int i = 0; i < 6; ++i) 14 | camPose[i] = 0; 15 | 16 | useVisibilityMask = settings.visibilityMask; 17 | 18 | setIntrinsicMatrix(K); 19 | 20 | nRenderingLevel = 0; 21 | m_nNumMeshLevels = settings.meshLevelList.size(); 22 | 23 | // a bit ugly 24 | nFrameStep = imageSourceSettings.frameStep; 25 | 26 | // loading meshes into buffer 27 | // outputInfoPyramidBuffer.resize(numTrackingFrames); 28 | // outputPropPyramidBuffer.resize(numTrackingFrames); 29 | int bufferSize = (numTrackingFrames - startFrameNo)/nFrameStep + 1; 30 | outputInfoPyramidBuffer.resize(bufferSize); 31 | outputPropPyramidBuffer.resize(bufferSize); 32 | 33 | m_nGoodFrames = 0; 34 | 35 | TICK("loadingMeshBuffer"); 36 | 37 | for(int i = startFrameNo; i <= numTrackingFrames; i = i + nFrameStep) 38 | { 39 | 40 | // TICK("loadingOneFrame"); 41 | 42 | if(!existenceTest(settings.meshPath, settings.meshLevelFormat, 43 | i, settings.meshLevelList)) 44 | break; 45 | 46 | ++m_nGoodFrames; 47 | currentMeshPyramid = std::move(PangaeaMeshPyramid(settings.meshPath, 48 | settings.meshLevelFormat, i, settings.meshLevelList)); 49 | 50 | // TOCK("loadingOneFrame"); 51 | 52 | if(settings.loadProp) 53 | { 54 | propMeshPyramid = std::move(PangaeaMeshPyramid(settings.meshPath, 55 | settings.propLevelFormat, i, settings.meshLevelList)); 56 | } 57 | if(!settings.fastLoading) 58 | propMeshPyramid = currentMeshPyramid; 59 | 60 | 61 | // TICK("setOneFrame"); 62 | 63 | setMeshPyramid(); 64 | 65 | int bufferPos = (i-startFrameNo)/nFrameStep; 66 | outputInfoPyramidBuffer[ bufferPos ] = std::move(outputInfoPyramid); 67 | outputPropPyramidBuffer[ bufferPos ] = std::move(outputPropPyramid); 68 | 69 | // TOCK("setOneFrame"); 70 | 71 | cout << "loading frame " << i << endl; 72 | } 73 | 74 | TOCK("loadingMeshBuffer"); 75 | 76 | } 77 | 78 | MeshBufferReader::~MeshBufferReader(){ 79 | delete[] pCurrentColorImageRGB; 80 | } 81 | 82 | void MeshBufferReader::setIntrinsicMatrix(double K[3][3]) 83 | { 84 | for(int i = 0; i < 3; ++i) 85 | { 86 | for(int j = 0; j < 3; ++j) 87 | { 88 | KK[i][j] = K[i][j]; 89 | } 90 | } 91 | } 92 | 93 | bool MeshBufferReader::setCurrentFrame(int curFrame) 94 | { 95 | double bufferPos = (curFrame - startFrameNo) / nFrameStep; 96 | if(bufferPos + 1 <= m_nGoodFrames){ 97 | currentFrameNo = curFrame; 98 | return true; 99 | }else 100 | return false; 101 | } 102 | 103 | bool MeshBufferReader::trackFrame(int nFrame, unsigned char* pColorImageRGB, 104 | TrackerOutputInfo** pOutputInfo) 105 | { 106 | memcpy(pCurrentColorImageRGB, pColorImageRGB, 3*m_nWidth*m_nHeight); 107 | cv::Mat tempColorImageRGB(m_nHeight, m_nWidth, CV_8UC3, pCurrentColorImageRGB); 108 | tempColorImageRGB.convertTo(colorImage, cv::DataType::type, 1./255); 109 | cv::split(colorImage, colorImageSplit); 110 | 111 | if(!setCurrentFrame(nFrame)) 112 | return false; 113 | 114 | int bufferPos = (currentFrameNo - startFrameNo) / nFrameStep; 115 | *pOutputInfo = &outputInfoPyramidBuffer[ bufferPos ][ nRenderingLevel ]; 116 | 117 | return true; 118 | } 119 | 120 | void MeshBufferReader::setMeshPyramid() 121 | { 122 | TICK("setupMeshBufferRendering"); 123 | 124 | visibilityMaskPyramid.resize(m_nNumMeshLevels); 125 | outputInfoPyramid.resize(m_nNumMeshLevels); 126 | outputPropPyramid.resize(m_nNumMeshLevels); 127 | 128 | for(int i = 0; i < m_nNumMeshLevels; ++i) 129 | { 130 | int numVertices = currentMeshPyramid.levels[i].numVertices; 131 | visibilityMaskPyramid[i].resize(numVertices,true); 132 | 133 | vector proj2D; 134 | proj2D.resize(2); proj2D[0] = 0; proj2D[1] = 0; 135 | 136 | outputInfoPyramid[i].meshData = std::move(currentMeshPyramid.levels[i]); 137 | // outputInfoPyramid[i].meshDataGT = currentMeshPyramid.levels[i]; 138 | // outputInfoPyramid[i].meshDataColorDiff = currentMeshPyramid.levels[i]; 139 | outputInfoPyramid[i].nRenderLevel = i; 140 | outputInfoPyramid[i].meshProj.resize(numVertices, proj2D); 141 | // outputInfoPyramid[i].meshProjGT = outputInfoPyramid[i].meshProj; 142 | outputInfoPyramid[i].visibilityMask.resize(numVertices, true); 143 | memset(outputInfoPyramid[i].camPose, 0, 6*sizeof(double)); 144 | 145 | UpdateRenderingData(outputInfoPyramid[i], KK, camPose, outputInfoPyramid[i].meshData); 146 | 147 | //////////////////////////// outputPropPyramid 148 | 149 | if(meshLoadingSettings.loadProp) 150 | { 151 | outputPropPyramid[i].meshData = std::move(propMeshPyramid.levels[i]); 152 | // outputPropPyramid[i].meshDataGT = propMeshPyramid.levels[i]; 153 | // outputPropPyramid[i].meshDataColorDiff = propMeshPyramid.levels[i]; 154 | outputPropPyramid[i].nRenderLevel = i; 155 | outputPropPyramid[i].meshProj.resize(numVertices, proj2D); 156 | // outputPropPyramid[i].meshProjGT = outputPropPyramid[i].meshProj; 157 | outputPropPyramid[i].visibilityMask.resize(numVertices, true); 158 | memset(outputPropPyramid[i].camPose, 0, 6*sizeof(double)); 159 | 160 | UpdateRenderingData(outputPropPyramid[i], KK, camPose, outputPropPyramid[i].meshData); 161 | } 162 | 163 | // update the visibility of each vertex 164 | if(useVisibilityMask) 165 | { 166 | long long int ii = i; 167 | 168 | TICK( "visibilityMask" + std::to_string(ii) ); 169 | 170 | UpdateVisibilityMaskGL(outputInfoPyramid[i], visibilityMaskPyramid[i], KK, camPose, m_nWidth, m_nHeight); 171 | if(meshLoadingSettings.loadProp) 172 | UpdateVisibilityMaskGL(outputPropPyramid[i], visibilityMaskPyramid[i], KK, camPose, m_nWidth, m_nHeight); 173 | 174 | TOCK( "visibilityMask" + std::to_string(ii) ); 175 | 176 | } 177 | } 178 | 179 | TOCK("setupMeshBufferRendering"); 180 | 181 | } 182 | 183 | void MeshBufferReader::updateRenderingLevel(TrackerOutputInfo** pOutputInfoRendering, 184 | int nRenderLevel, bool renderType) 185 | { 186 | cout << "changing rendering type" << endl; 187 | int bufferPos = (currentFrameNo - startFrameNo) / nFrameStep; 188 | 189 | if(renderType) 190 | *pOutputInfoRendering = &(outputPropPyramidBuffer[ bufferPos ][ nRenderLevel ]); 191 | else 192 | *pOutputInfoRendering = &(outputInfoPyramidBuffer[ bufferPos ][ nRenderLevel ]); 193 | 194 | nRenderingLevel = nRenderLevel; 195 | } 196 | -------------------------------------------------------------------------------- /src/main_engine/tracker/MeshSequenceReader.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/tracker/TrackingEngine.h" 2 | 3 | MeshSequenceReader::MeshSequenceReader(MeshLoadingSettings& settings, int width, 4 | int height, double K[3][3], int startFrame, int numTrackingFrames): 5 | trackerInitialized(false) 6 | { 7 | m_nWidth = width; 8 | m_nHeight = height; 9 | startFrameNo = startFrame; 10 | currentFrameNo = startFrame; 11 | 12 | pCurrentColorImageRGB = new unsigned char[3*width*height]; 13 | // in this case camPose will always be zero 14 | for(int i = 0; i < 6; ++i) 15 | camPose[i] = 0; 16 | 17 | useVisibilityMask = settings.visibilityMask; 18 | 19 | // visibilityFacesTest.resize(width*height); 20 | // // 5 reserved faces for each pixel, to make stuff a bit faster 21 | // for(int i = 0; i < width*height; ++i) 22 | // visibilityFacesTest[i].reserve(5); 23 | 24 | setIntrinsicMatrix(K); 25 | 26 | // load the first frame 27 | loadMesh(meshLoadingSettings.meshPath, meshLoadingSettings.meshFormat, 28 | currentFrameNo); 29 | } 30 | 31 | MeshSequenceReader::~MeshSequenceReader() { 32 | delete[] pCurrentColorImageRGB; 33 | } 34 | 35 | bool MeshSequenceReader::setCurrentFrame(int curFrame) 36 | { 37 | if(currentFrameNo != curFrame) 38 | { 39 | currentFrameNo = curFrame; 40 | 41 | // changing new frame time 42 | TICK("setCurrentFrame"); 43 | 44 | if(!loadMesh(meshLoadingSettings.meshPath, 45 | meshLoadingSettings.meshFormat,currentFrameNo)) 46 | return false; 47 | 48 | TOCK("setCurrentFrame"); 49 | } 50 | return true; 51 | } 52 | 53 | void MeshSequenceReader::setIntrinsicMatrix(double K[3][3]) 54 | { 55 | for(int i = 0; i < 3; ++i) 56 | { 57 | for(int j = 0; j < 3; ++j) 58 | { 59 | KK[i][j] = K[i][j]; 60 | } 61 | } 62 | } 63 | 64 | bool MeshSequenceReader::loadMesh(std::string& meshPath, 65 | std::string& meshFormat, int curFrame) 66 | { 67 | char buffer[BUFFER_SIZE]; 68 | std::stringstream meshFile; 69 | sprintf(buffer, meshFormat.c_str(), curFrame); 70 | meshFile << meshPath << buffer; 71 | if(!bfs::exists(meshFile.str())) 72 | { 73 | cout << "File not existing: " << meshFile.str() << endl; 74 | return false; 75 | } 76 | 77 | if(!trackerInitialized) 78 | PangaeaMeshIO::loadfromFile(meshFile.str(), currentMesh, trackerSettings.clockwise); 79 | else 80 | PangaeaMeshIO::updateFromFile(meshFile.str(), currentMesh); 81 | 82 | return true; 83 | } 84 | 85 | bool MeshSequenceReader::trackFrame(int nFrame, unsigned char* pColorImageRGB, 86 | TrackerOutputInfo** pOutputInfo) 87 | { 88 | *pOutputInfo = &outputInfo; 89 | // 90 | if(!setCurrentFrame(nFrame)) 91 | return false; 92 | 93 | // set up the color used later 94 | memcpy(pCurrentColorImageRGB, pColorImageRGB, 3*m_nWidth*m_nHeight); 95 | cv::Mat tempColorImageRGB(m_nHeight, m_nWidth, CV_8UC3, pCurrentColorImageRGB); 96 | tempColorImageRGB.convertTo(colorImage, cv::DataType::type, 1./255); 97 | cv::split(colorImage, colorImageSplit); 98 | 99 | if(!trackerInitialized) 100 | trackerInitSetup(outputInfo); 101 | else 102 | trackerUpdate(outputInfo); 103 | 104 | return true; 105 | } 106 | 107 | void MeshSequenceReader::trackerInitSetup(TrackerOutputInfo& outputInfo) 108 | { 109 | TICK("visualRenderingInit"); 110 | 111 | outputInfo.meshData = currentMesh; 112 | outputInfo.meshDataGT = outputInfo.meshData; 113 | 114 | // get 2d projections 115 | double X,Y,Z; 116 | double u,v,w; 117 | vector proj2D, proj2DGT; 118 | proj2D.resize(2); proj2DGT.resize(2); 119 | for(int vertex = 0; vertex < currentMesh.numVertices; ++vertex) 120 | { 121 | 122 | X = currentMesh.vertices[vertex][0]; 123 | Y = currentMesh.vertices[vertex][1]; 124 | Z = currentMesh.vertices[vertex][2]; 125 | 126 | if(KK[0][2] == 0) // this is orthographic camera 127 | { 128 | proj2D[0] = X; proj2D[1] = Y; 129 | proj2DGT[0] = X; proj2DGT[1] = Y; 130 | } 131 | else 132 | { 133 | u = KK[0][0] * X + KK[0][1] * Y + KK[0][2] * Z; 134 | v = KK[1][0] * X + KK[1][1] * Y + KK[1][2] * Z; 135 | w = KK[2][0] * X + KK[2][1] * Y + KK[2][2] * Z; 136 | 137 | if(w != 0) 138 | { 139 | u = u/w; 140 | v = v/w; 141 | } 142 | 143 | proj2D[0] = u; proj2D[1] = v; 144 | proj2DGT[0] = u; proj2DGT[1] = v; 145 | 146 | } 147 | 148 | outputInfo.meshProj.push_back(proj2D); 149 | outputInfo.meshProjGT.push_back(proj2DGT); 150 | 151 | } 152 | 153 | outputInfo.visibilityMask.resize(outputInfo.meshData.numVertices,true); 154 | // update the visiblity mask 155 | if(useVisibilityMask) 156 | { 157 | UpdateVisibilityMaskGL(outputInfo, visibilityMask, KK, camPose, m_nWidth, m_nHeight); 158 | //UpdateVisibilityMask(outputInfo, visibilityMask, m_nWidth, m_nHeight); 159 | outputInfo.meshDataColorDiff = outputInfo.meshData; 160 | UpdateColorDiff(outputInfo, visibilityMask, colorImageSplit); 161 | } 162 | 163 | // camera pose is always 0 in this case 164 | for(int i = 0; i < 6; ++i) 165 | outputInfo.camPose[i] = 0; 166 | 167 | trackerInitialized = true; 168 | 169 | TOCK("visualRenderingInit"); 170 | 171 | } 172 | 173 | void MeshSequenceReader::trackerUpdate(TrackerOutputInfo& outputInfo) 174 | { 175 | TICK("visualRenderingUpdate"); 176 | 177 | UpdateRenderingData(outputInfo, KK, camPose, currentMesh); 178 | UpdateRenderingDataFast(outputInfo, KK, currentMesh); 179 | 180 | if(useVisibilityMask) 181 | { 182 | UpdateVisibilityMaskGL(outputInfo, visibilityMask, KK, camPose, m_nWidth, m_nHeight); 183 | //UpdateVisibilityMask(outputInfo, visibilityMask, m_nWidth, m_nHeight); 184 | UpdateColorDiff(outputInfo, visibilityMask, colorImageSplit); 185 | } 186 | 187 | TOCK("visualRenderingUpdate"); 188 | } 189 | -------------------------------------------------------------------------------- /src/main_engine/utils/global.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/utils/global.h" 2 | 3 | bool existenceTest(string meshPath, string meshLevelFormat, int frame, 4 | IntegerContainerType& meshLevelList) 5 | { 6 | char buffer[BUFFER_SIZE]; 7 | for(int i = 0; i < meshLevelList.size(); ++i) 8 | { 9 | std::stringstream meshFile; 10 | sprintf(buffer, meshLevelFormat.c_str(), frame, meshLevelList[i]); 11 | meshFile << meshPath << buffer; 12 | if(!bfs::exists(meshFile.str())){ 13 | cout << "File not existing: " << meshFile.str() << endl; 14 | return false; 15 | } 16 | } 17 | return true; 18 | } 19 | 20 | bool pointInTriangleTest2(double* pointP, double* pointA, double* pointB, double* pointC) 21 | { 22 | double A, B , D, E, C, F; 23 | A = pointA[0] - pointC[0]; 24 | B = pointB[0] - pointC[0]; 25 | D = pointA[1] - pointC[1]; 26 | E = pointB[1] - pointC[1]; 27 | 28 | C = pointC[0] - pointP[0]; 29 | F = pointC[1] - pointP[1]; 30 | 31 | double alpha, beta, gamma; 32 | alpha = (B*(F)-C*(E))/(A*(E)-B*(D)); 33 | beta = (A*(F)-C*(D))/(B*(D)-A*(E)); 34 | gamma = 1 - alpha - beta; 35 | 36 | return (alpha >=0 && beta >=0 && gamma >= 0); 37 | 38 | } 39 | 40 | double triangleArea(double* pointA, double* pointB, double* pointC) 41 | { 42 | double a = sqrt((pointB[0] - pointC[0])*(pointB[0] - pointC[0]) + 43 | (pointB[1] - pointC[1])*(pointB[1] - pointC[1])); 44 | 45 | double b = sqrt((pointA[0] - pointC[0])*(pointA[0] - pointC[0]) + 46 | (pointA[1] - pointC[1])*(pointA[1] - pointC[1])); 47 | 48 | double c = sqrt((pointB[0] - pointA[0])*(pointB[0] - pointA[0]) + 49 | (pointB[1] - pointA[1])*(pointB[1] - pointA[1])); 50 | 51 | double s = (a + b + c)/2; 52 | 53 | double area = sqrt(s*(s-a)*(s-b)*(s-c)); 54 | 55 | return area; 56 | 57 | } 58 | 59 | bool pointInTriangleTest(double* pointP, double* pointA, double* pointB, double* pointC) 60 | { 61 | // get the barycentric coordinate 62 | double AB[3], AC[3], PB[3], PC[3], PA[3]; 63 | double areaABC = triangleArea(pointA, pointB, pointC); 64 | double areaABP = triangleArea(pointA, pointB, pointP); 65 | double areaACP = triangleArea(pointA, pointC, pointP); 66 | double alpha = areaABP / areaABC; 67 | double beta = areaACP / areaABC; 68 | double gamma = 1 - alpha - beta; 69 | return (alpha >=0 && beta >=0 && gamma >= 0); 70 | } 71 | 72 | bool visibilityTest(double* vertex, double* center, double* normal, 73 | double* vertex1, double* vertex2, double* vertex3) 74 | { 75 | // tell if a point is in front of a triangle face 76 | double faceDist, pointDist, scale; 77 | faceDist = center[0]*normal[0] + center[1]*normal[1] + center[2]*normal[2]; 78 | pointDist = vertex[0]*normal[0] + vertex[1]*normal[1] + vertex[2]*normal[2]; 79 | scale = faceDist / pointDist; 80 | 81 | double intersection[3]; 82 | intersection[0] = scale*vertex[0]; 83 | intersection[1] = scale*vertex[1]; 84 | intersection[2] = scale*vertex[2]; 85 | 86 | // if the intersection is in front, 87 | // then the test vertex is occluded by 88 | // the front face, we give up 89 | if(intersection[2] < vertex[2]) 90 | return false; 91 | else 92 | return true; 93 | 94 | } 95 | 96 | ImageSourceType mapImageSourceType(std::string const& inString) 97 | { 98 | cout << inString << endl; 99 | 100 | if (inString == "ImagesBuffer") return ALLIMAGESBUFFER; 101 | if (inString == "ImageSequence") return IMAGESEQUENCE; 102 | if (inString == "Video") return VIDEO; 103 | if (inString == "Camera") return CAMERA; 104 | } 105 | 106 | TrackingType mapTrackingType(std::string const& inString) 107 | { 108 | cout << inString << endl; 109 | 110 | if (inString == "ShapesBuffer") return ALLSHAPESBUFFER; 111 | if (inString == "ShapeSequence") return SHAPESEQUENCE; 112 | if (inString == "DeformNRSFMTracker") return DEFORMNRSFM; 113 | if (inString == "MeshSequence") return MESHSEQUENCE; 114 | if (inString == "MeshPyramid") return MESHPYRAMID; 115 | if (inString == "MeshBuffer") return MESHBUFFER; 116 | } 117 | 118 | int typeConvert(string dataType) 119 | { 120 | if(dataType == "unsigned char") return CV_8U; 121 | if(dataType == "float") return CV_32F; 122 | if(dataType == "double") return CV_64F; 123 | } 124 | 125 | int typeSize(string dataType) 126 | { 127 | if(dataType == "unsigned char") return 1; 128 | if(dataType == "float") return 4; 129 | if(dataType == "double") return 8; 130 | } 131 | 132 | // points1, points2, 133 | void computeRot(vector& template_vextex, vector& vertex, 134 | vector >& template_nbor_vertices, vector >& nbor_vertices, 135 | vector& neighbors, vector& weights, 136 | vector& output_rot, bool deform) 137 | { 138 | int num_neighbors = neighbors.size(); 139 | MatrixXd Xt(3, num_neighbors), X(3, num_neighbors); 140 | for(int i = 0; i < num_neighbors; ++i) 141 | { 142 | for(int j = 0; j < 3; ++j) 143 | { 144 | Xt(j,i) = weights[i] * ( 145 | template_vextex[j] - template_nbor_vertices[ neighbors[i] ][j]); 146 | 147 | X(j,i) = weights[i] * ( 148 | vertex[j] - nbor_vertices[ neighbors[i] ][j]); 149 | 150 | if(deform) 151 | X(j,i) += Xt(j,i); 152 | } 153 | } 154 | 155 | Eigen::Matrix3d sigma = Xt * X.transpose(); 156 | Eigen::JacobiSVD svd(sigma, Eigen::ComputeFullU | Eigen::ComputeFullV); 157 | Eigen::Matrix3d Rot; 158 | 159 | if(svd.matrixU().determinant()*svd.matrixV().determinant() < 0.0) { 160 | Eigen::Vector3d S = Eigen::Vector3d::Ones(); S(2) = -1.0; 161 | Rot = svd.matrixV()*S.asDiagonal()*svd.matrixU().transpose(); 162 | } else { 163 | Rot = svd.matrixV()*svd.matrixU().transpose(); 164 | } 165 | 166 | // ceres::RotationMatrixToAngleAxis(&Rot(0, 0), &output_rot[0]); 167 | 168 | Eigen::AngleAxisd angleAxis(Rot); 169 | for(int i = 0; i < 3; ++i) 170 | output_rot[i] = angleAxis.axis()[i] * angleAxis.angle(); 171 | 172 | } 173 | 174 | // memory logging 175 | void process_mem_usage(double& vm_usage, double& resident_set) 176 | { 177 | using std::ios_base; 178 | using std::ifstream; 179 | using std::string; 180 | 181 | vm_usage = 0.0; 182 | resident_set = 0.0; 183 | 184 | // 'file' stat seems to give the most reliable results 185 | // 186 | ifstream stat_stream("/proc/self/stat",ios_base::in); 187 | 188 | // dummy vars for leading entries in stat that we don't care about 189 | // 190 | string pid, comm, state, ppid, pgrp, session, tty_nr; 191 | string tpgid, flags, minflt, cminflt, majflt, cmajflt; 192 | string utime, stime, cutime, cstime, priority, nice; 193 | string O, itrealvalue, starttime; 194 | 195 | // the two fields we want 196 | // 197 | unsigned long vsize; 198 | long rss; 199 | 200 | stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr 201 | >> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt 202 | >> utime >> stime >> cutime >> cstime >> priority >> nice 203 | >> O >> itrealvalue >> starttime >> vsize >> rss; // don't care about the rest 204 | 205 | stat_stream.close(); 206 | 207 | long page_size_kb = (long)boost::interprocess::mapped_region::get_page_size() / 1024; // in case x86-64 is configured to use 2MB pages 208 | vm_usage = vsize / 1024.0; 209 | resident_set = rss * page_size_kb; 210 | } 211 | -------------------------------------------------------------------------------- /src/main_engine/utils/qx_basic.cpp: -------------------------------------------------------------------------------- 1 | /******************************************************* 2 | \Author: Qingxiong Yang 3 | \Function: Basic functions. 4 | ********************************************************/ 5 | #include "main_engine/utils/qx_basic.h" 6 | 7 | void qx_timer::start(){ m_begin = clock(); } 8 | float qx_timer::stop(){ m_end = clock(); return (float(m_end - m_begin) / CLOCKS_PER_SEC); } 9 | void qx_timer::time_display(char *disp){ printf("Running time (%s) is: %5.5f Seconds.\n", disp, stop()); } 10 | void qx_timer::fps_display(char *disp){ printf("Running time (%s) is: %5.5f frame per second.\n", disp, 1.0f / stop()); } 11 | void boxcar_sliding_window_x(double *out, double *in, int h, int w, int radius) 12 | { 13 | double scale = 1.0f / (2 * radius + 1); 14 | for (int y = 0; y < h; y++) { 15 | double t; 16 | // do left edge 17 | t = in[y*w] * radius; 18 | for (int x = 0; x < radius + 1; x++) { 19 | t += in[y*w + x]; 20 | } 21 | out[y*w] = t * scale; 22 | for (int x = 1; x < radius + 1; x++) { 23 | int c = y*w + x; 24 | t += in[c + radius]; 25 | t -= in[y*w]; 26 | out[c] = t * scale; 27 | } 28 | // main loop 29 | for (int x = radius + 1; x < w - radius; x++) { 30 | int c = y*w + x; 31 | t += in[c + radius]; 32 | t -= in[c - radius - 1]; 33 | out[c] = t * scale; 34 | } 35 | // do right edge 36 | for (int x = w - radius; x < w; x++) { 37 | int c = y*w + x; 38 | t += in[(y*w) + w - 1]; 39 | t -= in[c - radius - 1]; 40 | out[c] = t * scale; 41 | } 42 | 43 | } 44 | } 45 | void boxcar_sliding_window_y(double *out, double *in, int h, int w, int radius) 46 | { 47 | double scale = 1.0f / (2 * radius + 1); 48 | for (int x = 0; x < w; x++) 49 | { 50 | double t; 51 | // do left edge 52 | t = in[x] * radius; 53 | for (int y = 0; y < radius + 1; y++) { 54 | t += in[y*w + x]; 55 | } 56 | out[x] = t * scale; 57 | for (int y = 1; y < radius + 1; y++) { 58 | int c = y*w + x; 59 | t += in[c + radius*w]; 60 | t -= in[x]; 61 | out[c] = t * scale; 62 | } 63 | // main loop 64 | for (int y = radius + 1; y < h - radius; y++) { 65 | int c = y*w + x; 66 | t += in[c + radius*w]; 67 | t -= in[c - (radius*w) - w]; 68 | out[c] = t * scale; 69 | } 70 | // do right edge 71 | for (int y = h - radius; y < h; y++) { 72 | int c = y*w + x; 73 | t += in[(h - 1)*w + x]; 74 | t -= in[c - (radius*w) - w]; 75 | out[c] = t * scale; 76 | } 77 | } 78 | } 79 | void boxcar_sliding_window(double **out, double **in, double **temp, int h, int w, int radius) 80 | { 81 | boxcar_sliding_window_x(temp[0], in[0], h, w, radius); 82 | boxcar_sliding_window_y(out[0], temp[0], h, w, radius); 83 | } 84 | void gaussian_recursive_x(double **od, double **id, int w, int h, double a0, double a1, double a2, double a3, double b1, double b2, double coefp, double coefn) 85 | { 86 | double xp = 0.0f; // previous input 87 | double yp = 0.0f; // previous output 88 | double yb = 0.0f; // previous output by 2 89 | for (int y = 0; y= 0; x--) { 110 | double xc = id[y][x]; 111 | double yc = a2*xn + a3*xa - b1*yn - b2*ya; 112 | xa = xn; xn = xc; ya = yn; yn = yc; 113 | od[y][x] = od[y][x] + yc; 114 | } 115 | } 116 | } 117 | void gaussian_recursive_y(double **od, double **id, int w, int h, double a0, double a1, double a2, double a3, double b1, double b2, double coefp, double coefn) 118 | { 119 | double xp = 0.0f; // previous input 120 | double yp = 0.0f; // previous output 121 | double yb = 0.0f; // previous output by 2 122 | for (int x = 0; x < w; x++) 123 | { 124 | xp = id[0][x]; yb = coefp*xp; yp = yb; 125 | for (int y = 0; y= 0; y--) 145 | { 146 | double xc = id[y][x]; 147 | double yc = a2*xn + a3*xa - b1*yn - b2*ya; 148 | xa = xn; xn = xc; ya = yn; yn = yc; 149 | od[y][x] = od[y][x] + yc; 150 | } 151 | } 152 | } 153 | int gaussian_recursive(double **image, double **temp, double sigma, int order, int h, int w) 154 | { 155 | const double 156 | nsigma = sigma < 0.1f ? 0.1f : sigma, 157 | alpha = 1.695f / nsigma, 158 | ema = exp(-alpha), 159 | ema2 = exp(-2 * alpha), 160 | b1 = -2 * ema, 161 | b2 = ema2; 162 | double a0 = 0, a1 = 0, a2 = 0, a3 = 0, coefp = 0, coefn = 0; 163 | switch (order) { 164 | case 0: { 165 | const double k = (1 - ema)*(1 - ema) / (1 + 2 * alpha*ema - ema2); 166 | a0 = k; 167 | a1 = k*(alpha - 1)*ema; 168 | a2 = k*(alpha + 1)*ema; 169 | a3 = -k*ema2; 170 | } break; 171 | 172 | case 1: { 173 | const double k = (1 - ema)*(1 - ema) / ema; 174 | a0 = k*ema; 175 | a1 = a3 = 0; 176 | a2 = -a0; 177 | } break; 178 | 179 | case 2: { 180 | const double 181 | ea = exp(-alpha), 182 | k = -(ema2 - 1) / (2 * alpha*ema), 183 | kn = (-2 * (-1 + 3 * ea - 3 * ea*ea + ea*ea*ea) / (3 * ea + 1 + 3 * ea*ea + ea*ea*ea)); 184 | a0 = kn; 185 | a1 = -kn*(1 + k*alpha)*ema; 186 | a2 = kn*(1 - k*alpha)*ema; 187 | a3 = -kn*ema2; 188 | } break; 189 | 190 | default: 191 | fprintf(stderr, "gaussianFilter: invalid order parameter!\n"); 192 | return 0; 193 | } 194 | coefp = (a0 + a1) / (1 + b1 + b2); 195 | coefn = (a2 + a3) / (1 + b1 + b2); 196 | //timer.start(); 197 | gaussian_recursive_x(temp, image, w, h, a0, a1, a2, a3, b1, b2, coefp, coefn); 198 | //image_display(temp,h,w); 199 | gaussian_recursive_y(image, temp, w, h, a0, a1, a2, a3, b1, b2, coefp, coefn); 200 | //timer.fps_display(); 201 | return(0); 202 | } 203 | void qx_specular_free_image(unsigned char ***image_specular_free, unsigned char ***image_normalized, float **diffuse_chromaticity_max, int h, int w) 204 | { 205 | int y, x; 206 | unsigned char *image_specular_free_x, *image_normalized_x; float *diffuse_chromaticity_max_x; 207 | unsigned char r, g, b; double imax, isum; float rf, gf, bf, c, t0, t1, t2, t3, diffuse, specular; 208 | //*image_sum_x,*image_max_x,*chromaticity_max_x, 209 | image_specular_free_x = image_specular_free[0][0]; 210 | image_normalized_x = image_normalized[0][0]; 211 | diffuse_chromaticity_max_x = diffuse_chromaticity_max[0]; 212 | for (y = 0; y0) 222 | { 223 | isum = r + g + b; 224 | if (isum == 0) c = 0; 225 | else 226 | { 227 | imax = max(max(r, g), b); 228 | c = (float)(imax / isum); 229 | } 230 | t0 = t1*c; 231 | if (fabs(t0)255.f) rf = 255.f; 246 | if (gf<0.f) gf = 0.f; else if (gf>255.f) gf = 255.f; 247 | if (bf<0.f) bf = 0.f; else if (bf>255.f) bf = 255.f; 248 | *image_specular_free_x++ = (unsigned char) (rf + 0.5f); 249 | *image_specular_free_x++ = (unsigned char) (gf + 0.5f); 250 | *image_specular_free_x++ = (unsigned char) (bf + 0.5f); 251 | } 252 | } 253 | else 254 | { 255 | *image_specular_free_x++ = r; 256 | *image_specular_free_x++ = g; 257 | *image_specular_free_x++ = b; 258 | } 259 | } 260 | } 261 | } -------------------------------------------------------------------------------- /src/main_engine/utils/qx_highlight_removal_bf.cpp: -------------------------------------------------------------------------------- 1 | /*************************************************************************************************** 2 | \Author: Qingxiong Yang 3 | \Function: Highlight removal using bilateral filter. 4 | \reference: Qingxiong Yang, Shengnan Wang and Narendra Ahuja, Real-time Specular Highlight Removal 5 | Using Bilateral Filtering, European Conference on Computer Vision (ECCV) 2010. 6 | ****************************************************************************************************/ 7 | #include "main_engine/utils/qx_highlight_removal_bf.h" 8 | 9 | qx_highlight_removal_bf::qx_highlight_removal_bf() 10 | { 11 | m_image_sf = NULL; 12 | m_mask = NULL; 13 | m_mask_dark_pixel = NULL; 14 | m_image_backup = NULL; 15 | m_image_diffuse = NULL; 16 | m_max_chrom = NULL; 17 | m_max_chrom_backup = NULL; 18 | m_temp = NULL; 19 | } 20 | qx_highlight_removal_bf::~qx_highlight_removal_bf() 21 | { 22 | clean(); 23 | } 24 | void qx_highlight_removal_bf::clean() 25 | { 26 | qx_freeu(m_image_sf); m_image_sf = NULL; 27 | qx_freeu(m_mask); m_mask = NULL; 28 | qx_freeu(m_temp); m_temp = NULL; 29 | qx_freeu(m_mask_dark_pixel); m_mask_dark_pixel = NULL; 30 | qx_freeu_3(m_image_diffuse); m_image_diffuse = NULL; 31 | qx_freeu_3(m_image_backup); m_image_backup = NULL; 32 | qx_freef_3(m_max_chrom); m_max_chrom = NULL; 33 | qx_freef(m_max_chrom_backup); m_max_chrom_backup = NULL; 34 | } 35 | int qx_highlight_removal_bf::init(int h, int w, unsigned char threshold_dark_pixel, float threshold_sigma_change) 36 | { 37 | m_h = h; m_w = w; m_threshold_dark_pixel = threshold_dark_pixel; m_threshold_sigma_change = threshold_sigma_change; 38 | m_image_sf = qx_allocu(m_h, m_w); 39 | m_mask = qx_allocu(m_h, m_w); 40 | m_temp = qx_allocu(m_h, m_w); 41 | m_mask_dark_pixel = qx_allocu(m_h, m_w); 42 | m_image_diffuse = qx_allocu_3(m_h, m_w, 3); 43 | m_image_backup = qx_allocu_3(m_h, m_w, 3); 44 | m_max_chrom = qx_allocf_3(2, m_h, m_w); 45 | m_max_chrom_backup = qx_allocf(m_h, m_w); 46 | m_bf.init(m_h, m_w); 47 | return(0); 48 | } 49 | int qx_highlight_removal_bf::diffuse(unsigned char***image_diffuse, unsigned char***image, int nr_iter) 50 | { 51 | float **max_chrom_bf, **max_chrom; 52 | max_chrom = m_max_chrom[0]; 53 | max_chrom_bf = m_max_chrom[1]; 54 | m_nr_iteration = nr_iter; 55 | if (m_nr_iteration>0)/*if # of iteration is pre-define*/ 56 | { 57 | for (int i = 0; i= 0) 71 | { 72 | if (i == 0)/*compute our approximation ($\lambda_{max}$) of the maximum diffuse chromaticity ($\Lambda_{max}$)*/ 73 | { 74 | compute_approximated_maximum_diffuse_chromaticity(m_image_sf, image, max_chrom, m_mask_dark_pixel, m_threshold_dark_pixel, m_h, m_w); 75 | //image_display(m_image_sf,m_h,m_w); 76 | } 77 | //image_display(max_chrom,m_h,m_w); 78 | m_bf.joint_bilateral_filter(max_chrom_bf, max_chrom, m_image_sf, m_mask_dark_pixel, 10, 0.25, 10.f / 255);/*O(1) bilateral filtering*/ 79 | int counter = 0; 80 | for (int y = 0; ym_threshold_sigma_change) counter++; 83 | max_chrom[y][x] = max(max_chrom[y][x], max_chrom_bf[y][x]); 84 | } 85 | //image_display(max_chrom,m_h,m_w); 86 | i++; 87 | //if(counter>0) printf("[%02d] unconverge: [%010d: %010d]\n",i,counter,m_h*m_w); 88 | if (counter == 0) 89 | { 90 | m_nr_iteration = i; 91 | i = -1; 92 | } 93 | } 94 | } 95 | //image_display(image,m_h,m_w); 96 | compute_diffuse_reflection_from_maximum_diffuse_chromaticity(image_diffuse, image, max_chrom, m_mask_dark_pixel, m_h, m_w);/*compute diffuse reflection*/ 97 | //image_display(image_diffuse,m_h,m_w); 98 | return(m_nr_iteration); 99 | } 100 | void qx_highlight_removal_bf::compute_approximated_maximum_diffuse_chromaticity(unsigned char **image_approximated_max_diffuse_chromaticity, 101 | unsigned char ***image_normalized, float**image_max_chrom, unsigned char**mask, 102 | unsigned char threshold_dark_pixel, int h, int w) 103 | { 104 | int y, x; 105 | unsigned char *image_approximated_x, *image_normalized_x, *mask_x; 106 | unsigned char r, g, b; int imax, isum; 107 | //double t; float rf,gf,bf,c,diffuse,specular; 108 | float c;//diffuse,specular; 109 | image_approximated_x = image_approximated_max_diffuse_chromaticity[0]; 110 | image_normalized_x = image_normalized[0][0]; 111 | mask_x = mask[0]; 112 | for (y = 0; ythreshold_dark_pixel) 121 | { 122 | *mask_x++ = 255; 123 | isum = r + g + b; 124 | c = (float)imax / isum; 125 | if (isum>10) image_max_chrom[y][x] = c; 126 | else image_max_chrom[y][x] = 0; 127 | 128 | float chrom[3] = { (float)r / isum, (float)g / isum, (float)b / isum }; 129 | float cmin = qx_min_f3(chrom); 130 | float cmax = qx_max_f3(chrom); 131 | float cappro = (cmax - cmin) / (1 - 3 * cmin); 132 | *image_approximated_x++ = (unsigned char)(255 * (cappro*1.5 - 0.5) + 0.5); 133 | //m_mask[y][x]=unsigned char(rf*0.3+gf*0.6+bf*0.1+0.5); 134 | 135 | } 136 | else 137 | { 138 | *mask_x++ = 0; 139 | *image_approximated_x++ = 0; 140 | } 141 | } 142 | } 143 | } 144 | void qx_highlight_removal_bf::compute_diffuse_reflection_from_maximum_diffuse_chromaticity(unsigned char ***image_approximated_max_diffuse_chromaticity, unsigned char ***image_normalized, 145 | float **max_diffuse_chromaticity, unsigned char**mask, int h, int w) 146 | { 147 | int y, x; 148 | unsigned char *image_specular_free_x, *image_normalized_x; float *diffuse_chromaticity_max_x; 149 | unsigned char r, g, b; double imax, isum; float rf, gf, bf, c, t0, t1, t2, t3, diffuse, specular; 150 | //*image_sum_x,*image_max_x,*chromaticity_max_x, 151 | image_specular_free_x = image_approximated_max_diffuse_chromaticity[0][0]; 152 | image_normalized_x = image_normalized[0][0]; 153 | diffuse_chromaticity_max_x = max_diffuse_chromaticity[0]; 154 | t3 = 1.0f / 3.0f; 155 | for (y = 0; y0) 170 | { 171 | isum = r + g + b; 172 | if (isum == 0) c = 0; 173 | else 174 | { 175 | imax = max(max(r, g), b); 176 | c = float(imax / isum); 177 | } 178 | t0 = t1*c; 179 | if (fabs(t0)255.f) rf = 255.f; 198 | if (gf<0.f) gf = 0.f; else if (gf>255.f) gf = 255.f; 199 | if (bf<0.f) bf = 0.f; else if (bf>255.f) bf = 255.f; 200 | //*image_specular_free_x++=unsigned char(rf+0.5f); 201 | //*image_specular_free_x++=unsigned char(gf+0.5f); 202 | //*image_specular_free_x++=unsigned char(bf+0.5f); 203 | image_approximated_max_diffuse_chromaticity[y][x][0] = (unsigned char)(rf + 0.5f); 204 | image_approximated_max_diffuse_chromaticity[y][x][1] = (unsigned char)(gf + 0.5f); 205 | image_approximated_max_diffuse_chromaticity[y][x][2] = (unsigned char)(bf + 0.5f); 206 | } 207 | } 208 | } 209 | else 210 | { 211 | //*image_specular_free_x++=(*image_normalized_x++); 212 | //*image_specular_free_x++=(*image_normalized_x++); 213 | //*image_specular_free_x++=(*image_normalized_x++); 214 | //*diffuse_chromaticity_max_x++; 215 | 216 | for (int c = 0; c<3; c++) image_approximated_max_diffuse_chromaticity[y][x][c] = image_normalized[y][x][c]; 217 | } 218 | 219 | } 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/mesh_rescale/MeshRescale.cpp: -------------------------------------------------------------------------------- 1 | #include "main_engine/tracker/Mesh.h" 2 | #include "main_engine/utils/global.h" 3 | 4 | class ParameterReader 5 | { 6 | public: 7 | ParameterReader( string filename="./parameters.txt" ) 8 | { 9 | ifstream fin( filename.c_str() ); 10 | if (!fin) 11 | { 12 | cerr<<"parameter file does not exist."<::iterator iter = data.find(key); 38 | if (iter == data.end()) 39 | { 40 | cerr<<"Parameter name "<second; 44 | } 45 | public: 46 | map data; 47 | }; 48 | 49 | int main(int argc, char** args) 50 | { 51 | // check the range of the values in this mesh 52 | ParameterReader pd; 53 | bool use_mesh_file = atof(pd.getData("use_mesh_file").c_str()); 54 | 55 | bool clockwise = true; 56 | if(pd.getData("clockwise").compare("NOT_FOUND") != 0) 57 | clockwise = atof(pd.getData("clockwise").c_str()); 58 | 59 | if(use_mesh_file) 60 | { 61 | // this works for a single mesh 62 | PangaeaMeshData inputMesh; 63 | 64 | std::string input_file = pd.getData("input_mesh_file"); 65 | std::string output_file = pd.getData("output_mesh_file"); 66 | 67 | PangaeaMeshIO::loadfromFile(input_file, inputMesh, clockwise); 68 | vector > bbox; 69 | getMeshBoundingBox(inputMesh, bbox); 70 | 71 | double xrange = bbox[0][1] - bbox[0][0]; 72 | double yrange = bbox[1][1] - bbox[1][0]; 73 | double zrange = bbox[2][1] - bbox[2][0]; 74 | 75 | cout << "x_range: " << xrange << endl; 76 | cout << "y_range: " << yrange << endl; 77 | cout << "z_range: " << zrange << endl; 78 | 79 | double factor = 400.0 / zrange; 80 | cout << "scaling factor is: " << factor << endl; 81 | 82 | // scale the mesh up 83 | 84 | scaleMeshUp(inputMesh, factor); 85 | PangaeaMeshIO::writeToFile(output_file, inputMesh); 86 | 87 | }else 88 | { 89 | // this works for a heirachy of meshes 90 | PangaeaMeshData inputMesh; 91 | 92 | std::string input_format = pd.getData("input_mesh_format"); 93 | std::string output_format = pd.getData("output_mesh_format"); 94 | std::string input_list_str = pd.getData("input_list"); 95 | 96 | std::stringstream input_list(input_list_str); 97 | int number; vector input_list_vector; 98 | 99 | input_list >> number; 100 | while( !input_list.fail() ) 101 | { 102 | input_list_vector.push_back(number); 103 | input_list >> number; 104 | } 105 | 106 | double factor = 0; 107 | char buffer[BUFFER_SIZE]; 108 | for(int i = 0; i < input_list_vector.size(); ++i) 109 | { 110 | stringstream input_file; 111 | sprintf(buffer, input_format.c_str(), input_list_vector[i]); 112 | input_file << buffer; 113 | //memset(&buffer[0], 0, sizeof(buffer)); 114 | 115 | stringstream output_file; 116 | sprintf(buffer, output_format.c_str(), input_list_vector[i]); 117 | output_file << buffer; 118 | //memset(&buffer[0], 0, sizeof(buffer)); 119 | 120 | PangaeaMeshIO::loadfromFile(input_file.str(), inputMesh, clockwise); 121 | 122 | if(factor == 0) // if this is the first frame 123 | { 124 | vector > bbox; 125 | getMeshBoundingBox(inputMesh, bbox); 126 | double xrange = bbox[0][1] - bbox[0][0]; 127 | double yrange = bbox[1][1] - bbox[1][0]; 128 | double zrange = bbox[2][1] - bbox[2][0]; 129 | cout << "x_range: " << xrange << endl; 130 | cout << "y_range: " << yrange << endl; 131 | cout << "z_range: " << zrange << endl; 132 | factor = 400.0 / zrange; 133 | cout << "scaling factor is: " << factor << endl; 134 | } 135 | 136 | // scale the mesh up 137 | scaleMeshUp(inputMesh, factor); 138 | PangaeaMeshIO::writeToFile(output_file.str(), inputMesh); 139 | } 140 | 141 | } 142 | 143 | 144 | } 145 | --------------------------------------------------------------------------------