├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cmake ├── FindEigen3.cmake ├── FindGflags.cmake └── FindGlog.cmake ├── colmap2mvsnet_acm.py ├── exe ├── main_depth_estimation.cpp └── main_depth_fusion.cpp └── src ├── ACMP.cpp ├── ACMP.cu ├── ACMP.h ├── colmap_interface ├── colmap_interface.cpp ├── colmap_interface.h └── endian.h └── types.h /.gitignore: -------------------------------------------------------------------------------- 1 | build*/ 2 | 3 | popsift/playground/try-gauss 4 | popsift/playground/try-gauss-param 5 | popsift/playground/try-gauss-interpolate 6 | popsift/playground/try-libav 7 | oxford 8 | 9 | # Prerequisites 10 | *.d 11 | 12 | # Compiled Object files 13 | *.slo 14 | *.lo 15 | *.o 16 | *.obj 17 | 18 | # Precompiled Headers 19 | *.gch 20 | *.pch 21 | 22 | # Compiled Dynamic libraries 23 | *.so 24 | *.dylib 25 | *.dll 26 | 27 | # Fortran module files 28 | *.mod 29 | *.smod 30 | 31 | # Compiled Static libraries 32 | *.lai 33 | *.la 34 | *.a 35 | *.lib 36 | 37 | # Executables 38 | *.exe 39 | *.out 40 | *.app 41 | 42 | # Support files 43 | *.nvvp 44 | 45 | # Temporary files 46 | .DS_Store 47 | 48 | # Downloaded archives for tests. 49 | *.tgz 50 | 51 | # VS Code files 52 | .vscode 53 | 54 | # Export files 55 | export/EasyARPopSift.jar 56 | export/java/* 57 | 58 | *.user 59 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8) 2 | project (ACMP) 3 | 4 | #--- CMake configuration 5 | set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) 6 | 7 | find_package(CUDA 6.0 REQUIRED ) # For Cuda Managed Memory and c++11 8 | find_package(OpenCV REQUIRED ) 9 | find_package(Eigen3 REQUIRED) 10 | 11 | include_directories( 12 | ${OpenCV_INCLUDE_DIRS} 13 | ${EIGEN3_INCLUDE_DIRS} 14 | ) 15 | include_directories(. 16 | ${CMAKE_SOURCE_DIR} 17 | ${CMAKE_SOURCE_DIR}/src 18 | ) 19 | 20 | # glog 21 | find_package(Gflags REQUIRED) 22 | find_package(Glog REQUIRED) 23 | include_directories(${GFLAGS_INCLUDE_DIRS} ${GLOG_INCLUDE_DIRS}) 24 | 25 | set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-O3 --use_fast_math --maxrregcount=128 --ptxas-options=-v -std=c++11 --compiler-options -Wall) 26 | 27 | if(CMAKE_COMPILER_IS_GNUCXX) 28 | add_definitions(-std=c++11) 29 | add_definitions(-pthread) 30 | add_definitions(-Wall) 31 | add_definitions(-Wextra) 32 | add_definitions(-pedantic) 33 | add_definitions(-Wno-unused-function) 34 | add_definitions(-Wno-switch) 35 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -ffast-math -march=native") # extend release-profile with fast-math 36 | endif() 37 | 38 | find_package(OpenMP) 39 | if (OPENMP_FOUND) 40 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 41 | endif() 42 | 43 | cuda_add_library(ACMP 44 | src/ACMP.h 45 | src/ACMP.cpp 46 | src/ACMP.cu 47 | ) 48 | 49 | add_executable(main_depth_estimation 50 | exe/main_depth_estimation.cpp 51 | src/colmap_interface/colmap_interface.cpp 52 | ) 53 | 54 | target_link_libraries(main_depth_estimation 55 | ACMP 56 | ${OpenCV_LIBS} 57 | ${GFLAGS_LIBRARIES} 58 | ${GLOG_LIBRARIES} 59 | ) 60 | 61 | add_executable(main_depth_fusion 62 | exe/main_depth_fusion.cpp 63 | src/colmap_interface/colmap_interface.cpp 64 | ) 65 | 66 | target_link_libraries(main_depth_fusion 67 | ${OpenCV_LIBS} 68 | ${GFLAGS_LIBRARIES} 69 | ${GLOG_LIBRARIES} 70 | ) 71 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Qingshan Xu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NOTE 2 | *--------------------------------------------------------* 3 | 4 | This project is modified from [ACMP](https://github.com/GhiXu/ACMP.git) 5 | 6 | I just port it to COLMAP project type, thanks GhiXu for his/her great job! 7 | 8 | *--------------------------------------------------------* 9 | 10 | # ACMP 11 | [News] The code for [ACMH](https://github.com/GhiXu/ACMH) is released!!! 12 | [News] The code for [ACMM](https://github.com/GhiXu/ACMM) is released!!! 13 | ## About 14 | This repository contains the code for the paper [Planar Prior Assisted PatchMatch Multi-View Stereo](https://arxiv.org/abs/1912.11744), Qingshan Xu and Wenbing Tao, AAAI2020. If you find this project useful for your research, please cite: 15 | ``` 16 | @article{Xu2020ACMP, 17 | title={Planar Prior Assisted PatchMatch Multi-View Stereo}, 18 | author={Xu, Qingshan and Tao, Wenbing}, 19 | journal={AAAI Conference on Artificial Intelligence (AAAI)}, 20 | year={2020} 21 | } 22 | @article{Xu2019ACMM, 23 | title={Multi-Scale Geometric Consistency Guided Multi-View Stereo}, 24 | author={Xu, Qingshan and Tao, Wenbing}, 25 | journal={Computer Vision and Pattern Recognition (CVPR)}, 26 | year={2019} 27 | } 28 | ``` 29 | ## Dependencies 30 | The code has been tested on Ubuntu 14.04 with GTX Titan X. 31 | * [Cuda](https://developer.nvidia.com/zh-cn/cuda-downloads) >= 6.0 32 | * [OpenCV](https://opencv.org/) >= 2.4 33 | * [cmake](https://cmake.org/) 34 | * [Eigen](https://gitlab.com/libeigen/eigen) 35 | * [Glog](https://github.com/google/glog) 36 | * [Gflag](https://github.com/gflags/gflags.git) 37 | ## Usage 38 | * Complie ACMP 39 | ``` 40 | cmake . 41 | make 42 | ``` 43 | * Test 44 | ``` 45 | Use script colmap2mvsnet_acm.py to convert COLMAP SfM result to ACMP input 46 | Run ./ACMP $data_folder to get reconstruction results 47 | ``` 48 | ## Acknowledgemets 49 | This code largely benefits from the following repositories: [Gipuma](https://github.com/kysucix/gipuma) and [COLMAP](https://colmap.github.io/). Thanks to their authors for opening source of their excellent works. 50 | 51 | -------------------------------------------------------------------------------- /cmake/FindEigen3.cmake: -------------------------------------------------------------------------------- 1 | FIND_PATH( EIGEN3_INCLUDE_DIRS Eigen/Geometry 2 | $ENV{EIGEN3DIR}/include 3 | /usr/local/include/eigen3 4 | /usr/local/include 5 | /usr/local/X11R6/include 6 | /usr/X11R6/include 7 | /usr/X11/include 8 | /usr/include/X11 9 | /usr/include/eigen3/ 10 | /usr/include 11 | /opt/X11/include 12 | /opt/include 13 | ${CMAKE_SOURCE_DIR}/external/eigen/include) 14 | 15 | SET(EIGEN3_FOUND "NO") 16 | IF(EIGEN3_INCLUDE_DIRS) 17 | SET(EIGEN3_FOUND "YES") 18 | ENDIF() 19 | -------------------------------------------------------------------------------- /cmake/FindGflags.cmake: -------------------------------------------------------------------------------- 1 | # Ceres Solver - A fast non-linear least squares minimizer 2 | # Copyright 2015 Google Inc. All rights reserved. 3 | # http://ceres-solver.org/ 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions are met: 7 | # 8 | # * Redistributions of source code must retain the above copyright notice, 9 | # this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # * Neither the name of Google Inc. nor the names of its contributors may be 14 | # used to endorse or promote products derived from this software without 15 | # specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | # POSSIBILITY OF SUCH DAMAGE. 28 | # 29 | # Author: alexs.mac@gmail.com (Alex Stewart) 30 | # 31 | 32 | # FindGflags.cmake - Find Google gflags logging library. 33 | # 34 | # This module will attempt to find gflags, either via an exported CMake 35 | # configuration (generated by gflags >= 2.1 which are built with CMake), or 36 | # by performing a standard search for all gflags components. The order of 37 | # precedence for these two methods of finding gflags is controlled by: 38 | # GFLAGS_PREFER_EXPORTED_GFLAGS_CMAKE_CONFIGURATION. 39 | # 40 | # This module defines the following variables: 41 | # 42 | # GFLAGS_FOUND: TRUE iff gflags is found. 43 | # GFLAGS_INCLUDE_DIRS: Include directories for gflags. 44 | # GFLAGS_LIBRARIES: Libraries required to link gflags. 45 | # GFLAGS_NAMESPACE: The namespace in which gflags is defined. In versions of 46 | # gflags < 2.1, this was google, for versions >= 2.1 it is 47 | # by default gflags, although can be configured when building 48 | # gflags to be something else (i.e. google for legacy 49 | # compatibility). 50 | # FOUND_INSTALLED_GFLAGS_CMAKE_CONFIGURATION: True iff the version of gflags 51 | # found was built & installed / 52 | # exported as a CMake package. 53 | # 54 | # The following variables control the behaviour of this module when an exported 55 | # gflags CMake configuration is not found. 56 | # 57 | # GFLAGS_PREFER_EXPORTED_GFLAGS_CMAKE_CONFIGURATION: TRUE/FALSE, iff TRUE then 58 | # then prefer using an exported CMake configuration 59 | # generated by gflags >= 2.1 over searching for the 60 | # gflags components manually. Otherwise (FALSE) 61 | # ignore any exported gflags CMake configurations and 62 | # always perform a manual search for the components. 63 | # Default: TRUE iff user does not define this variable 64 | # before we are called, and does NOT specify either 65 | # GFLAGS_INCLUDE_DIR_HINTS or GFLAGS_LIBRARY_DIR_HINTS 66 | # otherwise FALSE. 67 | # GFLAGS_INCLUDE_DIR_HINTS: List of additional directories in which to 68 | # search for gflags includes, e.g: /timbuktu/include. 69 | # GFLAGS_LIBRARY_DIR_HINTS: List of additional directories in which to 70 | # search for gflags libraries, e.g: /timbuktu/lib. 71 | # 72 | # The following variables are also defined by this module, but in line with 73 | # CMake recommended FindPackage() module style should NOT be referenced directly 74 | # by callers (use the plural variables detailed above instead). These variables 75 | # do however affect the behaviour of the module via FIND_[PATH/LIBRARY]() which 76 | # are NOT re-called (i.e. search for library is not repeated) if these variables 77 | # are set with valid values _in the CMake cache_. This means that if these 78 | # variables are set directly in the cache, either by the user in the CMake GUI, 79 | # or by the user passing -DVAR=VALUE directives to CMake when called (which 80 | # explicitly defines a cache variable), then they will be used verbatim, 81 | # bypassing the HINTS variables and other hard-coded search locations. 82 | # 83 | # GFLAGS_INCLUDE_DIR: Include directory for gflags, not including the 84 | # include directory of any dependencies. 85 | # GFLAGS_LIBRARY: gflags library, not including the libraries of any 86 | # dependencies. 87 | 88 | # Reset CALLERS_CMAKE_FIND_LIBRARY_PREFIXES to its value when FindGflags was 89 | # invoked, necessary for MSVC. 90 | macro(GFLAGS_RESET_FIND_LIBRARY_PREFIX) 91 | if (MSVC AND CALLERS_CMAKE_FIND_LIBRARY_PREFIXES) 92 | set(CMAKE_FIND_LIBRARY_PREFIXES "${CALLERS_CMAKE_FIND_LIBRARY_PREFIXES}") 93 | endif() 94 | endmacro(GFLAGS_RESET_FIND_LIBRARY_PREFIX) 95 | 96 | # Called if we failed to find gflags or any of it's required dependencies, 97 | # unsets all public (designed to be used externally) variables and reports 98 | # error message at priority depending upon [REQUIRED/QUIET/] argument. 99 | macro(GFLAGS_REPORT_NOT_FOUND REASON_MSG) 100 | unset(GFLAGS_FOUND) 101 | unset(GFLAGS_INCLUDE_DIRS) 102 | unset(GFLAGS_LIBRARIES) 103 | # Do not use unset, as we want to keep GFLAGS_NAMESPACE in the cache, 104 | # but simply clear its value. 105 | set(GFLAGS_NAMESPACE "" CACHE STRING 106 | "gflags namespace (google or gflags)" FORCE) 107 | 108 | # Make results of search visible in the CMake GUI if gflags has not 109 | # been found so that user does not have to toggle to advanced view. 110 | mark_as_advanced(CLEAR GFLAGS_INCLUDE_DIR 111 | GFLAGS_LIBRARY 112 | GFLAGS_NAMESPACE) 113 | 114 | gflags_reset_find_library_prefix() 115 | 116 | # Note _FIND_[REQUIRED/QUIETLY] variables defined by FindPackage() 117 | # use the camelcase library name, not uppercase. 118 | if (Gflags_FIND_QUIETLY) 119 | message(STATUS "Failed to find gflags - " ${REASON_MSG} ${ARGN}) 120 | elseif (Gflags_FIND_REQUIRED) 121 | message(FATAL_ERROR "Failed to find gflags - " ${REASON_MSG} ${ARGN}) 122 | else() 123 | # Neither QUIETLY nor REQUIRED, use no priority which emits a message 124 | # but continues configuration and allows generation. 125 | message("-- Failed to find gflags - " ${REASON_MSG} ${ARGN}) 126 | endif () 127 | return() 128 | endmacro(GFLAGS_REPORT_NOT_FOUND) 129 | 130 | # Verify that all variable names passed as arguments are defined (can be empty 131 | # but must be defined) or raise a fatal error. 132 | macro(GFLAGS_CHECK_VARS_DEFINED) 133 | foreach(CHECK_VAR ${ARGN}) 134 | if (NOT DEFINED ${CHECK_VAR}) 135 | message(FATAL_ERROR "Ceres Bug: ${CHECK_VAR} is not defined.") 136 | endif() 137 | endforeach() 138 | endmacro(GFLAGS_CHECK_VARS_DEFINED) 139 | 140 | # Use check_cxx_source_compiles() to compile trivial test programs to determine 141 | # the gflags namespace. This works on all OSs except Windows. If using Visual 142 | # Studio, it fails because msbuild forces check_cxx_source_compiles() to use 143 | # CMAKE_BUILD_TYPE=Debug for the test project, which usually breaks detection 144 | # because MSVC requires that the test project use the same build type as gflags, 145 | # which would normally be built in Release. 146 | # 147 | # Defines: GFLAGS_NAMESPACE in the caller's scope with the detected namespace, 148 | # which is blank (empty string, will test FALSE is CMake conditionals) 149 | # if detection failed. 150 | function(GFLAGS_CHECK_GFLAGS_NAMESPACE_USING_TRY_COMPILE) 151 | # Verify that all required variables are defined. 152 | gflags_check_vars_defined( 153 | GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) 154 | # Ensure that GFLAGS_NAMESPACE is always unset on completion unless 155 | # we explicitly set if after having the correct namespace. 156 | set(GFLAGS_NAMESPACE "" PARENT_SCOPE) 157 | 158 | include(CheckCXXSourceCompiles) 159 | # Setup include path & link library for gflags for CHECK_CXX_SOURCE_COMPILES. 160 | set(CMAKE_REQUIRED_INCLUDES ${GFLAGS_INCLUDE_DIR}) 161 | set(CMAKE_REQUIRED_LIBRARIES ${GFLAGS_LIBRARY} ${GFLAGS_LINK_LIBRARIES}) 162 | # First try the (older) google namespace. Note that the output variable 163 | # MUST be unique to the build type as otherwise the test is not repeated as 164 | # it is assumed to have already been performed. 165 | check_cxx_source_compiles( 166 | "#include 167 | int main(int argc, char * argv[]) { 168 | google::ParseCommandLineFlags(&argc, &argv, true); 169 | return 0; 170 | }" 171 | GFLAGS_IN_GOOGLE_NAMESPACE) 172 | if (GFLAGS_IN_GOOGLE_NAMESPACE) 173 | set(GFLAGS_NAMESPACE google PARENT_SCOPE) 174 | return() 175 | endif() 176 | 177 | # Try (newer) gflags namespace instead. Note that the output variable 178 | # MUST be unique to the build type as otherwise the test is not repeated as 179 | # it is assumed to have already been performed. 180 | set(CMAKE_REQUIRED_INCLUDES ${GFLAGS_INCLUDE_DIR}) 181 | set(CMAKE_REQUIRED_LIBRARIES ${GFLAGS_LIBRARY} ${GFLAGS_LINK_LIBRARIES}) 182 | check_cxx_source_compiles( 183 | "#include 184 | int main(int argc, char * argv[]) { 185 | gflags::ParseCommandLineFlags(&argc, &argv, true); 186 | return 0; 187 | }" 188 | GFLAGS_IN_GFLAGS_NAMESPACE) 189 | if (GFLAGS_IN_GFLAGS_NAMESPACE) 190 | set(GFLAGS_NAMESPACE gflags PARENT_SCOPE) 191 | return() 192 | endif (GFLAGS_IN_GFLAGS_NAMESPACE) 193 | endfunction(GFLAGS_CHECK_GFLAGS_NAMESPACE_USING_TRY_COMPILE) 194 | 195 | # Use regex on the gflags headers to attempt to determine the gflags namespace. 196 | # Checks both gflags.h (contained namespace on versions < 2.1.2) and 197 | # gflags_declare.h, which contains the namespace on versions >= 2.1.2. 198 | # In general, this method should only be used when 199 | # GFLAGS_CHECK_GFLAGS_NAMESPACE_USING_TRY_COMPILE() cannot be used, or has 200 | # failed. 201 | # 202 | # Defines: GFLAGS_NAMESPACE in the caller's scope with the detected namespace, 203 | # which is blank (empty string, will test FALSE is CMake conditionals) 204 | # if detection failed. 205 | function(GFLAGS_CHECK_GFLAGS_NAMESPACE_USING_REGEX) 206 | # Verify that all required variables are defined. 207 | gflags_check_vars_defined(GFLAGS_INCLUDE_DIR) 208 | # Ensure that GFLAGS_NAMESPACE is always undefined on completion unless 209 | # we explicitly set if after having the correct namespace. 210 | set(GFLAGS_NAMESPACE "" PARENT_SCOPE) 211 | 212 | # Scan gflags.h to identify what namespace gflags was built with. On 213 | # versions of gflags < 2.1.2, gflags.h was configured with the namespace 214 | # directly, on >= 2.1.2, gflags.h uses the GFLAGS_NAMESPACE #define which 215 | # is defined in gflags_declare.h, we try each location in turn. 216 | set(GFLAGS_HEADER_FILE ${GFLAGS_INCLUDE_DIR}/gflags/gflags.h) 217 | if (NOT EXISTS ${GFLAGS_HEADER_FILE}) 218 | gflags_report_not_found( 219 | "Could not find file: ${GFLAGS_HEADER_FILE} " 220 | "containing namespace information in gflags install located at: " 221 | "${GFLAGS_INCLUDE_DIR}.") 222 | endif() 223 | file(READ ${GFLAGS_HEADER_FILE} GFLAGS_HEADER_FILE_CONTENTS) 224 | 225 | string(REGEX MATCH "namespace [A-Za-z]+" 226 | GFLAGS_NAMESPACE "${GFLAGS_HEADER_FILE_CONTENTS}") 227 | string(REGEX REPLACE "namespace ([A-Za-z]+)" "\\1" 228 | GFLAGS_NAMESPACE "${GFLAGS_NAMESPACE}") 229 | 230 | if (NOT GFLAGS_NAMESPACE) 231 | gflags_report_not_found( 232 | "Failed to extract gflags namespace from header file: " 233 | "${GFLAGS_HEADER_FILE}.") 234 | endif (NOT GFLAGS_NAMESPACE) 235 | 236 | if (GFLAGS_NAMESPACE STREQUAL "google" OR 237 | GFLAGS_NAMESPACE STREQUAL "gflags") 238 | # Found valid gflags namespace from gflags.h. 239 | set(GFLAGS_NAMESPACE "${GFLAGS_NAMESPACE}" PARENT_SCOPE) 240 | return() 241 | endif() 242 | 243 | # Failed to find gflags namespace from gflags.h, gflags is likely a new 244 | # version, check gflags_declare.h, which in newer versions (>= 2.1.2) contains 245 | # the GFLAGS_NAMESPACE #define, which is then referenced in gflags.h. 246 | set(GFLAGS_DECLARE_FILE ${GFLAGS_INCLUDE_DIR}/gflags/gflags_declare.h) 247 | if (NOT EXISTS ${GFLAGS_DECLARE_FILE}) 248 | gflags_report_not_found( 249 | "Could not find file: ${GFLAGS_DECLARE_FILE} " 250 | "containing namespace information in gflags install located at: " 251 | "${GFLAGS_INCLUDE_DIR}.") 252 | endif() 253 | file(READ ${GFLAGS_DECLARE_FILE} GFLAGS_DECLARE_FILE_CONTENTS) 254 | 255 | string(REGEX MATCH "#define GFLAGS_NAMESPACE [A-Za-z]+" 256 | GFLAGS_NAMESPACE "${GFLAGS_DECLARE_FILE_CONTENTS}") 257 | string(REGEX REPLACE "#define GFLAGS_NAMESPACE ([A-Za-z]+)" "\\1" 258 | GFLAGS_NAMESPACE "${GFLAGS_NAMESPACE}") 259 | 260 | if (NOT GFLAGS_NAMESPACE) 261 | gflags_report_not_found( 262 | "Failed to extract gflags namespace from declare file: " 263 | "${GFLAGS_DECLARE_FILE}.") 264 | endif (NOT GFLAGS_NAMESPACE) 265 | 266 | if (GFLAGS_NAMESPACE STREQUAL "google" OR 267 | GFLAGS_NAMESPACE STREQUAL "gflags") 268 | # Found valid gflags namespace from gflags.h. 269 | set(GFLAGS_NAMESPACE "${GFLAGS_NAMESPACE}" PARENT_SCOPE) 270 | return() 271 | endif() 272 | endfunction(GFLAGS_CHECK_GFLAGS_NAMESPACE_USING_REGEX) 273 | 274 | # Protect against any alternative find_package scripts for this library having 275 | # been called previously (in a client project) which set GFLAGS_FOUND, but not 276 | # the other variables we require / set here which could cause the search logic 277 | # here to fail. 278 | unset(GFLAGS_FOUND) 279 | 280 | # ----------------------------------------------------------------- 281 | # By default, if the user has expressed no preference for using an exported 282 | # gflags CMake configuration over performing a search for the installed 283 | # components, and has not specified any hints for the search locations, then 284 | # prefer a gflags exported configuration if available. 285 | if (NOT DEFINED GFLAGS_PREFER_EXPORTED_GFLAGS_CMAKE_CONFIGURATION 286 | AND NOT GFLAGS_INCLUDE_DIR_HINTS 287 | AND NOT GFLAGS_LIBRARY_DIR_HINTS) 288 | message(STATUS "No preference for use of exported gflags CMake configuration " 289 | "set, and no hints for include/library directories provided. " 290 | "Defaulting to preferring an installed/exported gflags CMake configuration " 291 | "if available.") 292 | set(GFLAGS_PREFER_EXPORTED_GFLAGS_CMAKE_CONFIGURATION TRUE) 293 | endif() 294 | 295 | if (GFLAGS_PREFER_EXPORTED_GFLAGS_CMAKE_CONFIGURATION) 296 | # Try to find an exported CMake configuration for gflags, as generated by 297 | # gflags versions >= 2.1. 298 | # 299 | # We search twice, s/t we can invert the ordering of precedence used by 300 | # find_package() for exported package build directories, and installed 301 | # packages (found via CMAKE_SYSTEM_PREFIX_PATH), listed as items 6) and 7) 302 | # respectively in [1]. 303 | # 304 | # By default, exported build directories are (in theory) detected first, and 305 | # this is usually the case on Windows. However, on OS X & Linux, the install 306 | # path (/usr/local) is typically present in the PATH environment variable 307 | # which is checked in item 4) in [1] (i.e. before both of the above, unless 308 | # NO_SYSTEM_ENVIRONMENT_PATH is passed). As such on those OSs installed 309 | # packages are usually detected in preference to exported package build 310 | # directories. 311 | # 312 | # To ensure a more consistent response across all OSs, and as users usually 313 | # want to prefer an installed version of a package over a locally built one 314 | # where both exist (esp. as the exported build directory might be removed 315 | # after installation), we first search with NO_CMAKE_PACKAGE_REGISTRY which 316 | # means any build directories exported by the user are ignored, and thus 317 | # installed directories are preferred. If this fails to find the package 318 | # we then research again, but without NO_CMAKE_PACKAGE_REGISTRY, so any 319 | # exported build directories will now be detected. 320 | # 321 | # To prevent confusion on Windows, we also pass NO_CMAKE_BUILDS_PATH (which 322 | # is item 5) in [1]), to not preferentially use projects that were built 323 | # recently with the CMake GUI to ensure that we always prefer an installed 324 | # version if available. 325 | # 326 | # [1] http://www.cmake.org/cmake/help/v2.8.11/cmake.html#command:find_package 327 | find_package(gflags QUIET 328 | NO_MODULE 329 | NO_CMAKE_PACKAGE_REGISTRY 330 | NO_CMAKE_BUILDS_PATH) 331 | if (gflags_FOUND) 332 | message(STATUS "Found installed version of gflags: ${gflags_DIR}") 333 | else(gflags_FOUND) 334 | # Failed to find an installed version of gflags, repeat search allowing 335 | # exported build directories. 336 | message(STATUS "Failed to find installed gflags CMake configuration, " 337 | "searching for gflags build directories exported with CMake.") 338 | # Again pass NO_CMAKE_BUILDS_PATH, as we know that gflags is exported and 339 | # do not want to treat projects built with the CMake GUI preferentially. 340 | find_package(gflags QUIET 341 | NO_MODULE 342 | NO_CMAKE_BUILDS_PATH) 343 | if (gflags_FOUND) 344 | message(STATUS "Found exported gflags build directory: ${gflags_DIR}") 345 | endif(gflags_FOUND) 346 | endif(gflags_FOUND) 347 | 348 | set(FOUND_INSTALLED_GFLAGS_CMAKE_CONFIGURATION ${gflags_FOUND}) 349 | 350 | # gflags v2.1 - 2.1.2 shipped with a bug in their gflags-config.cmake [1] 351 | # whereby gflags_LIBRARIES = "gflags", but there was no imported target 352 | # called "gflags", they were called: gflags[_nothreads]-[static/shared]. 353 | # As this causes linker errors when gflags is not installed in a location 354 | # on the current library paths, detect if this problem is present and 355 | # fix it. 356 | # 357 | # [1] https://github.com/gflags/gflags/issues/110 358 | if (gflags_FOUND) 359 | # NOTE: This is not written as additional conditions in the outer 360 | # if (gflags_FOUND) as the NOT TARGET "${gflags_LIBRARIES}" 361 | # condition causes problems if gflags is not found. 362 | if (${gflags_VERSION} VERSION_LESS 2.1.3 AND 363 | NOT TARGET "${gflags_LIBRARIES}") 364 | message(STATUS "Detected broken gflags install in: ${gflags_DIR}, " 365 | "version: ${gflags_VERSION} <= 2.1.2 which defines gflags_LIBRARIES = " 366 | "${gflags_LIBRARIES} which is not an imported CMake target, see: " 367 | "https://github.com/gflags/gflags/issues/110. Attempting to fix by " 368 | "detecting correct gflags target.") 369 | # Ordering here expresses preference for detection, specifically we do not 370 | # want to use the _nothreads variants if the full library is available. 371 | list(APPEND CHECK_GFLAGS_IMPORTED_TARGET_NAMES 372 | gflags-shared gflags-static 373 | gflags_nothreads-shared gflags_nothreads-static) 374 | foreach(CHECK_GFLAGS_TARGET ${CHECK_GFLAGS_IMPORTED_TARGET_NAMES}) 375 | if (TARGET ${CHECK_GFLAGS_TARGET}) 376 | message(STATUS "Found valid gflags target: ${CHECK_GFLAGS_TARGET}, " 377 | "updating gflags_LIBRARIES.") 378 | set(gflags_LIBRARIES ${CHECK_GFLAGS_TARGET}) 379 | break() 380 | endif() 381 | endforeach() 382 | if (NOT TARGET ${gflags_LIBRARIES}) 383 | message(STATUS "Failed to fix detected broken gflags install in: " 384 | "${gflags_DIR}, version: ${gflags_VERSION} <= 2.1.2, none of the " 385 | "imported targets for gflags: ${CHECK_GFLAGS_IMPORTED_TARGET_NAMES} " 386 | "are defined. Will continue with a manual search for gflags " 387 | "components. We recommend you build/install a version of gflags > " 388 | "2.1.2 (or master).") 389 | set(FOUND_INSTALLED_GFLAGS_CMAKE_CONFIGURATION FALSE) 390 | endif() 391 | endif() 392 | endif() 393 | 394 | if (FOUND_INSTALLED_GFLAGS_CMAKE_CONFIGURATION) 395 | message(STATUS "Detected gflags version: ${gflags_VERSION}") 396 | set(GFLAGS_FOUND ${gflags_FOUND}) 397 | set(GFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR}) 398 | set(GFLAGS_LIBRARY ${gflags_LIBRARIES}) 399 | 400 | # gflags does not export the namespace in their CMake configuration, so 401 | # use our function to determine what it should be, as it can be either 402 | # gflags or google dependent upon version & configuration. 403 | # 404 | # NOTE: We use the regex method to determine the namespace here, as 405 | # check_cxx_source_compiles() will not use imported targets, which 406 | # is what gflags will be in this case. 407 | gflags_check_gflags_namespace_using_regex() 408 | 409 | if (NOT GFLAGS_NAMESPACE) 410 | gflags_report_not_found( 411 | "Failed to determine gflags namespace using regex for gflags " 412 | "version: ${gflags_VERSION} exported here: ${gflags_DIR} using CMake.") 413 | endif (NOT GFLAGS_NAMESPACE) 414 | else (FOUND_INSTALLED_GFLAGS_CMAKE_CONFIGURATION) 415 | message(STATUS "Failed to find an installed/exported CMake configuration " 416 | "for gflags, will perform search for installed gflags components.") 417 | endif (FOUND_INSTALLED_GFLAGS_CMAKE_CONFIGURATION) 418 | endif(GFLAGS_PREFER_EXPORTED_GFLAGS_CMAKE_CONFIGURATION) 419 | 420 | if (NOT GFLAGS_FOUND) 421 | # Either failed to find an exported gflags CMake configuration, or user 422 | # told us not to use one. Perform a manual search for all gflags components. 423 | 424 | # Handle possible presence of lib prefix for libraries on MSVC, see 425 | # also GFLAGS_RESET_FIND_LIBRARY_PREFIX(). 426 | if (MSVC) 427 | # Preserve the caller's original values for CMAKE_FIND_LIBRARY_PREFIXES 428 | # s/t we can set it back before returning. 429 | set(CALLERS_CMAKE_FIND_LIBRARY_PREFIXES "${CMAKE_FIND_LIBRARY_PREFIXES}") 430 | # The empty string in this list is important, it represents the case when 431 | # the libraries have no prefix (shared libraries / DLLs). 432 | set(CMAKE_FIND_LIBRARY_PREFIXES "lib" "" "${CMAKE_FIND_LIBRARY_PREFIXES}") 433 | endif (MSVC) 434 | 435 | # Search user-installed locations first, so that we prefer user installs 436 | # to system installs where both exist. 437 | list(APPEND GFLAGS_CHECK_INCLUDE_DIRS 438 | /usr/local/include 439 | /usr/local/homebrew/include # Mac OS X 440 | /opt/local/var/macports/software # Mac OS X. 441 | /opt/local/include 442 | /usr/include) 443 | list(APPEND GFLAGS_CHECK_PATH_SUFFIXES 444 | gflags/include # Windows (for C:/Program Files prefix). 445 | gflags/Include ) # Windows (for C:/Program Files prefix). 446 | 447 | list(APPEND GFLAGS_CHECK_LIBRARY_DIRS 448 | /usr/local/lib 449 | /usr/local/homebrew/lib # Mac OS X. 450 | /opt/local/lib 451 | /usr/lib) 452 | list(APPEND GFLAGS_CHECK_LIBRARY_SUFFIXES 453 | gflags/lib # Windows (for C:/Program Files prefix). 454 | gflags/Lib ) # Windows (for C:/Program Files prefix). 455 | 456 | # Search supplied hint directories first if supplied. 457 | find_path(GFLAGS_INCLUDE_DIR 458 | NAMES gflags/gflags.h 459 | HINTS ${GFLAGS_INCLUDE_DIR_HINTS} 460 | PATHS ${GFLAGS_CHECK_INCLUDE_DIRS} 461 | PATH_SUFFIXES ${GFLAGS_CHECK_PATH_SUFFIXES}) 462 | if (NOT GFLAGS_INCLUDE_DIR OR 463 | NOT EXISTS ${GFLAGS_INCLUDE_DIR}) 464 | gflags_report_not_found( 465 | "Could not find gflags include directory, set GFLAGS_INCLUDE_DIR " 466 | "to directory containing gflags/gflags.h") 467 | endif (NOT GFLAGS_INCLUDE_DIR OR 468 | NOT EXISTS ${GFLAGS_INCLUDE_DIR}) 469 | 470 | find_library(GFLAGS_LIBRARY NAMES gflags 471 | HINTS ${GFLAGS_LIBRARY_DIR_HINTS} 472 | PATHS ${GFLAGS_CHECK_LIBRARY_DIRS} 473 | PATH_SUFFIXES ${GFLAGS_CHECK_LIBRARY_SUFFIXES}) 474 | if (NOT GFLAGS_LIBRARY OR 475 | NOT EXISTS ${GFLAGS_LIBRARY}) 476 | gflags_report_not_found( 477 | "Could not find gflags library, set GFLAGS_LIBRARY " 478 | "to full path to libgflags.") 479 | endif (NOT GFLAGS_LIBRARY OR 480 | NOT EXISTS ${GFLAGS_LIBRARY}) 481 | 482 | # gflags typically requires a threading library (which is OS dependent), note 483 | # that this defines the CMAKE_THREAD_LIBS_INIT variable. If we are able to 484 | # detect threads, we assume that gflags requires it. 485 | find_package(Threads QUIET) 486 | set(GFLAGS_LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) 487 | # On Windows (including MinGW), the Shlwapi library is used by gflags if 488 | # available. 489 | if (WIN32) 490 | include(CheckIncludeFileCXX) 491 | check_include_file_cxx("shlwapi.h" HAVE_SHLWAPI) 492 | if (HAVE_SHLWAPI) 493 | list(APPEND GFLAGS_LINK_LIBRARIES shlwapi.lib) 494 | endif(HAVE_SHLWAPI) 495 | endif (WIN32) 496 | 497 | # Mark internally as found, then verify. GFLAGS_REPORT_NOT_FOUND() unsets 498 | # if called. 499 | set(GFLAGS_FOUND TRUE) 500 | 501 | # Identify what namespace gflags was built with. 502 | if (GFLAGS_INCLUDE_DIR AND NOT GFLAGS_NAMESPACE) 503 | # To handle Windows peculiarities / CMake bugs on MSVC we try two approaches 504 | # to detect the gflags namespace: 505 | # 506 | # 1) Try to use check_cxx_source_compiles() to compile a trivial program 507 | # with the two choices for the gflags namespace. 508 | # 509 | # 2) [In the event 1) fails] Use regex on the gflags headers to try to 510 | # determine the gflags namespace. Whilst this is less robust than 1), 511 | # it does avoid any interaction with msbuild. 512 | gflags_check_gflags_namespace_using_try_compile() 513 | 514 | if (NOT GFLAGS_NAMESPACE) 515 | # Failed to determine gflags namespace using check_cxx_source_compiles() 516 | # method, try and obtain it using regex on the gflags headers instead. 517 | message(STATUS "Failed to find gflags namespace using using " 518 | "check_cxx_source_compiles(), trying namespace regex instead, " 519 | "this is expected on Windows.") 520 | gflags_check_gflags_namespace_using_regex() 521 | 522 | if (NOT GFLAGS_NAMESPACE) 523 | gflags_report_not_found( 524 | "Failed to determine gflags namespace either by " 525 | "check_cxx_source_compiles(), or namespace regex.") 526 | endif (NOT GFLAGS_NAMESPACE) 527 | endif (NOT GFLAGS_NAMESPACE) 528 | endif (GFLAGS_INCLUDE_DIR AND NOT GFLAGS_NAMESPACE) 529 | 530 | # Make the GFLAGS_NAMESPACE a cache variable s/t the user can view it, and could 531 | # overwrite it in the CMake GUI. 532 | set(GFLAGS_NAMESPACE "${GFLAGS_NAMESPACE}" CACHE STRING 533 | "gflags namespace (google or gflags)" FORCE) 534 | 535 | # gflags does not seem to provide any record of the version in its 536 | # source tree, thus cannot extract version. 537 | 538 | # Catch case when caller has set GFLAGS_NAMESPACE in the cache / GUI 539 | # with an invalid value. 540 | if (GFLAGS_NAMESPACE AND 541 | NOT GFLAGS_NAMESPACE STREQUAL "google" AND 542 | NOT GFLAGS_NAMESPACE STREQUAL "gflags") 543 | gflags_report_not_found( 544 | "Caller defined GFLAGS_NAMESPACE:" 545 | " ${GFLAGS_NAMESPACE} is not valid, not google or gflags.") 546 | endif () 547 | # Catch case when caller has set GFLAGS_INCLUDE_DIR in the cache / GUI and 548 | # thus FIND_[PATH/LIBRARY] are not called, but specified locations are 549 | # invalid, otherwise we would report the library as found. 550 | if (GFLAGS_INCLUDE_DIR AND 551 | NOT EXISTS ${GFLAGS_INCLUDE_DIR}/gflags/gflags.h) 552 | gflags_report_not_found( 553 | "Caller defined GFLAGS_INCLUDE_DIR:" 554 | " ${GFLAGS_INCLUDE_DIR} does not contain gflags/gflags.h header.") 555 | endif (GFLAGS_INCLUDE_DIR AND 556 | NOT EXISTS ${GFLAGS_INCLUDE_DIR}/gflags/gflags.h) 557 | # TODO: This regex for gflags library is pretty primitive, we use lowercase 558 | # for comparison to handle Windows using CamelCase library names, could 559 | # this check be better? 560 | string(TOLOWER "${GFLAGS_LIBRARY}" LOWERCASE_GFLAGS_LIBRARY) 561 | if (GFLAGS_LIBRARY AND 562 | NOT "${LOWERCASE_GFLAGS_LIBRARY}" MATCHES ".*gflags[^/]*") 563 | gflags_report_not_found( 564 | "Caller defined GFLAGS_LIBRARY: " 565 | "${GFLAGS_LIBRARY} does not match gflags.") 566 | endif (GFLAGS_LIBRARY AND 567 | NOT "${LOWERCASE_GFLAGS_LIBRARY}" MATCHES ".*gflags[^/]*") 568 | 569 | gflags_reset_find_library_prefix() 570 | 571 | endif(NOT GFLAGS_FOUND) 572 | 573 | # Set standard CMake FindPackage variables if found. 574 | if (GFLAGS_FOUND) 575 | set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) 576 | set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY} ${GFLAGS_LINK_LIBRARIES}) 577 | endif (GFLAGS_FOUND) 578 | 579 | # Handle REQUIRED / QUIET optional arguments. 580 | include(FindPackageHandleStandardArgs) 581 | find_package_handle_standard_args(Gflags DEFAULT_MSG 582 | GFLAGS_INCLUDE_DIRS GFLAGS_LIBRARIES GFLAGS_NAMESPACE) 583 | 584 | # Only mark internal variables as advanced if we found gflags, otherwise 585 | # leave them visible in the standard GUI for the user to set manually. 586 | if (GFLAGS_FOUND) 587 | mark_as_advanced(FORCE GFLAGS_INCLUDE_DIR 588 | GFLAGS_LIBRARY 589 | GFLAGS_NAMESPACE 590 | gflags_DIR) # Autogenerated by find_package(gflags) 591 | endif (GFLAGS_FOUND) 592 | -------------------------------------------------------------------------------- /cmake/FindGlog.cmake: -------------------------------------------------------------------------------- 1 | # Ceres Solver - A fast non-linear least squares minimizer 2 | # Copyright 2015 Google Inc. All rights reserved. 3 | # http://ceres-solver.org/ 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions are met: 7 | # 8 | # * Redistributions of source code must retain the above copyright notice, 9 | # this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # * Neither the name of Google Inc. nor the names of its contributors may be 14 | # used to endorse or promote products derived from this software without 15 | # specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | # POSSIBILITY OF SUCH DAMAGE. 28 | # 29 | # Author: alexs.mac@gmail.com (Alex Stewart) 30 | # 31 | 32 | # FindGlog.cmake - Find Google glog logging library. 33 | # 34 | # This module defines the following variables: 35 | # 36 | # GLOG_FOUND: TRUE iff glog is found. 37 | # GLOG_INCLUDE_DIRS: Include directories for glog. 38 | # GLOG_LIBRARIES: Libraries required to link glog. 39 | # FOUND_INSTALLED_GLOG_CMAKE_CONFIGURATION: True iff the version of glog found 40 | # was built & installed / exported 41 | # as a CMake package. 42 | # 43 | # The following variables control the behaviour of this module: 44 | # 45 | # GLOG_PREFER_EXPORTED_GLOG_CMAKE_CONFIGURATION: TRUE/FALSE, iff TRUE then 46 | # then prefer using an exported CMake configuration 47 | # generated by glog > 0.3.4 over searching for the 48 | # glog components manually. Otherwise (FALSE) 49 | # ignore any exported glog CMake configurations and 50 | # always perform a manual search for the components. 51 | # Default: TRUE iff user does not define this variable 52 | # before we are called, and does NOT specify either 53 | # GLOG_INCLUDE_DIR_HINTS or GLOG_LIBRARY_DIR_HINTS 54 | # otherwise FALSE. 55 | # GLOG_INCLUDE_DIR_HINTS: List of additional directories in which to 56 | # search for glog includes, e.g: /timbuktu/include. 57 | # GLOG_LIBRARY_DIR_HINTS: List of additional directories in which to 58 | # search for glog libraries, e.g: /timbuktu/lib. 59 | # 60 | # The following variables are also defined by this module, but in line with 61 | # CMake recommended FindPackage() module style should NOT be referenced directly 62 | # by callers (use the plural variables detailed above instead). These variables 63 | # do however affect the behaviour of the module via FIND_[PATH/LIBRARY]() which 64 | # are NOT re-called (i.e. search for library is not repeated) if these variables 65 | # are set with valid values _in the CMake cache_. This means that if these 66 | # variables are set directly in the cache, either by the user in the CMake GUI, 67 | # or by the user passing -DVAR=VALUE directives to CMake when called (which 68 | # explicitly defines a cache variable), then they will be used verbatim, 69 | # bypassing the HINTS variables and other hard-coded search locations. 70 | # 71 | # GLOG_INCLUDE_DIR: Include directory for glog, not including the 72 | # include directory of any dependencies. 73 | # GLOG_LIBRARY: glog library, not including the libraries of any 74 | # dependencies. 75 | 76 | # Reset CALLERS_CMAKE_FIND_LIBRARY_PREFIXES to its value when 77 | # FindGlog was invoked. 78 | macro(GLOG_RESET_FIND_LIBRARY_PREFIX) 79 | if (MSVC AND CALLERS_CMAKE_FIND_LIBRARY_PREFIXES) 80 | set(CMAKE_FIND_LIBRARY_PREFIXES "${CALLERS_CMAKE_FIND_LIBRARY_PREFIXES}") 81 | endif() 82 | endmacro(GLOG_RESET_FIND_LIBRARY_PREFIX) 83 | 84 | # Called if we failed to find glog or any of it's required dependencies, 85 | # unsets all public (designed to be used externally) variables and reports 86 | # error message at priority depending upon [REQUIRED/QUIET/] argument. 87 | macro(GLOG_REPORT_NOT_FOUND REASON_MSG) 88 | unset(GLOG_FOUND) 89 | unset(GLOG_INCLUDE_DIRS) 90 | unset(GLOG_LIBRARIES) 91 | # Make results of search visible in the CMake GUI if glog has not 92 | # been found so that user does not have to toggle to advanced view. 93 | mark_as_advanced(CLEAR GLOG_INCLUDE_DIR 94 | GLOG_LIBRARY) 95 | 96 | glog_reset_find_library_prefix() 97 | 98 | # Note _FIND_[REQUIRED/QUIETLY] variables defined by FindPackage() 99 | # use the camelcase library name, not uppercase. 100 | if (Glog_FIND_QUIETLY) 101 | message(STATUS "Failed to find glog - " ${REASON_MSG} ${ARGN}) 102 | elseif (Glog_FIND_REQUIRED) 103 | message(FATAL_ERROR "Failed to find glog - " ${REASON_MSG} ${ARGN}) 104 | else() 105 | # Neither QUIETLY nor REQUIRED, use no priority which emits a message 106 | # but continues configuration and allows generation. 107 | message("-- Failed to find glog - " ${REASON_MSG} ${ARGN}) 108 | endif () 109 | return() 110 | endmacro(GLOG_REPORT_NOT_FOUND) 111 | 112 | # Protect against any alternative find_package scripts for this library having 113 | # been called previously (in a client project) which set GLOG_FOUND, but not 114 | # the other variables we require / set here which could cause the search logic 115 | # here to fail. 116 | unset(GLOG_FOUND) 117 | 118 | # ----------------------------------------------------------------- 119 | # By default, if the user has expressed no preference for using an exported 120 | # glog CMake configuration over performing a search for the installed 121 | # components, and has not specified any hints for the search locations, then 122 | # prefer a glog exported configuration if available. 123 | if (NOT DEFINED GLOG_PREFER_EXPORTED_GLOG_CMAKE_CONFIGURATION 124 | AND NOT GLOG_INCLUDE_DIR_HINTS 125 | AND NOT GLOG_LIBRARY_DIR_HINTS) 126 | message(STATUS "No preference for use of exported glog CMake configuration " 127 | "set, and no hints for include/library directories provided. " 128 | "Defaulting to preferring an installed/exported glog CMake configuration " 129 | "if available.") 130 | set(GLOG_PREFER_EXPORTED_GLOG_CMAKE_CONFIGURATION TRUE) 131 | endif() 132 | 133 | if (GLOG_PREFER_EXPORTED_GLOG_CMAKE_CONFIGURATION) 134 | # Try to find an exported CMake configuration for glog, as generated by 135 | # glog versions > 0.3.4 136 | # 137 | # We search twice, s/t we can invert the ordering of precedence used by 138 | # find_package() for exported package build directories, and installed 139 | # packages (found via CMAKE_SYSTEM_PREFIX_PATH), listed as items 6) and 7) 140 | # respectively in [1]. 141 | # 142 | # By default, exported build directories are (in theory) detected first, and 143 | # this is usually the case on Windows. However, on OS X & Linux, the install 144 | # path (/usr/local) is typically present in the PATH environment variable 145 | # which is checked in item 4) in [1] (i.e. before both of the above, unless 146 | # NO_SYSTEM_ENVIRONMENT_PATH is passed). As such on those OSs installed 147 | # packages are usually detected in preference to exported package build 148 | # directories. 149 | # 150 | # To ensure a more consistent response across all OSs, and as users usually 151 | # want to prefer an installed version of a package over a locally built one 152 | # where both exist (esp. as the exported build directory might be removed 153 | # after installation), we first search with NO_CMAKE_PACKAGE_REGISTRY which 154 | # means any build directories exported by the user are ignored, and thus 155 | # installed directories are preferred. If this fails to find the package 156 | # we then research again, but without NO_CMAKE_PACKAGE_REGISTRY, so any 157 | # exported build directories will now be detected. 158 | # 159 | # To prevent confusion on Windows, we also pass NO_CMAKE_BUILDS_PATH (which 160 | # is item 5) in [1]), to not preferentially use projects that were built 161 | # recently with the CMake GUI to ensure that we always prefer an installed 162 | # version if available. 163 | # 164 | # NOTE: We use the NAMES option as glog erroneously uses 'google-glog' as its 165 | # project name when built with CMake, but exports itself as just 'glog'. 166 | # On Linux/OS X this does not break detection as the project name is 167 | # not used as part of the install path for the CMake package files, 168 | # e.g. /usr/local/lib/cmake/glog, where the suffix is hardcoded 169 | # in glog's CMakeLists. However, on Windows the project name *is* 170 | # part of the install prefix: C:/Program Files/google-glog/[include,lib]. 171 | # However, by default CMake checks: 172 | # C:/Program Files/ which does not 173 | # exist and thus detection fails. Thus we use the NAMES to force the 174 | # search to use both google-glog & glog. 175 | # 176 | # [1] http://www.cmake.org/cmake/help/v2.8.11/cmake.html#command:find_package 177 | find_package(glog QUIET 178 | NAMES google-glog glog 179 | NO_MODULE 180 | NO_CMAKE_PACKAGE_REGISTRY 181 | NO_CMAKE_BUILDS_PATH) 182 | if (glog_FOUND) 183 | message(STATUS "Found installed version of glog: ${glog_DIR}") 184 | else() 185 | # Failed to find an installed version of glog, repeat search allowing 186 | # exported build directories. 187 | message(STATUS "Failed to find installed glog CMake configuration, " 188 | "searching for glog build directories exported with CMake.") 189 | # Again pass NO_CMAKE_BUILDS_PATH, as we know that glog is exported and 190 | # do not want to treat projects built with the CMake GUI preferentially. 191 | find_package(glog QUIET 192 | NAMES google-glog glog 193 | NO_MODULE 194 | NO_CMAKE_BUILDS_PATH) 195 | if (glog_FOUND) 196 | message(STATUS "Found exported glog build directory: ${glog_DIR}") 197 | endif(glog_FOUND) 198 | endif(glog_FOUND) 199 | 200 | set(FOUND_INSTALLED_GLOG_CMAKE_CONFIGURATION ${glog_FOUND}) 201 | 202 | if (FOUND_INSTALLED_GLOG_CMAKE_CONFIGURATION) 203 | message(STATUS "Detected glog version: ${glog_VERSION}") 204 | set(GLOG_FOUND ${glog_FOUND}) 205 | # glog wraps the include directories into the exported glog::glog target. 206 | set(GLOG_INCLUDE_DIR "") 207 | set(GLOG_LIBRARY glog::glog) 208 | else (FOUND_INSTALLED_GLOG_CMAKE_CONFIGURATION) 209 | message(STATUS "Failed to find an installed/exported CMake configuration " 210 | "for glog, will perform search for installed glog components.") 211 | endif (FOUND_INSTALLED_GLOG_CMAKE_CONFIGURATION) 212 | endif(GLOG_PREFER_EXPORTED_GLOG_CMAKE_CONFIGURATION) 213 | 214 | if (NOT GLOG_FOUND) 215 | # Either failed to find an exported glog CMake configuration, or user 216 | # told us not to use one. Perform a manual search for all glog components. 217 | 218 | # Handle possible presence of lib prefix for libraries on MSVC, see 219 | # also GLOG_RESET_FIND_LIBRARY_PREFIX(). 220 | if (MSVC) 221 | # Preserve the caller's original values for CMAKE_FIND_LIBRARY_PREFIXES 222 | # s/t we can set it back before returning. 223 | set(CALLERS_CMAKE_FIND_LIBRARY_PREFIXES "${CMAKE_FIND_LIBRARY_PREFIXES}") 224 | # The empty string in this list is important, it represents the case when 225 | # the libraries have no prefix (shared libraries / DLLs). 226 | set(CMAKE_FIND_LIBRARY_PREFIXES "lib" "" "${CMAKE_FIND_LIBRARY_PREFIXES}") 227 | endif (MSVC) 228 | 229 | # Search user-installed locations first, so that we prefer user installs 230 | # to system installs where both exist. 231 | list(APPEND GLOG_CHECK_INCLUDE_DIRS 232 | /usr/local/include 233 | /usr/local/homebrew/include # Mac OS X 234 | /opt/local/var/macports/software # Mac OS X. 235 | /opt/local/include 236 | /usr/include) 237 | # Windows (for C:/Program Files prefix). 238 | list(APPEND GLOG_CHECK_PATH_SUFFIXES 239 | glog/include 240 | glog/Include 241 | Glog/include 242 | Glog/Include 243 | google-glog/include # CMake installs with project name prefix. 244 | google-glog/Include) 245 | 246 | list(APPEND GLOG_CHECK_LIBRARY_DIRS 247 | /usr/local/lib 248 | /usr/local/homebrew/lib # Mac OS X. 249 | /opt/local/lib 250 | /usr/lib) 251 | # Windows (for C:/Program Files prefix). 252 | list(APPEND GLOG_CHECK_LIBRARY_SUFFIXES 253 | glog/lib 254 | glog/Lib 255 | Glog/lib 256 | Glog/Lib 257 | google-glog/lib # CMake installs with project name prefix. 258 | google-glog/Lib) 259 | 260 | # Search supplied hint directories first if supplied. 261 | find_path(GLOG_INCLUDE_DIR 262 | NAMES glog/logging.h 263 | HINTS ${GLOG_INCLUDE_DIR_HINTS} 264 | PATHS ${GLOG_CHECK_INCLUDE_DIRS} 265 | PATH_SUFFIXES ${GLOG_CHECK_PATH_SUFFIXES}) 266 | if (NOT GLOG_INCLUDE_DIR OR 267 | NOT EXISTS ${GLOG_INCLUDE_DIR}) 268 | glog_report_not_found( 269 | "Could not find glog include directory, set GLOG_INCLUDE_DIR " 270 | "to directory containing glog/logging.h") 271 | endif (NOT GLOG_INCLUDE_DIR OR 272 | NOT EXISTS ${GLOG_INCLUDE_DIR}) 273 | 274 | find_library(GLOG_LIBRARY NAMES glog 275 | HINTS ${GLOG_LIBRARY_DIR_HINTS} 276 | PATHS ${GLOG_CHECK_LIBRARY_DIRS} 277 | PATH_SUFFIXES ${GLOG_CHECK_LIBRARY_SUFFIXES}) 278 | if (NOT GLOG_LIBRARY OR 279 | NOT EXISTS ${GLOG_LIBRARY}) 280 | glog_report_not_found( 281 | "Could not find glog library, set GLOG_LIBRARY " 282 | "to full path to libglog.") 283 | endif (NOT GLOG_LIBRARY OR 284 | NOT EXISTS ${GLOG_LIBRARY}) 285 | 286 | # Mark internally as found, then verify. GLOG_REPORT_NOT_FOUND() unsets 287 | # if called. 288 | set(GLOG_FOUND TRUE) 289 | 290 | # Glog does not seem to provide any record of the version in its 291 | # source tree, thus cannot extract version. 292 | 293 | # Catch case when caller has set GLOG_INCLUDE_DIR in the cache / GUI and 294 | # thus FIND_[PATH/LIBRARY] are not called, but specified locations are 295 | # invalid, otherwise we would report the library as found. 296 | if (GLOG_INCLUDE_DIR AND 297 | NOT EXISTS ${GLOG_INCLUDE_DIR}/glog/logging.h) 298 | glog_report_not_found( 299 | "Caller defined GLOG_INCLUDE_DIR:" 300 | " ${GLOG_INCLUDE_DIR} does not contain glog/logging.h header.") 301 | endif (GLOG_INCLUDE_DIR AND 302 | NOT EXISTS ${GLOG_INCLUDE_DIR}/glog/logging.h) 303 | # TODO: This regex for glog library is pretty primitive, we use lowercase 304 | # for comparison to handle Windows using CamelCase library names, could 305 | # this check be better? 306 | string(TOLOWER "${GLOG_LIBRARY}" LOWERCASE_GLOG_LIBRARY) 307 | if (GLOG_LIBRARY AND 308 | NOT "${LOWERCASE_GLOG_LIBRARY}" MATCHES ".*glog[^/]*") 309 | glog_report_not_found( 310 | "Caller defined GLOG_LIBRARY: " 311 | "${GLOG_LIBRARY} does not match glog.") 312 | endif (GLOG_LIBRARY AND 313 | NOT "${LOWERCASE_GLOG_LIBRARY}" MATCHES ".*glog[^/]*") 314 | 315 | glog_reset_find_library_prefix() 316 | 317 | endif(NOT GLOG_FOUND) 318 | 319 | # Set standard CMake FindPackage variables if found. 320 | if (GLOG_FOUND) 321 | set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) 322 | set(GLOG_LIBRARIES ${GLOG_LIBRARY}) 323 | endif (GLOG_FOUND) 324 | 325 | # If we are using an exported CMake glog target, the include directories are 326 | # wrapped into the target itself, and do not have to be (and are not) 327 | # separately specified. In which case, we should not add GLOG_INCLUDE_DIRS 328 | # to the list of required variables in order that glog be reported as found. 329 | if (FOUND_INSTALLED_GLOG_CMAKE_CONFIGURATION) 330 | set(GLOG_REQUIRED_VARIABLES GLOG_LIBRARIES) 331 | else() 332 | set(GLOG_REQUIRED_VARIABLES GLOG_INCLUDE_DIRS GLOG_LIBRARIES) 333 | endif() 334 | 335 | # Handle REQUIRED / QUIET optional arguments. 336 | include(FindPackageHandleStandardArgs) 337 | find_package_handle_standard_args(Glog DEFAULT_MSG 338 | ${GLOG_REQUIRED_VARIABLES}) 339 | 340 | # Only mark internal variables as advanced if we found glog, otherwise 341 | # leave them visible in the standard GUI for the user to set manually. 342 | if (GLOG_FOUND) 343 | mark_as_advanced(FORCE GLOG_INCLUDE_DIR 344 | GLOG_LIBRARY 345 | glog_DIR) # Autogenerated by find_package(glog) 346 | endif (GLOG_FOUND) 347 | -------------------------------------------------------------------------------- /colmap2mvsnet_acm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Copyright 2019, Jingyang Zhang and Yao Yao, HKUST. Model reading is provided by COLMAP. 4 | Preprocess script. 5 | View selection is modified according to COLMAP's strategy, Qingshan Xu 6 | """ 7 | 8 | from __future__ import print_function 9 | import collections 10 | import struct 11 | import numpy as np 12 | import multiprocessing as mp 13 | from functools import partial 14 | import os 15 | import argparse 16 | import shutil 17 | import cv2 18 | 19 | #============================ read_model.py ============================# 20 | CameraModel = collections.namedtuple( 21 | "CameraModel", ["model_id", "model_name", "num_params"]) 22 | Camera = collections.namedtuple( 23 | "Camera", ["id", "model", "width", "height", "params"]) 24 | BaseImage = collections.namedtuple( 25 | "Image", ["id", "qvec", "tvec", "camera_id", "name", "xys", "point3D_ids"]) 26 | Point3D = collections.namedtuple( 27 | "Point3D", ["id", "xyz", "rgb", "error", "image_ids", "point2D_idxs"]) 28 | 29 | class Image(BaseImage): 30 | def qvec2rotmat(self): 31 | return qvec2rotmat(self.qvec) 32 | 33 | CAMERA_MODELS = { 34 | CameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3), 35 | CameraModel(model_id=1, model_name="PINHOLE", num_params=4), 36 | CameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4), 37 | CameraModel(model_id=3, model_name="RADIAL", num_params=5), 38 | CameraModel(model_id=4, model_name="OPENCV", num_params=8), 39 | CameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8), 40 | CameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12), 41 | CameraModel(model_id=7, model_name="FOV", num_params=5), 42 | CameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4), 43 | CameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5), 44 | CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12) 45 | } 46 | CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) \ 47 | for camera_model in CAMERA_MODELS]) 48 | 49 | 50 | def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"): 51 | """Read and unpack the next bytes from a binary file. 52 | :param fid: 53 | :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. 54 | :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. 55 | :param endian_character: Any of {@, =, <, >, !} 56 | :return: Tuple of read and unpacked values. 57 | """ 58 | data = fid.read(num_bytes) 59 | return struct.unpack(endian_character + format_char_sequence, data) 60 | 61 | 62 | def read_cameras_text(path): 63 | """ 64 | see: src/base/reconstruction.cc 65 | void Reconstruction::WriteCamerasText(const std::string& path) 66 | void Reconstruction::ReadCamerasText(const std::string& path) 67 | """ 68 | cameras = {} 69 | with open(path, "r") as fid: 70 | while True: 71 | line = fid.readline() 72 | if not line: 73 | break 74 | line = line.strip() 75 | if len(line) > 0 and line[0] != "#": 76 | elems = line.split() 77 | camera_id = int(elems[0]) 78 | model = elems[1] 79 | width = int(elems[2]) 80 | height = int(elems[3]) 81 | params = np.array(tuple(map(float, elems[4:]))) 82 | cameras[camera_id] = Camera(id=camera_id, model=model, 83 | width=width, height=height, 84 | params=params) 85 | return cameras 86 | 87 | 88 | def read_cameras_binary(path_to_model_file): 89 | """ 90 | see: src/base/reconstruction.cc 91 | void Reconstruction::WriteCamerasBinary(const std::string& path) 92 | void Reconstruction::ReadCamerasBinary(const std::string& path) 93 | """ 94 | cameras = {} 95 | with open(path_to_model_file, "rb") as fid: 96 | num_cameras = read_next_bytes(fid, 8, "Q")[0] 97 | for camera_line_index in range(num_cameras): 98 | camera_properties = read_next_bytes( 99 | fid, num_bytes=24, format_char_sequence="iiQQ") 100 | camera_id = camera_properties[0] 101 | model_id = camera_properties[1] 102 | model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name 103 | width = camera_properties[2] 104 | height = camera_properties[3] 105 | num_params = CAMERA_MODEL_IDS[model_id].num_params 106 | params = read_next_bytes(fid, num_bytes=8*num_params, 107 | format_char_sequence="d"*num_params) 108 | cameras[camera_id] = Camera(id=camera_id, 109 | model=model_name, 110 | width=width, 111 | height=height, 112 | params=np.array(params)) 113 | assert len(cameras) == num_cameras 114 | return cameras 115 | 116 | 117 | def read_images_text(path): 118 | """ 119 | see: src/base/reconstruction.cc 120 | void Reconstruction::ReadImagesText(const std::string& path) 121 | void Reconstruction::WriteImagesText(const std::string& path) 122 | """ 123 | images = {} 124 | with open(path, "r") as fid: 125 | while True: 126 | line = fid.readline() 127 | if not line: 128 | break 129 | line = line.strip() 130 | if len(line) > 0 and line[0] != "#": 131 | elems = line.split() 132 | image_id = int(elems[0]) 133 | qvec = np.array(tuple(map(float, elems[1:5]))) 134 | tvec = np.array(tuple(map(float, elems[5:8]))) 135 | camera_id = int(elems[8]) 136 | image_name = elems[9] 137 | elems = fid.readline().split() 138 | xys = np.column_stack([tuple(map(float, elems[0::3])), 139 | tuple(map(float, elems[1::3]))]) 140 | point3D_ids = np.array(tuple(map(int, elems[2::3]))) 141 | images[image_id] = Image( 142 | id=image_id, qvec=qvec, tvec=tvec, 143 | camera_id=camera_id, name=image_name, 144 | xys=xys, point3D_ids=point3D_ids) 145 | return images 146 | 147 | 148 | def read_images_binary(path_to_model_file): 149 | """ 150 | see: src/base/reconstruction.cc 151 | void Reconstruction::ReadImagesBinary(const std::string& path) 152 | void Reconstruction::WriteImagesBinary(const std::string& path) 153 | """ 154 | images = {} 155 | with open(path_to_model_file, "rb") as fid: 156 | num_reg_images = read_next_bytes(fid, 8, "Q")[0] 157 | for image_index in range(num_reg_images): 158 | binary_image_properties = read_next_bytes( 159 | fid, num_bytes=64, format_char_sequence="idddddddi") 160 | image_id = binary_image_properties[0] 161 | qvec = np.array(binary_image_properties[1:5]) 162 | tvec = np.array(binary_image_properties[5:8]) 163 | camera_id = binary_image_properties[8] 164 | image_name = "" 165 | current_char = read_next_bytes(fid, 1, "c")[0] 166 | while current_char != b"\x00": # look for the ASCII 0 entry 167 | image_name += current_char.decode("utf-8") 168 | current_char = read_next_bytes(fid, 1, "c")[0] 169 | num_points2D = read_next_bytes(fid, num_bytes=8, 170 | format_char_sequence="Q")[0] 171 | x_y_id_s = read_next_bytes(fid, num_bytes=24*num_points2D, 172 | format_char_sequence="ddq"*num_points2D) 173 | xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])), 174 | tuple(map(float, x_y_id_s[1::3]))]) 175 | point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3]))) 176 | images[image_id] = Image( 177 | id=image_id, qvec=qvec, tvec=tvec, 178 | camera_id=camera_id, name=image_name, 179 | xys=xys, point3D_ids=point3D_ids) 180 | return images 181 | 182 | 183 | def read_points3D_text(path): 184 | """ 185 | see: src/base/reconstruction.cc 186 | void Reconstruction::ReadPoints3DText(const std::string& path) 187 | void Reconstruction::WritePoints3DText(const std::string& path) 188 | """ 189 | points3D = {} 190 | with open(path, "r") as fid: 191 | while True: 192 | line = fid.readline() 193 | if not line: 194 | break 195 | line = line.strip() 196 | if len(line) > 0 and line[0] != "#": 197 | elems = line.split() 198 | point3D_id = int(elems[0]) 199 | xyz = np.array(tuple(map(float, elems[1:4]))) 200 | rgb = np.array(tuple(map(int, elems[4:7]))) 201 | error = float(elems[7]) 202 | image_ids = np.array(tuple(map(int, elems[8::2]))) 203 | point2D_idxs = np.array(tuple(map(int, elems[9::2]))) 204 | points3D[point3D_id] = Point3D(id=point3D_id, xyz=xyz, rgb=rgb, 205 | error=error, image_ids=image_ids, 206 | point2D_idxs=point2D_idxs) 207 | return points3D 208 | 209 | 210 | def read_points3d_binary(path_to_model_file): 211 | """ 212 | see: src/base/reconstruction.cc 213 | void Reconstruction::ReadPoints3DBinary(const std::string& path) 214 | void Reconstruction::WritePoints3DBinary(const std::string& path) 215 | """ 216 | points3D = {} 217 | with open(path_to_model_file, "rb") as fid: 218 | num_points = read_next_bytes(fid, 8, "Q")[0] 219 | for point_line_index in range(num_points): 220 | binary_point_line_properties = read_next_bytes( 221 | fid, num_bytes=43, format_char_sequence="QdddBBBd") 222 | point3D_id = binary_point_line_properties[0] 223 | xyz = np.array(binary_point_line_properties[1:4]) 224 | rgb = np.array(binary_point_line_properties[4:7]) 225 | error = np.array(binary_point_line_properties[7]) 226 | track_length = read_next_bytes( 227 | fid, num_bytes=8, format_char_sequence="Q")[0] 228 | track_elems = read_next_bytes( 229 | fid, num_bytes=8*track_length, 230 | format_char_sequence="ii"*track_length) 231 | image_ids = np.array(tuple(map(int, track_elems[0::2]))) 232 | point2D_idxs = np.array(tuple(map(int, track_elems[1::2]))) 233 | points3D[point3D_id] = Point3D( 234 | id=point3D_id, xyz=xyz, rgb=rgb, 235 | error=error, image_ids=image_ids, 236 | point2D_idxs=point2D_idxs) 237 | return points3D 238 | 239 | 240 | def read_model(path, ext): 241 | if ext == ".txt": 242 | cameras = read_cameras_text(os.path.join(path, "cameras" + ext)) 243 | images = read_images_text(os.path.join(path, "images" + ext)) 244 | points3D = read_points3D_text(os.path.join(path, "points3D") + ext) 245 | else: 246 | cameras = read_cameras_binary(os.path.join(path, "cameras" + ext)) 247 | images = read_images_binary(os.path.join(path, "images" + ext)) 248 | points3D = read_points3d_binary(os.path.join(path, "points3D") + ext) 249 | return cameras, images, points3D 250 | 251 | 252 | def qvec2rotmat(qvec): 253 | return np.array([ 254 | [1 - 2 * qvec[2]**2 - 2 * qvec[3]**2, 255 | 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], 256 | 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2]], 257 | [2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], 258 | 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2, 259 | 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1]], 260 | [2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], 261 | 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], 262 | 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2]]) 263 | 264 | 265 | def rotmat2qvec(R): 266 | Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat 267 | K = np.array([ 268 | [Rxx - Ryy - Rzz, 0, 0, 0], 269 | [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0], 270 | [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0], 271 | [Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0 272 | eigvals, eigvecs = np.linalg.eigh(K) 273 | qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)] 274 | if qvec[0] < 0: 275 | qvec *= -1 276 | return qvec 277 | 278 | 279 | 280 | def calc_score(inputs, images, points3d, extrinsic, args): 281 | i, j = inputs 282 | id_i = images[i+1].point3D_ids 283 | id_j = images[j+1].point3D_ids 284 | id_intersect = [it for it in id_i if it in id_j] 285 | cam_center_i = -np.matmul(extrinsic[i+1][:3, :3].transpose(), extrinsic[i+1][:3, 3:4])[:, 0] 286 | cam_center_j = -np.matmul(extrinsic[j+1][:3, :3].transpose(), extrinsic[j+1][:3, 3:4])[:, 0] 287 | score = 0 288 | angles = [] 289 | for pid in id_intersect: 290 | if pid == -1: 291 | continue 292 | p = points3d[pid].xyz 293 | theta = (180 / np.pi) * np.arccos(np.dot(cam_center_i - p, cam_center_j - p) / np.linalg.norm(cam_center_i - p) / np.linalg.norm(cam_center_j - p)) # triangulation angle 294 | # score += np.exp(-(theta - args.theta0) * (theta - args.theta0) / (2 * (args.sigma1 if theta <= args.theta0 else args.sigma2) ** 2)) 295 | angles.append(theta) 296 | score += 1 297 | if len(angles) > 0: 298 | angles_sorted = sorted(angles) 299 | triangulationangle = angles_sorted[int(len(angles_sorted) * 0.75)] 300 | if triangulationangle < 1: 301 | score = 0.0 302 | return i, j, score 303 | 304 | def processing_single_scene(args): 305 | 306 | image_dir = os.path.join(args.dense_folder, 'images') 307 | model_dir = os.path.join(args.dense_folder, 'sparse') 308 | cam_dir = os.path.join(args.save_folder, 'cams') 309 | image_converted_dir = os.path.join(args.save_folder, 'images') 310 | 311 | if os.path.exists(image_converted_dir): 312 | print("remove:{}".format(image_converted_dir)) 313 | shutil.rmtree(image_converted_dir) 314 | os.makedirs(image_converted_dir) 315 | if os.path.exists(cam_dir): 316 | print("remove:{}".format(cam_dir)) 317 | shutil.rmtree(cam_dir) 318 | 319 | cameras, images, points3d = read_model(model_dir, args.model_ext) 320 | num_images = len(list(images.items())) 321 | 322 | param_type = { 323 | 'SIMPLE_PINHOLE': ['f', 'cx', 'cy'], 324 | 'PINHOLE': ['fx', 'fy', 'cx', 'cy'], 325 | 'SIMPLE_RADIAL': ['f', 'cx', 'cy', 'k'], 326 | 'SIMPLE_RADIAL_FISHEYE': ['f', 'cx', 'cy', 'k'], 327 | 'RADIAL': ['f', 'cx', 'cy', 'k1', 'k2'], 328 | 'RADIAL_FISHEYE': ['f', 'cx', 'cy', 'k1', 'k2'], 329 | 'OPENCV': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2'], 330 | 'OPENCV_FISHEYE': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'k3', 'k4'], 331 | 'FULL_OPENCV': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2', 'k3', 'k4', 'k5', 'k6'], 332 | 'FOV': ['fx', 'fy', 'cx', 'cy', 'omega'], 333 | 'THIN_PRISM_FISHEYE': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2', 'k3', 'k4', 'sx1', 'sy1'] 334 | } 335 | 336 | # intrinsic 337 | intrinsic = {} 338 | for camera_id, cam in cameras.items(): 339 | params_dict = {key: value for key, value in zip(param_type[cam.model], cam.params)} 340 | if 'f' in param_type[cam.model]: 341 | params_dict['fx'] = params_dict['f'] 342 | params_dict['fy'] = params_dict['f'] 343 | i = np.array([ 344 | [params_dict['fx'], 0, params_dict['cx']], 345 | [0, params_dict['fy'], params_dict['cy']], 346 | [0, 0, 1] 347 | ]) 348 | intrinsic[camera_id] = i 349 | print('intrinsic\n', intrinsic, end='\n\n') 350 | 351 | new_images = {} 352 | for i, image_id in enumerate(sorted(images.keys())): 353 | new_images[i+1] = images[image_id] 354 | images = new_images 355 | 356 | # extrinsic 357 | extrinsic = {} 358 | for image_id, image in images.items(): 359 | e = np.zeros((4, 4)) 360 | e[:3, :3] = qvec2rotmat(image.qvec) 361 | e[:3, 3] = image.tvec 362 | e[3, 3] = 1 363 | extrinsic[image_id] = e 364 | print('extrinsic[1]\n', extrinsic[1], end='\n\n') 365 | 366 | # depth range and interval 367 | depth_ranges = {} 368 | for i in range(num_images): 369 | zs = [] 370 | for p3d_id in images[i+1].point3D_ids: 371 | if p3d_id == -1: 372 | continue 373 | transformed = np.matmul(extrinsic[i+1], [points3d[p3d_id].xyz[0], points3d[p3d_id].xyz[1], points3d[p3d_id].xyz[2], 1]) 374 | zs.append(np.asscalar(transformed[2])) 375 | 376 | if(len(zs) < 10): 377 | depth_min = 0.01 378 | depth_max = 100 379 | depth_num = 192 if args.max_d == 0 else args.max_d 380 | depth_interval = (depth_max - depth_min) / (depth_num - 1) / args.interval_scale 381 | depth_ranges[i+1] = (depth_min, depth_interval, depth_num, depth_max) 382 | print('img ' + str(i) + ' has no point! use default parameters: %f %f %f %f' % (depth_min, depth_interval, depth_num, depth_max)) 383 | continue 384 | 385 | zs_sorted = sorted(zs) 386 | # relaxed depth range 387 | depth_min = zs_sorted[int(len(zs) * .01)] * 0.75 388 | depth_max = zs_sorted[int(len(zs) * .99)] * 1.25 389 | # determine depth number by inverse depth setting, see supplementary material 390 | if args.max_d == 0: 391 | image_int = intrinsic[images[i+1].camera_id] 392 | image_ext = extrinsic[i+1] 393 | image_r = image_ext[0:3, 0:3] 394 | image_t = image_ext[0:3, 3] 395 | p1 = [image_int[0, 2], image_int[1, 2], 1] 396 | p2 = [image_int[0, 2] + 1, image_int[1, 2], 1] 397 | P1 = np.matmul(np.linalg.inv(image_int), p1) * depth_min 398 | P1 = np.matmul(np.linalg.inv(image_r), (P1 - image_t)) 399 | P2 = np.matmul(np.linalg.inv(image_int), p2) * depth_min 400 | P2 = np.matmul(np.linalg.inv(image_r), (P2 - image_t)) 401 | depth_num = (1 / depth_min - 1 / depth_max) / (1 / depth_min - 1 / (depth_min + np.linalg.norm(P2 - P1))) 402 | else: 403 | depth_num = args.max_d 404 | depth_interval = (depth_max - depth_min) / (depth_num - 1) / args.interval_scale 405 | depth_ranges[i+1] = (depth_min, depth_interval, depth_num, depth_max) 406 | print('depth_ranges[1]\n', depth_ranges[1], end='\n\n') 407 | 408 | # view selection 409 | score = np.zeros((len(images), len(images))) 410 | queue = [] 411 | for i in range(len(images)): 412 | for j in range(i + 1, len(images)): 413 | queue.append((i, j)) 414 | 415 | p = mp.Pool(processes=mp.cpu_count()) 416 | func = partial(calc_score, images=images, points3d=points3d, args=args, extrinsic=extrinsic) 417 | result = p.map(func, queue) 418 | for i, j, s in result: 419 | score[i, j] = s 420 | score[j, i] = s 421 | view_sel = [] 422 | num_view = min(20, len(images) - 1) 423 | for i in range(len(images)): 424 | sorted_score = np.argsort(score[i])[::-1] 425 | view_sel.append([(k, score[i, k]) for k in sorted_score[:num_view]]) 426 | print('view_sel[0]\n', view_sel[0], end='\n\n') 427 | 428 | # write 429 | try: 430 | os.makedirs(cam_dir) 431 | except os.error: 432 | print(cam_dir + ' already exist.') 433 | for i in range(num_images): 434 | with open(os.path.join(cam_dir, '%08d_cam.txt' % i), 'w') as f: 435 | f.write('extrinsic\n') 436 | for j in range(4): 437 | for k in range(4): 438 | f.write(str(extrinsic[i+1][j, k]) + ' ') 439 | f.write('\n') 440 | f.write('\nintrinsic\n') 441 | for j in range(3): 442 | for k in range(3): 443 | f.write(str(intrinsic[images[i+1].camera_id][j, k]) + ' ') 444 | f.write('\n') 445 | f.write('\n%f %f %f %f\n' % (depth_ranges[i+1][0], depth_ranges[i+1][1], depth_ranges[i+1][2], depth_ranges[i+1][3])) 446 | with open(os.path.join(args.save_folder, 'pair.txt'), 'w') as f: 447 | f.write('%d\n' % len(images)) 448 | for i, sorted_score in enumerate(view_sel): 449 | f.write('%d\n%d ' % (i, len(sorted_score))) 450 | for image_id, s in sorted_score: 451 | f.write('%d %d ' % (image_id, s)) 452 | f.write('\n') 453 | 454 | #convert to jpg 455 | for i in range(num_images): 456 | img_path = os.path.join(image_dir, images[i + 1].name) 457 | if not img_path.endswith(".jpg"): 458 | img = cv2.imread(img_path) 459 | cv2.imwrite(os.path.join(image_converted_dir, '%08d.jpg' % i), img) 460 | else: 461 | shutil.copyfile(os.path.join(image_dir, images[i+1].name), os.path.join(image_converted_dir, '%08d.jpg' % i)) 462 | 463 | 464 | if __name__ == '__main__': 465 | parser = argparse.ArgumentParser(description='Convert colmap camera') 466 | 467 | parser.add_argument('--dense_folder', required=True, type=str, help='dense_folder.') 468 | parser.add_argument('--save_folder', required=True, type=str, help='save_folder.') 469 | 470 | parser.add_argument('--max_d', type=int, default=192) 471 | parser.add_argument('--interval_scale', type=float, default=1) 472 | 473 | parser.add_argument('--theta0', type=float, default=5) 474 | parser.add_argument('--sigma1', type=float, default=1) 475 | parser.add_argument('--sigma2', type=float, default=10) 476 | parser.add_argument('--model_ext', type=str, default=".txt", choices=[".txt", ".bin"], help='sparse model ext') 477 | 478 | args = parser.parse_args() 479 | 480 | os.makedirs(os.path.join(args.save_folder), exist_ok=True) 481 | processing_single_scene(args) 482 | -------------------------------------------------------------------------------- /exe/main_depth_estimation.cpp: -------------------------------------------------------------------------------- 1 | #include "colmap_interface/colmap_interface.h" 2 | #include "ACMP.h" 3 | 4 | void CreateProblems(const Model& model, std::vector& problems) { 5 | for (const auto& pair : model.covis_vec) { 6 | Problem problem; 7 | problem.src_image_ids.clear(); 8 | 9 | problem.ref_image_id = pair.first; 10 | for (const int id : pair.second) { 11 | problem.src_image_ids.push_back(id); 12 | } 13 | problems.push_back(problem); 14 | } 15 | } 16 | 17 | void ProcessProblem(const Problem &problem, const Model& model, bool geom_consistency, bool planar_prior, bool multi_geometry = false) 18 | { 19 | std::string dense_folder = model.root_folder; 20 | std::string imageName = model.image_id_to_image_name.at(problem.ref_image_id); 21 | std::cout << "Processing image " << imageName << "..." << std::endl; 22 | cudaSetDevice(0); 23 | 24 | ACMP acmp(model.image_id_to_image_name, model.image_id_to_camera, model.depth_folder, model.normal_folder, model.image_folder); 25 | if (geom_consistency) { 26 | acmp.SetGeomConsistencyParams(multi_geometry); 27 | } 28 | acmp.InuputInitialization(dense_folder, problem); 29 | 30 | acmp.CudaSpaceInitialization(dense_folder, problem); 31 | acmp.RunPatchMatch(); 32 | 33 | const int width = acmp.GetReferenceImageWidth(); 34 | const int height = acmp.GetReferenceImageHeight(); 35 | 36 | cv::Mat_ depths = cv::Mat::zeros(height, width, CV_32FC1); 37 | cv::Mat_ normals = cv::Mat::zeros(height, width, CV_32FC3); 38 | // cv::Mat_ costs = cv::Mat::zeros(height, width, CV_32FC1); 39 | 40 | for (int col = 0; col < width; ++col) { 41 | for (int row = 0; row < height; ++row) { 42 | int center = row * width + col; 43 | float4 plane_hypothesis = acmp.GetPlaneHypothesis(center); 44 | depths(row, col) = plane_hypothesis.w; 45 | normals(row, col) = cv::Vec3f(plane_hypothesis.x, plane_hypothesis.y, plane_hypothesis.z); 46 | // costs(row, col) = acmp.GetCost(center); 47 | } 48 | } 49 | 50 | if (planar_prior) { 51 | std::cout << "Run Planar Prior Assisted PatchMatch MVS ..." << std::endl; 52 | acmp.SetPlanarPriorParams(); 53 | 54 | const cv::Rect imageRC(0, 0, width, height); 55 | std::vector support2DPoints; 56 | 57 | acmp.GetSupportPoints(support2DPoints); 58 | const auto triangles = acmp.DelaunayTriangulation(imageRC, support2DPoints); 59 | cv::Mat refImage = acmp.GetReferenceImage().clone(); 60 | std::vector mbgr(3); 61 | mbgr[0] = refImage.clone(); 62 | mbgr[1] = refImage.clone(); 63 | mbgr[2] = refImage.clone(); 64 | cv::Mat srcImage; 65 | cv::merge(mbgr, srcImage); 66 | for (const auto& triangle : triangles) { 67 | if (imageRC.contains(triangle.pt1) && imageRC.contains(triangle.pt2) && imageRC.contains(triangle.pt3)) { 68 | cv::line(srcImage, triangle.pt1, triangle.pt2, cv::Scalar(0, 0, 255)); 69 | cv::line(srcImage, triangle.pt1, triangle.pt3, cv::Scalar(0, 0, 255)); 70 | cv::line(srcImage, triangle.pt2, triangle.pt3, cv::Scalar(0, 0, 255)); 71 | } 72 | } 73 | std::string triangulation_path = dense_folder + "/" + model.depth_folder + "/" + imageName + ".png"; 74 | cv::imwrite(triangulation_path, srcImage); 75 | 76 | cv::Mat_ mask_tri = cv::Mat::zeros(height, width, CV_32FC1); 77 | std::vector planeParams_tri; 78 | planeParams_tri.clear(); 79 | 80 | uint32_t idx = 0; 81 | for (const auto& triangle : triangles) { 82 | if (imageRC.contains(triangle.pt1) && imageRC.contains(triangle.pt2) && imageRC.contains(triangle.pt3)) { 83 | float L01 = sqrt(pow(triangle.pt1.x - triangle.pt2.x, 2) + pow(triangle.pt1.y - triangle.pt2.y, 2)); 84 | float L02 = sqrt(pow(triangle.pt1.x - triangle.pt3.x, 2) + pow(triangle.pt1.y - triangle.pt3.y, 2)); 85 | float L12 = sqrt(pow(triangle.pt2.x - triangle.pt3.x, 2) + pow(triangle.pt2.y - triangle.pt3.y, 2)); 86 | 87 | float max_edge_length = std::max(L01, std::max(L02, L12)); 88 | float step = 1.0 / max_edge_length; 89 | 90 | for (float p = 0; p < 1.0; p += step) { 91 | for (float q = 0; q < 1.0 - p; q += step) { 92 | int x = p * triangle.pt1.x + q * triangle.pt2.x + (1.0 - p - q) * triangle.pt3.x; 93 | int y = p * triangle.pt1.y + q * triangle.pt2.y + (1.0 - p - q) * triangle.pt3.y; 94 | mask_tri(y, x) = idx + 1.0; // To distinguish from the label of non-triangulated areas 95 | } 96 | } 97 | 98 | // estimate plane parameter 99 | float4 n4 = acmp.GetPriorPlaneParams(triangle, depths); 100 | planeParams_tri.push_back(n4); 101 | idx++; 102 | } 103 | } 104 | 105 | cv::Mat_ priordepths = cv::Mat::zeros(height, width, CV_32FC1); 106 | for (int i = 0; i < width; ++i) { 107 | for (int j = 0; j < height; ++j) { 108 | if (mask_tri(j, i) > 0) { 109 | float d = acmp.GetDepthFromPlaneParam(planeParams_tri[mask_tri(j, i) - 1], i, j); 110 | if (d <= acmp.GetMaxDepth() && d >= acmp.GetMinDepth()) { 111 | priordepths(j, i) = d; 112 | } 113 | else { 114 | mask_tri(j, i) = 0; 115 | } 116 | } 117 | } 118 | } 119 | // std::string depth_path = result_folder + "/depths_prior.dmb"; 120 | // writeDepthDmb(depth_path, priordepths); 121 | 122 | acmp.CudaPlanarPriorInitialization(planeParams_tri, mask_tri); 123 | acmp.RunPatchMatch(); 124 | 125 | for (int col = 0; col < width; ++col) { 126 | for (int row = 0; row < height; ++row) { 127 | int center = row * width + col; 128 | float4 plane_hypothesis = acmp.GetPlaneHypothesis(center); 129 | depths(row, col) = plane_hypothesis.w; 130 | normals(row, col) = cv::Vec3f(plane_hypothesis.x, plane_hypothesis.y, plane_hypothesis.z); 131 | // costs(row, col) = acmp.GetCost(center); 132 | } 133 | } 134 | } 135 | 136 | std::string suffix = ".photometric.bin"; 137 | if (geom_consistency) { 138 | suffix = ".geometric.bin"; 139 | } 140 | std::string depth_path = dense_folder + "/" + model.depth_folder + "/" + imageName + suffix; 141 | std::string normal_path = dense_folder + "/" + model.normal_folder + "/" + imageName + suffix; 142 | WriteMap(depth_path, depths); 143 | WriteMap(normal_path, normals); 144 | std::cout << "Processing image " << imageName << " done!" << std::endl; 145 | } 146 | 147 | int main(int argc, char** argv) 148 | { 149 | double t_start = cv::getTickCount(); 150 | 151 | google::InitGoogleLogging(argv[0]); 152 | FLAGS_alsologtostderr = true; 153 | FLAGS_colorlogtostderr = true; 154 | FLAGS_v = 2; 155 | 156 | if (argc < 2) { 157 | std::cout << "USAGE: ACMP dense_folder" << std::endl; 158 | return -1; 159 | } 160 | 161 | std::string dense_folder = argv[1]; 162 | Model model(dense_folder, "sparse", "stereo/depth_maps", "stereo/normal_maps", "images"); 163 | model.Read(); 164 | model.ReduceMemory(); 165 | 166 | std::vector problems; 167 | CreateProblems(model, problems); 168 | 169 | size_t num_images = problems.size(); 170 | std::cout << "There are " << num_images << " problems needed to be processed!" << std::endl; 171 | 172 | bool geom_consistency = false; 173 | bool planar_prior = true; 174 | for (size_t i = 0; i < num_images; ++i) { 175 | ProcessProblem(problems[i], model, geom_consistency, planar_prior); 176 | } 177 | 178 | bool multi_geometry = false; 179 | int geom_iteration = 2; 180 | geom_consistency = true; 181 | planar_prior = false; 182 | for(int geom_iter = 0; geom_iter < geom_iteration; ++geom_iter) { 183 | if(geom_iter == 0) { 184 | multi_geometry = false; 185 | } else { 186 | multi_geometry = true; 187 | } 188 | for (size_t i = 0; i < num_images; ++i) { 189 | ProcessProblem(problems[i], model, geom_consistency, planar_prior, multi_geometry); 190 | } 191 | } 192 | 193 | double t_end = cv::getTickCount(); 194 | double t_used = (t_end - t_start) / cv::getTickFrequency() / 60; 195 | std::cout << "Total time: " << t_used << " min" << std::endl; 196 | 197 | return 0; 198 | } 199 | -------------------------------------------------------------------------------- /exe/main_depth_fusion.cpp: -------------------------------------------------------------------------------- 1 | #include "colmap_interface/colmap_interface.h" 2 | 3 | void StoreColorPlyFileBinaryPointCloud (const std::string &plyFilePath, const std::vector &pc) 4 | { 5 | std::cout << "store 3D points to ply file" << std::endl; 6 | 7 | FILE *outputPly; 8 | outputPly=fopen(plyFilePath.c_str(), "wb"); 9 | 10 | /*write header*/ 11 | fprintf(outputPly, "ply\n"); 12 | fprintf(outputPly, "format binary_little_endian 1.0\n"); 13 | fprintf(outputPly, "element vertex %lu\n",pc.size()); 14 | fprintf(outputPly, "property float x\n"); 15 | fprintf(outputPly, "property float y\n"); 16 | fprintf(outputPly, "property float z\n"); 17 | fprintf(outputPly, "property float nx\n"); 18 | fprintf(outputPly, "property float ny\n"); 19 | fprintf(outputPly, "property float nz\n"); 20 | fprintf(outputPly, "property uchar red\n"); 21 | fprintf(outputPly, "property uchar green\n"); 22 | fprintf(outputPly, "property uchar blue\n"); 23 | fprintf(outputPly, "end_header\n"); 24 | 25 | //write data 26 | #pragma omp parallel for 27 | for(size_t i = 0; i < pc.size(); i++) { 28 | const PointList &p = pc[i]; 29 | float3 X = p.coord; 30 | const float3 normal = p.normal; 31 | const float3 color = p.color; 32 | const char b_color = (int)color.x; 33 | const char g_color = (int)color.y; 34 | const char r_color = (int)color.z; 35 | 36 | if(!(X.x < FLT_MAX && X.x > -FLT_MAX) || !(X.y < FLT_MAX && X.y > -FLT_MAX) || !(X.z < FLT_MAX && X.z >= -FLT_MAX)){ 37 | X.x = 0.0f; 38 | X.y = 0.0f; 39 | X.z = 0.0f; 40 | } 41 | #pragma omp critical 42 | { 43 | fwrite(&X.x, sizeof(X.x), 1, outputPly); 44 | fwrite(&X.y, sizeof(X.y), 1, outputPly); 45 | fwrite(&X.z, sizeof(X.z), 1, outputPly); 46 | fwrite(&normal.x, sizeof(normal.x), 1, outputPly); 47 | fwrite(&normal.y, sizeof(normal.y), 1, outputPly); 48 | fwrite(&normal.z, sizeof(normal.z), 1, outputPly); 49 | fwrite(&r_color, sizeof(char), 1, outputPly); 50 | fwrite(&g_color, sizeof(char), 1, outputPly); 51 | fwrite(&b_color, sizeof(char), 1, outputPly); 52 | } 53 | 54 | } 55 | fclose(outputPly); 56 | } 57 | 58 | void RescaleImageAndCamera(cv::Mat_ &src, cv::Mat_ &dst, cv::Mat_ &depth, Camera &camera) 59 | { 60 | const int cols = depth.cols; 61 | const int rows = depth.rows; 62 | 63 | if (cols == src.cols && rows == src.rows) { 64 | dst = src.clone(); 65 | return; 66 | } 67 | 68 | const float scale_x = cols / static_cast(src.cols); 69 | const float scale_y = rows / static_cast(src.rows); 70 | 71 | cv::resize(src, dst, cv::Size(cols,rows), 0, 0, cv::INTER_LINEAR); 72 | 73 | camera.K[0] *= scale_x; 74 | camera.K[2] *= scale_x; 75 | camera.K[4] *= scale_y; 76 | camera.K[5] *= scale_y; 77 | camera.width = cols; 78 | camera.height = rows; 79 | } 80 | 81 | float3 Get3DPointonWorld(const int x, const int y, const float depth, const Camera camera) 82 | { 83 | float3 pointX; 84 | float3 tmpX; 85 | // Reprojection 86 | pointX.x = depth * (x - camera.K[2]) / camera.K[0]; 87 | pointX.y = depth * (y - camera.K[5]) / camera.K[4]; 88 | pointX.z = depth; 89 | 90 | // Rotation 91 | tmpX.x = camera.R[0] * pointX.x + camera.R[3] * pointX.y + camera.R[6] * pointX.z; 92 | tmpX.y = camera.R[1] * pointX.x + camera.R[4] * pointX.y + camera.R[7] * pointX.z; 93 | tmpX.z = camera.R[2] * pointX.x + camera.R[5] * pointX.y + camera.R[8] * pointX.z; 94 | 95 | // Transformation 96 | float3 C; 97 | C.x = -(camera.R[0] * camera.t[0] + camera.R[3] * camera.t[1] + camera.R[6] * camera.t[2]); 98 | C.y = -(camera.R[1] * camera.t[0] + camera.R[4] * camera.t[1] + camera.R[7] * camera.t[2]); 99 | C.z = -(camera.R[2] * camera.t[0] + camera.R[5] * camera.t[1] + camera.R[8] * camera.t[2]); 100 | pointX.x = tmpX.x + C.x; 101 | pointX.y = tmpX.y + C.y; 102 | pointX.z = tmpX.z + C.z; 103 | 104 | return pointX; 105 | } 106 | 107 | void ProjectonCamera(const float3 PointX, const Camera camera, float2 &point, float &depth) 108 | { 109 | float3 tmp; 110 | tmp.x = camera.R[0] * PointX.x + camera.R[1] * PointX.y + camera.R[2] * PointX.z + camera.t[0]; 111 | tmp.y = camera.R[3] * PointX.x + camera.R[4] * PointX.y + camera.R[5] * PointX.z + camera.t[1]; 112 | tmp.z = camera.R[6] * PointX.x + camera.R[7] * PointX.y + camera.R[8] * PointX.z + camera.t[2]; 113 | 114 | depth = camera.K[6] * tmp.x + camera.K[7] * tmp.y + camera.K[8] * tmp.z; 115 | point.x = (camera.K[0] * tmp.x + camera.K[1] * tmp.y + camera.K[2] * tmp.z) / depth; 116 | point.y = (camera.K[3] * tmp.x + camera.K[4] * tmp.y + camera.K[5] * tmp.z) / depth; 117 | } 118 | 119 | float GetAngle( const cv::Vec3f &v1, const cv::Vec3f &v2 ) 120 | { 121 | float dot_product = v1[0] * v2[0] + v1[1] * v2[1] + v1[2] * v2[2]; 122 | float angle = acosf(dot_product); 123 | //if angle is not a number the dot product was 1 and thus the two vectors should be identical --> return 0 124 | if ( angle != angle ) 125 | return 0.0f; 126 | 127 | return angle; 128 | } 129 | 130 | cv::Vec3f TransformNormal(const Camera& camera, cv::Vec3f normal_local) 131 | { 132 | cv::Vec3f normal_global; 133 | normal_global[0] = camera.R[0] * normal_local[0] + camera.R[3] * normal_local[1] + camera.R[6] * normal_local[2]; 134 | normal_global[1] = camera.R[1] * normal_local[0] + camera.R[4] * normal_local[1] + camera.R[7] * normal_local[2]; 135 | normal_global[2] = camera.R[2] * normal_local[0] + camera.R[5] * normal_local[1] + camera.R[8] * normal_local[2]; 136 | return normal_global; 137 | } 138 | 139 | void RunFusion(const Model& model, bool geom_consistency) 140 | { 141 | std::unordered_map images; 142 | std::unordered_map cameras; 143 | std::unordered_map> depths; 144 | std::unordered_map> normals; 145 | std::unordered_map masks; 146 | images.clear(); 147 | cameras.clear(); 148 | depths.clear(); 149 | normals.clear(); 150 | masks.clear(); 151 | 152 | // read all data 153 | for (const auto& pair : model.covis_vec) { 154 | int ref_image_id = pair.first; 155 | std::string image_name = model.image_id_to_image_name.at(ref_image_id); 156 | std::string image_path = model.root_folder + "/" + model.image_folder + "/" + image_name; 157 | cv::Mat_ image = cv::imread (image_path, cv::IMREAD_COLOR); 158 | 159 | Camera camera = model.image_id_to_camera.at(ref_image_id); 160 | 161 | std::string suffix = ".photometric.bin"; 162 | if (geom_consistency) { 163 | suffix = ".geometric.bin"; 164 | } 165 | std::string depth_path = model.root_folder + "/" + model.depth_folder + "/" + image_name + suffix; 166 | std::string normal_path = model.root_folder + "/" + model.normal_folder + "/" + image_name + suffix; 167 | cv::Mat_ depth; 168 | cv::Mat_ normal; 169 | ReadMap(depth_path, depth); 170 | ReadMap(normal_path, normal); 171 | 172 | cv::Mat_ scaled_image; 173 | RescaleImageAndCamera(image, scaled_image, depth, camera); 174 | images[ref_image_id] = scaled_image; 175 | cameras[ref_image_id] = camera; 176 | depths[ref_image_id] = depth; 177 | normals[ref_image_id] = normal; 178 | cv::Mat mask = cv::Mat::zeros(depth.rows, depth.cols, CV_8UC1); 179 | masks[ref_image_id] = mask; 180 | } 181 | 182 | std::vector PointCloud; 183 | PointCloud.clear(); 184 | 185 | for (const auto& pair : model.covis_vec) { 186 | int ref_image_id = pair.first; 187 | std::vector src_image_ids = pair.second; 188 | std::cout << "Fusing image " << model.image_id_to_image_name.at(ref_image_id) << std::endl; 189 | 190 | const auto& ref_image = images.at(ref_image_id); 191 | const auto& ref_camera = cameras.at(ref_image_id); 192 | const auto& ref_depthmap = depths.at(ref_image_id); 193 | const auto& ref_normalmap = normals.at(ref_image_id); 194 | const auto& ref_mask = masks.at(ref_image_id); 195 | 196 | const int cols = ref_depthmap.cols; 197 | const int rows = ref_depthmap.rows; 198 | int num_ngb = src_image_ids.size(); 199 | std::vector used_list(num_ngb, make_int2(-1, -1)); 200 | for (int r =0; r < rows; ++r) { 201 | for (int c = 0; c < cols; ++c) { 202 | if (ref_mask.at(r, c) == 1) 203 | continue; 204 | float ref_depth = ref_depthmap.at(r, c); 205 | cv::Vec3f ref_normal = ref_normalmap.at(r, c); 206 | ref_normal = TransformNormal(ref_camera, ref_normal); 207 | 208 | if (ref_depth <= 0.0) 209 | continue; 210 | 211 | float3 PointX = Get3DPointonWorld(c, r, ref_depth, ref_camera); 212 | float3 consistent_Point = PointX; 213 | cv::Vec3f consistent_normal = ref_normal; 214 | cv::Vec3b ref_color = ref_image.at(r, c); 215 | float consistent_Color[3] = {(float)ref_color[0], (float)ref_color[1], (float)ref_color[2]}; 216 | int num_consistent = 0; 217 | 218 | for (int j = 0; j < num_ngb; ++j) { 219 | int src_image_id = src_image_ids[j]; 220 | const auto& src_image = images.at(src_image_id); 221 | const auto& src_camera = cameras.at(src_image_id); 222 | const auto& src_depthmap = depths.at(src_image_id); 223 | const auto& src_normalmap = normals.at(src_image_id); 224 | const auto& src_mask = masks.at(src_image_id); 225 | 226 | const int src_cols = src_depthmap.cols; 227 | const int src_rows = src_depthmap.rows; 228 | float2 point; 229 | float proj_depth; 230 | ProjectonCamera(PointX, src_camera, point, proj_depth); 231 | int src_r = int(point.y + 0.5f); 232 | int src_c = int(point.x + 0.5f); 233 | if (src_c >= 0 && src_c < src_cols && src_r >= 0 && src_r < src_rows) { 234 | if (src_mask.at(src_r, src_c) == 1) 235 | continue; 236 | 237 | float src_depth = src_depthmap.at(src_r, src_c); 238 | cv::Vec3f src_normal = src_normalmap.at(src_r, src_c); 239 | src_normal = TransformNormal(src_camera, src_normal); 240 | if (src_depth <= 0.0) 241 | continue; 242 | 243 | float3 tmp_X = Get3DPointonWorld(src_c, src_r, src_depth, src_camera); 244 | float2 tmp_pt; 245 | ProjectonCamera(tmp_X, ref_camera, tmp_pt, proj_depth); 246 | float reproj_error = sqrt(pow(c - tmp_pt.x, 2) + pow(r - tmp_pt.y, 2)); 247 | float relative_depth_diff = fabs(proj_depth - ref_depth) / ref_depth; 248 | float angle = GetAngle(ref_normal, src_normal); 249 | 250 | if (reproj_error < 2.0f && relative_depth_diff < 0.01f && angle < 0.174533f) { 251 | consistent_Point.x += tmp_X.x; 252 | consistent_Point.y += tmp_X.y; 253 | consistent_Point.z += tmp_X.z; 254 | consistent_normal = consistent_normal + src_normal; 255 | consistent_Color[0] += src_image.at(src_r, src_c)[0]; 256 | consistent_Color[1] += src_image.at(src_r, src_c)[1]; 257 | consistent_Color[2] += src_image.at(src_r, src_c)[2]; 258 | 259 | used_list[j].x = src_c; 260 | used_list[j].y = src_r; 261 | num_consistent++; 262 | } 263 | } 264 | } 265 | 266 | if (num_consistent >= 4) { 267 | consistent_Point.x /= (num_consistent + 1.0f); 268 | consistent_Point.y /= (num_consistent + 1.0f); 269 | consistent_Point.z /= (num_consistent + 1.0f); 270 | consistent_normal /= (num_consistent + 1.0f); 271 | consistent_Color[0] /= (num_consistent + 1.0f); 272 | consistent_Color[1] /= (num_consistent + 1.0f); 273 | consistent_Color[2] /= (num_consistent + 1.0f); 274 | 275 | PointList point3D; 276 | point3D.coord = consistent_Point; 277 | point3D.normal = make_float3(consistent_normal[0], consistent_normal[1], consistent_normal[2]); 278 | point3D.color = make_float3(consistent_Color[0], consistent_Color[1], consistent_Color[2]); 279 | PointCloud.push_back(point3D); 280 | 281 | for (int j = 0; j < num_ngb; ++j) { 282 | if (used_list[j].x == -1) 283 | continue; 284 | masks.at(src_image_ids[j]).at(used_list[j].y, used_list[j].x) = 1; 285 | } 286 | } 287 | } 288 | } 289 | } 290 | 291 | std::string ply_path = model.root_folder + "/ACMP_model.ply"; 292 | StoreColorPlyFileBinaryPointCloud (ply_path, PointCloud); 293 | } 294 | 295 | int main(int argc, char** argv) 296 | { 297 | double t_start = cv::getTickCount(); 298 | 299 | google::InitGoogleLogging(argv[0]); 300 | FLAGS_alsologtostderr = true; 301 | FLAGS_colorlogtostderr = true; 302 | FLAGS_v = 2; 303 | 304 | if (argc < 2) { 305 | std::cout << "USAGE: ACMP dense_folder" << std::endl; 306 | return -1; 307 | } 308 | 309 | std::string dense_folder = argv[1]; 310 | Model model(dense_folder, "sparse", "stereo/depth_maps", "stereo/normal_maps", "images"); 311 | model.Read(); 312 | model.ReduceMemory(); 313 | 314 | RunFusion(model, true); 315 | 316 | double t_end = cv::getTickCount(); 317 | double t_used = (t_end - t_start) / cv::getTickFrequency() / 60; 318 | std::cout << "Total time: " << t_used << " min" << std::endl; 319 | 320 | return 0; 321 | } 322 | -------------------------------------------------------------------------------- /src/ACMP.cpp: -------------------------------------------------------------------------------- 1 | #include "ACMP.h" 2 | 3 | #include 4 | 5 | void StringAppendV(std::string* dst, const char* format, va_list ap) { 6 | // First try with a small fixed size buffer. 7 | static const int kFixedBufferSize = 1024; 8 | char fixed_buffer[kFixedBufferSize]; 9 | 10 | // It is possible for methods that use a va_list to invalidate 11 | // the data in it upon use. The fix is to make a copy 12 | // of the structure before using it and use that copy instead. 13 | va_list backup_ap; 14 | va_copy(backup_ap, ap); 15 | int result = vsnprintf(fixed_buffer, kFixedBufferSize, format, backup_ap); 16 | va_end(backup_ap); 17 | 18 | if (result < kFixedBufferSize) { 19 | if (result >= 0) { 20 | // Normal case - everything fits. 21 | dst->append(fixed_buffer, result); 22 | return; 23 | } 24 | 25 | #ifdef _MSC_VER 26 | // Error or MSVC running out of space. MSVC 8.0 and higher 27 | // can be asked about space needed with the special idiom below: 28 | va_copy(backup_ap, ap); 29 | result = vsnprintf(nullptr, 0, format, backup_ap); 30 | va_end(backup_ap); 31 | #endif 32 | 33 | if (result < 0) { 34 | // Just an error. 35 | return; 36 | } 37 | } 38 | 39 | // Increase the buffer size to the size requested by vsnprintf, 40 | // plus one for the closing \0. 41 | const int variable_buffer_size = result + 1; 42 | std::unique_ptr variable_buffer(new char[variable_buffer_size]); 43 | 44 | // Restore the va_list before we use it again. 45 | va_copy(backup_ap, ap); 46 | result = 47 | vsnprintf(variable_buffer.get(), variable_buffer_size, format, backup_ap); 48 | va_end(backup_ap); 49 | 50 | if (result >= 0 && result < variable_buffer_size) { 51 | dst->append(variable_buffer.get(), result); 52 | } 53 | } 54 | 55 | std::string StringPrintf(const char* format, ...) { 56 | va_list ap; 57 | va_start(ap, format); 58 | std::string result; 59 | StringAppendV(&result, format, ap); 60 | va_end(ap); 61 | return result; 62 | } 63 | 64 | void CudaSafeCall(const cudaError_t error, const std::string& file, 65 | const int line) { 66 | if (error != cudaSuccess) { 67 | std::cerr << StringPrintf("%s in %s at line %i", cudaGetErrorString(error), 68 | file.c_str(), line) 69 | << std::endl; 70 | exit(EXIT_FAILURE); 71 | } 72 | } 73 | 74 | void CudaCheckError(const char* file, const int line) { 75 | cudaError error = cudaGetLastError(); 76 | if (error != cudaSuccess) { 77 | std::cerr << StringPrintf("cudaCheckError() failed at %s:%i : %s", file, 78 | line, cudaGetErrorString(error)) 79 | << std::endl; 80 | exit(EXIT_FAILURE); 81 | } 82 | 83 | // More careful checking. However, this will affect performance. 84 | // Comment away if needed. 85 | error = cudaDeviceSynchronize(); 86 | if (cudaSuccess != error) { 87 | std::cerr << StringPrintf("cudaCheckError() with sync failed at %s:%i : %s", 88 | file, line, cudaGetErrorString(error)) 89 | << std::endl; 90 | std::cerr 91 | << "This error is likely caused by the graphics card timeout " 92 | "detection mechanism of your operating system. Please refer to " 93 | "the FAQ in the documentation on how to solve this problem." 94 | << std::endl; 95 | exit(EXIT_FAILURE); 96 | } 97 | } 98 | 99 | ACMP::ACMP(const std::unordered_map& _image_id_to_image_name, const std::unordered_map &_image_id_to_camera, const std::string &_depth_folder, const std::string &_normal_folder, const std::string &_image_folder) 100 | :image_id_to_image_name(_image_id_to_image_name), image_id_to_camera(_image_id_to_camera), depth_folder(_depth_folder), normal_folder(_normal_folder), image_folder(_image_folder){} 101 | 102 | ACMP::~ACMP() 103 | { 104 | delete[] plane_hypotheses_host; 105 | delete[] costs_host; 106 | 107 | for (int i = 0; i < num_images; ++i) { 108 | cudaDestroyTextureObject(texture_objects_host.images[i]); 109 | cudaFreeArray(cuArray[i]); 110 | } 111 | cudaFree(texture_objects_cuda); 112 | cudaFree(cameras_cuda); 113 | cudaFree(plane_hypotheses_cuda); 114 | cudaFree(costs_cuda); 115 | cudaFree(rand_states_cuda); 116 | cudaFree(selected_views_cuda); 117 | cudaFree(depths_cuda); 118 | 119 | if (params.geom_consistency) { 120 | for (int i = 0; i < num_images; ++i) { 121 | cudaDestroyTextureObject(texture_depths_host.images[i]); 122 | cudaFreeArray(cuDepthArray[i]); 123 | } 124 | cudaFree(texture_depths_cuda); 125 | } 126 | 127 | if (params.planar_prior) { 128 | delete[] prior_planes_host; 129 | delete[] plane_masks_host; 130 | 131 | cudaFree(prior_planes_cuda); 132 | cudaFree(plane_masks_cuda); 133 | } 134 | } 135 | 136 | 137 | float3 Get3DPointonRefCam(const int x, const int y, const float depth, const Camera camera) 138 | { 139 | float3 pointX; 140 | // Reprojection 141 | pointX.x = depth * (x - camera.K[2]) / camera.K[0]; 142 | pointX.y = depth * (y - camera.K[5]) / camera.K[4]; 143 | pointX.z = depth; 144 | 145 | return pointX; 146 | } 147 | 148 | 149 | void ACMP::SetGeomConsistencyParams(bool multi_geometry) 150 | { 151 | params.geom_consistency = true; 152 | params.max_iterations = 2; 153 | if(multi_geometry) { 154 | params.multi_geometry = true; 155 | } 156 | } 157 | 158 | void ACMP::SetPlanarPriorParams() 159 | { 160 | params.planar_prior = true; 161 | } 162 | 163 | void ACMP::InuputInitialization(const std::string &dense_folder, const Problem &problem) 164 | { 165 | images.clear(); 166 | cameras.clear(); 167 | 168 | // ref image 169 | std::string image_path = dense_folder + "/" + image_folder + "/" + image_id_to_image_name.at(problem.ref_image_id); 170 | cv::Mat_ image_uint = cv::imread(image_path, cv::IMREAD_GRAYSCALE); 171 | cv::Mat image_float; 172 | image_uint.convertTo(image_float, CV_32FC1); 173 | images.push_back(image_float); 174 | 175 | Camera camera = image_id_to_camera.at(problem.ref_image_id); 176 | cameras.push_back(camera); 177 | 178 | // target images 179 | size_t num_src_images = problem.src_image_ids.size(); 180 | for (size_t i = 0; i < num_src_images; ++i) { 181 | std::string image_path = dense_folder + "/" + image_folder + "/" + image_id_to_image_name.at(problem.src_image_ids[i]); 182 | cv::Mat_ image_uint = cv::imread(image_path, cv::IMREAD_GRAYSCALE); 183 | cv::Mat image_float; 184 | image_uint.convertTo(image_float, CV_32FC1); 185 | images.push_back(image_float); 186 | 187 | Camera camera = image_id_to_camera.at(problem.src_image_ids[i]); 188 | cameras.push_back(camera); 189 | } 190 | 191 | // Scale cameras and images 192 | for (size_t i = 0; i < images.size(); ++i) { 193 | if (images[i].cols <= params.max_image_size && images[i].rows <= params.max_image_size) { 194 | continue; 195 | } 196 | 197 | const float factor_x = static_cast(params.max_image_size) / images[i].cols; 198 | const float factor_y = static_cast(params.max_image_size) / images[i].rows; 199 | const float factor = std::min(factor_x, factor_y); 200 | 201 | const int new_cols = std::round(images[i].cols * factor); 202 | const int new_rows = std::round(images[i].rows * factor); 203 | 204 | const float scale_x = new_cols / static_cast(images[i].cols); 205 | const float scale_y = new_rows / static_cast(images[i].rows); 206 | 207 | cv::Mat_ scaled_image_float; 208 | cv::resize(images[i], scaled_image_float, cv::Size(new_cols,new_rows), 0, 0, cv::INTER_LINEAR); 209 | images[i] = scaled_image_float.clone(); 210 | 211 | cameras[i].K[0] *= scale_x; 212 | cameras[i].K[2] *= scale_x; 213 | cameras[i].K[4] *= scale_y; 214 | cameras[i].K[5] *= scale_y; 215 | cameras[i].height = scaled_image_float.rows; 216 | cameras[i].width = scaled_image_float.cols; 217 | } 218 | 219 | params.depth_min = cameras[0].depth_min * 0.6f; 220 | params.depth_max = cameras[0].depth_max * 1.2f; 221 | std::cout << "depthe range: " << params.depth_min << " " << params.depth_max << std::endl; 222 | params.num_images = (int)images.size(); 223 | std::cout << "num images: " << params.num_images << std::endl; 224 | params.disparity_min = cameras[0].K[0] * params.baseline / params.depth_max; 225 | params.disparity_max = cameras[0].K[0] * params.baseline / params.depth_min; 226 | 227 | if (params.geom_consistency) { 228 | depths.clear(); 229 | 230 | std::string suffix = params.multi_geometry ? ".geometric.bin" : ".photometric.bin"; 231 | std::string depth_path = dense_folder + "/" + depth_folder + "/" + image_id_to_image_name.at(problem.ref_image_id) + suffix; 232 | cv::Mat_ ref_depth; 233 | ReadMap(depth_path, ref_depth); 234 | depths.push_back(ref_depth); 235 | 236 | size_t num_src_images = problem.src_image_ids.size(); 237 | for (size_t i = 0; i < num_src_images; ++i) { 238 | std::string depth_path = dense_folder + "/" + depth_folder + "/" + image_id_to_image_name.at(problem.src_image_ids[i]) + suffix; 239 | cv::Mat_ depth; 240 | ReadMap(depth_path, depth); 241 | depths.push_back(depth); 242 | } 243 | } 244 | } 245 | 246 | void ACMP::CudaSpaceInitialization(const std::string &dense_folder, const Problem &problem) 247 | { 248 | num_images = (int)images.size(); 249 | 250 | for (int i = 0; i < num_images; ++i) { 251 | int rows = images[i].rows; 252 | int cols = images[i].cols; 253 | 254 | cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat); 255 | cudaMallocArray(&cuArray[i], &channelDesc, cols, rows); 256 | cudaMemcpy2DToArray (cuArray[i], 0, 0, images[i].ptr(), images[i].step[0], cols*sizeof(float), rows, cudaMemcpyHostToDevice); 257 | 258 | struct cudaResourceDesc resDesc; 259 | memset(&resDesc, 0, sizeof(cudaResourceDesc)); 260 | resDesc.resType = cudaResourceTypeArray; 261 | resDesc.res.array.array = cuArray[i]; 262 | 263 | struct cudaTextureDesc texDesc; 264 | memset(&texDesc, 0, sizeof(cudaTextureDesc)); 265 | texDesc.addressMode[0] = cudaAddressModeWrap; 266 | texDesc.addressMode[1] = cudaAddressModeWrap; 267 | texDesc.filterMode = cudaFilterModeLinear; 268 | texDesc.readMode = cudaReadModeElementType; 269 | texDesc.normalizedCoords = 0; 270 | 271 | cudaCreateTextureObject(&(texture_objects_host.images[i]), &resDesc, &texDesc, NULL); 272 | } 273 | cudaMalloc((void**)&texture_objects_cuda, sizeof(cudaTextureObjects)); 274 | cudaMemcpy(texture_objects_cuda, &texture_objects_host, sizeof(cudaTextureObjects), cudaMemcpyHostToDevice); 275 | 276 | cudaMalloc((void**)&cameras_cuda, sizeof(Camera) * (num_images)); 277 | cudaMemcpy(cameras_cuda, &cameras[0], sizeof(Camera) * (num_images), cudaMemcpyHostToDevice); 278 | 279 | plane_hypotheses_host = new float4[cameras[0].height * cameras[0].width]; 280 | cudaMalloc((void**)&plane_hypotheses_cuda, sizeof(float4) * (cameras[0].height * cameras[0].width)); 281 | 282 | costs_host = new float[cameras[0].height * cameras[0].width]; 283 | cudaMalloc((void**)&costs_cuda, sizeof(float) * (cameras[0].height * cameras[0].width)); 284 | 285 | cudaMalloc((void**)&rand_states_cuda, sizeof(curandState) * (cameras[0].height * cameras[0].width)); 286 | cudaMalloc((void**)&selected_views_cuda, sizeof(unsigned int) * (cameras[0].height * cameras[0].width)); 287 | 288 | cudaMalloc((void**)&depths_cuda, sizeof(float) * (cameras[0].height * cameras[0].width)); // Updated by Qingshan 2020-01-15 289 | 290 | if (params.geom_consistency) { 291 | for (int i = 0; i < num_images; ++i) { 292 | int rows = depths[i].rows; 293 | int cols = depths[i].cols; 294 | 295 | cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat); 296 | cudaMallocArray(&cuDepthArray[i], &channelDesc, cols, rows); 297 | cudaMemcpy2DToArray (cuDepthArray[i], 0, 0, depths[i].ptr(), depths[i].step[0], cols*sizeof(float), rows, cudaMemcpyHostToDevice); 298 | 299 | struct cudaResourceDesc resDesc; 300 | memset(&resDesc, 0, sizeof(cudaResourceDesc)); 301 | resDesc.resType = cudaResourceTypeArray; 302 | resDesc.res.array.array = cuDepthArray[i]; 303 | 304 | struct cudaTextureDesc texDesc; 305 | memset(&texDesc, 0, sizeof(cudaTextureDesc)); 306 | texDesc.addressMode[0] = cudaAddressModeWrap; 307 | texDesc.addressMode[1] = cudaAddressModeWrap; 308 | texDesc.filterMode = cudaFilterModeLinear; 309 | texDesc.readMode = cudaReadModeElementType; 310 | texDesc.normalizedCoords = 0; 311 | 312 | cudaCreateTextureObject(&(texture_depths_host.images[i]), &resDesc, &texDesc, NULL); 313 | } 314 | cudaMalloc((void**)&texture_depths_cuda, sizeof(cudaTextureObjects)); 315 | cudaMemcpy(texture_depths_cuda, &texture_depths_host, sizeof(cudaTextureObjects), cudaMemcpyHostToDevice); 316 | 317 | std::string suffix = params.multi_geometry ? ".geometric.bin" : ".photometric.bin"; 318 | std::string depth_path = dense_folder + "/" + depth_folder + "/" + image_id_to_image_name.at(problem.ref_image_id) + suffix; 319 | std::string normal_path = dense_folder + "/" + normal_folder + "/" + image_id_to_image_name.at(problem.ref_image_id) + suffix; 320 | cv::Mat_ ref_depth; 321 | cv::Mat_ ref_normal; 322 | ReadMap(depth_path, ref_depth); 323 | depths.push_back(ref_depth); 324 | ReadMap(normal_path, ref_normal); 325 | int width = ref_depth.cols; 326 | int height = ref_depth.rows; 327 | for (int col = 0; col < width; ++col) { 328 | for (int row = 0; row < height; ++row) { 329 | int center = row * width + col; 330 | float4 plane_hypothesis; 331 | plane_hypothesis.x = ref_normal(row, col)[0]; 332 | plane_hypothesis.y = ref_normal(row, col)[1]; 333 | plane_hypothesis.z = ref_normal(row, col)[2]; 334 | plane_hypothesis.w = ref_depth(row, col); 335 | plane_hypotheses_host[center] = plane_hypothesis; 336 | } 337 | } 338 | 339 | cudaMemcpy(plane_hypotheses_cuda, plane_hypotheses_host, sizeof(float4) * width * height, cudaMemcpyHostToDevice); 340 | } 341 | } 342 | 343 | void ACMP::CudaPlanarPriorInitialization(const std::vector &PlaneParams, const cv::Mat_ &masks) 344 | { 345 | prior_planes_host = new float4[cameras[0].height * cameras[0].width]; 346 | cudaMalloc((void**)&prior_planes_cuda, sizeof(float4) * (cameras[0].height * cameras[0].width)); 347 | 348 | plane_masks_host = new unsigned int[cameras[0].height * cameras[0].width]; 349 | cudaMalloc((void**)&plane_masks_cuda, sizeof(unsigned int) * (cameras[0].height * cameras[0].width)); 350 | 351 | for (int i = 0; i < cameras[0].width; ++i) { 352 | for (int j = 0; j < cameras[0].height; ++j) { 353 | int center = j * cameras[0].width + i; 354 | plane_masks_host[center] = (unsigned int)masks(j, i); 355 | if (masks(j, i) > 0) { 356 | prior_planes_host[center] = PlaneParams[masks(j, i) - 1]; 357 | } 358 | } 359 | } 360 | 361 | cudaMemcpy(prior_planes_cuda, prior_planes_host, sizeof(float4) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice); 362 | cudaMemcpy(plane_masks_cuda, plane_masks_host, sizeof(unsigned int) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice); 363 | } 364 | 365 | int ACMP::GetReferenceImageWidth() 366 | { 367 | return cameras[0].width; 368 | } 369 | 370 | int ACMP::GetReferenceImageHeight() 371 | { 372 | return cameras[0].height; 373 | } 374 | 375 | cv::Mat ACMP::GetReferenceImage() 376 | { 377 | return images[0]; 378 | } 379 | 380 | float4 ACMP::GetPlaneHypothesis(const int index) 381 | { 382 | return plane_hypotheses_host[index]; 383 | } 384 | 385 | float ACMP::GetCost(const int index) 386 | { 387 | return costs_host[index]; 388 | } 389 | 390 | float ACMP::GetMinDepth() 391 | { 392 | return params.depth_min; 393 | } 394 | 395 | float ACMP::GetMaxDepth() 396 | { 397 | return params.depth_max; 398 | } 399 | 400 | void ACMP::GetSupportPoints(std::vector& support2DPoints) 401 | { 402 | support2DPoints.clear(); 403 | const int step_size = 5; 404 | const int width = GetReferenceImageWidth(); 405 | const int height = GetReferenceImageHeight(); 406 | for (int col = 0; col < width; col += step_size) { 407 | for (int row = 0; row < height; row += step_size) { 408 | float min_cost = 2.0f; 409 | cv::Point temp_point; 410 | int c_bound = std::min(width, col + step_size); 411 | int r_bound = std::min(height, row + step_size); 412 | for (int c = col; c < c_bound; ++c) { 413 | for (int r = row; r < r_bound; ++r) { 414 | int center = r * width + c; 415 | if (GetCost(center) < 2.0f && min_cost > GetCost(center)) { 416 | temp_point = cv::Point(c, r); 417 | min_cost = GetCost(center); 418 | } 419 | } 420 | } 421 | if (min_cost < 0.1f) { 422 | support2DPoints.push_back(temp_point); 423 | } 424 | } 425 | } 426 | } 427 | 428 | std::vector ACMP::DelaunayTriangulation(const cv::Rect boundRC, const std::vector& points) 429 | { 430 | if (points.empty()) { 431 | return std::vector(); 432 | } 433 | 434 | std::vector results; 435 | 436 | std::vector temp_results; 437 | cv::Subdiv2D subdiv2d(boundRC); 438 | for (const auto point : points) { 439 | subdiv2d.insert(cv::Point2f((float)point.x, (float)point.y)); 440 | } 441 | subdiv2d.getTriangleList(temp_results); 442 | 443 | for (const auto temp_vec : temp_results) { 444 | cv::Point pt1((int)temp_vec[0], (int)temp_vec[1]); 445 | cv::Point pt2((int)temp_vec[2], (int)temp_vec[3]); 446 | cv::Point pt3((int)temp_vec[4], (int)temp_vec[5]); 447 | results.push_back(Triangle(pt1, pt2, pt3)); 448 | } 449 | return results; 450 | } 451 | 452 | float4 ACMP::GetPriorPlaneParams(const Triangle triangle, const cv::Mat_ depths) 453 | { 454 | cv::Mat A(3, 4, CV_32FC1); 455 | cv::Mat B(4, 1, CV_32FC1); 456 | 457 | float3 ptX1 = Get3DPointonRefCam(triangle.pt1.x, triangle.pt1.y, depths(triangle.pt1.y, triangle.pt1.x), cameras[0]); 458 | float3 ptX2 = Get3DPointonRefCam(triangle.pt2.x, triangle.pt2.y, depths(triangle.pt2.y, triangle.pt2.x), cameras[0]); 459 | float3 ptX3 = Get3DPointonRefCam(triangle.pt3.x, triangle.pt3.y, depths(triangle.pt3.y, triangle.pt3.x), cameras[0]); 460 | 461 | A.at(0, 0) = ptX1.x; 462 | A.at(0, 1) = ptX1.y; 463 | A.at(0, 2) = ptX1.z; 464 | A.at(0, 3) = 1.0; 465 | A.at(1, 0) = ptX2.x; 466 | A.at(1, 1) = ptX2.y; 467 | A.at(1, 2) = ptX2.z; 468 | A.at(1, 3) = 1.0; 469 | A.at(2, 0) = ptX3.x; 470 | A.at(2, 1) = ptX3.y; 471 | A.at(2, 2) = ptX3.z; 472 | A.at(2, 3) = 1.0; 473 | cv::SVD::solveZ(A, B); 474 | float4 n4 = make_float4(B.at(0, 0), B.at(1, 0), B.at(2, 0), B.at(3, 0)); 475 | float norm2 = sqrt(pow(n4.x, 2) + pow(n4.y, 2) + pow(n4.z, 2)); 476 | if (n4.w < 0) { 477 | norm2 *= -1; 478 | } 479 | n4.x /= norm2; 480 | n4.y /= norm2; 481 | n4.z /= norm2; 482 | n4.w /= norm2; 483 | 484 | return n4; 485 | } 486 | 487 | float ACMP::GetDepthFromPlaneParam(const float4 plane_hypothesis, const int x, const int y) 488 | { 489 | return -plane_hypothesis.w * cameras[0].K[0] / ((x - cameras[0].K[2]) * plane_hypothesis.x + (cameras[0].K[0] / cameras[0].K[4]) * (y - cameras[0].K[5]) * plane_hypothesis.y + cameras[0].K[0] * plane_hypothesis.z); 490 | } 491 | -------------------------------------------------------------------------------- /src/ACMP.cu: -------------------------------------------------------------------------------- 1 | #include "ACMP.h" 2 | 3 | __device__ void sort_small(float *d, const int n) 4 | { 5 | int j; 6 | for (int i = 1; i < n; i++) { 7 | float tmp = d[i]; 8 | for (j = i; j >= 1 && tmp < d[j-1]; j--) 9 | d[j] = d[j-1]; 10 | d[j] = tmp; 11 | } 12 | } 13 | 14 | __device__ void sort_small_weighted(float *d, float *w, int n) 15 | { 16 | int j; 17 | for (int i = 1; i < n; i++) { 18 | float tmp = d[i]; 19 | float tmp_w = w[i]; 20 | for (j = i; j >= 1 && tmp < d[j - 1]; j--) { 21 | d[j] = d[j - 1]; 22 | w[j] = w[j - 1]; 23 | } 24 | d[j] = tmp; 25 | w[j] = tmp_w; 26 | } 27 | } 28 | 29 | __device__ int FindMinCostIndex(const float *costs, const int n) 30 | { 31 | float min_cost = costs[0]; 32 | int min_cost_idx = 0; 33 | for (int idx = 1; idx < n; ++idx) { 34 | if (costs[idx] <= min_cost) { 35 | min_cost = costs[idx]; 36 | min_cost_idx = idx; 37 | } 38 | } 39 | return min_cost_idx; 40 | } 41 | 42 | __device__ int FindMaxCostIndex(const float *costs, const int n) 43 | { 44 | float max_cost = costs[0]; 45 | int max_cost_idx = 0; 46 | for (int idx = 1; idx < n; ++idx) { 47 | if (costs[idx] >= max_cost) { 48 | max_cost = costs[idx]; 49 | max_cost_idx = idx; 50 | } 51 | } 52 | return max_cost_idx; 53 | } 54 | 55 | __device__ void setBit(unsigned int &input, const unsigned int n) 56 | { 57 | input |= (unsigned int)(1 << n); 58 | } 59 | 60 | __device__ int isSet(unsigned int input, const unsigned int n) 61 | { 62 | return (input >> n) & 1; 63 | } 64 | 65 | __device__ void Mat33DotVec3(const float mat[9], const float4 vec, float4 *result) 66 | { 67 | result->x = mat[0] * vec.x + mat[1] * vec.y + mat[2] * vec.z; 68 | result->y = mat[3] * vec.x + mat[4] * vec.y + mat[5] * vec.z; 69 | result->z = mat[6] * vec.x + mat[7] * vec.y + mat[8] * vec.z; 70 | } 71 | 72 | __device__ float Vec3DotVec3(const float4 vec1, const float4 vec2) 73 | { 74 | return vec1.x * vec2.x + vec1.y * vec2.y + vec1.z * vec2.z; 75 | } 76 | 77 | __device__ void NormalizeVec3 (float4 *vec) 78 | { 79 | const float normSquared = vec->x * vec->x + vec->y * vec->y + vec->z * vec->z; 80 | const float inverse_sqrt = rsqrtf (normSquared); 81 | vec->x *= inverse_sqrt; 82 | vec->y *= inverse_sqrt; 83 | vec->z *= inverse_sqrt; 84 | } 85 | 86 | __device__ void TransformPDFToCDF(float* probs, const int num_probs) 87 | { 88 | float prob_sum = 0.0f; 89 | for (int i = 0; i < num_probs; ++i) { 90 | prob_sum += probs[i]; 91 | } 92 | const float inv_prob_sum = 1.0f / prob_sum; 93 | 94 | float cum_prob = 0.0f; 95 | for (int i = 0; i < num_probs; ++i) { 96 | const float prob = probs[i] * inv_prob_sum; 97 | cum_prob += prob; 98 | probs[i] = cum_prob; 99 | } 100 | } 101 | 102 | __device__ void Get3DPoint(const Camera camera, const int2 p, const float depth, float *X) 103 | { 104 | X[0] = depth * (p.x - camera.K[2]) / camera.K[0]; 105 | X[1] = depth * (p.y - camera.K[5]) / camera.K[4]; 106 | X[2] = depth; 107 | } 108 | 109 | __device__ float4 GetViewDirection(const Camera camera, const int2 p, const float depth) 110 | { 111 | float X[3]; 112 | Get3DPoint(camera, p, depth, X); 113 | float norm = sqrt(X[0] * X[0] + X[1] * X[1] + X[2] * X[2]); 114 | 115 | float4 view_direction; 116 | view_direction.x = X[0] / norm; 117 | view_direction.y = X[1] / norm; 118 | view_direction.z = X[2] / norm; 119 | view_direction.w = 0; 120 | return view_direction; 121 | } 122 | 123 | __device__ float GetDistance2Origin(const Camera camera, const int2 p, const float depth, const float4 normal) 124 | { 125 | float X[3]; 126 | Get3DPoint(camera, p, depth, X); 127 | return -(normal.x * X[0] + normal.y * X[1] + normal.z * X[2]); 128 | } 129 | 130 | __device__ float ComputeDepthfromPlaneHypothesis(const Camera camera, const float4 plane_hypothesis, const int2 p) 131 | { 132 | return -plane_hypothesis.w * camera.K[0] / ((p.x - camera.K[2]) * plane_hypothesis.x + (camera.K[0] / camera.K[4]) * (p.y - camera.K[5]) * plane_hypothesis.y + camera.K[0] * plane_hypothesis.z); 133 | } 134 | 135 | __device__ float4 GenerateRandomNormal(const Camera camera, const int2 p, curandState *rand_state, const float depth) 136 | { 137 | float4 normal; 138 | float q1 = 1.0f; 139 | float q2 = 1.0f; 140 | float s = 2.0f; 141 | while (s >= 1.0f) { 142 | q1 = 2.0f * curand_uniform(rand_state) -1.0f; 143 | q2 = 2.0f * curand_uniform(rand_state) - 1.0f; 144 | s = q1 * q1 + q2 * q2; 145 | } 146 | const float sq = sqrt(1.0f - s); 147 | normal.x = 2.0f * q1 * sq; 148 | normal.y = 2.0f * q2 * sq; 149 | normal.z = 1.0f - 2.0f * s; 150 | normal.w = 0; 151 | 152 | float4 view_direction = GetViewDirection(camera, p, depth); 153 | float dot_product = normal.x * view_direction.x + normal.y * view_direction.y + normal.z * view_direction.z; 154 | if (dot_product > 0.0f) { 155 | normal.x = -normal.x; 156 | normal.y = -normal.y; 157 | normal.z = - normal.z; 158 | } 159 | NormalizeVec3(&normal); 160 | return normal; 161 | } 162 | 163 | __device__ float4 GeneratePerturbedNormal(const Camera camera, const int2 p, const float4 normal, curandState *rand_state, const float perturbation) 164 | { 165 | float4 view_direction = GetViewDirection(camera, p, 1.0f); 166 | 167 | const float a1 = (curand_uniform(rand_state) - 0.5f) * perturbation; 168 | const float a2 = (curand_uniform(rand_state) - 0.5f) * perturbation; 169 | const float a3 = (curand_uniform(rand_state) - 0.5f) * perturbation; 170 | 171 | const float sin_a1 = sin(a1); 172 | const float sin_a2 = sin(a2); 173 | const float sin_a3 = sin(a3); 174 | const float cos_a1 = cos(a1); 175 | const float cos_a2 = cos(a2); 176 | const float cos_a3 = cos(a3); 177 | 178 | float R[9]; 179 | R[0] = cos_a2 * cos_a3; 180 | R[1] = cos_a3 * sin_a1 * sin_a2 - cos_a1 * sin_a3; 181 | R[2] = sin_a1 * sin_a3 + cos_a1 * cos_a3 * sin_a2; 182 | R[3] = cos_a2 * sin_a3; 183 | R[4] = cos_a1 * cos_a3 + sin_a1 * sin_a2 * sin_a3; 184 | R[5] = cos_a1 * sin_a2 * sin_a3 - cos_a3 * sin_a1; 185 | R[6] = -sin_a2; 186 | R[7] = cos_a2 * sin_a1; 187 | R[8] = cos_a1 * cos_a2; 188 | 189 | float4 normal_perturbed; 190 | Mat33DotVec3(R, normal, &normal_perturbed); 191 | 192 | if (Vec3DotVec3(normal_perturbed, view_direction) >= 0.0f) { 193 | normal_perturbed = normal; 194 | } 195 | 196 | NormalizeVec3(&normal_perturbed); 197 | return normal_perturbed; 198 | } 199 | 200 | __device__ float4 GenerateRandomPlaneHypothesis(const Camera camera, const int2 p, curandState *rand_state, const float depth_min, const float depth_max) 201 | { 202 | float depth = curand_uniform(rand_state) * (depth_max - depth_min) + depth_min; 203 | float4 plane_hypothesis = GenerateRandomNormal(camera, p, rand_state, depth); 204 | plane_hypothesis.w = GetDistance2Origin(camera, p, depth, plane_hypothesis); 205 | return plane_hypothesis; 206 | } 207 | 208 | __device__ float4 GeneratePertubedPlaneHypothesis(const Camera camera, const int2 p, curandState *rand_state, const float perturbation, const float4 plane_hypothesis_now, const float depth_now, const float depth_min, const float depth_max) 209 | { 210 | float depth_perturbed = depth_now; 211 | 212 | float dist_perturbed = plane_hypothesis_now.w; 213 | const float dist_min_perturbed = (1 - perturbation) * dist_perturbed; 214 | const float dist_max_perturbed = (1 + perturbation) * dist_perturbed; 215 | float4 plane_hypothesis_temp = plane_hypothesis_now; 216 | do { 217 | dist_perturbed = curand_uniform(rand_state) * (dist_max_perturbed - dist_min_perturbed) + dist_min_perturbed; 218 | plane_hypothesis_temp.w = dist_perturbed; 219 | depth_perturbed = ComputeDepthfromPlaneHypothesis(camera, plane_hypothesis_temp, p); 220 | } while (depth_perturbed < depth_min && depth_perturbed > depth_max); 221 | 222 | float4 plane_hypothesis = GeneratePerturbedNormal(camera, p, plane_hypothesis_now, rand_state, perturbation * M_PI); 223 | plane_hypothesis.w = dist_perturbed; 224 | return plane_hypothesis; 225 | } 226 | 227 | __device__ void ComputeHomography(const Camera ref_camera, const Camera src_camera, const float4 plane_hypothesis, float *H) 228 | { 229 | float ref_C[3]; 230 | float src_C[3]; 231 | ref_C[0] = -(ref_camera.R[0] * ref_camera.t[0] + ref_camera.R[3] * ref_camera.t[1] + ref_camera.R[6] * ref_camera.t[2]); 232 | ref_C[1] = -(ref_camera.R[1] * ref_camera.t[0] + ref_camera.R[4] * ref_camera.t[1] + ref_camera.R[7] * ref_camera.t[2]); 233 | ref_C[2] = -(ref_camera.R[2] * ref_camera.t[0] + ref_camera.R[5] * ref_camera.t[1] + ref_camera.R[8] * ref_camera.t[2]); 234 | src_C[0] = -(src_camera.R[0] * src_camera.t[0] + src_camera.R[3] * src_camera.t[1] + src_camera.R[6] * src_camera.t[2]); 235 | src_C[1] = -(src_camera.R[1] * src_camera.t[0] + src_camera.R[4] * src_camera.t[1] + src_camera.R[7] * src_camera.t[2]); 236 | src_C[2] = -(src_camera.R[2] * src_camera.t[0] + src_camera.R[5] * src_camera.t[1] + src_camera.R[8] * src_camera.t[2]); 237 | 238 | float R_relative[9]; 239 | float C_relative[3]; 240 | float t_relative[3]; 241 | R_relative[0] = src_camera.R[0] * ref_camera.R[0] + src_camera.R[1] * ref_camera.R[1] + src_camera.R[2] *ref_camera.R[2]; 242 | R_relative[1] = src_camera.R[0] * ref_camera.R[3] + src_camera.R[1] * ref_camera.R[4] + src_camera.R[2] *ref_camera.R[5]; 243 | R_relative[2] = src_camera.R[0] * ref_camera.R[6] + src_camera.R[1] * ref_camera.R[7] + src_camera.R[2] *ref_camera.R[8]; 244 | R_relative[3] = src_camera.R[3] * ref_camera.R[0] + src_camera.R[4] * ref_camera.R[1] + src_camera.R[5] *ref_camera.R[2]; 245 | R_relative[4] = src_camera.R[3] * ref_camera.R[3] + src_camera.R[4] * ref_camera.R[4] + src_camera.R[5] *ref_camera.R[5]; 246 | R_relative[5] = src_camera.R[3] * ref_camera.R[6] + src_camera.R[4] * ref_camera.R[7] + src_camera.R[5] *ref_camera.R[8]; 247 | R_relative[6] = src_camera.R[6] * ref_camera.R[0] + src_camera.R[7] * ref_camera.R[1] + src_camera.R[8] *ref_camera.R[2]; 248 | R_relative[7] = src_camera.R[6] * ref_camera.R[3] + src_camera.R[7] * ref_camera.R[4] + src_camera.R[8] *ref_camera.R[5]; 249 | R_relative[8] = src_camera.R[6] * ref_camera.R[6] + src_camera.R[7] * ref_camera.R[7] + src_camera.R[8] *ref_camera.R[8]; 250 | C_relative[0] = (ref_C[0] - src_C[0]); 251 | C_relative[1] = (ref_C[1] - src_C[1]); 252 | C_relative[2] = (ref_C[2] - src_C[2]); 253 | t_relative[0] = src_camera.R[0] * C_relative[0] + src_camera.R[1] * C_relative[1] + src_camera.R[2] * C_relative[2]; 254 | t_relative[1] = src_camera.R[3] * C_relative[0] + src_camera.R[4] * C_relative[1] + src_camera.R[5] * C_relative[2]; 255 | t_relative[2] = src_camera.R[6] * C_relative[0] + src_camera.R[7] * C_relative[1] + src_camera.R[8] * C_relative[2]; 256 | 257 | H[0] = R_relative[0] - t_relative[0] * plane_hypothesis.x / plane_hypothesis.w; 258 | H[1] = R_relative[1] - t_relative[0] * plane_hypothesis.y / plane_hypothesis.w; 259 | H[2] = R_relative[2] - t_relative[0] * plane_hypothesis.z / plane_hypothesis.w; 260 | H[3] = R_relative[3] - t_relative[1] * plane_hypothesis.x / plane_hypothesis.w; 261 | H[4] = R_relative[4] - t_relative[1] * plane_hypothesis.y / plane_hypothesis.w; 262 | H[5] = R_relative[5] - t_relative[1] * plane_hypothesis.z / plane_hypothesis.w; 263 | H[6] = R_relative[6] - t_relative[2] * plane_hypothesis.x / plane_hypothesis.w; 264 | H[7] = R_relative[7] - t_relative[2] * plane_hypothesis.y / plane_hypothesis.w; 265 | H[8] = R_relative[8] - t_relative[2] * plane_hypothesis.z / plane_hypothesis.w; 266 | 267 | float tmp[9]; 268 | tmp[0] = H[0] / ref_camera.K[0]; 269 | tmp[1] = H[1] / ref_camera.K[4]; 270 | tmp[2] = -H[0] * ref_camera.K[2] / ref_camera.K[0] - H[1] * ref_camera.K[5] / ref_camera.K[4] + H[2]; 271 | tmp[3] = H[3] / ref_camera.K[0]; 272 | tmp[4] = H[4] / ref_camera.K[4]; 273 | tmp[5] = -H[3] * ref_camera.K[2] / ref_camera.K[0] - H[4] * ref_camera.K[5] / ref_camera.K[4] + H[5]; 274 | tmp[6] = H[6] / ref_camera.K[0]; 275 | tmp[7] = H[7] / ref_camera.K[4]; 276 | tmp[8] = -H[6] * ref_camera.K[2] / ref_camera.K[0] - H[7] * ref_camera.K[5] / ref_camera.K[4] + H[8]; 277 | 278 | H[0] = src_camera.K[0] * tmp[0] + src_camera.K[2] * tmp[6]; 279 | H[1] = src_camera.K[0] * tmp[1] + src_camera.K[2] * tmp[7]; 280 | H[2] = src_camera.K[0] * tmp[2] + src_camera.K[2] * tmp[8]; 281 | H[3] = src_camera.K[4] * tmp[3] + src_camera.K[5] * tmp[6]; 282 | H[4] = src_camera.K[4] * tmp[4] + src_camera.K[5] * tmp[7]; 283 | H[5] = src_camera.K[4] * tmp[5] + src_camera.K[5] * tmp[8]; 284 | H[6] = src_camera.K[8] * tmp[6]; 285 | H[7] = src_camera.K[8] * tmp[7]; 286 | H[8] = src_camera.K[8] * tmp[8]; 287 | } 288 | 289 | __device__ float2 ComputeCorrespondingPoint(const float *H, const int2 p) 290 | { 291 | float3 pt; 292 | pt.x = H[0] * p.x + H[1] * p.y + H[2]; 293 | pt.y = H[3] * p.x + H[4] * p.y + H[5]; 294 | pt.z = H[6] * p.x + H[7] * p.y + H[8]; 295 | return make_float2(pt.x / pt.z, pt.y / pt.z); 296 | } 297 | 298 | __device__ float ComputeBilateralWeight(const float x_dist, const float y_dist, const float pix, const float center_pix, const float sigma_spatial, const float sigma_color) 299 | { 300 | const float spatial_dist = sqrt(x_dist * x_dist + y_dist * y_dist); 301 | const float color_dist = fabs(pix - center_pix); 302 | return exp(-spatial_dist / (2.0f * sigma_spatial* sigma_spatial) - color_dist / (2.0f * sigma_color * sigma_color)); 303 | } 304 | 305 | __device__ float ComputeBilateralNCC(const cudaTextureObject_t ref_image, const Camera ref_camera, const cudaTextureObject_t src_image, const Camera src_camera, const int2 p, const float4 plane_hypothesis, const PatchMatchParams params) 306 | { 307 | const float cost_max = 2.0f; 308 | int radius = params.patch_size / 2; 309 | 310 | float H[9]; 311 | ComputeHomography(ref_camera, src_camera, plane_hypothesis, H); 312 | float2 pt = ComputeCorrespondingPoint(H, p); 313 | if (pt.x >= src_camera.width || pt.x < 0.0f || pt.y >= src_camera.height || pt.y < 0.0f) { 314 | return cost_max; 315 | } 316 | 317 | float cost = 0.0f; 318 | { 319 | float sum_ref = 0.0f; 320 | float sum_ref_ref = 0.0f; 321 | float sum_src = 0.0f; 322 | float sum_src_src = 0.0f; 323 | float sum_ref_src = 0.0f; 324 | float bilateral_weight_sum = 0.0f; 325 | const float ref_center_pix = tex2D(ref_image, p.x + 0.5f, p.y + 0.5f); 326 | 327 | for (int i = -radius; i < radius + 1; i += params.radius_increment) { 328 | float sum_ref_row = 0.0f; 329 | float sum_src_row = 0.0f; 330 | float sum_ref_ref_row = 0.0f; 331 | float sum_src_src_row = 0.0f; 332 | float sum_ref_src_row = 0.0f; 333 | float bilateral_weight_sum_row = 0.0f; 334 | 335 | for (int j = -radius; j < radius + 1; j += params.radius_increment) { 336 | const int2 ref_pt = make_int2(p.x + i, p.y + j); 337 | const float ref_pix = tex2D(ref_image, ref_pt.x + 0.5f, ref_pt.y + 0.5f); 338 | float2 src_pt = ComputeCorrespondingPoint(H, ref_pt); 339 | const float src_pix = tex2D(src_image, src_pt.x + 0.5f, src_pt.y + 0.5f); 340 | 341 | float weight = ComputeBilateralWeight(i, j, ref_pix, ref_center_pix, params.sigma_spatial, params.sigma_color); 342 | 343 | sum_ref_row += weight * ref_pix; 344 | sum_ref_ref_row += weight * ref_pix * ref_pix; 345 | sum_src_row += weight * src_pix; 346 | sum_src_src_row += weight * src_pix * src_pix; 347 | sum_ref_src_row += weight * ref_pix * src_pix; 348 | bilateral_weight_sum_row += weight; 349 | } 350 | 351 | sum_ref += sum_ref_row; 352 | sum_ref_ref += sum_ref_ref_row; 353 | sum_src += sum_src_row; 354 | sum_src_src += sum_src_src_row; 355 | sum_ref_src += sum_ref_src_row; 356 | bilateral_weight_sum += bilateral_weight_sum_row; 357 | } 358 | const float inv_bilateral_weight_sum = 1.0f / bilateral_weight_sum; 359 | sum_ref *= inv_bilateral_weight_sum; 360 | sum_ref_ref *= inv_bilateral_weight_sum; 361 | sum_src *= inv_bilateral_weight_sum; 362 | sum_src_src *= inv_bilateral_weight_sum; 363 | sum_ref_src *= inv_bilateral_weight_sum; 364 | 365 | const float var_ref = sum_ref_ref - sum_ref * sum_ref; 366 | const float var_src = sum_src_src - sum_src * sum_src; 367 | 368 | const float kMinVar = 1e-5f; 369 | if (var_ref < kMinVar || var_src < kMinVar) { 370 | return cost = cost_max; 371 | } else { 372 | const float covar_src_ref = sum_ref_src - sum_ref * sum_src; 373 | const float var_ref_src = sqrt(var_ref * var_src); 374 | return cost = max(0.0f, min(cost_max, 1.0f - covar_src_ref / var_ref_src)); 375 | } 376 | } 377 | } 378 | 379 | __device__ float ComputeMultiViewInitialCostandSelectedViews(const cudaTextureObject_t *images, const Camera *cameras, const int2 p, const float4 plane_hypothesis, unsigned int *selected_views, const PatchMatchParams params) 380 | { 381 | float cost_max = 2.0f; 382 | float cost_vector[32] = {2.0f}; 383 | float cost_vector_copy[32] = {2.0f}; 384 | int cost_count = 0; 385 | int num_valid_views = 0; 386 | 387 | for (int i = 1; i < params.num_images; ++i) { 388 | float c = ComputeBilateralNCC(images[0], cameras[0], images[i], cameras[i], p, plane_hypothesis, params); 389 | cost_vector[i - 1] = c; 390 | cost_vector_copy[i - 1] = c; 391 | cost_count++; 392 | if (c < cost_max) { 393 | num_valid_views++; 394 | } 395 | } 396 | 397 | sort_small(cost_vector, cost_count); 398 | *selected_views = 0; 399 | 400 | int top_k = min(num_valid_views, params.top_k); 401 | if (top_k > 0) { 402 | float cost = 0.0f; 403 | for (int i = 0; i < top_k; ++i) { 404 | cost += cost_vector[i]; 405 | } 406 | float cost_threshold = cost_vector[top_k - 1]; 407 | for (int i = 0; i < params.num_images - 1; ++i) { 408 | if (cost_vector_copy[i] <= cost_threshold) { 409 | setBit(*selected_views, i); 410 | } 411 | } 412 | return cost / top_k; 413 | } else { 414 | return cost_max; 415 | } 416 | } 417 | 418 | __device__ void ComputeMultiViewCostVector(const cudaTextureObject_t *images, const Camera *cameras, const int2 p, const float4 plane_hypothesis, float *cost_vector, const PatchMatchParams params) 419 | { 420 | for (int i = 1; i < params.num_images; ++i) { 421 | cost_vector[i - 1] = ComputeBilateralNCC(images[0], cameras[0], images[i], cameras[i], p, plane_hypothesis, params); 422 | } 423 | } 424 | 425 | __device__ float3 Get3DPointonWorld_cu(const float x, const float y, const float depth, const Camera camera) 426 | { 427 | float3 pointX; 428 | float3 tmpX; 429 | // Reprojection 430 | pointX.x = depth * (x - camera.K[2]) / camera.K[0]; 431 | pointX.y = depth * (y - camera.K[5]) / camera.K[4]; 432 | pointX.z = depth; 433 | 434 | // Rotation 435 | tmpX.x = camera.R[0] * pointX.x + camera.R[3] * pointX.y + camera.R[6] * pointX.z; 436 | tmpX.y = camera.R[1] * pointX.x + camera.R[4] * pointX.y + camera.R[7] * pointX.z; 437 | tmpX.z = camera.R[2] * pointX.x + camera.R[5] * pointX.y + camera.R[8] * pointX.z; 438 | 439 | // Transformation 440 | float3 C; 441 | C.x = -(camera.R[0] * camera.t[0] + camera.R[3] * camera.t[1] + camera.R[6] * camera.t[2]); 442 | C.y = -(camera.R[1] * camera.t[0] + camera.R[4] * camera.t[1] + camera.R[7] * camera.t[2]); 443 | C.z = -(camera.R[2] * camera.t[0] + camera.R[5] * camera.t[1] + camera.R[8] * camera.t[2]); 444 | pointX.x = tmpX.x + C.x; 445 | pointX.y = tmpX.y + C.y; 446 | pointX.z = tmpX.z + C.z; 447 | 448 | return pointX; 449 | } 450 | 451 | __device__ void ProjectonCamera_cu(const float3 PointX, const Camera camera, float2 &point, float &depth) 452 | { 453 | float3 tmp; 454 | tmp.x = camera.R[0] * PointX.x + camera.R[1] * PointX.y + camera.R[2] * PointX.z + camera.t[0]; 455 | tmp.y = camera.R[3] * PointX.x + camera.R[4] * PointX.y + camera.R[5] * PointX.z + camera.t[1]; 456 | tmp.z = camera.R[6] * PointX.x + camera.R[7] * PointX.y + camera.R[8] * PointX.z + camera.t[2]; 457 | 458 | depth = camera.K[6] * tmp.x + camera.K[7] * tmp.y + camera.K[8] * tmp.z; 459 | point.x = (camera.K[0] * tmp.x + camera.K[1] * tmp.y + camera.K[2] * tmp.z) / depth; 460 | point.y = (camera.K[3] * tmp.x + camera.K[4] * tmp.y + camera.K[5] * tmp.z) / depth; 461 | } 462 | 463 | __device__ float ComputeGeomConsistencyCost(const cudaTextureObject_t depth_image, const Camera ref_camera, const Camera src_camera, const float4 plane_hypothesis, const int2 p) 464 | { 465 | const float max_cost = 5.0f; 466 | 467 | float depth = ComputeDepthfromPlaneHypothesis(ref_camera, plane_hypothesis, p); 468 | float3 forward_point = Get3DPointonWorld_cu(p.x, p.y, depth, ref_camera); 469 | 470 | float2 src_pt; 471 | float src_d; 472 | ProjectonCamera_cu(forward_point, src_camera, src_pt, src_d); 473 | const float src_depth = tex2D(depth_image, (int)src_pt.x + 0.5f, (int)src_pt.y + 0.5f); 474 | 475 | if (src_depth == 0.0f) { 476 | return max_cost; 477 | } 478 | 479 | float3 src_3D_pt = Get3DPointonWorld_cu(src_pt.x, src_pt.y, src_depth, src_camera); 480 | 481 | float2 backward_point; 482 | float ref_d; 483 | ProjectonCamera_cu(src_3D_pt, ref_camera, backward_point, ref_d); 484 | 485 | const float diff_col = p.x - backward_point.x; 486 | const float diff_row = p.y - backward_point.y; 487 | return min(max_cost, sqrt(diff_col * diff_col + diff_row * diff_row)); 488 | } 489 | 490 | __global__ void RandomInitialization(cudaTextureObjects *texture_objects, Camera *cameras, float4 *plane_hypotheses, float *costs, curandState *rand_states, unsigned int *selected_views, float4 *prior_planes, unsigned int *plane_masks, const PatchMatchParams params) 491 | { 492 | const int2 p = make_int2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); 493 | int width = cameras[0].width; 494 | int height = cameras[0].height; 495 | 496 | if (p.x >= width || p.y >= height) { 497 | return; 498 | } 499 | 500 | const int center = p.y * width + p.x; 501 | curand_init(clock64(), p.y, p.x, &rand_states[center]); 502 | 503 | if (params.geom_consistency) { 504 | float4 plane_hypothesis = plane_hypotheses[center]; 505 | float depth = plane_hypothesis.w; 506 | plane_hypothesis.w = GetDistance2Origin(cameras[0], p, depth, plane_hypothesis); 507 | plane_hypotheses[center] = plane_hypothesis; 508 | costs[center] = ComputeMultiViewInitialCostandSelectedViews(texture_objects[0].images, cameras, p, plane_hypotheses[center], &selected_views[center], params); 509 | } 510 | else if (params.planar_prior) { 511 | if (plane_masks[center] > 0 && costs[center] >= 0.1f) { 512 | float perturbation = 0.02f; 513 | 514 | float4 plane_hypothesis = prior_planes[center]; 515 | float depth_perturbed = plane_hypothesis.w; 516 | const float depth_min_perturbed = (1 - 3 * perturbation) * depth_perturbed; 517 | const float depth_max_perturbed = (1 + 3 * perturbation) * depth_perturbed; 518 | depth_perturbed = curand_uniform(&rand_states[center]) * (depth_max_perturbed - depth_min_perturbed) + depth_min_perturbed; 519 | float4 plane_hypothesis_perturbed = GeneratePerturbedNormal(cameras[0], p, plane_hypothesis, &rand_states[center], 3 * perturbation * M_PI); 520 | plane_hypothesis_perturbed.w = depth_perturbed; 521 | plane_hypotheses[center] = plane_hypothesis_perturbed; 522 | costs[center] = ComputeMultiViewInitialCostandSelectedViews(texture_objects[0].images, cameras, p, plane_hypotheses[center], &selected_views[center], params); 523 | } 524 | else { 525 | float4 plane_hypothesis = plane_hypotheses[center]; 526 | float depth = plane_hypothesis.w; 527 | plane_hypothesis.w = GetDistance2Origin(cameras[0], p, depth, plane_hypothesis); 528 | plane_hypotheses[center] = plane_hypothesis; 529 | costs[center] = ComputeMultiViewInitialCostandSelectedViews(texture_objects[0].images, cameras, p, plane_hypotheses[center], &selected_views[center], params); 530 | } 531 | } 532 | else { 533 | plane_hypotheses[center] = GenerateRandomPlaneHypothesis(cameras[0], p, &rand_states[center], params.depth_min, params.depth_max); 534 | costs[center] = ComputeMultiViewInitialCostandSelectedViews(texture_objects[0].images, cameras, p, plane_hypotheses[center], &selected_views[center], params); 535 | } 536 | } 537 | 538 | __device__ void PlaneHypothesisRefinement(const cudaTextureObject_t *images, const cudaTextureObject_t *depth_images, const Camera *cameras, float4 *plane_hypothesis, float *depth, float *cost, curandState *rand_state, const float *view_weights, const float weight_norm, float4 *prior_planes, unsigned int *plane_masks, float *restricted_cost, const int2 p, const PatchMatchParams params) 539 | { 540 | float perturbation = 0.02f; 541 | const int center = p.y * cameras[0].width + p.x; 542 | 543 | float gamma = 0.5f; 544 | float depth_sigma = (params.depth_max - params.depth_min) / 64.0f; 545 | float two_depth_sigma_squared = 2 * depth_sigma * depth_sigma; 546 | float angle_sigma = M_PI * (5.0f / 180.0f); 547 | float two_angle_sigma_squared = 2 * angle_sigma * angle_sigma; 548 | float beta = 0.18f; 549 | float depth_prior = 0.0f; 550 | 551 | float depth_rand; 552 | float4 plane_hypothesis_rand; 553 | if (params.planar_prior && plane_masks[center] > 0) { 554 | depth_prior = ComputeDepthfromPlaneHypothesis(cameras[0], prior_planes[center], p); 555 | depth_rand = curand_uniform(rand_state) * 6 * depth_sigma + (depth_prior - 3 * depth_sigma); 556 | plane_hypothesis_rand = GeneratePerturbedNormal(cameras[0], p, prior_planes[center], rand_state, angle_sigma); 557 | } 558 | else { 559 | depth_rand = curand_uniform(rand_state) * (params.depth_max - params.depth_min) + params.depth_min; 560 | plane_hypothesis_rand = GenerateRandomNormal(cameras[0], p, rand_state, *depth); 561 | } 562 | float depth_perturbed = *depth; 563 | const float depth_min_perturbed = (1 - perturbation) * depth_perturbed; 564 | const float depth_max_perturbed = (1 + perturbation) * depth_perturbed; 565 | do { 566 | depth_perturbed = curand_uniform(rand_state) * (depth_max_perturbed - depth_min_perturbed) + depth_min_perturbed; 567 | } while (depth_perturbed < params.depth_min && depth_perturbed > params.depth_max); 568 | float4 plane_hypothesis_perturbed = GeneratePerturbedNormal(cameras[0], p, *plane_hypothesis, rand_state, perturbation * M_PI); // GeneratePertubedPlaneHypothesis(cameras[0], p, rand_state, perturbation, *plane_hypothesis, *depth, params.depth_min, params.depth_max); 569 | 570 | const int num_planes = 5; 571 | float depths[num_planes] = {depth_rand, *depth, depth_rand, *depth, depth_perturbed}; 572 | float4 normals[num_planes] = {*plane_hypothesis, plane_hypothesis_rand, plane_hypothesis_rand, plane_hypothesis_perturbed, *plane_hypothesis}; 573 | 574 | for (int i = 0; i < num_planes; ++i) { 575 | float cost_vector[32] = {2.0f}; 576 | float4 temp_plane_hypothesis = normals[i]; 577 | temp_plane_hypothesis.w = GetDistance2Origin(cameras[0], p, depths[i], temp_plane_hypothesis); // dists[i]; 578 | ComputeMultiViewCostVector(images, cameras, p, temp_plane_hypothesis, cost_vector, params); 579 | 580 | float temp_cost = 0.0f; 581 | for (int j = 0; j < params.num_images - 1; ++j) { 582 | if (view_weights[j] > 0) { 583 | if (params.geom_consistency) { 584 | temp_cost += view_weights[j] * (cost_vector[j] + 0.1f * ComputeGeomConsistencyCost(depth_images[j+1], cameras[0], cameras[j+1], temp_plane_hypothesis, p)); 585 | } 586 | else { 587 | temp_cost += view_weights[j] * cost_vector[j]; 588 | } 589 | } 590 | } 591 | temp_cost /= weight_norm; 592 | 593 | float depth_before = ComputeDepthfromPlaneHypothesis(cameras[0], temp_plane_hypothesis, p); 594 | if (params.planar_prior && plane_masks[center] > 0) { 595 | float depth_diff = depths[i] - depth_prior; 596 | float angle_cos = Vec3DotVec3(prior_planes[center], temp_plane_hypothesis); 597 | float angle_diff = acos(angle_cos); 598 | float prior = gamma + exp(- depth_diff * depth_diff / two_depth_sigma_squared) * exp(- angle_diff * angle_diff / two_angle_sigma_squared); 599 | float restricted_temp_cost = exp(-temp_cost * temp_cost / beta) * prior; 600 | if (depth_before >= params.depth_min && depth_before <= params.depth_max && restricted_temp_cost > *restricted_cost) { 601 | *depth = depth_before; 602 | *plane_hypothesis = temp_plane_hypothesis; 603 | *cost = temp_cost; 604 | *restricted_cost = restricted_temp_cost; 605 | } 606 | } 607 | else { 608 | if (depth_before >= params.depth_min && depth_before <= params.depth_max && temp_cost < *cost) { 609 | *depth = depth_before; 610 | *plane_hypothesis = temp_plane_hypothesis; 611 | *cost = temp_cost; 612 | } 613 | } 614 | } 615 | } 616 | 617 | __device__ void CheckerboardPropagation(const cudaTextureObject_t *images, const cudaTextureObject_t *depths, const Camera *cameras, float4 *plane_hypotheses, float *costs, curandState *rand_states, unsigned int *selected_views, float4 *prior_planes, unsigned int *plane_masks, const int2 p, const PatchMatchParams params, const int iter) 618 | { 619 | int width = cameras[0].width; 620 | int height = cameras[0].height; 621 | if (p.x >= width || p.y >= height) { 622 | return; 623 | } 624 | 625 | const int center = p.y * width + p.x; 626 | int left_near = center - 1; 627 | int left_far = center - 3; 628 | int right_near = center + 1; 629 | int right_far = center + 3; 630 | int up_near = center - width; 631 | int up_far = center - 3 * width; 632 | int down_near = center + width; 633 | int down_far = center + 3 * width; 634 | 635 | // Adaptive Checkerboard Sampling 636 | float cost_array[8][32] = {2.0f}; 637 | // 0 -- up_near, 1 -- up_far, 2 -- down_near, 3 -- down_far, 4 -- left_near, 5 -- left_far, 6 -- right_near, 7 -- right_far 638 | bool flag[8] = {false}; 639 | int num_valid_pixels = 0; 640 | 641 | float costMin; 642 | int costMinPoint; 643 | 644 | // up_far 645 | if (p.y > 2) { 646 | flag[1] = true; 647 | num_valid_pixels++; 648 | costMin = costs[up_far]; 649 | costMinPoint = up_far; 650 | for (int i = 1; i < 11; ++i) { 651 | if (p.y > 2 + 2 * i) { 652 | int pointTemp = up_far - 2 * i * width; 653 | if (costs[pointTemp] < costMin) { 654 | costMin = costs[pointTemp]; 655 | costMinPoint = pointTemp; 656 | } 657 | } 658 | } 659 | up_far = costMinPoint; 660 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[up_far], cost_array[1], params); 661 | } 662 | 663 | // dwon_far 664 | if (p.y < height - 3) { 665 | flag[3] = true; 666 | num_valid_pixels++; 667 | costMin = costs[down_far]; 668 | costMinPoint = down_far; 669 | for (int i = 1; i < 11; ++i) { 670 | if (p.y < height - 3 - 2 * i) { 671 | int pointTemp = down_far + 2 * i * width; 672 | if (costs[pointTemp] < costMin) { 673 | costMin = costs[pointTemp]; 674 | costMinPoint = pointTemp; 675 | } 676 | } 677 | } 678 | down_far = costMinPoint; 679 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[down_far], cost_array[3], params); 680 | } 681 | 682 | // left_far 683 | if (p.x > 2) { 684 | flag[5] = true; 685 | num_valid_pixels++; 686 | costMin = costs[left_far]; 687 | costMinPoint = left_far; 688 | for (int i = 1; i < 11; ++i) { 689 | if (p.x > 2 + 2 * i) { 690 | int pointTemp = left_far - 2 * i; 691 | if (costs[pointTemp] < costMin) { 692 | costMin = costs[pointTemp]; 693 | costMinPoint = pointTemp; 694 | } 695 | } 696 | } 697 | left_far = costMinPoint; 698 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[left_far], cost_array[5], params); 699 | } 700 | 701 | // right_far 702 | if (p.x < width - 3) { 703 | flag[7] = true; 704 | num_valid_pixels++; 705 | costMin = costs[right_far]; 706 | costMinPoint = right_far; 707 | for (int i = 1; i < 11; ++i) { 708 | if (p.x < width - 3 - 2 * i) { 709 | int pointTemp = right_far + 2 * i; 710 | if (costMin < costs[pointTemp]) { 711 | costMin = costs[pointTemp]; 712 | costMinPoint = pointTemp; 713 | } 714 | } 715 | } 716 | right_far = costMinPoint; 717 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[right_far], cost_array[7], params); 718 | } 719 | 720 | // up_near 721 | if (p.y > 0) { 722 | flag[0] = true; 723 | num_valid_pixels++; 724 | costMin = costs[up_near]; 725 | costMinPoint = up_near; 726 | for (int i = 0; i < 3; ++i) { 727 | if (p.y > 1 + i && p.x > i) { 728 | int pointTemp = up_near - (1 + i) * width - i; 729 | if (costs[pointTemp] < costMin) { 730 | costMin = costs[pointTemp]; 731 | costMinPoint = pointTemp; 732 | } 733 | } 734 | if (p.y > 1 + i && p.x < width - 1 - i) { 735 | int pointTemp = up_near - (1 + i) * width + i; 736 | if (costs[pointTemp] < costMin) { 737 | costMin = costs[pointTemp]; 738 | costMinPoint = pointTemp; 739 | } 740 | } 741 | } 742 | up_near = costMinPoint; 743 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[up_near], cost_array[0], params); 744 | } 745 | 746 | // down_near 747 | if (p.y < height - 1) { 748 | flag[2] = true; 749 | num_valid_pixels++; 750 | costMin = costs[down_near]; 751 | costMinPoint = down_near; 752 | for (int i = 0; i < 3; ++i) { 753 | if (p.y < height - 2 - i && p.x > i) { 754 | int pointTemp = down_near + (1 + i) * width - i; 755 | if (costs[pointTemp] < costMin) { 756 | costMin = costs[pointTemp]; 757 | costMinPoint = pointTemp; 758 | } 759 | } 760 | if (p.y < height - 2 - i && p.x < width - 1 - i) { 761 | int pointTemp = down_near + (1 + i) * width + i; 762 | if (costs[pointTemp] < costMin) { 763 | costMin = costs[pointTemp]; 764 | costMinPoint = pointTemp; 765 | } 766 | } 767 | } 768 | down_near = costMinPoint; 769 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[down_near], cost_array[2], params); 770 | } 771 | 772 | // left_near 773 | if (p.x > 0) { 774 | flag[4] = true; 775 | num_valid_pixels++; 776 | costMin = costs[left_near]; 777 | costMinPoint = left_near; 778 | for (int i = 0; i < 3; ++i) { 779 | if (p.x > 1 + i && p.y > i) { 780 | int pointTemp = left_near - (1 + i) - i * width; 781 | if (costs[pointTemp] < costMin) { 782 | costMin = costs[pointTemp]; 783 | costMinPoint = pointTemp; 784 | } 785 | } 786 | if (p.x > 1 + i && p.y < height - 1 - i) { 787 | int pointTemp = left_near - (1 + i) + i * width; 788 | if (costs[pointTemp] < costMin) { 789 | costMin = costs[pointTemp]; 790 | costMinPoint = pointTemp; 791 | } 792 | } 793 | } 794 | left_near = costMinPoint; 795 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[left_near], cost_array[4], params); 796 | } 797 | 798 | // right_near 799 | if (p.x < width - 1) { 800 | flag[6] = true; 801 | num_valid_pixels++; 802 | costMin = costs[right_near]; 803 | costMinPoint = right_near; 804 | for (int i = 0; i < 3; ++i) { 805 | if (p.x < width - 2 - i && p.y > i) { 806 | int pointTemp = right_near + (1 + i) - i * width; 807 | if (costs[pointTemp] < costMin) { 808 | costMin = costs[pointTemp]; 809 | costMinPoint = pointTemp; 810 | } 811 | } 812 | if (p.x < width - 2 - i && p.y < height - 1- i) { 813 | int pointTemp = right_near + (1 + i) + i * width; 814 | if (costs[pointTemp] < costMin) { 815 | costMin = costs[pointTemp]; 816 | costMinPoint = pointTemp; 817 | } 818 | } 819 | } 820 | right_near = costMinPoint; 821 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[right_near], cost_array[6], params); 822 | } 823 | const int positions[8] = {up_near, up_far, down_near, down_far, left_near, left_far, right_near, right_far}; 824 | 825 | // Multi-hypothesis Joint View Selection 826 | float view_weights[32] = {0.0f}; 827 | float view_selection_priors[32] = {0.0f}; 828 | int neighbor_positions[4] = {center - width, center + width, center - 1, center + 1}; 829 | for (int i = 0; i < 4; ++i) { 830 | if (flag[2 * i]) { 831 | for (int j = 0; j < params.num_images - 1; ++j) { 832 | if (isSet(selected_views[neighbor_positions[i]], j) == 1) { 833 | view_selection_priors[j] += 0.9f; 834 | } else { 835 | view_selection_priors[j] += 0.1f; 836 | } 837 | } 838 | } 839 | } 840 | 841 | float sampling_probs[32] = {0.0f}; 842 | for (int i = 0; i < params.num_images - 1; ++i) { 843 | float temp_w = 0.0f; 844 | for (int j = 0; j < 8; ++j) { 845 | if (flag[j]) { 846 | temp_w += exp(cost_array[j][i] * cost_array[j][i] / -0.18f); 847 | } 848 | } 849 | sampling_probs[i] = temp_w / num_valid_pixels * view_selection_priors[i]; 850 | } 851 | 852 | TransformPDFToCDF(sampling_probs, params.num_images - 1); 853 | for (int sample = 0; sample < 15; ++sample) { 854 | const float rand_prob = curand_uniform(&rand_states[center]) - FLT_EPSILON; 855 | 856 | for (int image_id = 0; image_id < params.num_images - 1; ++image_id) { 857 | const float prob = sampling_probs[image_id]; 858 | if (prob > rand_prob) { 859 | view_weights[image_id] += 1.0f; 860 | break; 861 | } 862 | } 863 | } 864 | 865 | unsigned int temp_selected_views = 0; 866 | int num_selected_view = 0; 867 | float weight_norm = 0; 868 | for (int i = 0; i < params.num_images - 1; ++i) { 869 | if (view_weights[i] > 0) { 870 | setBit(temp_selected_views, i); 871 | weight_norm += view_weights[i]; 872 | num_selected_view++; 873 | } 874 | } 875 | 876 | float final_costs[8] = {0.0f}; 877 | for (int i = 0; i < 8; ++i) { 878 | for (int j = 0; j < params.num_images - 1; ++j) { 879 | if (view_weights[j] > 0) { 880 | if (params.geom_consistency) { 881 | if (flag[i]) { 882 | final_costs[i] += view_weights[j] * (cost_array[i][j] + 0.1f * ComputeGeomConsistencyCost(depths[j+1], cameras[0], cameras[j+1], plane_hypotheses[positions[i]], p)); 883 | } 884 | else { 885 | final_costs[i] += view_weights[j] * (cost_array[i][j] + 0.1f * 5.0f); 886 | } 887 | } 888 | else { 889 | final_costs[i] += view_weights[j] * cost_array[i][j]; 890 | } 891 | } 892 | } 893 | final_costs[i] /= weight_norm; 894 | } 895 | 896 | const int min_cost_idx = FindMinCostIndex(final_costs, 8); 897 | 898 | float cost_vector_now[32] = {2.0f}; 899 | ComputeMultiViewCostVector(images, cameras, p, plane_hypotheses[center], cost_vector_now, params); 900 | float cost_now = 0.0f; 901 | for (int i = 0; i < params.num_images - 1; ++i) { 902 | if (params.geom_consistency) { 903 | cost_now += view_weights[i] * (cost_vector_now[i] + 0.1f * ComputeGeomConsistencyCost(depths[i+1], cameras[0], cameras[i+1], plane_hypotheses[center], p)); 904 | } 905 | else { 906 | cost_now += view_weights[i] * cost_vector_now[i]; 907 | } 908 | } 909 | cost_now /= weight_norm; 910 | costs[center] = cost_now; 911 | float depth_now = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[center], p); 912 | 913 | float restricted_cost = 0.0f; 914 | if (params.planar_prior) { 915 | float restricted_final_costs[8] = {0.0f}; 916 | float gamma = 0.5f; 917 | float depth_sigma = (params.depth_max - params.depth_min) / 64.0f; 918 | float two_depth_sigma_squared = 2 * depth_sigma * depth_sigma; 919 | float angle_sigma = M_PI * (5.0f / 180.0f); 920 | float two_angle_sigma_squared = 2 * angle_sigma * angle_sigma; 921 | float depth_prior = ComputeDepthfromPlaneHypothesis(cameras[0], prior_planes[center], p); 922 | float beta = 0.18f; 923 | 924 | if (plane_masks[center] > 0) { 925 | for (int i = 0; i < 8; i++) { 926 | if (flag[i]) { 927 | float depth_now = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[positions[i]], p); 928 | float depth_diff = depth_now - depth_prior; 929 | float angle_cos = Vec3DotVec3(prior_planes[center], plane_hypotheses[positions[i]]); 930 | float angle_diff = acos(angle_cos); 931 | float prior = gamma + exp(- depth_diff * depth_diff / two_depth_sigma_squared) * exp(- angle_diff * angle_diff / two_angle_sigma_squared); 932 | restricted_final_costs[i] = exp(-final_costs[i] * final_costs[i] / beta) * prior; 933 | } 934 | } 935 | const int max_cost_idx = FindMaxCostIndex(restricted_final_costs, 8); 936 | 937 | float restricted_cost_now = 0.0f; 938 | float depth_now = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[center], p); 939 | float depth_diff = depth_now - depth_prior; 940 | float angle_cos = Vec3DotVec3(prior_planes[center], plane_hypotheses[center]); 941 | float angle_diff = acos(angle_cos); 942 | float prior = gamma + exp(- depth_diff * depth_diff / two_depth_sigma_squared) * exp(- angle_diff * angle_diff / two_angle_sigma_squared); 943 | restricted_cost_now = exp(-cost_now * cost_now / beta) * prior; 944 | 945 | if (flag[max_cost_idx]) { 946 | float depth_before = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[positions[max_cost_idx]], p); 947 | 948 | if (depth_before >= params.depth_min && depth_before <= params.depth_max && restricted_final_costs[max_cost_idx] > restricted_cost_now) { 949 | depth_now = depth_before; 950 | plane_hypotheses[center] = plane_hypotheses[positions[max_cost_idx]]; 951 | costs[center] = final_costs[max_cost_idx]; 952 | restricted_cost = restricted_final_costs[max_cost_idx]; 953 | selected_views[center] = temp_selected_views; 954 | } 955 | } 956 | } 957 | else if (flag[min_cost_idx]) { 958 | float depth_before = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[positions[min_cost_idx]], p); 959 | 960 | if (depth_before >= params.depth_min && depth_before <= params.depth_max && final_costs[min_cost_idx] < cost_now) { 961 | depth_now = depth_before; 962 | plane_hypotheses[center] = plane_hypotheses[positions[min_cost_idx]]; 963 | costs[center] = final_costs[min_cost_idx]; 964 | } 965 | } 966 | } 967 | 968 | if (!params.planar_prior && flag[min_cost_idx]) { 969 | float depth_before = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[positions[min_cost_idx]], p); 970 | 971 | if (depth_before >= params.depth_min && depth_before <= params.depth_max && final_costs[min_cost_idx] < cost_now) { 972 | depth_now = depth_before; 973 | plane_hypotheses[center] = plane_hypotheses[positions[min_cost_idx]]; 974 | costs[center] = final_costs[min_cost_idx]; 975 | selected_views[center] = temp_selected_views; 976 | } 977 | } 978 | 979 | PlaneHypothesisRefinement(images, depths, cameras, &plane_hypotheses[center], &depth_now, &costs[center], &rand_states[center], view_weights, weight_norm, prior_planes, plane_masks, &restricted_cost, p, params); 980 | } 981 | 982 | __global__ void BlackPixelUpdate(cudaTextureObjects *texture_objects, cudaTextureObjects *texture_depths, Camera *cameras, float4 *plane_hypotheses, float *costs, curandState *rand_states, unsigned int *selected_views, float4 *prior_planes, unsigned int *plane_masks, const PatchMatchParams params, const int iter) 983 | { 984 | int2 p = make_int2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); 985 | if (threadIdx.x % 2 == 0) { 986 | p.y = p.y * 2; 987 | } else { 988 | p.y = p.y * 2 + 1; 989 | } 990 | 991 | CheckerboardPropagation(texture_objects[0].images, texture_depths[0].images, cameras, plane_hypotheses, costs, rand_states, selected_views, prior_planes, plane_masks, p, params, iter); 992 | } 993 | 994 | __global__ void RedPixelUpdate(cudaTextureObjects *texture_objects, cudaTextureObjects *texture_depths, Camera *cameras, float4 *plane_hypotheses, float *costs, curandState *rand_states, unsigned int *selected_views, float4 *prior_planes, unsigned int *plane_masks, const PatchMatchParams params, const int iter) 995 | { 996 | int2 p = make_int2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); 997 | if (threadIdx.x % 2 == 0) { 998 | p.y = p.y * 2 + 1; 999 | } else { 1000 | p.y = p.y * 2; 1001 | } 1002 | 1003 | CheckerboardPropagation(texture_objects[0].images, texture_depths[0].images, cameras, plane_hypotheses, costs, rand_states, selected_views, prior_planes, plane_masks, p, params, iter); 1004 | } 1005 | 1006 | __global__ void GetDepthandNormal(Camera *cameras, float4 *plane_hypotheses, const PatchMatchParams params) 1007 | { 1008 | const int2 p = make_int2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); 1009 | const int width = cameras[0].width; 1010 | const int height = cameras[0].height; 1011 | 1012 | if (p.x >= width || p.y >= height) { 1013 | return; 1014 | } 1015 | 1016 | const int center = p.y * width + p.x; 1017 | plane_hypotheses[center].w = ComputeDepthfromPlaneHypothesis(cameras[0], plane_hypotheses[center], p); 1018 | } 1019 | 1020 | __device__ void CheckerboardFilter(const Camera *cameras, float4 *plane_hypotheses, float *costs, const int2 p) 1021 | { 1022 | int width = cameras[0].width; 1023 | int height = cameras[0].height; 1024 | if (p.x >= width || p.y >= height) { 1025 | return; 1026 | } 1027 | 1028 | const int center = p.y * width + p.x; 1029 | 1030 | float filter[21]; 1031 | int index = 0; 1032 | 1033 | filter[index++] = plane_hypotheses[center].w; 1034 | 1035 | // Left 1036 | const int left = center - 1; 1037 | const int leftleft = center - 3; 1038 | 1039 | // Up 1040 | const int up = center - width; 1041 | const int upup = center - 3 * width; 1042 | 1043 | // Down 1044 | const int down = center + width; 1045 | const int downdown = center + 3 * width; 1046 | 1047 | // Right 1048 | const int right = center + 1; 1049 | const int rightright = center + 3; 1050 | 1051 | if (costs[center] < 0.001f) { 1052 | return; 1053 | } 1054 | 1055 | if (p.y>0) { 1056 | filter[index++] = plane_hypotheses[up].w; 1057 | } 1058 | if (p.y>2) { 1059 | filter[index++] = plane_hypotheses[upup].w; 1060 | } 1061 | if (p.y>4) { 1062 | filter[index++] = plane_hypotheses[upup-width*2].w; 1063 | } 1064 | if (p.y0) { 1074 | filter[index++] = plane_hypotheses[left].w; 1075 | } 1076 | if (p.x>2) { 1077 | filter[index++] = plane_hypotheses[leftleft].w; 1078 | } 1079 | if (p.x>4) { 1080 | filter[index++] = plane_hypotheses[leftleft-2].w; 1081 | } 1082 | if (p.x0 && 1092 | p.x0 && 1100 | p.x>1) 1101 | { 1102 | filter[index++] = plane_hypotheses[up-2].w; 1103 | } 1104 | if (p.y1) { 1106 | filter[index++] = plane_hypotheses[down-2].w; 1107 | } 1108 | if (p.x>0 && 1109 | p.y>2) 1110 | { 1111 | filter[index++] = plane_hypotheses[left - width*2].w; 1112 | } 1113 | if (p.x2) 1115 | { 1116 | filter[index++] = plane_hypotheses[right - width*2].w; 1117 | } 1118 | if (p.x>0 && 1119 | p.y>>(texture_objects_cuda, cameras_cuda, plane_hypotheses_cuda, costs_cuda, rand_states_cuda, selected_views_cuda, prior_planes_cuda, plane_masks_cuda, params); 1190 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1191 | 1192 | for (int i = 0; i < max_iterations; ++i) { 1193 | BlackPixelUpdate<<>>(texture_objects_cuda, texture_depths_cuda, cameras_cuda, plane_hypotheses_cuda, costs_cuda, rand_states_cuda, selected_views_cuda, prior_planes_cuda, plane_masks_cuda, params, i); 1194 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1195 | RedPixelUpdate<<>>(texture_objects_cuda, texture_depths_cuda, cameras_cuda, plane_hypotheses_cuda, costs_cuda, rand_states_cuda, selected_views_cuda, prior_planes_cuda, plane_masks_cuda, params, i); 1196 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1197 | printf("iteration: %d\n", i); 1198 | } 1199 | 1200 | GetDepthandNormal<<>>(cameras_cuda, plane_hypotheses_cuda, params); 1201 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1202 | 1203 | BlackPixelFilter<<>>(cameras_cuda, plane_hypotheses_cuda, costs_cuda); 1204 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1205 | RedPixelFilter<<>>(cameras_cuda, plane_hypotheses_cuda, costs_cuda); 1206 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1207 | 1208 | cudaMemcpy(plane_hypotheses_host, plane_hypotheses_cuda, sizeof(float4) * width * height, cudaMemcpyDeviceToHost); 1209 | cudaMemcpy(costs_host, costs_cuda, sizeof(float) * width * height, cudaMemcpyDeviceToHost); 1210 | CUDA_SAFE_CALL(cudaDeviceSynchronize()); 1211 | } 1212 | -------------------------------------------------------------------------------- /src/ACMP.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "colmap_interface/endian.h" 7 | #include "./types.h" 8 | 9 | float3 Get3DPointonRefCam(const int x, const int y, const float depth, const Camera camera); 10 | 11 | #define CUDA_SAFE_CALL(error) CudaSafeCall(error, __FILE__, __LINE__) 12 | #define CUDA_CHECK_ERROR() CudaCheckError(__FILE__, __LINE__) 13 | 14 | void CudaSafeCall(const cudaError_t error, const std::string& file, const int line); 15 | void CudaCheckError(const char* file, const int line); 16 | 17 | struct cudaTextureObjects { 18 | cudaTextureObject_t images[MAX_IMAGES]; 19 | }; 20 | 21 | struct PatchMatchParams { 22 | int max_iterations = 3; 23 | int patch_size = 11; 24 | int num_images = 5; 25 | int max_image_size=3200; 26 | int radius_increment = 2; 27 | float sigma_spatial = 5.0f; 28 | float sigma_color = 3.0f; 29 | int top_k = 4; 30 | float baseline = 0.54f; 31 | float depth_min = 0.0f; 32 | float depth_max = 1.0f; 33 | float disparity_min = 0.0f; 34 | float disparity_max = 1.0f; 35 | // 下面有三个参数会使得程序处在很多中模式下，一般 36 | // 1. 用平面先验，去掉几何一致 37 | // 2. 不用平面先验，加上几何一致，但是读入的先验是上一次photometric得到的深度图 38 | // 3. 不用平面先验，加上几何一致，并且读入的是上一次geometric得到的深度图 39 | bool geom_consistency = false; 40 | bool multi_geometry = false; // 如果之前有过使用几何一致性才需要使用这个参数，这个参数会控制预先读入的数据是带几何一致的还是不带。 41 | bool planar_prior = false; 42 | }; 43 | 44 | class ACMP { 45 | public: 46 | ACMP(const std::unordered_map& _image_id_to_image_name, const std::unordered_map& _image_id_to_camera, const std::string& _depth_folder, const std::string& _normal_folder, const std::string& _image_folder); 47 | ~ACMP(); 48 | 49 | void InuputInitialization(const std::string &dense_folder, const Problem &problem); 50 | void Colmap2MVS(const std::string &dense_folder, std::vector &problems); 51 | void CudaSpaceInitialization(const std::string &dense_folder, const Problem &problem); 52 | void RunPatchMatch(); 53 | void SetGeomConsistencyParams(bool multi_geometry = false); 54 | void SetPlanarPriorParams(); 55 | int GetReferenceImageWidth(); 56 | int GetReferenceImageHeight(); 57 | cv::Mat GetReferenceImage(); 58 | float4 GetPlaneHypothesis(const int index); 59 | float GetCost(const int index); 60 | void GetSupportPoints(std::vector& support2DPoints); 61 | std::vector DelaunayTriangulation(const cv::Rect boundRC, const std::vector& points); 62 | float4 GetPriorPlaneParams(const Triangle triangle, const cv::Mat_ depths); 63 | float GetDepthFromPlaneParam(const float4 plane_hypothesis, const int x, const int y); 64 | float GetMinDepth(); 65 | float GetMaxDepth(); 66 | void CudaPlanarPriorInitialization(const std::vector &PlaneParams, const cv::Mat_ &masks); 67 | private: 68 | int num_images; 69 | std::vector images; 70 | std::vector depths; 71 | std::vector cameras; 72 | cudaTextureObjects texture_objects_host; 73 | cudaTextureObjects texture_depths_host; 74 | float4 *plane_hypotheses_host; 75 | float *costs_host; 76 | float4 *prior_planes_host; 77 | unsigned int *plane_masks_host; 78 | PatchMatchParams params; 79 | 80 | Camera *cameras_cuda; 81 | cudaArray *cuArray[MAX_IMAGES]; 82 | cudaArray *cuDepthArray[MAX_IMAGES]; 83 | cudaTextureObjects *texture_objects_cuda; 84 | cudaTextureObjects *texture_depths_cuda; 85 | float4 *plane_hypotheses_cuda; 86 | float *costs_cuda; 87 | curandState *rand_states_cuda; 88 | unsigned int *selected_views_cuda; 89 | float *depths_cuda; 90 | float4 *prior_planes_cuda; 91 | unsigned int *plane_masks_cuda; 92 | 93 | public: 94 | const std::unordered_map& image_id_to_image_name; 95 | const std::unordered_map& image_id_to_camera; 96 | const std::string depth_folder, normal_folder, image_folder; 97 | }; 98 | -------------------------------------------------------------------------------- /src/colmap_interface/colmap_interface.cpp: -------------------------------------------------------------------------------- 1 | #include "./colmap_interface.h" 2 | 3 | namespace colmap { 4 | 5 | bool IsNotWhiteSpace(const int character) { 6 | return character != ' ' && character != '\n' && character != '\r' && 7 | character != '\t'; 8 | } 9 | 10 | void StringLeftTrim(std::string* str) { 11 | str->erase(str->begin(), 12 | std::find_if(str->begin(), str->end(), IsNotWhiteSpace)); 13 | } 14 | 15 | void StringRightTrim(std::string* str) { 16 | str->erase(std::find_if(str->rbegin(), str->rend(), IsNotWhiteSpace).base(), 17 | str->end()); 18 | } 19 | 20 | void StringTrim(std::string* str) { 21 | StringLeftTrim(str); 22 | StringRightTrim(str); 23 | } 24 | 25 | void ReadCamerasBinary(std::unordered_map& cameras, const std::string& path){ 26 | cameras.clear(); 27 | 28 | std::ifstream file(path, std::ios::binary); 29 | CHECK(file.is_open()) << path; 30 | 31 | const size_t num_cameras = ReadBinaryLittleEndian(&file); 32 | cameras.reserve(num_cameras); 33 | 34 | Camera camera; 35 | for (size_t i = 0; i < num_cameras; ++i) { 36 | camera.camera_id = ReadBinaryLittleEndian(&file); 37 | camera.model_id = ReadBinaryLittleEndian(&file); 38 | camera.width = ReadBinaryLittleEndian(&file); 39 | camera.height = ReadBinaryLittleEndian(&file); 40 | CHECK(camera.model_id == 1); 41 | camera.params.resize(4); 42 | ReadBinaryLittleEndian(&file, &camera.params); 43 | cameras.emplace(camera.camera_id, camera); 44 | } 45 | } 46 | 47 | void ReadImagesBinary(std::unordered_map& images, const std::string& path) { 48 | std::ifstream file(path, std::ios::binary); 49 | CHECK(file.is_open()) << path; 50 | 51 | const size_t num_reg_images = ReadBinaryLittleEndian(&file); 52 | images.reserve(num_reg_images); 53 | 54 | Image image; 55 | for (size_t i = 0; i < num_reg_images; ++i) { 56 | image.image_id = ReadBinaryLittleEndian(&file); 57 | 58 | image.qcw[0] = ReadBinaryLittleEndian(&file); 59 | image.qcw[1] = ReadBinaryLittleEndian(&file); 60 | image.qcw[2] = ReadBinaryLittleEndian(&file); 61 | image.qcw[3] = ReadBinaryLittleEndian(&file); 62 | image.qcw.normalize(); 63 | image.Rcw = Eigen::Quaterniond(image.qcw[0], image.qcw[1], image.qcw[2], image.qcw[3]).matrix(); 64 | 65 | image.tcw[0] = ReadBinaryLittleEndian(&file); 66 | image.tcw[1] = ReadBinaryLittleEndian(&file); 67 | image.tcw[2] = ReadBinaryLittleEndian(&file); 68 | 69 | image.camera_id = ReadBinaryLittleEndian(&file); 70 | 71 | image.name.clear(); 72 | char name_char; 73 | do { 74 | file.read(&name_char, 1); 75 | if (name_char != '\0') { 76 | image.name += name_char; 77 | } 78 | } while (name_char != '\0'); 79 | 80 | const size_t num_points2D = ReadBinaryLittleEndian(&file); 81 | image.keypoints.resize(num_points2D); 82 | for(size_t j = 0; j < num_points2D; ++j) { 83 | const double x = ReadBinaryLittleEndian(&file); 84 | const double y = ReadBinaryLittleEndian(&file); 85 | point3D_t point3D_idx = ReadBinaryLittleEndian(&file); 86 | image.keypoints[j] = std::pair(Eigen::Vector2d(x, y), point3D_idx); 87 | } 88 | 89 | images.emplace(image.image_id, image); 90 | } 91 | } 92 | 93 | void ReadPoints3DBinary(std::unordered_map& points3D, const std::string& path) { 94 | std::ifstream file(path, std::ios::binary); 95 | CHECK(file.is_open()) << path; 96 | 97 | const size_t num_points3D = ReadBinaryLittleEndian(&file); 98 | points3D.reserve(num_points3D); 99 | 100 | Point3D point3D; 101 | for (size_t i = 0; i < num_points3D; ++i) { 102 | const point3D_t point3D_id = ReadBinaryLittleEndian(&file); 103 | 104 | point3D.xyz[0] = ReadBinaryLittleEndian(&file); 105 | point3D.xyz[1] = ReadBinaryLittleEndian(&file); 106 | point3D.xyz[2] = ReadBinaryLittleEndian(&file); 107 | point3D.color[0] = ReadBinaryLittleEndian(&file); 108 | point3D.color[1] = ReadBinaryLittleEndian(&file); 109 | point3D.color[2] = ReadBinaryLittleEndian(&file); 110 | point3D.error = ReadBinaryLittleEndian(&file); 111 | 112 | const size_t track_length = ReadBinaryLittleEndian(&file); 113 | point3D.track.resize(track_length); 114 | for (size_t j = 0; j < track_length; ++j) { 115 | const image_t image_id = ReadBinaryLittleEndian(&file); 116 | const point2D_t point2D_idx = ReadBinaryLittleEndian(&file); 117 | point3D.track[j] = std::pair(image_id, point2D_idx); 118 | } 119 | point3D.track.shrink_to_fit(); 120 | 121 | points3D.emplace(point3D_id, point3D); 122 | } 123 | } 124 | 125 | void ReadCamerasTxt(std::unordered_map& cameras, const std::string& path) { 126 | cameras.clear(); 127 | 128 | std::ifstream file(path); 129 | CHECK(file.is_open()) << path; 130 | 131 | std::string line; 132 | std::string item; 133 | 134 | while (std::getline(file, line)) { 135 | StringTrim(&line); 136 | 137 | if (line.empty() || line[0] == '#') { 138 | continue; 139 | } 140 | 141 | std::stringstream line_stream(line); 142 | 143 | Camera camera; 144 | camera.params.resize(4); 145 | std::string model_type; 146 | line_stream >> camera.camera_id >> model_type; 147 | CHECK(model_type == "PINHOLE"); 148 | 149 | line_stream >> camera.width >> camera.height >> camera.params[0] >> camera.params[1] >> camera.params[2] >> camera.params[3]; 150 | 151 | cameras.emplace(camera.camera_id, camera); 152 | } 153 | } 154 | 155 | void ReadImagesTxt(std::unordered_map& images, const std::string& path) { 156 | images.clear(); 157 | 158 | std::ifstream file(path); 159 | CHECK(file.is_open()) << path; 160 | 161 | std::string line; 162 | std::string item; 163 | 164 | constexpr point3D_t kInvalidPoint3DId = std::numeric_limits::max(); 165 | while (std::getline(file, line)) { 166 | StringTrim(&line); 167 | 168 | if (line.empty() || line[0] == '#') { 169 | continue; 170 | } 171 | 172 | std::stringstream line_stream1(line); 173 | 174 | // ID 175 | std::getline(line_stream1, item, ' '); 176 | const image_t image_id = std::stoul(item); 177 | 178 | Image image; 179 | image.image_id = image_id; 180 | 181 | // QVEC (qw, qx, qy, qz) 182 | std::getline(line_stream1, item, ' '); 183 | image.qcw[0] = std::stod(item); 184 | 185 | std::getline(line_stream1, item, ' '); 186 | image.qcw[1] = std::stod(item); 187 | 188 | std::getline(line_stream1, item, ' '); 189 | image.qcw[2] = std::stod(item); 190 | 191 | std::getline(line_stream1, item, ' '); 192 | image.qcw[3] = std::stod(item); 193 | 194 | image.qcw.normalize(); 195 | image.Rcw = Eigen::Quaterniond(image.qcw[0], image.qcw[1], image.qcw[2], image.qcw[3]).matrix(); 196 | 197 | // TVEC 198 | std::getline(line_stream1, item, ' '); 199 | image.tcw[0] = std::stod(item); 200 | 201 | std::getline(line_stream1, item, ' '); 202 | image.tcw[1] = std::stod(item); 203 | 204 | std::getline(line_stream1, item, ' '); 205 | image.tcw[2] = std::stod(item); 206 | 207 | // CAMERA_ID 208 | std::getline(line_stream1, item, ' '); 209 | image.camera_id = std::stoul(item); 210 | 211 | // NAME 212 | std::getline(line_stream1, item, ' '); 213 | image.name = item; 214 | 215 | // POINTS2D 216 | if (!std::getline(file, line)) { 217 | break; 218 | } 219 | 220 | StringTrim(&line); 221 | std::stringstream line_stream2(line); 222 | 223 | if (!line.empty()) { 224 | while (!line_stream2.eof()) { 225 | Eigen::Vector2d point; 226 | 227 | std::getline(line_stream2, item, ' '); 228 | point.x() = std::stod(item); 229 | 230 | std::getline(line_stream2, item, ' '); 231 | point.y() = std::stod(item); 232 | 233 | std::getline(line_stream2, item, ' '); 234 | if (item == "-1") { 235 | image.keypoints.emplace_back(point, kInvalidPoint3DId); 236 | } else { 237 | image.keypoints.emplace_back(point, std::stoul(item)); 238 | } 239 | } 240 | } 241 | 242 | images.emplace(image.image_id, image); 243 | } 244 | } 245 | 246 | void ReadPoints3DTxt(std::unordered_map& points3D, const std::string& path) { 247 | points3D.clear(); 248 | 249 | std::ifstream file(path); 250 | CHECK(file.is_open()) << path; 251 | 252 | std::string line; 253 | std::string item; 254 | 255 | Point3D point3D; 256 | while (std::getline(file, line)) { 257 | StringTrim(&line); 258 | 259 | if (line.empty() || line[0] == '#') { 260 | continue; 261 | } 262 | 263 | std::stringstream line_stream(line); 264 | 265 | // ID 266 | std::getline(line_stream, item, ' '); 267 | const point3D_t point3D_id = std::stol(item); 268 | 269 | // XYZ 270 | std::getline(line_stream, item, ' '); 271 | point3D.xyz[0] = std::stod(item); 272 | 273 | std::getline(line_stream, item, ' '); 274 | point3D.xyz[1] = std::stod(item); 275 | 276 | std::getline(line_stream, item, ' '); 277 | point3D.xyz[2] = std::stod(item); 278 | 279 | // Color 280 | std::getline(line_stream, item, ' '); 281 | point3D.color[0] = static_cast(std::stoi(item)); 282 | 283 | std::getline(line_stream, item, ' '); 284 | point3D.color[1] = static_cast(std::stoi(item)); 285 | 286 | std::getline(line_stream, item, ' '); 287 | point3D.color[2] = static_cast(std::stoi(item)); 288 | 289 | // ERROR 290 | std::getline(line_stream, item, ' '); 291 | point3D.error = std::stod(item); 292 | 293 | // TRACK 294 | point3D.track.clear(); 295 | while (!line_stream.eof()) { 296 | std::getline(line_stream, item, ' '); 297 | StringTrim(&item); 298 | if (item.empty()) { 299 | break; 300 | } 301 | image_t image_id = std::stoul(item); 302 | 303 | std::getline(line_stream, item, ' '); 304 | point2D_t point2D_idx = std::stoul(item); 305 | 306 | point3D.track.emplace_back(image_id, point2D_idx); 307 | } 308 | 309 | point3D.track.shrink_to_fit(); 310 | points3D.emplace(point3D_id, point3D); 311 | } 312 | } 313 | } // namespace colmap 314 | 315 | 316 | bool Model::Read(){ 317 | std::string path = root_folder + "/" + model_folder; 318 | if (ExistsFile(path + "/cameras.bin") && 319 | ExistsFile(path + "/images.bin") && 320 | ExistsFile(path + "/points3D.bin")) { 321 | VLOG(2) << "reading binary model..."; 322 | colmap::ReadCamerasBinary(cameras, path + "/cameras.bin"); 323 | colmap::ReadImagesBinary(images, path + "/images.bin"); 324 | colmap::ReadPoints3DBinary(points, path + "/points3D.bin"); 325 | } else if (ExistsFile(path + "/cameras.txt") && 326 | ExistsFile(path + "/images.txt") && 327 | ExistsFile(path + "/points3D.txt")) { 328 | VLOG(2) << "reading txt model..."; 329 | colmap::ReadCamerasTxt(cameras, path + "/cameras.txt"); 330 | colmap::ReadImagesTxt(images, path + "/images.txt"); 331 | colmap::ReadPoints3DTxt(points, path + "/points3D.txt"); 332 | } else { 333 | LOG(FATAL) << "cameras, images, points3D files do not exist at " << path; 334 | } 335 | 336 | VLOG(2) << "computing depth_ranges..."; 337 | 338 | depth_ranges = ComputeDepthRanges(); 339 | 340 | for(auto iter = images.begin(); iter != images.end();) { 341 | if(!depth_ranges.count(iter->first)) { 342 | std::cout << "image " << iter->second.name << " has no depth, so it will be erased" << std::endl; 343 | iter = images.erase(iter); 344 | } else { 345 | iter++; 346 | } 347 | } 348 | 349 | VLOG(2) << "computing covis_vec..."; 350 | 351 | const size_t num_images = 20; 352 | const double min_triangulation_angle = 2; 353 | covis_vec = GetMaxOverlappingImages(num_images, min_triangulation_angle); 354 | 355 | for(const auto& pair : images) { 356 | colmap::Image image = pair.second; 357 | auto image_id = image.image_id; 358 | image_id_to_image_name[image_id] = image.name; 359 | 360 | colmap::Camera camera_colmap = cameras.at(image.camera_id); 361 | Camera camera; 362 | camera.width = camera_colmap.width; 363 | camera.height = camera_colmap.height; 364 | const auto& range = depth_ranges.at(image_id); 365 | camera.depth_min = range.first; 366 | camera.depth_max = range.second; 367 | 368 | auto& K = camera.K; 369 | K[0] = camera_colmap.params[0]; K[1] = 0; K[2] = camera_colmap.params[2]; 370 | K[3] = 0; K[4] = camera_colmap.params[1]; K[5] = camera_colmap.params[3]; 371 | K[6] = 0; K[7] = 0; K[8] = 1; 372 | 373 | auto& Rcw = camera.R; 374 | Rcw[0] = image.Rcw(0, 0); Rcw[1] = image.Rcw(0, 1); Rcw[2] = image.Rcw(0, 2); 375 | Rcw[3] = image.Rcw(1, 0); Rcw[4] = image.Rcw(1, 1); Rcw[5] = image.Rcw(1, 2); 376 | Rcw[6] = image.Rcw(2, 0); Rcw[7] = image.Rcw(2, 1); Rcw[8] = image.Rcw(2, 2); 377 | 378 | auto& tcw = camera.t; 379 | tcw[0] = image.tcw[0]; tcw[1] = image.tcw[1]; tcw[2] = image.tcw[2]; 380 | 381 | image_id_to_camera[image_id] = camera; 382 | } 383 | 384 | // create folders 385 | std::unordered_set subfolders; 386 | for(const auto& pair : images) { 387 | std::string name = pair.second.name; 388 | std::string subfolder = GetFolder(name); 389 | if(subfolder != name) { 390 | subfolders.insert(subfolder); 391 | } 392 | } 393 | subfolders.insert(""); 394 | std::string outputfolder; 395 | for(const auto& subfolder : subfolders) { 396 | for(const auto& folder2 : {depth_folder, normal_folder}) { 397 | outputfolder = root_folder + "/" + folder2 + "/" + subfolder; 398 | CreateFolder(outputfolder); 399 | } 400 | } 401 | return true; 402 | } 403 | 404 | 405 | void Model::ReduceMemory(){ 406 | images.clear(); 407 | points.clear(); 408 | cameras.clear(); 409 | depth_ranges.clear(); 410 | } 411 | 412 | 413 | std::unordered_map> Model::GetMaxOverlappingImages(const size_t num_images, const double min_triangulation_angle) const { 414 | std::unordered_map> overlapping_images(images.size()); 415 | 416 | const double min_triangulation_angle_rad = min_triangulation_angle * 0.0174532925199432954743716805978692718781530857086181640625; 417 | 418 | const auto shared_num_points = ComputeSharedPoints(); 419 | 420 | const double kTriangulationAnglePercentile = 75; 421 | const auto triangulation_angles = ComputeTriangulationAngles(kTriangulationAnglePercentile); 422 | 423 | for (const auto& pair : images) { 424 | auto image_id = pair.first; 425 | if(!shared_num_points.count(image_id) || !triangulation_angles.count(image_id)) { 426 | std::cout << "image name: " << images.at(image_id).name << std::endl; 427 | if(depth_ranges.count(image_id)) { 428 | std::cout << "depth range: " << depth_ranges.at(image_id).first << " to " << depth_ranges.at(image_id).second << std::endl; 429 | } else { 430 | std::cerr << "how does this come?" << std::endl; 431 | } 432 | 433 | continue; 434 | } 435 | const auto& shared_images = shared_num_points.at(image_id); 436 | const auto& overlapping_triangulation_angles = triangulation_angles.at(image_id); 437 | 438 | std::vector> ordered_images; 439 | ordered_images.reserve(shared_images.size()); 440 | for (const auto& image : shared_images) { 441 | if (overlapping_triangulation_angles.at(image.first) >= 442 | min_triangulation_angle_rad) { 443 | ordered_images.emplace_back(image.first, image.second); 444 | } 445 | } 446 | 447 | const size_t eff_num_images = std::min(ordered_images.size(), num_images); 448 | if (eff_num_images < shared_images.size()) { 449 | std::partial_sort(ordered_images.begin(), 450 | ordered_images.begin() + eff_num_images, 451 | ordered_images.end(), 452 | [](const std::pair image1, 453 | const std::pair image2) { 454 | return image1.second > image2.second; 455 | }); 456 | } else { 457 | std::sort(ordered_images.begin(), ordered_images.end(), 458 | [](const std::pair image1, 459 | const std::pair image2) { 460 | return image1.second > image2.second; 461 | }); 462 | } 463 | 464 | overlapping_images[image_id].reserve(eff_num_images); 465 | for (size_t i = 0; i < eff_num_images; ++i) { 466 | overlapping_images[image_id].push_back(ordered_images[i].first); 467 | } 468 | } 469 | 470 | return overlapping_images; 471 | } 472 | 473 | std::unordered_map> Model::ComputeSharedPoints() const { 474 | std::unordered_map> shared_points(images.size()); 475 | for (const auto& pair : points) { 476 | const auto& point = pair.second; 477 | for (size_t i = 0; i < point.track.size(); ++i) { 478 | const int image_idx1 = point.track[i].first; 479 | for (size_t j = 0; j < i; ++j) { 480 | const int image_idx2 = point.track[j].first; 481 | if (image_idx1 != image_idx2) { 482 | shared_points[image_idx1][image_idx2] += 1; 483 | shared_points[image_idx2][image_idx1] += 1; 484 | } 485 | } 486 | } 487 | } 488 | return shared_points; 489 | } 490 | 491 | double CalculateTriangulationAngle(const Eigen::Vector3d& proj_center1, 492 | const Eigen::Vector3d& proj_center2, 493 | const Eigen::Vector3d& point3D) { 494 | const double baseline_length_squared = 495 | (proj_center1 - proj_center2).squaredNorm(); 496 | 497 | const double ray_length_squared1 = (point3D - proj_center1).squaredNorm(); 498 | const double ray_length_squared2 = (point3D - proj_center2).squaredNorm(); 499 | 500 | // Using "law of cosines" to compute the enclosing angle between rays. 501 | const double denominator = 502 | 2.0 * std::sqrt(ray_length_squared1 * ray_length_squared2); 503 | if (denominator == 0.0) { 504 | return 0.0; 505 | } 506 | const double nominator = 507 | ray_length_squared1 + ray_length_squared2 - baseline_length_squared; 508 | const double angle = std::abs(std::acos(nominator / denominator)); 509 | 510 | // Triangulation is unstable for acute angles (far away points) and 511 | // obtuse angles (close points), so always compute the minimum angle 512 | // between the two intersecting rays. 513 | return std::min(angle, M_PI - angle); 514 | } 515 | 516 | std::unordered_map> Model::ComputeTriangulationAngles( 517 | const double percentile) const { 518 | std::unordered_map proj_centers(images.size()); 519 | for(const auto& pair : images) { 520 | const auto& image = pair.second; 521 | Eigen::Vector3d C = image.GetCenter(); 522 | proj_centers[pair.first] = C.cast(); 523 | } 524 | 525 | std::unordered_map>> all_triangulation_angles(images.size()); 526 | for(const auto& id_point_pair : points) { 527 | const auto& point = id_point_pair.second; 528 | for (size_t i = 0; i < point.track.size(); ++i) { 529 | const int image_id1 = point.track[i].first; 530 | for (size_t j = 0; j < i; ++j) { 531 | const int image_id2 = point.track[j].first; 532 | if (image_id1 != image_id2) { 533 | const double angle = CalculateTriangulationAngle( 534 | proj_centers.at(image_id1), proj_centers.at(image_id2), point.xyz); 535 | all_triangulation_angles[image_id1][image_id2].push_back(angle); 536 | all_triangulation_angles[image_id2][image_id1].push_back(angle); 537 | } 538 | } 539 | } 540 | } 541 | 542 | std::unordered_map> triangulation_angles(images.size()); 543 | for(const auto& pair : all_triangulation_angles) { 544 | auto image_id1 = pair.first; 545 | const auto& overlapping_images = pair.second; 546 | for (const auto& image : overlapping_images) { 547 | triangulation_angles[image_id1].emplace(image.first, Percentile(image.second, percentile)); 548 | } 549 | } 550 | 551 | return triangulation_angles; 552 | } 553 | 554 | std::unordered_map > Model::ComputeDepthRanges() const { 555 | std::unordered_map> depths(images.size()); 556 | for (const auto& pair : points) { 557 | auto point = pair.second; 558 | for (const auto& track : point.track) { 559 | auto image_id = track.first; 560 | const auto& image = images.at(image_id); 561 | const double depth = image.Project(point.xyz)[2]; 562 | if (depth > 0) { 563 | depths[image_id].push_back(depth); 564 | } 565 | } 566 | } 567 | 568 | std::unordered_map> depth_ranges(depths.size()); 569 | for(auto& pair : depths) { 570 | int image_id = pair.first; 571 | auto& image_depths = pair.second; 572 | std::sort(image_depths.begin(), image_depths.end()); 573 | 574 | auto& depth_range = depth_ranges[image_id]; 575 | 576 | const double kMinPercentile = 0.01; 577 | const double kMaxPercentile = 0.99; 578 | depth_range.first = image_depths[image_depths.size() * kMinPercentile]; 579 | depth_range.second = image_depths[image_depths.size() * kMaxPercentile]; 580 | 581 | const double kStretchRatio = 0.25; 582 | depth_range.first *= (1.0 - kStretchRatio); 583 | depth_range.second *= (1.0 + kStretchRatio); 584 | } 585 | 586 | return depth_ranges; 587 | } 588 | 589 | 590 | 591 | // *************************************************** 592 | // ************ util functions ********************* 593 | // *************************************************** 594 | // *************************************************** 595 | 596 | 597 | std::string GetFolder(std::string name) { 598 | std::string folder = name; 599 | size_t pos = name.find_last_of('/'); 600 | if(pos < name.size()) { 601 | folder = name.substr(0, pos); 602 | } 603 | return folder; 604 | } 605 | 606 | bool ExistsFile(const std::string& path) { 607 | std::ifstream ifs; 608 | ifs.open(path); 609 | if(ifs.is_open()) { 610 | return true; 611 | } else { 612 | return false; 613 | } 614 | } 615 | 616 | void CreateFolder(std::string abspath) { 617 | if(abspath[abspath.size() - 1] == '/') { 618 | abspath = abspath.substr(0, abspath.size() - 1); 619 | } 620 | 621 | struct stat info; 622 | std::string parent_path = GetFolder(abspath); 623 | if(parent_path == abspath) { 624 | std::cout << "Warn: this should not happen in CreateFolder " << abspath << std::endl; 625 | return; 626 | } 627 | 628 | if(stat(parent_path.c_str(), &info) != 0) { 629 | CreateFolder(parent_path); 630 | } 631 | 632 | mode_t target_mode = 0777; 633 | if (mkdir(abspath.c_str(), target_mode) == 0) 634 | chmod(abspath.c_str(), target_mode); 635 | } 636 | -------------------------------------------------------------------------------- /src/colmap_interface/colmap_interface.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | #include "./endian.h" 17 | 18 | #include "types.h" 19 | 20 | namespace Eigen { 21 | typedef Eigen::Matrix Vector3ub; 22 | } 23 | 24 | namespace colmap { 25 | 26 | // See colmap/src/util/types.h 27 | typedef uint32_t camera_t; 28 | typedef uint32_t image_t; 29 | typedef uint64_t image_pair_t; 30 | typedef uint32_t point2D_t; 31 | typedef uint64_t point3D_t; 32 | 33 | struct Camera { 34 | camera_t camera_id; 35 | int model_id = 1; 36 | int width, height; 37 | std::vector params; 38 | }; 39 | 40 | struct Image { 41 | std::string name; 42 | image_t image_id; 43 | camera_t camera_id; 44 | Eigen::Vector4d qcw; 45 | Eigen::Vector3d tcw; 46 | std::vector> keypoints; 47 | 48 | Eigen::Matrix3d Rcw; 49 | Eigen::Vector3d Project(const Eigen::Vector3d& point3D) const { 50 | return Rcw * point3D + tcw; 51 | } 52 | 53 | Eigen::Vector3d GetCenter() const{ 54 | auto twc = -Rcw.transpose() * tcw; 55 | return twc; 56 | } 57 | }; 58 | 59 | struct Point3D { 60 | Eigen::Vector3d xyz; 61 | Eigen::Vector3ub color; 62 | double error; 63 | std::vector> track; 64 | }; 65 | 66 | void ReadCamerasBinary(std::unordered_map& cameras, const std::string& path); 67 | void ReadImagesBinary(std::unordered_map& images, const std::string& path); 68 | void ReadPoints3DBinary(std::unordered_map& points3D, const std::string& path); 69 | 70 | void ReadCamerasTxt(std::unordered_map& cameras, const std::string& path); 71 | void ReadImagesTxt(std::unordered_map& images, const std::string& path); 72 | void ReadPoints3DTxt(std::unordered_map& points3D, const std::string& path); 73 | } // namespace colmap 74 | 75 | class Model { 76 | public: 77 | std::unordered_map image_id_to_image_name; 78 | std::unordered_map image_id_to_camera; 79 | std::unordered_map> covis_vec; 80 | 81 | std::string root_folder; 82 | const std::string model_folder = "sparse"; 83 | const std::string depth_folder = "stereo/depth_maps"; 84 | const std::string normal_folder = "stereo/normal_maps"; 85 | const std::string image_folder = "images"; 86 | 87 | Model(const std::string& _root_folder, const std::string& _model_folder, const std::string& _depth_folder, const std::string& _normal_folder, const std::string& _image_folder) 88 | :root_folder(_root_folder), model_folder(_model_folder), depth_folder(_depth_folder), normal_folder(_normal_folder), image_folder(_image_folder) 89 | {} 90 | 91 | // For each image, determine the maximally overlapping images, sorted based on 92 | // the number of shared points subject to a minimum robust average 93 | // triangulation angle of the points. 94 | std::unordered_map> GetMaxOverlappingImages(const size_t num_images, const double min_triangulation_angle) const; 95 | 96 | // Compute the number of shared points between all overlapping images. 97 | std::unordered_map> ComputeSharedPoints() const; 98 | 99 | // Compute the median triangulation angles between all overlapping images. 100 | std::unordered_map> ComputeTriangulationAngles(const double percentile = 50) const; 101 | 102 | // Compute the robust minimum and maximum depths from the sparse point cloud. 103 | std::unordered_map> ComputeDepthRanges() const; 104 | 105 | bool Read(); 106 | 107 | // release model contents 108 | void ReduceMemory(); 109 | 110 | private: 111 | std::unordered_map cameras; 112 | std::unordered_map images; 113 | std::unordered_map points; 114 | std::unordered_map> depth_ranges; 115 | }; 116 | 117 | 118 | // *************************************************** 119 | // ************ util functions ********************* 120 | // *************************************************** 121 | // *************************************************** 122 | 123 | bool ExistsFile(const std::string& path); 124 | std::string GetFolder(std::string name); 125 | void CreateFolder(std::string abspath); // recursively 126 | 127 | template 128 | T Percentile(const std::vector& elems, const double p) { 129 | CHECK(!elems.empty()); 130 | CHECK_GE(p, 0); 131 | CHECK_LE(p, 100); 132 | 133 | const int idx = static_cast(std::round(p / 100 * (elems.size() - 1))); 134 | const size_t percentile_idx = 135 | std::max(0, std::min(static_cast(elems.size() - 1), idx)); 136 | 137 | std::vector ordered_elems = elems; 138 | std::nth_element(ordered_elems.begin(), 139 | ordered_elems.begin() + percentile_idx, ordered_elems.end()); 140 | 141 | return ordered_elems.at(percentile_idx); 142 | } 143 | -------------------------------------------------------------------------------- /src/colmap_interface/endian.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright 8 | // notice, this list of conditions and the following disclaimer. 9 | // 10 | // * Redistributions in binary form must reproduce the above copyright 11 | // notice, this list of conditions and the following disclaimer in the 12 | // documentation and/or other materials provided with the distribution. 13 | // 14 | // * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 | // its contributors may be used to endorse or promote products derived 16 | // from this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | // POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 | 32 | #ifndef COLMAP_SRC_UTIL_ENDIAN_H_ 33 | #define COLMAP_SRC_UTIL_ENDIAN_H_ 34 | 35 | #include 36 | #include 37 | #include 38 | 39 | namespace colmap { 40 | 41 | // Reverse the order of each byte. 42 | template 43 | T ReverseBytes(const T& data); 44 | 45 | // Check the order in which bytes are stored in computer memory. 46 | bool IsLittleEndian(); 47 | bool IsBigEndian(); 48 | 49 | // Convert data between endianness and the native format. Note that, for float 50 | // and double types, these functions are only valid if the format is IEEE-754. 51 | // This is the case for pretty much most processors. 52 | template 53 | T LittleEndianToNative(const T x); 54 | template 55 | T BigEndianToNative(const T x); 56 | template 57 | T NativeToLittleEndian(const T x); 58 | template 59 | T NativeToBigEndian(const T x); 60 | 61 | // Read data in little endian format for cross-platform support. 62 | template 63 | T ReadBinaryLittleEndian(std::istream* stream); 64 | template 65 | void ReadBinaryLittleEndian(std::istream* stream, std::vector* data); 66 | 67 | // Write data in little endian format for cross-platform support. 68 | template 69 | void WriteBinaryLittleEndian(std::ostream* stream, const T& data); 70 | template 71 | void WriteBinaryLittleEndian(std::ostream* stream, const std::vector& data); 72 | 73 | //////////////////////////////////////////////////////////////////////////////// 74 | // Implementation 75 | //////////////////////////////////////////////////////////////////////////////// 76 | 77 | template 78 | T ReverseBytes(const T& data) { 79 | T data_reversed = data; 80 | std::reverse(reinterpret_cast(&data_reversed), 81 | reinterpret_cast(&data_reversed) + sizeof(T)); 82 | return data_reversed; 83 | } 84 | 85 | inline bool IsLittleEndian() { 86 | #ifdef BOOST_BIG_ENDIAN 87 | return false; 88 | #else 89 | return true; 90 | #endif 91 | } 92 | 93 | inline bool IsBigEndian() { 94 | #ifdef BOOST_BIG_ENDIAN 95 | return true; 96 | #else 97 | return false; 98 | #endif 99 | } 100 | 101 | template 102 | T LittleEndianToNative(const T x) { 103 | if (IsLittleEndian()) { 104 | return x; 105 | } else { 106 | return ReverseBytes(x); 107 | } 108 | } 109 | 110 | template 111 | T BigEndianToNative(const T x) { 112 | if (IsBigEndian()) { 113 | return x; 114 | } else { 115 | return ReverseBytes(x); 116 | } 117 | } 118 | 119 | template 120 | T NativeToLittleEndian(const T x) { 121 | if (IsLittleEndian()) { 122 | return x; 123 | } else { 124 | return ReverseBytes(x); 125 | } 126 | } 127 | 128 | template 129 | T NativeToBigEndian(const T x) { 130 | if (IsBigEndian()) { 131 | return x; 132 | } else { 133 | return ReverseBytes(x); 134 | } 135 | } 136 | 137 | template 138 | T ReadBinaryLittleEndian(std::istream* stream) { 139 | T data_little_endian; 140 | stream->read(reinterpret_cast(&data_little_endian), sizeof(T)); 141 | return LittleEndianToNative(data_little_endian); 142 | } 143 | 144 | template 145 | void ReadBinaryLittleEndian(std::istream* stream, std::vector* data) { 146 | for (size_t i = 0; i < data->size(); ++i) { 147 | (*data)[i] = ReadBinaryLittleEndian(stream); 148 | } 149 | } 150 | 151 | template 152 | void WriteBinaryLittleEndian(std::ostream* stream, const T& data) { 153 | const T data_little_endian = NativeToLittleEndian(data); 154 | stream->write(reinterpret_cast(&data_little_endian), sizeof(T)); 155 | } 156 | 157 | template 158 | void WriteBinaryLittleEndian(std::ostream* stream, const std::vector& data) { 159 | for (const auto& elem : data) { 160 | WriteBinaryLittleEndian(stream, elem); 161 | } 162 | } 163 | 164 | } // namespace colmap 165 | 166 | #endif // COLMAP_SRC_UTIL_ENDIAN_H_ 167 | 168 | -------------------------------------------------------------------------------- /src/types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "opencv2/calib3d/calib3d.hpp" 4 | #include "opencv2/imgproc/imgproc.hpp" 5 | #include "opencv2/core/core.hpp" 6 | #include "opencv2/highgui/highgui.hpp" 7 | 8 | // Includes CUDA 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "iomanip" 25 | 26 | #include // mkdir 27 | #include // mkdir 28 | 29 | #define MAX_IMAGES 256 30 | 31 | struct Camera { 32 | float K[9]; 33 | float R[9]; // Rcw 34 | float t[3]; // tcw 35 | int height; 36 | int width; 37 | float depth_min; 38 | float depth_max; 39 | }; 40 | 41 | struct Problem { 42 | int ref_image_id; 43 | std::vector src_image_ids; 44 | }; 45 | 46 | struct Triangle { 47 | cv::Point pt1, pt2, pt3; 48 | Triangle (const cv::Point _pt1, const cv::Point _pt2, const cv::Point _pt3) : pt1(_pt1) , pt2(_pt2), pt3(_pt3) {} 49 | }; 50 | 51 | struct PointList { 52 | float3 coord; 53 | float3 normal; 54 | float3 color; 55 | }; 56 | 57 | 58 | #include "colmap_interface/endian.h" 59 | #include 60 | 61 | template 62 | void ReadMap(const std::string& path, cv::Mat_ &data) { 63 | std::fstream text_file(path, std::ios::in | std::ios::binary); 64 | CHECK(text_file.is_open()) << path; 65 | 66 | char unused_char; 67 | size_t width, height, depth; 68 | text_file >> width >> unused_char >> height >> unused_char >> depth >> 69 | unused_char; 70 | std::streampos pos = text_file.tellg(); 71 | text_file.close(); 72 | 73 | CHECK_GT(width, 0); 74 | CHECK_GT(height, 0); 75 | CHECK_EQ(depth, data.channels()); 76 | std::vector raw_data(width * height * sizeof (T)); 77 | 78 | std::fstream binary_file(path, std::ios::in | std::ios::binary); 79 | CHECK(binary_file.is_open()) << path; 80 | binary_file.seekg(pos); 81 | colmap::ReadBinaryLittleEndian(&binary_file, &raw_data); 82 | binary_file.close(); 83 | 84 | size_t one_channel_bytes = width * height * sizeof (T) / depth; 85 | std::vector channels(depth); 86 | for(int i = 0; i < depth; i++) { 87 | channels[i] = cv::Mat::zeros(height, width, CV_MAKETYPE(data.depth(), 1)); 88 | memcpy(channels[i].data, raw_data.data() + i * one_channel_bytes, one_channel_bytes); 89 | } 90 | cv::merge(channels, data); 91 | } 92 | 93 | template 94 | void WriteMap(const std::string& path, cv::Mat_ &data) { 95 | size_t width = data.cols, height = data.rows, depth = data.channels(); 96 | 97 | std::vector channels; 98 | cv::split(data, channels); 99 | 100 | size_t one_channel_bytes = width * height * sizeof (T) / depth; 101 | std::vector raw_data(width * height * sizeof (T)); 102 | for(size_t i = 0; i < depth; i++) { 103 | uchar* p_raw_data = &(raw_data[i * one_channel_bytes]); 104 | memcpy(p_raw_data, channels[i].data, one_channel_bytes); 105 | } 106 | 107 | std::fstream text_file(path, std::ios::out); 108 | CHECK(text_file.is_open()) << path; 109 | text_file << width << "&" << height << "&" << depth << "&"; 110 | text_file.close(); 111 | 112 | std::fstream binary_file(path, std::ios::out | std::ios::binary | std::ios::app); 113 | CHECK(binary_file.is_open()) << path; 114 | colmap::WriteBinaryLittleEndian(&binary_file, raw_data); 115 | binary_file.close(); 116 | } 117 | --------------------------------------------------------------------------------