├── .gitignore ├── CMakeLists.txt ├── CreateVoca ├── CMakeLists.txt ├── include │ ├── BowVector.h │ ├── DBoW2.h │ ├── FClass.h │ ├── FSuperpoint.h │ ├── FeatureVector.h │ ├── QueryResults.h │ ├── ScoringObject.h │ ├── TemplatedDatabase.h │ └── TemplatedVocabulary.h └── src │ ├── BowVector.cpp │ ├── FSuperpoint.cpp │ ├── FeatureVector.cpp │ ├── QueryResults.cpp │ └── ScoringObject.cpp ├── Dataset ├── icl_snippet │ ├── 250.png │ ├── 254.png │ ├── 258.png │ ├── 262.png │ ├── 266.png │ ├── 270.png │ ├── 274.png │ ├── 278.png │ ├── 282.png │ ├── 286.png │ ├── 290.png │ ├── 294.png │ ├── 298.png │ ├── 302.png │ ├── 306.png │ ├── 310.png │ ├── 314.png │ ├── 318.png │ ├── 322.png │ ├── 326.png │ ├── 330.png │ ├── 334.png │ ├── 338.png │ ├── 342.png │ ├── 346.png │ ├── 350.png │ ├── 354.png │ ├── 358.png │ ├── 362.png │ ├── 366.png │ └── magicleap.png ├── magicleap.png └── nyu_snippet.mp4 ├── INSTALL_LibTorch.sh ├── INSTALL_OpenCV.sh ├── README.md ├── Weights └── superpoint.pt ├── gitpush.sh ├── include ├── SPDetector.hpp ├── SuperPoint.hpp ├── Tools.hpp └── zed.hpp ├── main_CreateVoca.cpp ├── main_SuperPoint.cpp ├── runSuperPoint.sh ├── src ├── CMakeLists.txt ├── SPDetector.cpp ├── SuperPoint.cpp └── Tools.cpp └── zed ├── CMakeLists.txt └── zed.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | bin 3 | lib 4 | SuperPointPretrainedNetwork 5 | Voca/ 6 | .vscode 7 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.2) 2 | 3 | project( 4 | SuperPointCpp 5 | VERSION 20.08 6 | LANGUAGES CXX 7 | DESCRIPTION "C++ Implementation of Superpoint." 8 | ) 9 | 10 | set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build." FORCE) 11 | 12 | ### LIBTORCH 13 | set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/usr/local/libtorch") 14 | find_package(Torch REQUIRED QUIET) 15 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") 16 | include_directories(${TORCH_INCLUDE_DIRS}) 17 | ### 18 | 19 | ### OPENCV 20 | find_package(OpenCV 3.4.11 QUIET) 21 | message(STATUS "Found OpenCV version is ${OpenCV_VERSION}") 22 | message(STATUS "Found OpenCV include is ${OpenCV_INCLUDE_DIRS}") 23 | message(STATUS "Found OpenCV libraries is ${OpenCV_LIBS}") 24 | include_directories(${OpenCV_INCLUDE_DIRS}) 25 | ### 26 | 27 | # ### ZED & CUDA 28 | # set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/usr/local/zed") 29 | # find_package(ZED 3 REQUIRED) 30 | # find_package(CUDA ${ZED_CUDA_VERSION} EXACT REQUIRED) 31 | # include_directories(${ZED_INCLUDE_DIRS}) 32 | # ### 33 | 34 | ### SUB_DIRECTORIES 35 | add_subdirectory(src) 36 | add_subdirectory(CreateVoca) 37 | # add_subdirectory(zed) 38 | 39 | # ### Deprecated ### 40 | # add_executable(main main_SuperPoint.cpp) 41 | # target_compile_features(main PUBLIC cxx_std_14) 42 | # target_link_libraries(main PUBLIC 43 | # superpoint 44 | # #libzed 45 | # ${TORCH_LIBRARIES} 46 | # ${OpenCV_LIBS} 47 | # ) 48 | # set_target_properties(main PROPERTIES 49 | # RUNTIME_OUTPUT_DIRECTORY "${CMAKE_HOME_DIRECTORY}/bin" 50 | # ) 51 | 52 | ### Runfile to test SuperPoint 53 | add_executable(SuperPoint main_SuperPoint.cpp) 54 | target_compile_features(SuperPoint PUBLIC cxx_std_14) 55 | target_link_libraries(SuperPoint PUBLIC 56 | Super 57 | ${TORCH_LIBRARIES} 58 | ${OpenCV_LIBS} 59 | ) 60 | set_target_properties(SuperPoint PROPERTIES 61 | RUNTIME_OUTPUT_DIRECTORY "${CMAKE_HOME_DIRECTORY}/bin" 62 | ) 63 | 64 | ### Runfile to create Vocabulary 65 | add_executable(CreateVoca main_CreateVoca.cpp) 66 | target_link_libraries(CreateVoca PUBLIC 67 | Super 68 | DBoW2 69 | # ${TORCH_LIBRARIES} 70 | # ${OpenCV_LIBS} 71 | ) 72 | set_target_properties(CreateVoca PROPERTIES 73 | CXX_STANDARD 11 74 | RUNTIME_OUTPUT_DIRECTORY "${CMAKE_HOME_DIRECTORY}/bin" 75 | ) 76 | -------------------------------------------------------------------------------- /CreateVoca/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # cmake_minimum_required(VERSION 3.0) 2 | # project(CreateVaca 3 | # VERSION 1.0.0 4 | # LANGUAGES CXX 5 | # DESCRIPTION "Create Vocabulary file using Superpoint features." 6 | 7 | option(BUILD_DBoW2 "Build DBoW2" ON) 8 | option(BUILD_Test "Build test application" OFF) 9 | 10 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 11 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" 12 | "MinSizeRel" "RelWithDebInfo") 13 | endif() 14 | 15 | # set(HDRS 16 | # include/BowVector.h 17 | # include/QueryResults.h include/TemplatedDatabase.h 18 | # include/FClass.h include/FeatureVector.h 19 | # include/ScoringObject.h include/TemplatedVocabulary.h include/FSuperpoint.h) 20 | set(SRCS 21 | src/BowVector.cpp src/FSuperpoint.cpp 22 | src/FeatureVector.cpp src/QueryResults.cpp src/ScoringObject.cpp) 23 | 24 | # set(DEPENDENCY_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies) 25 | # set(DEPENDENCY_INSTALL_DIR ${DEPENDENCY_DIR}/install) 26 | 27 | ### Root CMakeLists.txt already find OpenCV. 28 | # find_package(OpenCV 3.4.11 REQUIRED) 29 | 30 | message("CreateVoca's home directory is ${CMAKE_HOME_DIRECTORY}") 31 | 32 | add_library(DBoW2 SHARED ${SRCS}) 33 | target_include_directories(DBoW2 PUBLIC 34 | ${CMAKE_HOME_DIRECTORY}/CreateVoca/include 35 | ) 36 | target_link_libraries(DBoW2 ${OpenCV_LIBS}) 37 | set_target_properties(DBoW2 PROPERTIES 38 | CXX_STANDARD 11 39 | LIBRARY_OUTPUT_DIRECTORY "${CMAKE_HOME_DIRECTORY}/lib" 40 | ) 41 | 42 | # if(BUILD_Test) 43 | # add_executable(test test/test.cpp) 44 | # target_link_libraries(test ${PROJECT_NAME} ${OpenCV_LIBS}) 45 | # set_target_properties(test PROPERTIES CXX_STANDARD 11) 46 | # endif(BUILD_Test) 47 | 48 | 49 | 50 | # configure_file(src/DBoW2.cmake.in 51 | # "${PROJECT_BINARY_DIR}/DBoW2Config.cmake" @ONLY) 52 | 53 | # install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) 54 | # if(BUILD_DBoW2) 55 | # install(DIRECTORY include/DBoW2 DESTINATION ${CMAKE_INSTALL_PREFIX}/include) 56 | # endif() 57 | # install(FILES "${CMAKE_CURRENT_BINARY_DIR}/DBoW2Config.cmake" 58 | # DESTINATION ${CMAKE_INSTALL_PREFIX}/include/${PROJECT_NAME}) 59 | # install(FILES "${PROJECT_BINARY_DIR}/DBoW2Config.cmake" 60 | # DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/cmake/DBoW2/) 61 | # install(DIRECTORY ${DEPENDENCY_INSTALL_DIR}/ DESTINATION ${CMAKE_INSTALL_PREFIX} OPTIONAL) 62 | 63 | -------------------------------------------------------------------------------- /CreateVoca/include/BowVector.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: BowVector.h 3 | * Date: March 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: bag of words vector 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_BOW_VECTOR__ 11 | #define __D_T_BOW_VECTOR__ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace DBoW2 { 18 | 19 | /// Id of words 20 | typedef unsigned int WordId; 21 | 22 | /// Value of a word 23 | typedef double WordValue; 24 | 25 | /// Id of nodes in the vocabulary treee 26 | typedef unsigned int NodeId; 27 | 28 | /// L-norms for normalization 29 | enum LNorm 30 | { 31 | L1, 32 | L2 33 | }; 34 | 35 | /// Weighting type 36 | enum WeightingType 37 | { 38 | TF_IDF, 39 | TF, 40 | IDF, 41 | BINARY 42 | }; 43 | 44 | /// Scoring type 45 | enum ScoringType 46 | { 47 | L1_NORM, 48 | L2_NORM, 49 | CHI_SQUARE, 50 | KL, 51 | BHATTACHARYYA, 52 | DOT_PRODUCT 53 | }; 54 | 55 | /// Vector of words to represent images 56 | 57 | /** 58 | * @brief BowVector는 std::map을 상속받은 class로서 59 | * WordId를 key로, WordValue를 value로 하여 map을 만든다. 60 | * 61 | */ 62 | class BowVector: 63 | public std::map 64 | { 65 | public: 66 | 67 | /** 68 | * Constructor 69 | */ 70 | BowVector(void); 71 | 72 | /** 73 | * Destructor 74 | */ 75 | ~BowVector(void); 76 | 77 | /** 78 | * Adds a value to a word value existing in the vector, or creates a new 79 | * word with the given value 80 | * @param id word id to look for 81 | * @param v value to create the word with, or to add to existing word 82 | */ 83 | void addWeight(WordId id, WordValue v); 84 | 85 | /** 86 | * @brief BowVector는 map구조를 가지고 있다. 이 map에 id를 키로 하는 Value가 존재하지 않으면 insert한다. 87 | * .etc) map이지만 this[id]로 존재를 찾지 않고, lower_bound함수를 이용하는 것이 인상깊었다. 더 빠르게 구현되어 있나? 88 | * 89 | * @param id 목표로 하는 word_id. 해당하는 value가 존재하지 않을 때만 v를 집어넣음 90 | * @param v world value 91 | */ 92 | void addIfNotExist(WordId id, WordValue v); 93 | 94 | /** 95 | * L1-Normalizes the values in the vector 96 | * @param norm_type norm used 97 | */ 98 | void normalize(LNorm norm_type); 99 | 100 | /** 101 | * Prints the content of the bow vector 102 | * @param out stream 103 | * @param v 104 | */ 105 | friend std::ostream& operator<<(std::ostream &out, const BowVector &v); 106 | 107 | /** 108 | * Saves the bow vector as a vector in a matlab file 109 | * @param filename 110 | * @param W number of words in the vocabulary 111 | */ 112 | void saveM(const std::string &filename, size_t W) const; 113 | }; 114 | 115 | } // namespace DBoW2 116 | 117 | #endif 118 | -------------------------------------------------------------------------------- /CreateVoca/include/DBoW2.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __D_T_DBOW2__ 3 | #define __D_T_DBOW2__ 4 | 5 | /// Includes all the data structures to manage vocabularies and image databases 6 | namespace DBoW2 7 | { 8 | } 9 | 10 | #include "TemplatedVocabulary.h" 11 | #include "TemplatedDatabase.h" 12 | #include "BowVector.h" 13 | #include "FeatureVector.h" 14 | #include "QueryResults.h" 15 | #include "FSuperpoint.h" 16 | 17 | /// ORB Vocabulary 18 | typedef DBoW2::TemplatedVocabulary 19 | SuperpointVocabulary; 20 | 21 | /// FORB Database 22 | typedef DBoW2::TemplatedDatabase 23 | SuperpointDatabase; 24 | 25 | #endif 26 | 27 | -------------------------------------------------------------------------------- /CreateVoca/include/FClass.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: FClass.h 3 | * Date: November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: generic FClass to instantiate templated classes 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_FCLASS__ 11 | #define __D_T_FCLASS__ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace DBoW2 { 18 | 19 | /// Generic class to encapsulate functions to manage descriptors. 20 | /** 21 | * This class must be inherited. Derived classes can be used as the 22 | * parameter F when creating Templated structures 23 | * (TemplatedVocabulary, TemplatedDatabase, ...) 24 | */ 25 | class FClass 26 | { 27 | class TDescriptor; 28 | typedef const TDescriptor* pDescriptor; 29 | 30 | /** 31 | * Calculates the mean value of a set of descriptors 32 | * @param descriptors 33 | * @param mean mean descriptor 34 | */ 35 | virtual void meanValue(const std::vector &descriptors, 36 | TDescriptor &mean) = 0; 37 | 38 | /** 39 | * Calculates the distance between two descriptors 40 | * @param a 41 | * @param b 42 | * @return distance 43 | */ 44 | static double distance(const TDescriptor &a, const TDescriptor &b); 45 | 46 | /** 47 | * Returns a string version of the descriptor 48 | * @param a descriptor 49 | * @return string version 50 | */ 51 | static std::string toString(const TDescriptor &a); 52 | 53 | /** 54 | * Returns a descriptor from a string 55 | * @param a descriptor 56 | * @param s string version 57 | */ 58 | static void fromString(TDescriptor &a, const std::string &s); 59 | 60 | /** 61 | * Returns a mat with the descriptors in float format 62 | * @param descriptors 63 | * @param mat (out) NxL 32F matrix 64 | */ 65 | static void toMat32F(const std::vector &descriptors, 66 | cv::Mat &mat); 67 | }; 68 | 69 | } // namespace DBoW2 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /CreateVoca/include/FSuperpoint.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: FSuperpoint.h 3 | * Date: September 2020 4 | * Author: Chanwoo Lee 5 | * Description: functions for SUPERPOINT descriptors 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_F_SUPERPOINT__ 11 | #define __D_T_F_SUPERPOINT__ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "FClass.h" 19 | 20 | namespace DBoW2 { 21 | 22 | //######################################################################## 23 | // Functions to manipulate vector CV_32FC1 [1, 256] descriptors 24 | 25 | class FSUPERPOINT: protected FClass 26 | { 27 | public: 28 | 29 | /// Descriptor type 30 | typedef cv::Mat TDescriptor; // CV_32F 31 | /// Pointer to a single descriptor 32 | typedef const TDescriptor* pDescriptor; 33 | /// Descriptor length (in float) 34 | static const int L; 35 | 36 | /** 37 | * Calculates the mean value of a set of descriptors 38 | * @param descriptors 39 | * @param mean mean descriptor 40 | */ 41 | static void meanValue(const std::vector &descriptors, 42 | TDescriptor &mean); 43 | 44 | /** 45 | * Calculates the distance between two descriptors using L1-norm method. 46 | * Since SuperPoint's descriptors have float type elements, 47 | * it was concluded that the result of L2 norm was not good for a larger amount of computation. 48 | */ 49 | static double distance(const TDescriptor &a, const TDescriptor &b); 50 | 51 | /** 52 | * Returns a string version of the descriptor 53 | * @param a descriptor 54 | * @return string version 55 | */ 56 | static std::string toString(const TDescriptor &a); 57 | 58 | /** 59 | * Returns a descriptor from a string 60 | * @param a descriptor 61 | * @param s string version 62 | */ 63 | static void fromString(TDescriptor &a, const std::string &s); 64 | 65 | /** 66 | * Returns a mat with the descriptors in float format 67 | * @param descriptors 68 | * @param mat (out) NxL 32F matrix 69 | */ 70 | static void toMat32F(const std::vector &descriptors, 71 | cv::Mat &mat); 72 | 73 | /** 74 | * Returns a mat with the descriptors in float format 75 | * @param descriptors NxL CV_8U matrix 76 | * @param mat (out) NxL 32F matrix 77 | */ 78 | static void toMat32F(const cv::Mat &descriptors, cv::Mat &mat); 79 | 80 | /** 81 | * Returns a matrix with the descriptor in OpenCV format 82 | * @param descriptors vector of N row descriptors 83 | * @param mat (out) NxL CV_8U matrix 84 | */ 85 | static void toMat8U(const std::vector &descriptors, 86 | cv::Mat &mat); 87 | 88 | }; 89 | 90 | } // namespace DBoW2 91 | 92 | #endif 93 | 94 | -------------------------------------------------------------------------------- /CreateVoca/include/FeatureVector.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: FeatureVector.h 3 | * Date: November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: feature vector 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_FEATURE_VECTOR__ 11 | #define __D_T_FEATURE_VECTOR__ 12 | 13 | #include "BowVector.h" 14 | #include 15 | #include 16 | #include 17 | 18 | namespace DBoW2 { 19 | 20 | /** 21 | * @brief Vector of nodes with indexes of local features 22 | * 하나의 이미지에 들어있는 feature들이 있을 때의 인덱스를 이용하여, 23 | * 를 가지는 map구조. 24 | * FeatureVector는 이미지당 하나의 객체를 가질 수 있으며, 어떤 NodeId에 어떠어떠한 feature가 포함되어 있는지 담는다. 25 | * 26 | */ 27 | class FeatureVector: 28 | public std::map > 29 | { 30 | public: 31 | 32 | /** 33 | * Constructor 34 | */ 35 | FeatureVector(void); 36 | 37 | /** 38 | * Destructor 39 | */ 40 | ~FeatureVector(void); 41 | 42 | /** 43 | * Adds a feature to an existing node, or adds a new node with an initial 44 | * feature 45 | * @param id node id to add or to modify 46 | * @param i_feature index of feature to add to the given node 47 | */ 48 | void addFeature(NodeId id, unsigned int i_feature); 49 | 50 | /** 51 | * Sends a string versions of the feature vector through the stream 52 | * @param out stream 53 | * @param v feature vector 54 | */ 55 | friend std::ostream& operator<<(std::ostream &out, const FeatureVector &v); 56 | 57 | }; 58 | 59 | } // namespace DBoW2 60 | 61 | #endif 62 | 63 | -------------------------------------------------------------------------------- /CreateVoca/include/QueryResults.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: QueryResults.h 3 | * Date: March, November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: structure to store results of database queries 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_QUERY_RESULTS__ 11 | #define __D_T_QUERY_RESULTS__ 12 | 13 | #include 14 | 15 | namespace DBoW2 { 16 | 17 | /// Id of entries of the database 18 | typedef unsigned int EntryId; 19 | 20 | /// Single result of a query 21 | class Result 22 | { 23 | public: 24 | 25 | /// Entry id 26 | EntryId Id; 27 | 28 | /// Score obtained 29 | double Score; 30 | 31 | /// debug 32 | int nWords; // words in common 33 | // !!! this is filled only by Bhatt score! 34 | // (and for BCMatching, BCThresholding then) 35 | 36 | double bhatScore, chiScore; 37 | /// debug 38 | 39 | // only done by ChiSq and BCThresholding 40 | double sumCommonVi; 41 | double sumCommonWi; 42 | double expectedChiScore; 43 | /// debug 44 | 45 | /** 46 | * Empty constructors 47 | */ 48 | inline Result(){} 49 | 50 | /** 51 | * Creates a result with the given data 52 | * @param _id entry id 53 | * @param _score score 54 | */ 55 | inline Result(EntryId _id, double _score): Id(_id), Score(_score){} 56 | 57 | /** 58 | * Compares the scores of two results 59 | * @return true iff this.score < r.score 60 | */ 61 | inline bool operator<(const Result &r) const 62 | { 63 | return this->Score < r.Score; 64 | } 65 | 66 | /** 67 | * Compares the scores of two results 68 | * @return true iff this.score > r.score 69 | */ 70 | inline bool operator>(const Result &r) const 71 | { 72 | return this->Score > r.Score; 73 | } 74 | 75 | /** 76 | * Compares the entry id of the result 77 | * @return true iff this.id == id 78 | */ 79 | inline bool operator==(EntryId id) const 80 | { 81 | return this->Id == id; 82 | } 83 | 84 | /** 85 | * Compares the score of this entry with a given one 86 | * @param s score to compare with 87 | * @return true iff this score < s 88 | */ 89 | inline bool operator<(double s) const 90 | { 91 | return this->Score < s; 92 | } 93 | 94 | /** 95 | * Compares the score of this entry with a given one 96 | * @param s score to compare with 97 | * @return true iff this score > s 98 | */ 99 | inline bool operator>(double s) const 100 | { 101 | return this->Score > s; 102 | } 103 | 104 | /** 105 | * Compares the score of two results 106 | * @param a 107 | * @param b 108 | * @return true iff a.Score > b.Score 109 | */ 110 | static inline bool gt(const Result &a, const Result &b) 111 | { 112 | return a.Score > b.Score; 113 | } 114 | 115 | /** 116 | * Compares the scores of two results 117 | * @return true iff a.Score > b.Score 118 | */ 119 | inline static bool ge(const Result &a, const Result &b) 120 | { 121 | return a.Score > b.Score; 122 | } 123 | 124 | /** 125 | * Returns true iff a.Score >= b.Score 126 | * @param a 127 | * @param b 128 | * @return true iff a.Score >= b.Score 129 | */ 130 | static inline bool geq(const Result &a, const Result &b) 131 | { 132 | return a.Score >= b.Score; 133 | } 134 | 135 | /** 136 | * Returns true iff a.Score >= s 137 | * @param a 138 | * @param s 139 | * @return true iff a.Score >= s 140 | */ 141 | static inline bool geqv(const Result &a, double s) 142 | { 143 | return a.Score >= s; 144 | } 145 | 146 | 147 | /** 148 | * Returns true iff a.Id < b.Id 149 | * @param a 150 | * @param b 151 | * @return true iff a.Id < b.Id 152 | */ 153 | static inline bool ltId(const Result &a, const Result &b) 154 | { 155 | return a.Id < b.Id; 156 | } 157 | 158 | /** 159 | * Prints a string version of the result 160 | * @param os ostream 161 | * @param ret Result to print 162 | */ 163 | friend std::ostream & operator<<(std::ostream& os, const Result& ret ); 164 | }; 165 | 166 | /// Multiple results from a query 167 | class QueryResults: public std::vector 168 | { 169 | public: 170 | 171 | /** 172 | * Multiplies all the scores in the vector by factor 173 | * @param factor 174 | */ 175 | inline void scaleScores(double factor); 176 | 177 | /** 178 | * Prints a string version of the results 179 | * @param os ostream 180 | * @param ret QueryResults to print 181 | */ 182 | friend std::ostream & operator<<(std::ostream& os, const QueryResults& ret ); 183 | 184 | /** 185 | * Saves a matlab file with the results 186 | * @param filename 187 | */ 188 | void saveM(const std::string &filename) const; 189 | 190 | }; 191 | 192 | // -------------------------------------------------------------------------- 193 | 194 | inline void QueryResults::scaleScores(double factor) 195 | { 196 | for(QueryResults::iterator qit = begin(); qit != end(); ++qit) 197 | qit->Score *= factor; 198 | } 199 | 200 | // -------------------------------------------------------------------------- 201 | 202 | } // namespace TemplatedBoW 203 | 204 | #endif 205 | 206 | -------------------------------------------------------------------------------- /CreateVoca/include/ScoringObject.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: ScoringObject.h 3 | * Date: November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: functions to compute bow scores 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_SCORING_OBJECT__ 11 | #define __D_T_SCORING_OBJECT__ 12 | 13 | #include "BowVector.h" 14 | 15 | namespace DBoW2 { 16 | 17 | /// Base class of scoring functions 18 | class GeneralScoring 19 | { 20 | public: 21 | /** 22 | * Computes the score between two vectors. Vectors must be sorted and 23 | * normalized if necessary 24 | * @param v (in/out) 25 | * @param w (in/out) 26 | * @return score 27 | */ 28 | virtual double score(const BowVector &v, const BowVector &w) const = 0; 29 | 30 | /** 31 | * Returns whether a vector must be normalized before scoring according 32 | * to the scoring scheme 33 | * @param norm norm to use 34 | * @return true iff must normalize 35 | */ 36 | virtual bool mustNormalize(LNorm &norm) const = 0; 37 | 38 | /// Log of epsilon 39 | static const double LOG_EPS; 40 | // If you change the type of WordValue, make sure you change also the 41 | // epsilon value (this is needed by the KL method) 42 | 43 | virtual ~GeneralScoring() {} //!< Required for virtual base classes 44 | }; 45 | 46 | /** 47 | * Macro for defining Scoring classes 48 | * @param NAME name of class 49 | * @param MUSTNORMALIZE if vectors must be normalized to compute the score 50 | * @param NORM type of norm to use when MUSTNORMALIZE 51 | */ 52 | #define __SCORING_CLASS(NAME, MUSTNORMALIZE, NORM) \ 53 | NAME: public GeneralScoring \ 54 | { public: \ 55 | /** \ 56 | * Computes score between two vectors \ 57 | * @param v \ 58 | * @param w \ 59 | * @return score between v and w \ 60 | */ \ 61 | virtual double score(const BowVector &v, const BowVector &w) const; \ 62 | \ 63 | /** \ 64 | * Says if a vector must be normalized according to the scoring function \ 65 | * @param norm (out) if true, norm to use 66 | * @return true iff vectors must be normalized \ 67 | */ \ 68 | virtual inline bool mustNormalize(LNorm &norm) const \ 69 | { norm = NORM; return MUSTNORMALIZE; } \ 70 | } 71 | 72 | /// L1 Scoring object 73 | class __SCORING_CLASS(L1Scoring, true, L1); 74 | 75 | /// L2 Scoring object 76 | class __SCORING_CLASS(L2Scoring, true, L2); 77 | 78 | /// Chi square Scoring object 79 | class __SCORING_CLASS(ChiSquareScoring, true, L1); 80 | 81 | /// KL divergence Scoring object 82 | class __SCORING_CLASS(KLScoring, true, L1); 83 | 84 | /// Bhattacharyya Scoring object 85 | class __SCORING_CLASS(BhattacharyyaScoring, true, L1); 86 | 87 | /// Dot product Scoring object 88 | class __SCORING_CLASS(DotProductScoring, false, L1); 89 | 90 | #undef __SCORING_CLASS 91 | 92 | } // namespace DBoW2 93 | 94 | #endif 95 | 96 | -------------------------------------------------------------------------------- /CreateVoca/include/TemplatedDatabase.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: TemplatedDatabase.h 3 | * Date: March 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: templated database of images 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_TEMPLATED_DATABASE__ 11 | #define __D_T_TEMPLATED_DATABASE__ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "TemplatedVocabulary.h" 21 | #include "QueryResults.h" 22 | #include "ScoringObject.h" 23 | #include "BowVector.h" 24 | #include "FeatureVector.h" 25 | 26 | namespace DBoW2 { 27 | 28 | // For query functions 29 | static int MIN_COMMON_WORDS = 5; 30 | 31 | /// @param TDescriptor class of descriptor 32 | /// @param F class of descriptor functions 33 | template 34 | /// Generic Database 35 | class TemplatedDatabase 36 | { 37 | public: 38 | 39 | /** 40 | * Creates an empty database without vocabulary 41 | * @param use_di a direct index is used to store feature indexes 42 | * @param di_levels levels to go up the vocabulary tree to select the 43 | * node id to store in the direct index when adding images 44 | */ 45 | explicit TemplatedDatabase(bool use_di = true, int di_levels = 0); 46 | 47 | /** 48 | * Creates a database with the given vocabulary 49 | * @param T class inherited from TemplatedVocabulary 50 | * @param voc vocabulary 51 | * @param use_di a direct index is used to store feature indexes 52 | * @param di_levels levels to go up the vocabulary tree to select the 53 | * node id to store in the direct index when adding images 54 | */ 55 | template 56 | explicit TemplatedDatabase(const T &voc, bool use_di = true, 57 | int di_levels = 0); 58 | 59 | /** 60 | * Copy constructor. Copies the vocabulary too 61 | * @param db object to copy 62 | */ 63 | TemplatedDatabase(const TemplatedDatabase &db); 64 | 65 | /** 66 | * Creates the database from a file 67 | * @param filename 68 | */ 69 | TemplatedDatabase(const std::string &filename); 70 | 71 | /** 72 | * Creates the database from a file 73 | * @param filename 74 | */ 75 | TemplatedDatabase(const char *filename); 76 | 77 | /** 78 | * Destructor 79 | */ 80 | virtual ~TemplatedDatabase(void); 81 | 82 | /** 83 | * Copies the given database and its vocabulary 84 | * @param db database to copy 85 | */ 86 | TemplatedDatabase& operator=( 87 | const TemplatedDatabase &db); 88 | 89 | /** 90 | * Sets the vocabulary to use and clears the content of the database. 91 | * @param T class inherited from TemplatedVocabulary 92 | * @param voc vocabulary to copy 93 | */ 94 | template 95 | inline void setVocabulary(const T &voc); 96 | 97 | /** 98 | * Sets the vocabulary to use and the direct index parameters, and clears 99 | * the content of the database 100 | * @param T class inherited from TemplatedVocabulary 101 | * @param voc vocabulary to copy 102 | * @param use_di a direct index is used to store feature indexes 103 | * @param di_levels levels to go up the vocabulary tree to select the 104 | * node id to store in the direct index when adding images 105 | */ 106 | template 107 | void setVocabulary(const T& voc, bool use_di, int di_levels = 0); 108 | 109 | /** 110 | * Returns a pointer to the vocabulary used 111 | * @return vocabulary 112 | */ 113 | inline const TemplatedVocabulary* getVocabulary() const; 114 | 115 | /** 116 | * Allocates some memory for the direct and inverted indexes 117 | * @param nd number of expected image entries in the database 118 | * @param ni number of expected words per image 119 | * @note Use 0 to ignore a parameter 120 | */ 121 | void allocate(int nd = 0, int ni = 0); 122 | 123 | /** 124 | * Adds an entry to the database and returns its index 125 | * @param features features of the new entry 126 | * @param bowvec if given, the bow vector of these features is returned 127 | * @param fvec if given, the vector of nodes and feature indexes is returned 128 | * @return id of new entry 129 | */ 130 | EntryId add(const std::vector &features, 131 | BowVector *bowvec = NULL, FeatureVector *fvec = NULL); 132 | 133 | /** 134 | * Adss an entry to the database and returns its index 135 | * @param vec bow vector 136 | * @param fec feature vector to add the entry. Only necessary if using the 137 | * direct index 138 | * @return id of new entry 139 | */ 140 | EntryId add(const BowVector &vec, 141 | const FeatureVector &fec = FeatureVector() ); 142 | 143 | /** 144 | * Empties the database 145 | */ 146 | inline void clear(); 147 | 148 | /** 149 | * Returns the number of entries in the database 150 | * @return number of entries in the database 151 | */ 152 | inline unsigned int size() const; 153 | 154 | /** 155 | * Checks if the direct index is being used 156 | * @return true iff using direct index 157 | */ 158 | inline bool usingDirectIndex() const; 159 | 160 | /** 161 | * Returns the di levels when using direct index 162 | * @return di levels 163 | */ 164 | inline int getDirectIndexLevels() const; 165 | 166 | /** 167 | * Queries the database with some features 168 | * @param features query features 169 | * @param ret (out) query results 170 | * @param max_results number of results to return. <= 0 means all 171 | * @param max_id only entries with id <= max_id are returned in ret. 172 | * < 0 means all 173 | */ 174 | void query(const std::vector &features, QueryResults &ret, 175 | int max_results = 1, int max_id = -1) const; 176 | 177 | /** 178 | * Queries the database with a vector 179 | * @param vec bow vector already normalized 180 | * @param ret results 181 | * @param max_results number of results to return. <= 0 means all 182 | * @param max_id only entries with id <= max_id are returned in ret. 183 | * < 0 means all 184 | */ 185 | void query(const BowVector &vec, QueryResults &ret, 186 | int max_results = 1, int max_id = -1) const; 187 | 188 | /** 189 | * Returns the a feature vector associated with a database entry 190 | * @param id entry id (must be < size()) 191 | * @return const reference to map of nodes and their associated features in 192 | * the given entry 193 | */ 194 | const FeatureVector& retrieveFeatures(EntryId id) const; 195 | 196 | /** 197 | * Stores the database in a file 198 | * @param filename 199 | */ 200 | void save(const std::string &filename) const; 201 | 202 | /** 203 | * Loads the database from a file 204 | * @param filename 205 | */ 206 | void load(const std::string &filename); 207 | 208 | /** 209 | * Stores the database in the given file storage structure 210 | * @param fs 211 | * @param name node name 212 | */ 213 | virtual void save(cv::FileStorage &fs, 214 | const std::string &name = "database") const; 215 | 216 | /** 217 | * Loads the database from the given file storage structure 218 | * @param fs 219 | * @param name node name 220 | */ 221 | virtual void load(const cv::FileStorage &fs, 222 | const std::string &name = "database"); 223 | 224 | protected: 225 | 226 | /// Query with L1 scoring 227 | void queryL1(const BowVector &vec, QueryResults &ret, 228 | int max_results, int max_id) const; 229 | 230 | /// Query with L2 scoring 231 | void queryL2(const BowVector &vec, QueryResults &ret, 232 | int max_results, int max_id) const; 233 | 234 | /// Query with Chi square scoring 235 | void queryChiSquare(const BowVector &vec, QueryResults &ret, 236 | int max_results, int max_id) const; 237 | 238 | /// Query with Bhattacharyya scoring 239 | void queryBhattacharyya(const BowVector &vec, QueryResults &ret, 240 | int max_results, int max_id) const; 241 | 242 | /// Query with KL divergence scoring 243 | void queryKL(const BowVector &vec, QueryResults &ret, 244 | int max_results, int max_id) const; 245 | 246 | /// Query with dot product scoring 247 | void queryDotProduct(const BowVector &vec, QueryResults &ret, 248 | int max_results, int max_id) const; 249 | 250 | protected: 251 | 252 | /* Inverted file declaration */ 253 | 254 | /// Item of IFRow 255 | struct IFPair 256 | { 257 | /// Entry id 258 | EntryId entry_id; 259 | 260 | /// Word weight in this entry 261 | WordValue word_weight; 262 | 263 | /** 264 | * Creates an empty pair 265 | */ 266 | IFPair(){} 267 | 268 | /** 269 | * Creates an inverted file pair 270 | * @param eid entry id 271 | * @param wv word weight 272 | */ 273 | IFPair(EntryId eid, WordValue wv): entry_id(eid), word_weight(wv) {} 274 | 275 | /** 276 | * Compares the entry ids 277 | * @param eid 278 | * @return true iff this entry id is the same as eid 279 | */ 280 | inline bool operator==(EntryId eid) const { return entry_id == eid; } 281 | }; 282 | 283 | /// Row of InvertedFile 284 | typedef std::list IFRow; 285 | // IFRows are sorted in ascending entry_id order 286 | 287 | /// Inverted index 288 | typedef std::vector InvertedFile; 289 | // InvertedFile[word_id] --> inverted file of that word 290 | 291 | /* Direct file declaration */ 292 | 293 | /// Direct index 294 | typedef std::vector DirectFile; 295 | // DirectFile[entry_id] --> [ directentry, ... ] 296 | 297 | protected: 298 | 299 | /// Associated vocabulary 300 | TemplatedVocabulary *m_voc; 301 | 302 | /// Flag to use direct index 303 | bool m_use_di; 304 | 305 | /// Levels to go up the vocabulary tree to select nodes to store 306 | /// in the direct index 307 | int m_dilevels; 308 | 309 | /// Inverted file (must have size() == |words|) 310 | InvertedFile m_ifile; 311 | 312 | /// Direct file (resized for allocation) 313 | DirectFile m_dfile; 314 | 315 | /// Number of valid entries in m_dfile 316 | int m_nentries; 317 | 318 | }; 319 | 320 | // -------------------------------------------------------------------------- 321 | 322 | template 323 | TemplatedDatabase::TemplatedDatabase 324 | (bool use_di, int di_levels) 325 | : m_voc(NULL), m_use_di(use_di), m_dilevels(di_levels), m_nentries(0) 326 | { 327 | } 328 | 329 | // -------------------------------------------------------------------------- 330 | 331 | template 332 | template 333 | TemplatedDatabase::TemplatedDatabase 334 | (const T &voc, bool use_di, int di_levels) 335 | : m_voc(NULL), m_use_di(use_di), m_dilevels(di_levels) 336 | { 337 | setVocabulary(voc); 338 | clear(); 339 | } 340 | 341 | // -------------------------------------------------------------------------- 342 | 343 | template 344 | TemplatedDatabase::TemplatedDatabase 345 | (const TemplatedDatabase &db) 346 | : m_voc(NULL) 347 | { 348 | *this = db; 349 | } 350 | 351 | // -------------------------------------------------------------------------- 352 | 353 | template 354 | TemplatedDatabase::TemplatedDatabase 355 | (const std::string &filename) 356 | : m_voc(NULL) 357 | { 358 | load(filename); 359 | } 360 | 361 | // -------------------------------------------------------------------------- 362 | 363 | template 364 | TemplatedDatabase::TemplatedDatabase 365 | (const char *filename) 366 | : m_voc(NULL) 367 | { 368 | load(filename); 369 | } 370 | 371 | // -------------------------------------------------------------------------- 372 | 373 | template 374 | TemplatedDatabase::~TemplatedDatabase(void) 375 | { 376 | delete m_voc; 377 | } 378 | 379 | // -------------------------------------------------------------------------- 380 | 381 | template 382 | TemplatedDatabase& TemplatedDatabase::operator= 383 | (const TemplatedDatabase &db) 384 | { 385 | if(this != &db) 386 | { 387 | m_dfile = db.m_dfile; 388 | m_dilevels = db.m_dilevels; 389 | m_ifile = db.m_ifile; 390 | m_nentries = db.m_nentries; 391 | m_use_di = db.m_use_di; 392 | setVocabulary(*db.m_voc); 393 | } 394 | return *this; 395 | } 396 | 397 | // -------------------------------------------------------------------------- 398 | 399 | template 400 | EntryId TemplatedDatabase::add( 401 | const std::vector &features, 402 | BowVector *bowvec, FeatureVector *fvec) 403 | { 404 | BowVector aux; 405 | BowVector& v = (bowvec ? *bowvec : aux); 406 | 407 | if(m_use_di && fvec != NULL) 408 | { 409 | m_voc->transform(features, v, *fvec, m_dilevels); // with features 410 | return add(v, *fvec); 411 | } 412 | else if(m_use_di) 413 | { 414 | FeatureVector fv; 415 | m_voc->transform(features, v, fv, m_dilevels); // with features 416 | return add(v, fv); 417 | } 418 | else if(fvec != NULL) 419 | { 420 | m_voc->transform(features, v, *fvec, m_dilevels); // with features 421 | return add(v); 422 | } 423 | else 424 | { 425 | m_voc->transform(features, v); // with features 426 | return add(v); 427 | } 428 | } 429 | 430 | // --------------------------------------------------------------------------- 431 | 432 | template 433 | EntryId TemplatedDatabase::add(const BowVector &v, 434 | const FeatureVector &fv) 435 | { 436 | EntryId entry_id = m_nentries++; 437 | 438 | BowVector::const_iterator vit; 439 | 440 | if(m_use_di) 441 | { 442 | // update direct file 443 | if(entry_id == m_dfile.size()) 444 | { 445 | m_dfile.push_back(fv); 446 | } 447 | else 448 | { 449 | m_dfile[entry_id] = fv; 450 | } 451 | } 452 | 453 | // update inverted file 454 | for(vit = v.begin(); vit != v.end(); ++vit) 455 | { 456 | const WordId& word_id = vit->first; 457 | const WordValue& word_weight = vit->second; 458 | 459 | IFRow& ifrow = m_ifile[word_id]; 460 | ifrow.push_back(IFPair(entry_id, word_weight)); 461 | } 462 | 463 | return entry_id; 464 | } 465 | 466 | // -------------------------------------------------------------------------- 467 | 468 | template 469 | template 470 | inline void TemplatedDatabase::setVocabulary 471 | (const T& voc) 472 | { 473 | delete m_voc; 474 | m_voc = new T(voc); 475 | clear(); 476 | } 477 | 478 | // -------------------------------------------------------------------------- 479 | 480 | template 481 | template 482 | inline void TemplatedDatabase::setVocabulary 483 | (const T& voc, bool use_di, int di_levels) 484 | { 485 | m_use_di = use_di; 486 | m_dilevels = di_levels; 487 | delete m_voc; 488 | m_voc = new T(voc); 489 | clear(); 490 | } 491 | 492 | // -------------------------------------------------------------------------- 493 | 494 | template 495 | inline const TemplatedVocabulary* 496 | TemplatedDatabase::getVocabulary() const 497 | { 498 | return m_voc; 499 | } 500 | 501 | // -------------------------------------------------------------------------- 502 | 503 | template 504 | inline void TemplatedDatabase::clear() 505 | { 506 | // resize vectors 507 | m_ifile.resize(0); 508 | m_ifile.resize(m_voc->size()); 509 | m_dfile.resize(0); 510 | m_nentries = 0; 511 | } 512 | 513 | // -------------------------------------------------------------------------- 514 | 515 | template 516 | void TemplatedDatabase::allocate(int nd, int ni) 517 | { 518 | // m_ifile already contains |words| items 519 | if(ni > 0) 520 | { 521 | typename std::vector::iterator rit; 522 | for(rit = m_ifile.begin(); rit != m_ifile.end(); ++rit) 523 | { 524 | int n = (int)rit->size(); 525 | if(ni > n) 526 | { 527 | rit->resize(ni); 528 | rit->resize(n); 529 | } 530 | } 531 | } 532 | 533 | if(m_use_di && (int)m_dfile.size() < nd) 534 | { 535 | m_dfile.resize(nd); 536 | } 537 | } 538 | 539 | // -------------------------------------------------------------------------- 540 | 541 | template 542 | inline unsigned int TemplatedDatabase::size() const 543 | { 544 | return m_nentries; 545 | } 546 | 547 | // -------------------------------------------------------------------------- 548 | 549 | template 550 | inline bool TemplatedDatabase::usingDirectIndex() const 551 | { 552 | return m_use_di; 553 | } 554 | 555 | // -------------------------------------------------------------------------- 556 | 557 | template 558 | inline int TemplatedDatabase::getDirectIndexLevels() const 559 | { 560 | return m_dilevels; 561 | } 562 | 563 | // -------------------------------------------------------------------------- 564 | 565 | template 566 | void TemplatedDatabase::query( 567 | const std::vector &features, 568 | QueryResults &ret, int max_results, int max_id) const 569 | { 570 | BowVector vec; 571 | m_voc->transform(features, vec); 572 | query(vec, ret, max_results, max_id); 573 | } 574 | 575 | // -------------------------------------------------------------------------- 576 | 577 | template 578 | void TemplatedDatabase::query( 579 | const BowVector &vec, 580 | QueryResults &ret, int max_results, int max_id) const 581 | { 582 | ret.resize(0); 583 | 584 | switch(m_voc->getScoringType()) 585 | { 586 | case L1_NORM: 587 | queryL1(vec, ret, max_results, max_id); 588 | break; 589 | 590 | case L2_NORM: 591 | queryL2(vec, ret, max_results, max_id); 592 | break; 593 | 594 | case CHI_SQUARE: 595 | queryChiSquare(vec, ret, max_results, max_id); 596 | break; 597 | 598 | case KL: 599 | queryKL(vec, ret, max_results, max_id); 600 | break; 601 | 602 | case BHATTACHARYYA: 603 | queryBhattacharyya(vec, ret, max_results, max_id); 604 | break; 605 | 606 | case DOT_PRODUCT: 607 | queryDotProduct(vec, ret, max_results, max_id); 608 | break; 609 | } 610 | } 611 | 612 | // -------------------------------------------------------------------------- 613 | 614 | template 615 | void TemplatedDatabase::queryL1(const BowVector &vec, 616 | QueryResults &ret, int max_results, int max_id) const 617 | { 618 | BowVector::const_iterator vit; 619 | typename IFRow::const_iterator rit; 620 | 621 | std::map pairs; 622 | std::map::iterator pit; 623 | 624 | for(vit = vec.begin(); vit != vec.end(); ++vit) 625 | { 626 | const WordId word_id = vit->first; 627 | const WordValue& qvalue = vit->second; 628 | 629 | const IFRow& row = m_ifile[word_id]; 630 | 631 | // IFRows are sorted in ascending entry_id order 632 | 633 | for(rit = row.begin(); rit != row.end(); ++rit) 634 | { 635 | const EntryId entry_id = rit->entry_id; 636 | const WordValue& dvalue = rit->word_weight; 637 | 638 | if((int)entry_id < max_id || max_id == -1) 639 | { 640 | double value = fabs(qvalue - dvalue) - fabs(qvalue) - fabs(dvalue); 641 | 642 | pit = pairs.lower_bound(entry_id); 643 | if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) 644 | { 645 | pit->second += value; 646 | } 647 | else 648 | { 649 | pairs.insert(pit, 650 | std::map::value_type(entry_id, value)); 651 | } 652 | } 653 | 654 | } // for each inverted row 655 | } // for each query word 656 | 657 | // move to vector 658 | ret.reserve(pairs.size()); 659 | for(pit = pairs.begin(); pit != pairs.end(); ++pit) 660 | { 661 | ret.push_back(Result(pit->first, pit->second)); 662 | } 663 | 664 | // resulting "scores" are now in [-2 best .. 0 worst] 665 | 666 | // sort vector in ascending order of score 667 | std::sort(ret.begin(), ret.end()); 668 | // (ret is inverted now --the lower the better--) 669 | 670 | // cut vector 671 | if(max_results > 0 && (int)ret.size() > max_results) 672 | ret.resize(max_results); 673 | 674 | // complete and scale score to [0 worst .. 1 best] 675 | // ||v - w||_{L1} = 2 + Sum(|v_i - w_i| - |v_i| - |w_i|) 676 | // for all i | v_i != 0 and w_i != 0 677 | // (Nister, 2006) 678 | // scaled_||v - w||_{L1} = 1 - 0.5 * ||v - w||_{L1} 679 | QueryResults::iterator qit; 680 | for(qit = ret.begin(); qit != ret.end(); qit++) 681 | qit->Score = -qit->Score/2.0; 682 | } 683 | 684 | // -------------------------------------------------------------------------- 685 | 686 | template 687 | void TemplatedDatabase::queryL2(const BowVector &vec, 688 | QueryResults &ret, int max_results, int max_id) const 689 | { 690 | BowVector::const_iterator vit; 691 | typename IFRow::const_iterator rit; 692 | 693 | std::map pairs; 694 | std::map::iterator pit; 695 | 696 | //map counters; 697 | //map::iterator cit; 698 | 699 | for(vit = vec.begin(); vit != vec.end(); ++vit) 700 | { 701 | const WordId word_id = vit->first; 702 | const WordValue& qvalue = vit->second; 703 | 704 | const IFRow& row = m_ifile[word_id]; 705 | 706 | // IFRows are sorted in ascending entry_id order 707 | 708 | for(rit = row.begin(); rit != row.end(); ++rit) 709 | { 710 | const EntryId entry_id = rit->entry_id; 711 | const WordValue& dvalue = rit->word_weight; 712 | 713 | if((int)entry_id < max_id || max_id == -1) 714 | { 715 | double value = - qvalue * dvalue; // minus sign for sorting trick 716 | 717 | pit = pairs.lower_bound(entry_id); 718 | //cit = counters.lower_bound(entry_id); 719 | if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) 720 | { 721 | pit->second += value; 722 | //cit->second += 1; 723 | } 724 | else 725 | { 726 | pairs.insert(pit, 727 | std::map::value_type(entry_id, value)); 728 | 729 | //counters.insert(cit, 730 | // map::value_type(entry_id, 1)); 731 | } 732 | } 733 | 734 | } // for each inverted row 735 | } // for each query word 736 | 737 | // move to vector 738 | ret.reserve(pairs.size()); 739 | //cit = counters.begin(); 740 | for(pit = pairs.begin(); pit != pairs.end(); ++pit)//, ++cit) 741 | { 742 | ret.push_back(Result(pit->first, pit->second));// / cit->second)); 743 | } 744 | 745 | // resulting "scores" are now in [-1 best .. 0 worst] 746 | 747 | // sort vector in ascending order of score 748 | std::sort(ret.begin(), ret.end()); 749 | // (ret is inverted now --the lower the better--) 750 | 751 | // cut vector 752 | if(max_results > 0 && (int)ret.size() > max_results) 753 | ret.resize(max_results); 754 | 755 | // complete and scale score to [0 worst .. 1 best] 756 | // ||v - w||_{L2} = sqrt( 2 - 2 * Sum(v_i * w_i) 757 | // for all i | v_i != 0 and w_i != 0 ) 758 | // (Nister, 2006) 759 | QueryResults::iterator qit; 760 | for(qit = ret.begin(); qit != ret.end(); qit++) 761 | { 762 | if(qit->Score <= -1.0) // rounding error 763 | qit->Score = 1.0; 764 | else 765 | qit->Score = 1.0 - sqrt(1.0 + qit->Score); // [0..1] 766 | // the + sign is ok, it is due to - sign in 767 | // value = - qvalue * dvalue 768 | } 769 | 770 | } 771 | 772 | // -------------------------------------------------------------------------- 773 | 774 | template 775 | void TemplatedDatabase::queryChiSquare(const BowVector &vec, 776 | QueryResults &ret, int max_results, int max_id) const 777 | { 778 | BowVector::const_iterator vit; 779 | typename IFRow::const_iterator rit; 780 | 781 | std::map > pairs; 782 | std::map >::iterator pit; 783 | 784 | std::map > sums; // < sum vi, sum wi > 785 | std::map >::iterator sit; 786 | 787 | // In the current implementation, we suppose vec is not normalized 788 | 789 | //map expected; 790 | //map::iterator eit; 791 | 792 | for(vit = vec.begin(); vit != vec.end(); ++vit) 793 | { 794 | const WordId word_id = vit->first; 795 | const WordValue& qvalue = vit->second; 796 | 797 | const IFRow& row = m_ifile[word_id]; 798 | 799 | // IFRows are sorted in ascending entry_id order 800 | 801 | for(rit = row.begin(); rit != row.end(); ++rit) 802 | { 803 | const EntryId entry_id = rit->entry_id; 804 | const WordValue& dvalue = rit->word_weight; 805 | 806 | if((int)entry_id < max_id || max_id == -1) 807 | { 808 | // (v-w)^2/(v+w) - v - w = -4 vw/(v+w) 809 | // we move the 4 out 810 | double value = 0; 811 | if(qvalue + dvalue != 0.0) // words may have weight zero 812 | value = - qvalue * dvalue / (qvalue + dvalue); 813 | 814 | pit = pairs.lower_bound(entry_id); 815 | sit = sums.lower_bound(entry_id); 816 | //eit = expected.lower_bound(entry_id); 817 | if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) 818 | { 819 | pit->second.first += value; 820 | pit->second.second += 1; 821 | //eit->second += dvalue; 822 | sit->second.first += qvalue; 823 | sit->second.second += dvalue; 824 | } 825 | else 826 | { 827 | pairs.insert(pit, 828 | std::map >::value_type(entry_id, 829 | std::make_pair(value, 1) )); 830 | //expected.insert(eit, 831 | // map::value_type(entry_id, dvalue)); 832 | 833 | sums.insert(sit, 834 | std::map >::value_type(entry_id, 835 | std::make_pair(qvalue, dvalue) )); 836 | } 837 | } 838 | 839 | } // for each inverted row 840 | } // for each query word 841 | 842 | // move to vector 843 | ret.reserve(pairs.size()); 844 | sit = sums.begin(); 845 | for(pit = pairs.begin(); pit != pairs.end(); ++pit, ++sit) 846 | { 847 | if(pit->second.second >= MIN_COMMON_WORDS) 848 | { 849 | ret.push_back(Result(pit->first, pit->second.first)); 850 | ret.back().nWords = pit->second.second; 851 | ret.back().sumCommonVi = sit->second.first; 852 | ret.back().sumCommonWi = sit->second.second; 853 | ret.back().expectedChiScore = 854 | 2 * sit->second.second / (1 + sit->second.second); 855 | } 856 | 857 | //ret.push_back(Result(pit->first, pit->second)); 858 | } 859 | 860 | // resulting "scores" are now in [-2 best .. 0 worst] 861 | // we have to add +2 to the scores to obtain the chi square score 862 | 863 | // sort vector in ascending order of score 864 | std::sort(ret.begin(), ret.end()); 865 | // (ret is inverted now --the lower the better--) 866 | 867 | // cut vector 868 | if(max_results > 0 && (int)ret.size() > max_results) 869 | ret.resize(max_results); 870 | 871 | // complete and scale score to [0 worst .. 1 best] 872 | QueryResults::iterator qit; 873 | for(qit = ret.begin(); qit != ret.end(); qit++) 874 | { 875 | // this takes the 4 into account 876 | qit->Score = - 2. * qit->Score; // [0..1] 877 | 878 | qit->chiScore = qit->Score; 879 | } 880 | 881 | } 882 | 883 | // -------------------------------------------------------------------------- 884 | 885 | template 886 | void TemplatedDatabase::queryKL(const BowVector &vec, 887 | QueryResults &ret, int max_results, int max_id) const 888 | { 889 | BowVector::const_iterator vit; 890 | typename IFRow::const_iterator rit; 891 | 892 | std::map pairs; 893 | std::map::iterator pit; 894 | 895 | for(vit = vec.begin(); vit != vec.end(); ++vit) 896 | { 897 | const WordId word_id = vit->first; 898 | const WordValue& vi = vit->second; 899 | 900 | const IFRow& row = m_ifile[word_id]; 901 | 902 | // IFRows are sorted in ascending entry_id order 903 | 904 | for(rit = row.begin(); rit != row.end(); ++rit) 905 | { 906 | const EntryId entry_id = rit->entry_id; 907 | const WordValue& wi = rit->word_weight; 908 | 909 | if((int)entry_id < max_id || max_id == -1) 910 | { 911 | double value = 0; 912 | if(vi != 0 && wi != 0) value = vi * log(vi/wi); 913 | 914 | pit = pairs.lower_bound(entry_id); 915 | if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) 916 | { 917 | pit->second += value; 918 | } 919 | else 920 | { 921 | pairs.insert(pit, 922 | std::map::value_type(entry_id, value)); 923 | } 924 | } 925 | 926 | } // for each inverted row 927 | } // for each query word 928 | 929 | // resulting "scores" are now in [-X worst .. 0 best .. X worst] 930 | // but we cannot make sure which ones are better without calculating 931 | // the complete score 932 | 933 | // complete scores and move to vector 934 | ret.reserve(pairs.size()); 935 | for(pit = pairs.begin(); pit != pairs.end(); ++pit) 936 | { 937 | EntryId eid = pit->first; 938 | double value = 0.0; 939 | 940 | for(vit = vec.begin(); vit != vec.end(); ++vit) 941 | { 942 | const WordValue &vi = vit->second; 943 | const IFRow& row = m_ifile[vit->first]; 944 | 945 | if(vi != 0) 946 | { 947 | if(row.end() == find(row.begin(), row.end(), eid )) 948 | { 949 | value += vi * (log(vi) - GeneralScoring::LOG_EPS); 950 | } 951 | } 952 | } 953 | 954 | pit->second += value; 955 | 956 | // to vector 957 | ret.push_back(Result(pit->first, pit->second)); 958 | } 959 | 960 | // real scores are now in [0 best .. X worst] 961 | 962 | // sort vector in ascending order 963 | // (scores are inverted now --the lower the better--) 964 | std::sort(ret.begin(), ret.end()); 965 | 966 | // cut vector 967 | if(max_results > 0 && (int)ret.size() > max_results) 968 | ret.resize(max_results); 969 | 970 | // cannot scale scores 971 | 972 | } 973 | 974 | // -------------------------------------------------------------------------- 975 | 976 | template 977 | void TemplatedDatabase::queryBhattacharyya( 978 | const BowVector &vec, QueryResults &ret, int max_results, int max_id) const 979 | { 980 | BowVector::const_iterator vit; 981 | typename IFRow::const_iterator rit; 982 | 983 | //map pairs; 984 | //map::iterator pit; 985 | 986 | std::map > pairs; // > 987 | std::map >::iterator pit; 988 | 989 | for(vit = vec.begin(); vit != vec.end(); ++vit) 990 | { 991 | const WordId word_id = vit->first; 992 | const WordValue& qvalue = vit->second; 993 | 994 | const IFRow& row = m_ifile[word_id]; 995 | 996 | // IFRows are sorted in ascending entry_id order 997 | 998 | for(rit = row.begin(); rit != row.end(); ++rit) 999 | { 1000 | const EntryId entry_id = rit->entry_id; 1001 | const WordValue& dvalue = rit->word_weight; 1002 | 1003 | if((int)entry_id < max_id || max_id == -1) 1004 | { 1005 | double value = sqrt(qvalue * dvalue); 1006 | 1007 | pit = pairs.lower_bound(entry_id); 1008 | if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) 1009 | { 1010 | pit->second.first += value; 1011 | pit->second.second += 1; 1012 | } 1013 | else 1014 | { 1015 | pairs.insert(pit, 1016 | std::map >::value_type(entry_id, 1017 | std::make_pair(value, 1))); 1018 | } 1019 | } 1020 | 1021 | } // for each inverted row 1022 | } // for each query word 1023 | 1024 | // move to vector 1025 | ret.reserve(pairs.size()); 1026 | for(pit = pairs.begin(); pit != pairs.end(); ++pit) 1027 | { 1028 | if(pit->second.second >= MIN_COMMON_WORDS) 1029 | { 1030 | ret.push_back(Result(pit->first, pit->second.first)); 1031 | ret.back().nWords = pit->second.second; 1032 | ret.back().bhatScore = pit->second.first; 1033 | } 1034 | } 1035 | 1036 | // scores are already in [0..1] 1037 | 1038 | // sort vector in descending order 1039 | std::sort(ret.begin(), ret.end(), Result::gt); 1040 | 1041 | // cut vector 1042 | if(max_results > 0 && (int)ret.size() > max_results) 1043 | ret.resize(max_results); 1044 | 1045 | } 1046 | 1047 | // --------------------------------------------------------------------------- 1048 | 1049 | template 1050 | void TemplatedDatabase::queryDotProduct( 1051 | const BowVector &vec, QueryResults &ret, int max_results, int max_id) const 1052 | { 1053 | BowVector::const_iterator vit; 1054 | typename IFRow::const_iterator rit; 1055 | 1056 | std::map pairs; 1057 | std::map::iterator pit; 1058 | 1059 | for(vit = vec.begin(); vit != vec.end(); ++vit) 1060 | { 1061 | const WordId word_id = vit->first; 1062 | const WordValue& qvalue = vit->second; 1063 | 1064 | const IFRow& row = m_ifile[word_id]; 1065 | 1066 | // IFRows are sorted in ascending entry_id order 1067 | 1068 | for(rit = row.begin(); rit != row.end(); ++rit) 1069 | { 1070 | const EntryId entry_id = rit->entry_id; 1071 | const WordValue& dvalue = rit->word_weight; 1072 | 1073 | if((int)entry_id < max_id || max_id == -1) 1074 | { 1075 | double value; 1076 | if(this->m_voc->getWeightingType() == BINARY) 1077 | value = 1; 1078 | else 1079 | value = qvalue * dvalue; 1080 | 1081 | pit = pairs.lower_bound(entry_id); 1082 | if(pit != pairs.end() && !(pairs.key_comp()(entry_id, pit->first))) 1083 | { 1084 | pit->second += value; 1085 | } 1086 | else 1087 | { 1088 | pairs.insert(pit, 1089 | std::map::value_type(entry_id, value)); 1090 | } 1091 | } 1092 | 1093 | } // for each inverted row 1094 | } // for each query word 1095 | 1096 | // move to vector 1097 | ret.reserve(pairs.size()); 1098 | for(pit = pairs.begin(); pit != pairs.end(); ++pit) 1099 | { 1100 | ret.push_back(Result(pit->first, pit->second)); 1101 | } 1102 | 1103 | // scores are the greater the better 1104 | 1105 | // sort vector in descending order 1106 | std::sort(ret.begin(), ret.end(), Result::gt); 1107 | 1108 | // cut vector 1109 | if(max_results > 0 && (int)ret.size() > max_results) 1110 | ret.resize(max_results); 1111 | 1112 | // these scores cannot be scaled 1113 | } 1114 | 1115 | // --------------------------------------------------------------------------- 1116 | 1117 | template 1118 | const FeatureVector& TemplatedDatabase::retrieveFeatures 1119 | (EntryId id) const 1120 | { 1121 | assert(id < size()); 1122 | return m_dfile[id]; 1123 | } 1124 | 1125 | // -------------------------------------------------------------------------- 1126 | 1127 | template 1128 | void TemplatedDatabase::save(const std::string &filename) const 1129 | { 1130 | cv::FileStorage fs(filename.c_str(), cv::FileStorage::WRITE); 1131 | if(!fs.isOpened()) throw std::string("Could not open file ") + filename; 1132 | 1133 | save(fs); 1134 | } 1135 | 1136 | // -------------------------------------------------------------------------- 1137 | 1138 | template 1139 | void TemplatedDatabase::save(cv::FileStorage &fs, 1140 | const std::string &name) const 1141 | { 1142 | // Format YAML: 1143 | // vocabulary { ... see TemplatedVocabulary::save } 1144 | // database 1145 | // { 1146 | // nEntries: 1147 | // usingDI: 1148 | // diLevels: 1149 | // invertedIndex 1150 | // [ 1151 | // [ 1152 | // { 1153 | // imageId: 1154 | // weight: 1155 | // } 1156 | // ] 1157 | // ] 1158 | // directIndex 1159 | // [ 1160 | // [ 1161 | // { 1162 | // nodeId: 1163 | // features: [ ] 1164 | // } 1165 | // ] 1166 | // ] 1167 | 1168 | // invertedIndex[i] is for the i-th word 1169 | // directIndex[i] is for the i-th entry 1170 | // directIndex may be empty if not using direct index 1171 | // 1172 | // imageId's and nodeId's must be stored in ascending order 1173 | // (according to the construction of the indexes) 1174 | 1175 | m_voc->save(fs); 1176 | 1177 | fs << name << "{"; 1178 | 1179 | fs << "nEntries" << m_nentries; 1180 | fs << "usingDI" << (m_use_di ? 1 : 0); 1181 | fs << "diLevels" << m_dilevels; 1182 | 1183 | fs << "invertedIndex" << "["; 1184 | 1185 | typename InvertedFile::const_iterator iit; 1186 | typename IFRow::const_iterator irit; 1187 | for(iit = m_ifile.begin(); iit != m_ifile.end(); ++iit) 1188 | { 1189 | fs << "["; // word of IF 1190 | for(irit = iit->begin(); irit != iit->end(); ++irit) 1191 | { 1192 | fs << "{:" 1193 | << "imageId" << (int)irit->entry_id 1194 | << "weight" << irit->word_weight 1195 | << "}"; 1196 | } 1197 | fs << "]"; // word of IF 1198 | } 1199 | 1200 | fs << "]"; // invertedIndex 1201 | 1202 | fs << "directIndex" << "["; 1203 | 1204 | typename DirectFile::const_iterator dit; 1205 | typename FeatureVector::const_iterator drit; 1206 | for(dit = m_dfile.begin(); dit != m_dfile.end(); ++dit) 1207 | { 1208 | fs << "["; // entry of DF 1209 | 1210 | for(drit = dit->begin(); drit != dit->end(); ++drit) 1211 | { 1212 | NodeId nid = drit->first; 1213 | const std::vector& features = drit->second; 1214 | 1215 | // save info of last_nid 1216 | fs << "{"; 1217 | fs << "nodeId" << (int)nid; 1218 | // msvc++ 2010 with opencv 2.3.1 does not allow FileStorage::operator<< 1219 | // with vectors of unsigned int 1220 | fs << "features" << "[" 1221 | << *(const std::vector*)(&features) << "]"; 1222 | fs << "}"; 1223 | } 1224 | 1225 | fs << "]"; // entry of DF 1226 | } 1227 | 1228 | fs << "]"; // directIndex 1229 | 1230 | fs << "}"; // database 1231 | } 1232 | 1233 | // -------------------------------------------------------------------------- 1234 | 1235 | template 1236 | void TemplatedDatabase::load(const std::string &filename) 1237 | { 1238 | cv::FileStorage fs(filename.c_str(), cv::FileStorage::READ); 1239 | if(!fs.isOpened()) throw std::string("Could not open file ") + filename; 1240 | 1241 | load(fs); 1242 | } 1243 | 1244 | // -------------------------------------------------------------------------- 1245 | 1246 | template 1247 | void TemplatedDatabase::load(const cv::FileStorage &fs, 1248 | const std::string &name) 1249 | { 1250 | // load voc first 1251 | // subclasses must instantiate m_voc before calling this ::load 1252 | if(!m_voc) m_voc = new TemplatedVocabulary; 1253 | 1254 | m_voc->load(fs); 1255 | 1256 | // load database now 1257 | clear(); // resizes inverted file 1258 | 1259 | cv::FileNode fdb = fs[name]; 1260 | 1261 | m_nentries = (int)fdb["nEntries"]; 1262 | m_use_di = (int)fdb["usingDI"] != 0; 1263 | m_dilevels = (int)fdb["diLevels"]; 1264 | 1265 | cv::FileNode fn = fdb["invertedIndex"]; 1266 | for(WordId wid = 0; wid < fn.size(); ++wid) 1267 | { 1268 | cv::FileNode fw = fn[wid]; 1269 | 1270 | for(unsigned int i = 0; i < fw.size(); ++i) 1271 | { 1272 | EntryId eid = (int)fw[i]["imageId"]; 1273 | WordValue v = fw[i]["weight"]; 1274 | 1275 | m_ifile[wid].push_back(IFPair(eid, v)); 1276 | } 1277 | } 1278 | 1279 | if(m_use_di) 1280 | { 1281 | fn = fdb["directIndex"]; 1282 | 1283 | m_dfile.resize(fn.size()); 1284 | assert(m_nentries == (int)fn.size()); 1285 | 1286 | FeatureVector::iterator dit; 1287 | for(EntryId eid = 0; eid < fn.size(); ++eid) 1288 | { 1289 | cv::FileNode fe = fn[eid]; 1290 | 1291 | m_dfile[eid].clear(); 1292 | for(unsigned int i = 0; i < fe.size(); ++i) 1293 | { 1294 | NodeId nid = (int)fe[i]["nodeId"]; 1295 | 1296 | dit = m_dfile[eid].insert(m_dfile[eid].end(), 1297 | make_pair(nid, std::vector() )); 1298 | 1299 | // this failed to compile with some opencv versions (2.3.1) 1300 | //fe[i]["features"] >> dit->second; 1301 | 1302 | // this was ok until OpenCV 2.4.1 1303 | //std::vector aux; 1304 | //fe[i]["features"] >> aux; // OpenCV < 2.4.1 1305 | //dit->second.resize(aux.size()); 1306 | //std::copy(aux.begin(), aux.end(), dit->second.begin()); 1307 | 1308 | cv::FileNode ff = fe[i]["features"][0]; 1309 | dit->second.reserve(ff.size()); 1310 | 1311 | cv::FileNodeIterator ffit; 1312 | for(ffit = ff.begin(); ffit != ff.end(); ++ffit) 1313 | { 1314 | dit->second.push_back((int)*ffit); 1315 | } 1316 | } 1317 | } // for each entry 1318 | } // if use_id 1319 | 1320 | } 1321 | 1322 | // -------------------------------------------------------------------------- 1323 | 1324 | /** 1325 | * Writes printable information of the database 1326 | * @param os stream to write to 1327 | * @param db 1328 | */ 1329 | template 1330 | std::ostream& operator<<(std::ostream &os, 1331 | const TemplatedDatabase &db) 1332 | { 1333 | os << "Database: Entries = " << db.size() << ", " 1334 | "Using direct index = " << (db.usingDirectIndex() ? "yes" : "no"); 1335 | 1336 | if(db.usingDirectIndex()) 1337 | os << ", Direct index levels = " << db.getDirectIndexLevels(); 1338 | 1339 | os << ". " << *db.getVocabulary(); 1340 | return os; 1341 | } 1342 | 1343 | // -------------------------------------------------------------------------- 1344 | 1345 | } // namespace DBoW2 1346 | 1347 | #endif 1348 | -------------------------------------------------------------------------------- /CreateVoca/include/TemplatedVocabulary.h: -------------------------------------------------------------------------------- 1 | /** 2 | * File: TemplatedVocabulary.h 3 | * Date: February 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: templated vocabulary 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #ifndef __D_T_TEMPLATED_VOCABULARY__ 11 | #define __D_T_TEMPLATED_VOCABULARY__ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "FeatureVector.h" 23 | #include "FSuperpoint.h" 24 | #include "BowVector.h" 25 | #include "ScoringObject.h" 26 | 27 | using namespace std; 28 | 29 | namespace DBoW2 { 30 | 31 | /// @param TDescriptor class of descriptor 32 | /// @param F class of descriptor functions 33 | template 34 | /// Generic Vocabulary 35 | class TemplatedVocabulary 36 | { 37 | public: 38 | int MaxIterationPerCluster = 800; 39 | 40 | /** 41 | * Initiates an empty vocabulary 42 | * @param k branching factor 43 | * @param L depth levels 44 | * @param weighting weighting type 45 | * @param scoring scoring type 46 | */ 47 | TemplatedVocabulary(int k = 10, int L = 5, 48 | WeightingType weighting = TF_IDF, ScoringType scoring = L1_NORM); 49 | 50 | /** 51 | * Creates the vocabulary by loading a file 52 | * @param filename 53 | */ 54 | TemplatedVocabulary(const std::string &filename); 55 | 56 | /** 57 | * Creates the vocabulary by loading a file 58 | * @param filename 59 | */ 60 | TemplatedVocabulary(const char *filename); 61 | 62 | /** 63 | * Copy constructor 64 | * @param voc 65 | */ 66 | TemplatedVocabulary(const TemplatedVocabulary &voc); 67 | 68 | /** 69 | * Destructor 70 | */ 71 | virtual ~TemplatedVocabulary(); 72 | 73 | /** 74 | * Assigns the given vocabulary to this by copying its data and removing 75 | * all the data contained by this vocabulary before 76 | * @param voc 77 | * @return reference to this vocabulary 78 | */ 79 | TemplatedVocabulary& operator=( 80 | const TemplatedVocabulary &voc); 81 | 82 | /** 83 | * Creates a vocabulary from the training features with the already 84 | * defined parameters 85 | * @param training_features 86 | */ 87 | virtual void create 88 | (const std::vector > &training_features); 89 | 90 | /** 91 | * Creates a vocabulary from the training features, setting the branching 92 | * factor and the depth levels of the tree 93 | * @param training_features 94 | * @param k branching factor 95 | * @param L depth levels 96 | */ 97 | virtual void create 98 | (const std::vector > &training_features, 99 | int k, int L); 100 | 101 | /** 102 | * Creates a vocabulary from the training features, setting the branching 103 | * factor nad the depth levels of the tree, and the weighting and scoring 104 | * schemes 105 | */ 106 | virtual void create 107 | (const std::vector > &training_features, 108 | int k, int L, WeightingType weighting, ScoringType scoring); 109 | 110 | /** 111 | * Returns the number of words in the vocabulary 112 | * @return number of words 113 | */ 114 | virtual inline unsigned int size() const; 115 | 116 | /** 117 | * Returns whether the vocabulary is empty (i.e. it has not been trained) 118 | * @return true iff the vocabulary is empty 119 | */ 120 | virtual inline bool empty() const; 121 | 122 | virtual void transform(const std::vector& features, BowVector &v) 123 | const; 124 | 125 | /** 126 | * @brief 127 | * 128 | * @tparam TDescriptor 129 | * @tparam F 130 | * @param features input - image's descriptors 131 | * @param v output - BowVector 132 | * @param fv output - FeatureVector 133 | * @param levelsup input - (m_L - levelsuppression)까지 탐색. 134 | */ 135 | virtual void transform(const std::vector& features, 136 | BowVector &v, FeatureVector &fv, int levelsup) const; 137 | 138 | /** 139 | * Transforms a single feature into a word (without weight) 140 | * @param feature 141 | * @return word id 142 | */ 143 | virtual WordId transform(const TDescriptor& feature) const; 144 | 145 | /** 146 | * Returns the score of two vectors 147 | * @param a vector 148 | * @param b vector 149 | * @return score between vectors 150 | * @note the vectors must be already sorted and normalized if necessary 151 | */ 152 | inline double score(const BowVector &a, const BowVector &b) const; 153 | 154 | /** 155 | * Returns the id of the node that is "levelsup" levels from the word given 156 | * @param wid word id 157 | * @param levelsup 0..L 158 | * @return node id. if levelsup is 0, returns the node id associated to the 159 | * word id 160 | */ 161 | virtual NodeId getParentNode(WordId wid, int levelsup) const; 162 | 163 | /** 164 | * Returns the ids of all the words that are under the given node id, 165 | * by traversing any of the branches that goes down from the node 166 | * @param nid starting node id 167 | * @param words ids of words 168 | */ 169 | void getWordsFromNode(NodeId nid, std::vector &words) const; 170 | 171 | /** 172 | * Returns the branching factor of the tree (k) 173 | * @return k 174 | */ 175 | inline int getBranchingFactor() const { return m_k; } 176 | 177 | /** 178 | * Returns the depth levels of the tree (L) 179 | * @return L 180 | */ 181 | inline int getDepthLevels() const { return m_L; } 182 | 183 | /** 184 | * Returns the real depth levels of the tree on average 185 | * @return average of depth levels of leaves 186 | */ 187 | float getEffectiveLevels() const; 188 | 189 | /** 190 | * Returns the descriptor of a word 191 | * @param wid word id 192 | * @return descriptor 193 | */ 194 | virtual inline TDescriptor getWord(WordId wid) const; 195 | 196 | /** 197 | * Returns the weight of a word 198 | * @param wid word id 199 | * @return weight 200 | */ 201 | virtual inline WordValue getWordWeight(WordId wid) const; 202 | 203 | /** 204 | * Returns the weighting method 205 | * @return weighting method 206 | */ 207 | inline WeightingType getWeightingType() const { return m_weighting; } 208 | 209 | /** 210 | * Returns the scoring method 211 | * @return scoring method 212 | */ 213 | inline ScoringType getScoringType() const { return m_scoring; } 214 | 215 | /** 216 | * Changes the weighting method 217 | * @param type new weighting type 218 | */ 219 | inline void setWeightingType(WeightingType type); 220 | 221 | /** 222 | * Changes the scoring method 223 | * @param type new scoring type 224 | */ 225 | void setScoringType(ScoringType type); 226 | 227 | /** 228 | * Loads the vocabulary from a text file 229 | * @param filename 230 | */ 231 | bool loadFromTextFile(const std::string &filename); 232 | 233 | /** 234 | * Saves the vocabulary into a text file 235 | * @param filename 236 | */ 237 | void saveToTextFile(const std::string &filename) const; 238 | 239 | /** 240 | * Saves the vocabulary into a file 241 | * @param filename 242 | */ 243 | void save(const std::string &filename) const; 244 | 245 | /** 246 | * Loads the vocabulary from a file 247 | * @param filename 248 | */ 249 | void load(const std::string &filename); 250 | 251 | /** 252 | * Saves the vocabulary to a file storage structure 253 | * @param fn node in file storage 254 | */ 255 | virtual void save(cv::FileStorage &fs, 256 | const std::string &name = "vocabulary") const; 257 | 258 | /** 259 | * Loads the vocabulary from a file storage node 260 | * @param fn first node 261 | * @param subname name of the child node of fn where the tree is stored. 262 | * If not given, the fn node is used instead 263 | */ 264 | virtual void load(const cv::FileStorage &fs, 265 | const std::string &name = "vocabulary"); 266 | 267 | /** 268 | * Stops those words whose weight is below minWeight. 269 | * Words are stopped by setting their weight to 0. There are not returned 270 | * later when transforming image features into vectors. 271 | * Note that when using IDF or TF_IDF, the weight is the idf part, which 272 | * is equivalent to -log(f), where f is the frequency of the word 273 | * (f = Ni/N, Ni: number of training images where the word is present, 274 | * N: number of training images). 275 | * Note that the old weight is forgotten, and subsequent calls to this 276 | * function with a lower minWeight have no effect. 277 | * @return number of words stopped now 278 | */ 279 | virtual int stopWords(double minWeight); 280 | 281 | protected: 282 | 283 | /// Pointer to descriptor 284 | typedef const TDescriptor *pDescriptor; 285 | 286 | /// Tree node 287 | struct Node 288 | { 289 | /// Node id 290 | NodeId id; 291 | /// Weight if the node is a word 292 | WordValue weight; 293 | /// Children 294 | std::vector children; 295 | /// Parent node (undefined in case of root) 296 | NodeId parent; 297 | /// Node descriptor 298 | TDescriptor descriptor; 299 | 300 | /// Word id if the node is a word 301 | WordId word_id; 302 | 303 | /** 304 | * Empty constructor 305 | */ 306 | Node(): id(0), weight(0), parent(0), word_id(0){} 307 | 308 | /** 309 | * Constructor 310 | * @param _id node id 311 | */ 312 | Node(NodeId _id): id(_id), weight(0), parent(0), word_id(0){} 313 | 314 | /** 315 | * Returns whether the node is a leaf node 316 | * @return true iff the node is a leaf 317 | */ 318 | inline bool isLeaf() const { return children.empty(); } 319 | }; 320 | 321 | protected: 322 | 323 | /** 324 | * Creates an instance of the scoring object accoring to m_scoring 325 | */ 326 | void createScoringObject(); 327 | 328 | /** 329 | * Returns a set of pointers to descriptores 330 | * @param training_features all the features 331 | * @param features (out) pointers to the training features 332 | */ 333 | void getFeatures( 334 | const std::vector > &training_features, 335 | std::vector &features) const; 336 | 337 | /** 338 | * Returns the word id associated to a feature 339 | * @param feature 340 | * @param id (out) word id 341 | * @param weight (out) word weight 342 | * @param nid (out) if given, id of the node "levelsup" levels up 343 | * @param levelsup 344 | */ 345 | virtual void transform(const TDescriptor &feature, 346 | WordId &id, WordValue &weight, NodeId* nid = NULL, int levelsup = 0) const; 347 | 348 | /** 349 | * Returns the word id associated to a feature 350 | * @param feature 351 | * @param id (out) word id 352 | */ 353 | virtual void transform(const TDescriptor &feature, WordId &id) const; 354 | 355 | /** 356 | * Creates a level in the tree, under the parent, by running k-means with 357 | * a descriptor set, and recursively creates the subsequent levels, too. 358 | * @param parent_id id of parent node 359 | * @param descriptors descriptors to run the kmeans on 360 | * @param current_level current level in the tree 361 | */ 362 | void HKmeansStep(NodeId parent_id, const std::vector &descriptors, 363 | int current_level); 364 | 365 | /** 366 | * Creates k clusters from the given descriptors with some seeding algorithm. 367 | * @note In this class, kmeans++ is used, but this function should be 368 | * overriden by inherited classes. 369 | */ 370 | virtual void initiateClusters(const std::vector &descriptors, 371 | std::vector &clusters) const; 372 | 373 | /** 374 | * Creates k clusters from the given descriptor sets by running the 375 | * initial step of kmeans++ 376 | * @param descriptors 377 | * @param clusters resulting clusters 378 | */ 379 | void initiateClustersKMpp(const std::vector &descriptors, 380 | std::vector &clusters) const; 381 | 382 | /** 383 | * Create the words of the vocabulary once the tree has been built 384 | */ 385 | void createWords(); 386 | 387 | /** 388 | * Sets the weights of the nodes of tree according to the given features. 389 | * Before calling this function, the nodes and the words must be already 390 | * created (by calling HKmeansStep and createWords) 391 | * @param features 392 | */ 393 | void setNodeWeights(const std::vector > &features); 394 | 395 | /** 396 | * Returns a random number in the range [min..max] 397 | * @param min 398 | * @param max 399 | * @return random T number in [min..max] 400 | */ 401 | template 402 | static T RandomValue(T min, T max){ 403 | return ((T)rand()/(T)RAND_MAX) * (max - min) + min; 404 | } 405 | 406 | /** 407 | * Returns a random int in the range [min..max] 408 | * @param min 409 | * @param max 410 | * @return random int in [min..max] 411 | */ 412 | static int RandomInt(int min, int max){ 413 | int d = max - min + 1; 414 | return int(((double)rand()/((double)RAND_MAX + 1.0)) * d) + min; 415 | } 416 | 417 | protected: 418 | 419 | /// Branching factor 420 | int m_k; 421 | 422 | /// Depth levels 423 | int m_L; 424 | 425 | /// Weighting method 426 | WeightingType m_weighting; 427 | 428 | /// Scoring method 429 | ScoringType m_scoring; 430 | 431 | /// Object for computing scores 432 | GeneralScoring* m_scoring_object; 433 | 434 | /// Tree nodes 435 | std::vector m_nodes; 436 | 437 | /// Words of the vocabulary (tree leaves) 438 | /// this condition holds: m_words[wid]->word_id == wid 439 | std::vector m_words; 440 | 441 | }; 442 | 443 | // -------------------------------------------------------------------------- 444 | 445 | template 446 | TemplatedVocabulary::TemplatedVocabulary 447 | (int k, int L, WeightingType weighting, ScoringType scoring) 448 | : m_k(k), m_L(L), m_weighting(weighting), m_scoring(scoring), 449 | m_scoring_object(NULL) 450 | { 451 | createScoringObject(); 452 | } 453 | 454 | // -------------------------------------------------------------------------- 455 | 456 | template 457 | TemplatedVocabulary::TemplatedVocabulary 458 | (const std::string &filename): m_scoring_object(NULL) 459 | { 460 | load(filename); 461 | } 462 | 463 | // -------------------------------------------------------------------------- 464 | 465 | template 466 | TemplatedVocabulary::TemplatedVocabulary 467 | (const char *filename): m_scoring_object(NULL) 468 | { 469 | load(filename); 470 | } 471 | 472 | // -------------------------------------------------------------------------- 473 | 474 | /** 475 | * @brief m_scoring에 따라 m_scoring_object를 초기화한다. 476 | * DotProductScoring을 제외하고는 대체적으로 normalize가 포함되어 있으며, 477 | * L2 norm을 이용하는 것은 L2Scoring 밖에 없다. 478 | * @tparam TDescriptor 479 | * @tparam F 480 | */ 481 | template 482 | void TemplatedVocabulary::createScoringObject() 483 | { 484 | delete m_scoring_object; 485 | m_scoring_object = NULL; 486 | 487 | switch(m_scoring) 488 | { 489 | case L1_NORM: 490 | m_scoring_object = new L1Scoring; 491 | break; 492 | 493 | case L2_NORM: 494 | m_scoring_object = new L2Scoring; 495 | break; 496 | 497 | case CHI_SQUARE: 498 | m_scoring_object = new ChiSquareScoring; 499 | break; 500 | 501 | case KL: 502 | m_scoring_object = new KLScoring; 503 | break; 504 | 505 | case BHATTACHARYYA: 506 | m_scoring_object = new BhattacharyyaScoring; 507 | break; 508 | 509 | case DOT_PRODUCT: 510 | m_scoring_object = new DotProductScoring; 511 | break; 512 | 513 | } 514 | } 515 | 516 | // -------------------------------------------------------------------------- 517 | 518 | template 519 | void TemplatedVocabulary::setScoringType(ScoringType type) 520 | { 521 | m_scoring = type; 522 | createScoringObject(); 523 | } 524 | 525 | // -------------------------------------------------------------------------- 526 | 527 | template 528 | void TemplatedVocabulary::setWeightingType(WeightingType type) 529 | { 530 | this->m_weighting = type; 531 | } 532 | 533 | // -------------------------------------------------------------------------- 534 | 535 | template 536 | TemplatedVocabulary::TemplatedVocabulary( 537 | const TemplatedVocabulary &voc) 538 | : m_scoring_object(NULL) 539 | { 540 | *this = voc; 541 | } 542 | 543 | // -------------------------------------------------------------------------- 544 | 545 | template 546 | TemplatedVocabulary::~TemplatedVocabulary() 547 | { 548 | delete m_scoring_object; 549 | } 550 | 551 | // -------------------------------------------------------------------------- 552 | 553 | template 554 | TemplatedVocabulary& 555 | TemplatedVocabulary::operator= 556 | (const TemplatedVocabulary &voc) 557 | { 558 | this->m_k = voc.m_k; 559 | this->m_L = voc.m_L; 560 | this->m_scoring = voc.m_scoring; 561 | this->m_weighting = voc.m_weighting; 562 | 563 | this->createScoringObject(); 564 | 565 | this->m_nodes.clear(); 566 | this->m_words.clear(); 567 | 568 | this->m_nodes = voc.m_nodes; 569 | this->createWords(); 570 | 571 | return *this; 572 | } 573 | 574 | // -------------------------------------------------------------------------- 575 | 576 | /** 577 | * @brief Vocabulary 생성 함수. m_nodes, m_word가 비어있는 상태에서 시작. 578 | * #1. feature pointer vector 생성(연산 효율 + 구조화) 579 | * #2. HKmeansStep() 클러스터링 >> m_nodes생성 580 | * #3. createWords() m_nodes가지고 m_words를 생성. 581 | * #4. setNodeWeights() m_words의 모든 Node 원소의 Weight업데이트(IDF only) 582 | * 583 | * @tparam TDescriptor 584 | * @tparam F 585 | * @param training_features 586 | */ 587 | template 588 | void TemplatedVocabulary::create( 589 | const std::vector > &training_features) 590 | { 591 | m_nodes.clear(); 592 | m_words.clear(); 593 | std::cout << "===[Create]=== \n"; 594 | // expected_nodes = Sum_{i=0..L} ( k^i ) 595 | int expected_nodes = 596 | (int)((pow((double)m_k, (double)m_L + 1) - 1)/(m_k - 1)); 597 | 598 | m_nodes.reserve(expected_nodes); // avoid allocations when creating the tree 599 | 600 | std::vector features; 601 | getFeatures(training_features, features); 602 | // for(int i=0; i<5; i++) 603 | // std::cout << *features[i] << std::endl; -> OK [-1,1] 사이의 값들로 잘 나옴. 604 | 605 | // create root 606 | m_nodes.push_back(Node(0)); // root 607 | 608 | // create the tree 609 | HKmeansStep(0, features, 1); 610 | 611 | // create the words 612 | createWords(); 613 | 614 | // and set the weight of each node of the tree 615 | setNodeWeights(training_features); 616 | 617 | } 618 | 619 | // -------------------------------------------------------------------------- 620 | 621 | template 622 | void TemplatedVocabulary::create( 623 | const std::vector > &training_features, 624 | int k, int L) 625 | { 626 | m_k = k; 627 | m_L = L; 628 | 629 | create(training_features); 630 | } 631 | 632 | // -------------------------------------------------------------------------- 633 | 634 | template 635 | void TemplatedVocabulary::create( 636 | const std::vector > &training_features, 637 | int k, int L, WeightingType weighting, ScoringType scoring) 638 | { 639 | m_k = k; 640 | m_L = L; 641 | m_weighting = weighting; 642 | m_scoring = scoring; 643 | createScoringObject(); 644 | 645 | create(training_features); 646 | } 647 | 648 | // -------------------------------------------------------------------------- 649 | 650 | /** 651 | * @brief vector> 형태로 가지고 있는 훈련 데이터를 변형할 일이 없기 때문에, 652 | * pointer만 싹 긁어서 vector 로 보관한다. 그 형태로 features로 반환해주는 함수. 653 | * 654 | * @tparam TDescriptor 655 | * @tparam F 656 | * @param training_features Input : Detection&Description 과정을 거친 이미지당 feature를 가지고 있는 이중 벡터. 657 | * @param features Output : training_features의 각 elem의 Pointer를 일차원 벡터로 모아놓음 658 | */ 659 | template 660 | void TemplatedVocabulary::getFeatures( 661 | const std::vector > &training_features, 662 | std::vector &features) const 663 | { 664 | features.resize(0); 665 | 666 | typename std::vector >::const_iterator vvit; 667 | typename std::vector::const_iterator vit; 668 | for(vvit = training_features.begin(); vvit != training_features.end(); ++vvit) 669 | { 670 | features.reserve(features.size() + vvit->size()); // size 늘림. 671 | for(vit = vvit->begin(); vit != vvit->end(); ++vit) // Descriptor's pointer를 한곳으로 몰아넣음 672 | { 673 | features.push_back(&(*vit)); 674 | } 675 | } 676 | std::cout << "getFeatures() -- Get " << features.size() << "Features.\n\n"; 677 | } 678 | 679 | // -------------------------------------------------------------------------- 680 | 681 | template 682 | void TemplatedVocabulary::HKmeansStep(NodeId parent_id, 683 | const std::vector &descriptors, int current_level) 684 | { 685 | if(descriptors.empty()) return; 686 | std::cout << "[HKmeansStep] [Lv. "<< current_level 687 | << "] [pFeatrues. " << descriptors.size() << "]\n"; 688 | int cnt=0; 689 | 690 | // features associated to each cluster 691 | std::vector clusters; 692 | std::vector > groups; 693 | clusters.reserve(m_k); // m_k 한 노드에서 가질 자식 개수. 694 | groups.reserve(m_k); // cluster[i]가 k-means의 mean역할을 하고, 695 | // 그 근처에 위치하는 descriptor들은 groups[i]에 인덱스로 저장. 696 | // 이 인덱스는 descriptors vector의 인덱스 의미. 697 | 698 | //#############################################################################// 699 | 700 | if((int)descriptors.size() <= m_k) 701 | { 702 | // A. Trivial case: one cluster per feature 703 | std::cout << "A "; 704 | groups.resize(descriptors.size()); 705 | 706 | for(unsigned int i = 0; i < descriptors.size(); i++) 707 | { 708 | groups[i].push_back(i); 709 | clusters.push_back(*(descriptors[i])); 710 | } cnt++; 711 | } 712 | else 713 | { 714 | // B. Select clusters and groups with kmeans 715 | std::cout << "B "; 716 | bool first_time = true; 717 | bool go_on = true; 718 | 719 | // to check if clusters move after iterations 720 | std::vector last_association, current_association; 721 | 722 | while(go_on) 723 | { 724 | /* 1. Calculate clusters */ 725 | 726 | if(first_time) 727 | { 728 | // random sample 729 | initiateClusters(descriptors, clusters); 730 | std::cout << "Let's calculate ...\n"; 731 | } 732 | else 733 | { 734 | // calculate cluster centres 735 | std::cout << "="; cnt++; 736 | if(cnt % 100 == 0) cout << cnt << endl; 737 | 738 | for(unsigned int c = 0; c < clusters.size(); ++c) 739 | { 740 | std::vector cluster_descriptors; 741 | cluster_descriptors.reserve(groups[c].size()); 742 | 743 | std::vector::const_iterator vit; 744 | for(vit = groups[c].begin(); vit != groups[c].end(); ++vit) 745 | { 746 | cluster_descriptors.push_back(descriptors[*vit]); 747 | } 748 | 749 | // clusters[c]에 cluster_descriptors의 평균이 들어감. 750 | F::meanValue(cluster_descriptors, clusters[c]); 751 | } 752 | } 753 | 754 | /* 2. Associate features with clusters */ 755 | 756 | // calculate distances to cluster centers 757 | groups.clear(); 758 | groups.resize(m_k, std::vector()); 759 | current_association.resize(descriptors.size()); 760 | 761 | // Group정보를 초기화하고 decriptors 각각을 가장 가까운 cluster에 할당. 762 | typename std::vector::const_iterator fit; 763 | for(fit = descriptors.begin(); fit != descriptors.end(); ++fit) 764 | { 765 | if((*fit)->empty()) 766 | { 767 | std::cout << "fit empty\n"; 768 | } 769 | else if(clusters[0].empty()) 770 | { 771 | std::cout << "cluster[0] empty\n"; 772 | } 773 | 774 | double best_dist = F::distance(*(*fit), clusters[0]); 775 | unsigned int icluster = 0; 776 | 777 | for(unsigned int c = 1; c < clusters.size(); ++c) 778 | { 779 | if((*fit)->empty()) 780 | { 781 | std::cout << "fit empty\n"; continue; 782 | } 783 | else if(clusters[c].empty()) 784 | { 785 | std::cout << "cluster[" << c << "] empty\n"; continue; 786 | } 787 | 788 | double dist = F::distance(*(*fit), clusters[c]); 789 | if(dist < best_dist) 790 | { 791 | best_dist = dist; 792 | icluster = c; 793 | } 794 | } 795 | 796 | int fit_index = fit - descriptors.begin(); 797 | groups[icluster].push_back(fit_index); 798 | current_association[fit_index] = icluster; 799 | } 800 | 801 | // kmeans++ ensures all the clusters has any feature associated with them 802 | 803 | // 3. check convergence 804 | if(first_time) 805 | { 806 | first_time = false; 807 | } 808 | else 809 | { 810 | go_on = false; 811 | for(unsigned int i = 0; i < current_association.size(); i++) 812 | { 813 | if((current_association[i] != last_association[i]) && cnt < MaxIterationPerCluster){ 814 | go_on = true; 815 | break; 816 | } 817 | } 818 | } 819 | 820 | if(go_on) 821 | { 822 | // copy last feature-cluster association 823 | last_association = current_association; 824 | } 825 | 826 | } //END while(go_on) 827 | 828 | } //END else B. 829 | std::cout << "> " << cnt << "s try\n\n"; 830 | 831 | // create nodes // m_nodes는 Vocabulary의 벡터. 832 | // m_k만큼 Node추가. 833 | // m_nodes는 부모 자식을 같이 쭉 push_back으로 넣고 자식들이 parent_id를 가지고 있게 함. 834 | // m_nodes[i].children (Type) = vector 835 | for(unsigned int i = 0; i < clusters.size(); ++i) 836 | { 837 | NodeId id = m_nodes.size(); //가장 끝 index를 가지게함. 838 | m_nodes.push_back(Node(id)); 839 | m_nodes.back().descriptor = clusters[i]; 840 | m_nodes.back().parent = parent_id; 841 | m_nodes[parent_id].children.push_back(id); 842 | } 843 | 844 | // go on with the next level 845 | if(current_level < m_L) 846 | { 847 | // iterate again with the resulting clusters 848 | const std::vector &children_ids = m_nodes[parent_id].children; 849 | for(unsigned int i = 0; i < clusters.size(); ++i) 850 | { 851 | NodeId id = children_ids[i]; 852 | 853 | std::vector child_features; 854 | child_features.reserve(groups[i].size()); 855 | 856 | std::vector::const_iterator vit; 857 | for(vit = groups[i].begin(); vit != groups[i].end(); ++vit) 858 | { 859 | child_features.push_back(descriptors[*vit]); 860 | } 861 | 862 | if(child_features.size() > 1) 863 | { 864 | /* Recursive */ 865 | HKmeansStep(id, child_features, current_level + 1); 866 | } 867 | } 868 | } 869 | } 870 | 871 | // -------------------------------------------------------------------------- 872 | 873 | /** 874 | * @brief Real implementation: initiateClustersKMpp -- 초기화 파트. 클러스터 없이 모여있는 descriptor 중 875 | * 랜덤으로 하나를 골라 첫번째 center로 삼고 모든 descriptor와의 min_distance를 계산한다. 876 | * 그 후 center를 랜덤으로 추가해가면서, 모든 descriptor의 min_dist값을 업데이트하고 877 | * (조금 알 수 없는) 기준을 넘어가는 descriptor를 골라 또 센터로 삼고, 위 과정을 m_k번 반복한다. 878 | * 879 | * @tparam TDescriptor - Descriptor Type 880 | * @tparam F - Function class 881 | * @param descriptors input Descriptors. 효율적 계산 위해, pointer vector로 활용. 882 | * @param clusters 코드 전체에 걸쳐 clusters[m_k]는 k 개의 cluster의 중심점 Descriptor역할을 함. 883 | */ 884 | template 885 | void TemplatedVocabulary::initiateClusters 886 | (const std::vector &descriptors, 887 | std::vector &clusters) const 888 | { 889 | initiateClustersKMpp(descriptors, clusters); 890 | } 891 | 892 | // -------------------------------------------------------------------------- 893 | 894 | template 895 | void TemplatedVocabulary::initiateClustersKMpp 896 | (const std::vector &pfeatures, 897 | std::vector &clusters) const 898 | { 899 | // Implements kmeans++ seeding algorithm 900 | // Algorithm: 901 | // 1. Choose one center uniformly at random from among the data points. 902 | // 2. For each data point x, compute D(x), the distance between x and the nearest 903 | // center that has already been chosen. 904 | // 3. Add one new data point as a center. Each point x is chosen with probability 905 | // proportional to D(x)^2. 906 | // 4. Repeat Steps 2 and 3 until k centers have been chosen. 907 | // 5. Now that the initial centers have been chosen, proceed using standard k-means 908 | // clustering. 909 | std::cout << "[initiateClustersKMpp] => "; 910 | 911 | clusters.resize(0); 912 | clusters.reserve(m_k); 913 | 914 | // 각 descriptor별로 어떠한 center와의 거리든 가장 가까운 거리를 저장. 915 | std::vector min_dists(pfeatures.size(), std::numeric_limits::max()); 916 | 917 | // 1. descriptor 중 랜덤하게 하나 선택. 918 | int ifeature = RandomInt(0, pfeatures.size() - 1); 919 | 920 | // create first cluster 921 | //clusters[i]는 각 클러스터의 중심점의 역할을 한다. 여기서는 초기화이기 때문에 랜덤 선택하는것. 922 | clusters.push_back(*(pfeatures[ifeature])); 923 | 924 | // compute the initial distances 925 | typename std::vector::const_iterator fit; 926 | std::vector::iterator dit; 927 | dit = min_dists.begin(); 928 | for(fit = pfeatures.begin(); fit != pfeatures.end(); ++fit, ++dit) 929 | { 930 | *dit = F::distance(*(*fit), clusters.back()); 931 | } 932 | 933 | 934 | while((int)clusters.size() < m_k) 935 | { 936 | // 2. (1. 과정)을 반복. min_dist를 업데이트해준다는 것이 차이. 937 | dit = min_dists.begin(); 938 | for(fit = pfeatures.begin(); fit != pfeatures.end(); ++fit, ++dit) 939 | { 940 | if(*dit > 0) 941 | { 942 | double dist = F::distance(*(*fit), clusters.back()); 943 | if(dist < *dit) *dit = dist; 944 | } 945 | } 946 | 947 | // 3. 948 | double dist_sum = std::accumulate(min_dists.begin(), min_dists.end(), 0.0); 949 | 950 | if(dist_sum > 0) 951 | { 952 | double cut_d; 953 | do 954 | { 955 | cut_d = RandomValue(0, dist_sum); 956 | } while(cut_d == 0.0); 957 | 958 | double d_up_now = 0; 959 | for(dit = min_dists.begin(); dit != min_dists.end(); ++dit) 960 | { 961 | d_up_now += *dit; 962 | if(d_up_now >= cut_d) break; 963 | } 964 | 965 | if(dit == min_dists.end()) 966 | ifeature = pfeatures.size()-1; 967 | else 968 | ifeature = dit - min_dists.begin(); 969 | 970 | clusters.push_back(*pfeatures[ifeature]); 971 | 972 | } // if dist_sum > 0 973 | else 974 | break; 975 | 976 | } // while(used_clusters < m_k) 977 | std::cout << " DONE\n"; 978 | } 979 | 980 | // -------------------------------------------------------------------------- 981 | 982 | /** 983 | * @brief HKmeansStep를 통해 완성된 m_nodes를 가지고 m_words를 생성하는 함수. 984 | * m_node는 parent와 children의 id로 구조가 짜여있긴하지만 우리가 사용하는건 985 | * 가장 아래의 node, 즉 leaf node들이다. m_words는 m_nodes에서 children을 더이상 가지고 있지 않은 986 | * Leaf node들의 pointer로 구성되어 있으며, m_words에 들어간 Node 타입 객체는 word_id를 부여받는다. 987 | * @tparam TDescriptor 988 | * @tparam F 989 | */ 990 | template 991 | void TemplatedVocabulary::createWords() 992 | { 993 | m_words.resize(0); 994 | 995 | if(!m_nodes.empty()) 996 | { 997 | m_words.reserve( (int)pow((double)m_k, (double)m_L) ); 998 | 999 | typename std::vector::iterator nit; 1000 | 1001 | nit = m_nodes.begin(); // ignore root 1002 | for(++nit; nit != m_nodes.end(); ++nit) 1003 | { 1004 | if(nit->isLeaf()) // children을 가지고 있지 않으면, true를 반환할 것으로 예상. 1005 | { 1006 | nit->word_id = m_words.size(); // Node.word_id를 1007 | m_words.push_back( &(*nit) ); 1008 | } 1009 | } 1010 | } 1011 | } 1012 | 1013 | // -------------------------------------------------------------------------- 1014 | 1015 | /** 1016 | * @brief m_words에 저장되어 있는 node는 (정확히는 node pointer) weight라는 변수를 1017 | * 멤버로 가진다. 이는 해당 Word가 얼마나 identity가 확실해서 image recognition에 도움을 주는지를 1018 | * 측정하여 가지는 점수라고 볼 수 있다. setNodeWeights() 는 training_features를 입력으로 받아 1019 | * m_words의 각 원소에 weight를 넣어주는 함수이다. 1020 | * 1021 | * @tparam TDescriptor 1022 | * @tparam F 1023 | * @param training_features 1024 | */ 1025 | template 1026 | void TemplatedVocabulary::setNodeWeights 1027 | (const std::vector > &training_features) 1028 | { 1029 | const unsigned int NWords = m_words.size(); 1030 | const unsigned int NDocs = training_features.size(); 1031 | 1032 | if(m_weighting == TF || m_weighting == BINARY) 1033 | { 1034 | // idf part must be 1 always 1035 | for(unsigned int i = 0; i < NWords; i++) 1036 | m_words[i]->weight = 1; 1037 | } 1038 | else if(m_weighting == IDF || m_weighting == TF_IDF) 1039 | { 1040 | // IDF and TF-IDF: we calculte the idf path now 1041 | 1042 | // Note: this actually calculates the idf part of the tf-idf score. 1043 | // The complete tf-idf score is calculated in ::transform 1044 | 1045 | // i-th word를 가지고 있는 image의 개수. 1046 | std::vector Ni(NWords, 0); 1047 | // 각 word가 체크되었는지 확인. 1048 | std::vector counted(NWords, false); 1049 | 1050 | typename std::vector >::const_iterator mit; 1051 | typename std::vector::const_iterator fit; 1052 | 1053 | // input으로 넣은 모든 descriptor에 대해서 word_id를 할당해줌. 1054 | 1055 | for(mit = training_features.begin(); mit != training_features.end(); ++mit) 1056 | { 1057 | fill(counted.begin(), counted.end(), false); 1058 | 1059 | for(fit = mit->begin(); fit < mit->end(); ++fit) 1060 | { 1061 | WordId word_id; 1062 | // 아직 각 word의 weights가 초기화되있지 않은 상태이므로, word_id만을 추출. 1063 | transform(*fit, word_id); 1064 | 1065 | if(!counted[word_id]) 1066 | { 1067 | Ni[word_id]++; // mit가 가리키는 이미지에서 현재 word_id가 처음나왔으면 횟수 UP. 1068 | counted[word_id] = true; 1069 | } 1070 | } 1071 | } 1072 | 1073 | // 모든 word의 Weight업데이트! set ln(N/Ni) 1074 | for(unsigned int i = 0; i < NWords; i++) 1075 | { 1076 | // TF-IDF가 아닌 IDF만이 구현되어 있음. 1077 | // kmeans++ 는 나타나지 않은 word는 m_word에 보관하지 않으므로 Ni[i] = 0인 경우는 나타나지 않음. 1078 | if(Ni[i] > 0) 1079 | { 1080 | m_words[i]->weight = log((double)NDocs / (double)Ni[i]); 1081 | }// else // This cannot occur if using kmeans++ 1082 | } 1083 | 1084 | } 1085 | 1086 | } 1087 | 1088 | // -------------------------------------------------------------------------- 1089 | 1090 | template 1091 | inline unsigned int TemplatedVocabulary::size() const 1092 | { 1093 | return m_words.size(); 1094 | } 1095 | 1096 | // -------------------------------------------------------------------------- 1097 | 1098 | template 1099 | inline bool TemplatedVocabulary::empty() const 1100 | { 1101 | return m_words.empty(); 1102 | } 1103 | 1104 | // -------------------------------------------------------------------------- 1105 | 1106 | template 1107 | float TemplatedVocabulary::getEffectiveLevels() const 1108 | { 1109 | long sum = 0; 1110 | typename std::vector::const_iterator wit; 1111 | for(wit = m_words.begin(); wit != m_words.end(); ++wit) 1112 | { 1113 | const Node *p = *wit; 1114 | 1115 | for(; p->id != 0; sum++) p = &m_nodes[p->parent]; 1116 | } 1117 | 1118 | return (float)((double)sum / (double)m_words.size()); 1119 | } 1120 | 1121 | // -------------------------------------------------------------------------- 1122 | 1123 | template 1124 | TDescriptor TemplatedVocabulary::getWord(WordId wid) const 1125 | { 1126 | return m_words[wid]->descriptor; 1127 | } 1128 | 1129 | // -------------------------------------------------------------------------- 1130 | 1131 | template 1132 | WordValue TemplatedVocabulary::getWordWeight(WordId wid) const 1133 | { 1134 | return m_words[wid]->weight; 1135 | } 1136 | 1137 | // -------------------------------------------------------------------------- 1138 | 1139 | template 1140 | WordId TemplatedVocabulary::transform 1141 | (const TDescriptor& feature) const 1142 | { 1143 | if(empty()) 1144 | { 1145 | return 0; 1146 | } 1147 | 1148 | WordId wid; 1149 | transform(feature, wid); 1150 | return wid; 1151 | } 1152 | 1153 | // -------------------------------------------------------------------------- 1154 | 1155 | /** 1156 | * @brief feature->Word 변환이 아닌 Image->BowVector변환을 수행. 1157 | * 1158 | * @tparam TDescriptor 1159 | * @tparam F 1160 | * @param features 1161 | * @param v 1162 | */ 1163 | template 1164 | void TemplatedVocabulary::transform( 1165 | const std::vector& features, BowVector &v) const 1166 | { 1167 | v.clear(); 1168 | 1169 | if(empty()) 1170 | { 1171 | return; 1172 | } 1173 | 1174 | // normalize 1175 | LNorm norm; 1176 | // createScoringObject()에서 초기화된 m_scoring_object의 Score방식이 normalize가 필요할 경우 수행. 1177 | bool must = m_scoring_object->mustNormalize(norm); 1178 | 1179 | typename std::vector::const_iterator fit; 1180 | 1181 | // We use TF_IDF. 1182 | if(m_weighting == TF || m_weighting == TF_IDF) 1183 | { 1184 | for(fit = features.begin(); fit < features.end(); ++fit) 1185 | { 1186 | WordId id; 1187 | WordValue w; 1188 | // w is the idf value if TF_IDF, 1 if TF 1189 | 1190 | transform(*fit, id, w); 1191 | 1192 | // not stopped 1193 | // TF의 경우 word가 나온 횟수에 따라 가중치를 줘야하기 때문에 addWeight()를 사용한다. 1194 | if(w > 0) v.addWeight(id, w); 1195 | } 1196 | 1197 | // elementwise dividing by n_d. => n_id / n_d Complete 1198 | if(!v.empty() && !must) 1199 | { 1200 | // unnecessary when normalizing 1201 | const double nd = v.size(); 1202 | for(BowVector::iterator vit = v.begin(); vit != v.end(); vit++) 1203 | vit->second /= nd; 1204 | } 1205 | 1206 | } 1207 | else // _IDF_ || _BINARY_ 1208 | { 1209 | for(fit = features.begin(); fit < features.end(); ++fit) 1210 | { 1211 | WordId id; 1212 | WordValue w; 1213 | // w is idf if IDF, or 1 if BINARY 1214 | 1215 | transform(*fit, id, w); 1216 | 1217 | // not stopped 1218 | // IDF에서는 word가 이미지에서 몇번이나 나왔는지는 중요치 않기 때문에 addIfNotExist함수 사용 1219 | if(w > 0) v.addIfNotExist(id, w); 1220 | 1221 | } // if add_features 1222 | } // if m_weighting == ... 1223 | 1224 | if(must) v.normalize(norm); 1225 | } 1226 | 1227 | // -------------------------------------------------------------------------- 1228 | 1229 | /** 1230 | * @brief 1231 | * 1232 | * @tparam TDescriptor 1233 | * @tparam F 1234 | * @param features input - image's descriptors 1235 | * @param v input&output - BowVector 1236 | * @param fv output - FeatureVector 1237 | * @param levelsup input - (m_L - levelsuppression)까지 탐색. 1238 | */ 1239 | template 1240 | void TemplatedVocabulary::transform( 1241 | const std::vector& features, 1242 | BowVector &v, FeatureVector &fv, int levelsup) const 1243 | { 1244 | v.clear(); 1245 | fv.clear(); 1246 | 1247 | if(empty()) // safe for subclasses 1248 | { 1249 | return; 1250 | } 1251 | 1252 | // normalize 1253 | LNorm norm; 1254 | bool must = m_scoring_object->mustNormalize(norm); 1255 | 1256 | typename std::vector::const_iterator fit; 1257 | 1258 | if(m_weighting == TF || m_weighting == TF_IDF) 1259 | { 1260 | unsigned int i_feature = 0; 1261 | for(fit = features.begin(); fit < features.end(); ++fit, ++i_feature) 1262 | { 1263 | WordId id; 1264 | NodeId nid; 1265 | WordValue w; 1266 | // w is the idf value if TF_IDF, 1 if TF 1267 | 1268 | transform(*fit, id, w, &nid, levelsup); 1269 | 1270 | if(w > 0) // not stopped 1271 | { 1272 | v.addWeight(id, w); 1273 | fv.addFeature(nid, i_feature); 1274 | } 1275 | } 1276 | 1277 | if(!v.empty() && !must) 1278 | { 1279 | // unnecessary when normalizing 1280 | const double nd = v.size(); 1281 | for(BowVector::iterator vit = v.begin(); vit != v.end(); vit++) 1282 | vit->second /= nd; 1283 | } 1284 | 1285 | } 1286 | else // IDF || BINARY 1287 | { 1288 | unsigned int i_feature = 0; 1289 | for(fit = features.begin(); fit < features.end(); ++fit, ++i_feature) 1290 | { 1291 | WordId id; 1292 | NodeId nid; 1293 | WordValue w; 1294 | // w is idf if IDF, or 1 if BINARY 1295 | 1296 | transform(*fit, id, w, &nid, levelsup); 1297 | 1298 | if(w > 0) // not stopped 1299 | { 1300 | v.addIfNotExist(id, w); 1301 | fv.addFeature(nid, i_feature); 1302 | } 1303 | } 1304 | } // if m_weighting == ... 1305 | 1306 | if(must) v.normalize(norm); 1307 | } 1308 | 1309 | // -------------------------------------------------------------------------- 1310 | 1311 | template 1312 | inline double TemplatedVocabulary::score 1313 | (const BowVector &v1, const BowVector &v2) const 1314 | { 1315 | return m_scoring_object->score(v1, v2); 1316 | } 1317 | 1318 | // -------------------------------------------------------------------------- 1319 | 1320 | 1321 | template 1322 | void TemplatedVocabulary::transform 1323 | (const TDescriptor &feature, WordId &id) const 1324 | { 1325 | WordValue weight; 1326 | transform(feature, id, weight); 1327 | } 1328 | 1329 | // -------------------------------------------------------------------------- 1330 | 1331 | /** 1332 | * @brief 하나의 Descriptor를 가지고, 만들어져 있는 m_words에서 어디에 해당되는지 찾아 WordId를 반환. 1333 | * 해당 word가 가지고 있는 weight도 가져올 수 있으며, 해당 Word가 m_nodes에서 어디에 위치하는지 알기 위해 1334 | * NodeId도 얻을 수 있음. 1335 | * [과정] level 1에서 클러스터링된 m_k개의 node와 distance비교: 가장 가까운 1개의 node 찾은 다음 1336 | * 한 level내려가서 level 2에 해당하는 자식들과 distance비교. 반복하여 (m_L - levelup)에 해당하는 level층까지 탐색. 1337 | * 만약 도중에 leafnode 이면 해당 노드에서 반환. 반환되는 중요 정보는 결국 word_id와 weight이다. 1338 | * @tparam TDescriptor 1339 | * @tparam F 1340 | * @param feature input 1341 | * @param id output 1342 | * @param weight output - word의 weight 반환. 1343 | * @param nid 반환하는 NodeId 1344 | * @param levelup Node가 위치한 level을 특정하여 찾고 싶을 때, 값을 줄 수 있다. 1345 | */ 1346 | template 1347 | void TemplatedVocabulary::transform(const TDescriptor &feature, 1348 | WordId &word_id, WordValue &weight, NodeId *nid, int levelsup) const 1349 | { 1350 | // propagate the feature down the tree 1351 | std::vector nodes; 1352 | typename std::vector::const_iterator nit; 1353 | 1354 | // level at which the node must be stored in nid, if given 1355 | const int nid_level = m_L - levelsup; 1356 | if(nid_level <= 0 && nid != NULL) *nid = 0; // root 1357 | 1358 | NodeId final_id = 0; // root 1359 | int current_level = 0; 1360 | 1361 | do 1362 | { 1363 | ++current_level; 1364 | nodes = m_nodes[final_id].children; 1365 | final_id = nodes[0]; 1366 | 1367 | double best_d = F::distance(feature, m_nodes[final_id].descriptor); 1368 | 1369 | for(nit = nodes.begin() + 1; nit != nodes.end(); ++nit) 1370 | { 1371 | NodeId id = *nit; 1372 | double d = F::distance(feature, m_nodes[id].descriptor); 1373 | if(d < best_d) 1374 | { 1375 | best_d = d; 1376 | final_id = id; 1377 | } 1378 | } 1379 | 1380 | if(nid != NULL && current_level == nid_level) 1381 | *nid = final_id; 1382 | 1383 | } while( !m_nodes[final_id].isLeaf() ); 1384 | 1385 | // turn node id into word id 1386 | word_id = m_nodes[final_id].word_id; 1387 | weight = m_nodes[final_id].weight; 1388 | } 1389 | 1390 | // -------------------------------------------------------------------------- 1391 | 1392 | template 1393 | NodeId TemplatedVocabulary::getParentNode 1394 | (WordId wid, int levelsup) const 1395 | { 1396 | NodeId ret = m_words[wid]->id; // node id 1397 | while(levelsup > 0 && ret != 0) // ret == 0 --> root 1398 | { 1399 | --levelsup; 1400 | ret = m_nodes[ret].parent; 1401 | } 1402 | return ret; 1403 | } 1404 | 1405 | // -------------------------------------------------------------------------- 1406 | 1407 | template 1408 | void TemplatedVocabulary::getWordsFromNode 1409 | (NodeId nid, std::vector &words) const 1410 | { 1411 | words.clear(); 1412 | 1413 | if(m_nodes[nid].isLeaf()) 1414 | { 1415 | words.push_back(m_nodes[nid].word_id); 1416 | } 1417 | else 1418 | { 1419 | words.reserve(m_k); // ^1, ^2, ... 1420 | 1421 | std::vector parents; 1422 | parents.push_back(nid); 1423 | 1424 | while(!parents.empty()) 1425 | { 1426 | NodeId parentid = parents.back(); 1427 | parents.pop_back(); 1428 | 1429 | const std::vector &child_ids = m_nodes[parentid].children; 1430 | std::vector::const_iterator cit; 1431 | 1432 | for(cit = child_ids.begin(); cit != child_ids.end(); ++cit) 1433 | { 1434 | const Node &child_node = m_nodes[*cit]; 1435 | 1436 | if(child_node.isLeaf()) 1437 | words.push_back(child_node.word_id); 1438 | else 1439 | parents.push_back(*cit); 1440 | 1441 | } // for each child 1442 | } // while !parents.empty 1443 | } 1444 | } 1445 | 1446 | // -------------------------------------------------------------------------- 1447 | 1448 | template 1449 | int TemplatedVocabulary::stopWords(double minWeight) 1450 | { 1451 | int c = 0; 1452 | typename std::vector::iterator wit; 1453 | for(wit = m_words.begin(); wit != m_words.end(); ++wit) 1454 | { 1455 | if((*wit)->weight < minWeight) 1456 | { 1457 | ++c; 1458 | (*wit)->weight = 0; 1459 | } 1460 | } 1461 | return c; 1462 | } 1463 | 1464 | // -------------------------------------------------------------------------- 1465 | 1466 | template 1467 | bool TemplatedVocabulary::loadFromTextFile(const std::string &filename) 1468 | { 1469 | ifstream f; 1470 | f.open(filename.c_str()); 1471 | 1472 | if(f.eof()) 1473 | return false; 1474 | 1475 | m_words.clear(); 1476 | m_nodes.clear(); 1477 | 1478 | string s; 1479 | getline(f,s); 1480 | stringstream ss; 1481 | ss << s; 1482 | ss >> m_k; 1483 | ss >> m_L; 1484 | int n1, n2; 1485 | ss >> n1; 1486 | ss >> n2; 1487 | 1488 | if(m_k<0 || m_k>20 || m_L<1 || m_L>10 || n1<0 || n1>5 || n2<0 || n2>3) 1489 | { 1490 | std::cerr << "Vocabulary loading failure: This is not a correct text file!" << endl; 1491 | return false; 1492 | } 1493 | 1494 | m_scoring = (ScoringType)n1; 1495 | m_weighting = (WeightingType)n2; 1496 | createScoringObject(); 1497 | 1498 | // nodes 1499 | int expected_nodes = 1500 | (int)((pow((double)m_k, (double)m_L + 1) - 1)/(m_k - 1)); 1501 | m_nodes.reserve(expected_nodes); 1502 | 1503 | m_words.reserve(pow((double)m_k, (double)m_L + 1)); 1504 | 1505 | m_nodes.resize(1); 1506 | m_nodes[0].id = 0; 1507 | while(!f.eof()) 1508 | { 1509 | string snode; 1510 | getline(f,snode); 1511 | stringstream ssnode; 1512 | ssnode << snode; 1513 | 1514 | int nid = m_nodes.size(); 1515 | m_nodes.resize(m_nodes.size()+1); 1516 | m_nodes[nid].id = nid; 1517 | 1518 | int pid ; 1519 | ssnode >> pid; 1520 | m_nodes[nid].parent = pid; 1521 | m_nodes[pid].children.push_back(nid); 1522 | 1523 | int nIsLeaf; 1524 | ssnode >> nIsLeaf; 1525 | 1526 | stringstream ssd; 1527 | for(int iD=0;iD> sElement; 1531 | ssd << sElement << " "; 1532 | } 1533 | F::fromString(m_nodes[nid].descriptor, ssd.str()); 1534 | 1535 | ssnode >> m_nodes[nid].weight; 1536 | 1537 | if(nIsLeaf>0) 1538 | { 1539 | int wid = m_words.size(); 1540 | m_words.resize(wid+1); 1541 | 1542 | m_nodes[nid].word_id = wid; 1543 | m_words[wid] = &m_nodes[nid]; 1544 | } 1545 | else 1546 | { 1547 | m_nodes[nid].children.reserve(m_k); 1548 | } 1549 | } 1550 | 1551 | return true; 1552 | 1553 | } 1554 | 1555 | // -------------------------------------------------------------------------- 1556 | 1557 | 1558 | template 1559 | void TemplatedVocabulary::saveToTextFile(const std::string &filename) const 1560 | { 1561 | fstream f; 1562 | f.open(filename.c_str(),ios_base::out); 1563 | f << m_k << " " << m_L << " " << " " << m_scoring << " " << m_weighting << endl; 1564 | 1565 | for(size_t i=1; i 1584 | void TemplatedVocabulary::save(const std::string &filename) const 1585 | { 1586 | cv::FileStorage fs(filename.c_str(), cv::FileStorage::WRITE); 1587 | if(!fs.isOpened()) throw std::string("Could not open file ") + filename; 1588 | 1589 | save(fs); 1590 | } 1591 | 1592 | // -------------------------------------------------------------------------- 1593 | 1594 | template 1595 | void TemplatedVocabulary::load(const std::string &filename) 1596 | { 1597 | cv::FileStorage fs(filename.c_str(), cv::FileStorage::READ); 1598 | if(!fs.isOpened()) throw std::string("Could not open file ") + filename; 1599 | 1600 | this->load(fs); 1601 | } 1602 | 1603 | // -------------------------------------------------------------------------- 1604 | 1605 | template 1606 | void TemplatedVocabulary::save(cv::FileStorage &f, 1607 | const std::string &name) const 1608 | { 1609 | // Format YAML: 1610 | // vocabulary 1611 | // { 1612 | // k: 1613 | // L: 1614 | // scoringType: 1615 | // weightingType: 1616 | // nodes 1617 | // [ 1618 | // { 1619 | // nodeId: 1620 | // parentId: 1621 | // weight: 1622 | // descriptor: 1623 | // } 1624 | // ] 1625 | // words 1626 | // [ 1627 | // { 1628 | // wordId: 1629 | // nodeId: 1630 | // } 1631 | // ] 1632 | // } 1633 | // 1634 | // The root node (index 0) is not included in the node vector 1635 | // 1636 | 1637 | f << name << "{"; 1638 | 1639 | f << "k" << m_k; 1640 | f << "L" << m_L; 1641 | f << "scoringType" << m_scoring; 1642 | f << "weightingType" << m_weighting; 1643 | 1644 | // tree 1645 | f << "nodes" << "["; 1646 | std::vector parents, children; 1647 | std::vector::const_iterator pit; 1648 | 1649 | parents.push_back(0); // root 1650 | 1651 | while(!parents.empty()) 1652 | { 1653 | NodeId pid = parents.back(); 1654 | parents.pop_back(); 1655 | 1656 | const Node& parent = m_nodes[pid]; 1657 | children = parent.children; 1658 | 1659 | for(pit = children.begin(); pit != children.end(); pit++) 1660 | { 1661 | const Node& child = m_nodes[*pit]; 1662 | 1663 | // save node data 1664 | f << "{:"; 1665 | f << "nodeId" << (int)child.id; 1666 | f << "parentId" << (int)pid; 1667 | f << "weight" << (double)child.weight; 1668 | f << "descriptor" << F::toString(child.descriptor); 1669 | f << "}"; 1670 | 1671 | // add to parent list 1672 | if(!child.isLeaf()) 1673 | { 1674 | parents.push_back(*pit); 1675 | } 1676 | } 1677 | } 1678 | 1679 | f << "]"; // nodes 1680 | 1681 | // words 1682 | f << "words" << "["; 1683 | 1684 | typename std::vector::const_iterator wit; 1685 | for(wit = m_words.begin(); wit != m_words.end(); wit++) 1686 | { 1687 | WordId id = wit - m_words.begin(); 1688 | f << "{:"; 1689 | f << "wordId" << (int)id; 1690 | f << "nodeId" << (int)(*wit)->id; 1691 | f << "}"; 1692 | } 1693 | 1694 | f << "]"; // words 1695 | 1696 | f << "}"; 1697 | 1698 | } 1699 | 1700 | // -------------------------------------------------------------------------- 1701 | 1702 | template 1703 | void TemplatedVocabulary::load(const cv::FileStorage &fs, 1704 | const std::string &name) 1705 | { 1706 | m_words.clear(); 1707 | m_nodes.clear(); 1708 | 1709 | cv::FileNode fvoc = fs[name]; 1710 | 1711 | m_k = (int)fvoc["k"]; 1712 | m_L = (int)fvoc["L"]; 1713 | m_scoring = (ScoringType)((int)fvoc["scoringType"]); 1714 | m_weighting = (WeightingType)((int)fvoc["weightingType"]); 1715 | 1716 | createScoringObject(); 1717 | 1718 | // nodes 1719 | cv::FileNode fn = fvoc["nodes"]; 1720 | 1721 | m_nodes.resize(fn.size() + 1); // +1 to include root 1722 | m_nodes[0].id = 0; 1723 | 1724 | for(unsigned int i = 0; i < fn.size(); ++i) 1725 | { 1726 | NodeId nid = (int)fn[i]["nodeId"]; 1727 | NodeId pid = (int)fn[i]["parentId"]; 1728 | WordValue weight = (WordValue)fn[i]["weight"]; 1729 | std::string d = (std::string)fn[i]["descriptor"]; 1730 | 1731 | m_nodes[nid].id = nid; 1732 | m_nodes[nid].parent = pid; 1733 | m_nodes[nid].weight = weight; 1734 | m_nodes[pid].children.push_back(nid); 1735 | 1736 | F::fromString(m_nodes[nid].descriptor, d); 1737 | } 1738 | 1739 | // words 1740 | fn = fvoc["words"]; 1741 | 1742 | m_words.resize(fn.size()); 1743 | 1744 | for(unsigned int i = 0; i < fn.size(); ++i) 1745 | { 1746 | NodeId wid = (int)fn[i]["wordId"]; 1747 | NodeId nid = (int)fn[i]["nodeId"]; 1748 | 1749 | m_nodes[nid].word_id = wid; 1750 | m_words[wid] = &m_nodes[nid]; 1751 | } 1752 | } 1753 | 1754 | // -------------------------------------------------------------------------- 1755 | 1756 | /** 1757 | * Writes printable information of the vocabulary 1758 | * @param os stream to write to 1759 | * @param voc 1760 | */ 1761 | template 1762 | std::ostream& operator<<(std::ostream &os, 1763 | const TemplatedVocabulary &voc) 1764 | { 1765 | os << "Vocabulary: k = " << voc.getBranchingFactor() 1766 | << ", L = " << voc.getDepthLevels() 1767 | << ", Weighting = "; 1768 | 1769 | switch(voc.getWeightingType()) 1770 | { 1771 | case TF_IDF: os << "tf-idf"; break; 1772 | case TF: os << "tf"; break; 1773 | case IDF: os << "idf"; break; 1774 | case BINARY: os << "binary"; break; 1775 | } 1776 | 1777 | os << ", Scoring = "; 1778 | switch(voc.getScoringType()) 1779 | { 1780 | case L1_NORM: os << "L1-norm"; break; 1781 | case L2_NORM: os << "L2-norm"; break; 1782 | case CHI_SQUARE: os << "Chi square distance"; break; 1783 | case KL: os << "KL-divergence"; break; 1784 | case BHATTACHARYYA: os << "Bhattacharyya coefficient"; break; 1785 | case DOT_PRODUCT: os << "Dot product"; break; 1786 | } 1787 | 1788 | os << ", Number of words = " << voc.size(); 1789 | 1790 | return os; 1791 | } 1792 | 1793 | } // namespace DBoW2 1794 | 1795 | #endif 1796 | -------------------------------------------------------------------------------- /CreateVoca/src/BowVector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * File: BowVector.cpp 3 | * Date: March 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: bag of words vector 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "BowVector.h" 17 | 18 | namespace DBoW2 { 19 | 20 | // -------------------------------------------------------------------------- 21 | 22 | BowVector::BowVector(void) 23 | { 24 | } 25 | 26 | // -------------------------------------------------------------------------- 27 | 28 | BowVector::~BowVector(void) 29 | { 30 | } 31 | 32 | // -------------------------------------------------------------------------- 33 | 34 | /** 35 | * @brief WordId id가 이미 BowVector에 존재하면, v값을 덧셈. 36 | * 존재하지 않으면, 새로 생성. 37 | * 38 | * @param id 39 | * @param v 40 | */ 41 | void BowVector::addWeight(WordId id, WordValue v) 42 | { 43 | BowVector::iterator vit = this->lower_bound(id); 44 | 45 | if(vit != this->end() && !(this->key_comp()(id, vit->first))) 46 | { 47 | vit->second += v; 48 | } 49 | else 50 | { 51 | this->insert(vit, BowVector::value_type(id, v)); 52 | } 53 | } 54 | 55 | // -------------------------------------------------------------------------- 56 | 57 | /** 58 | * @brief BowVector는 map구조를 가지고 있다. 이 map에 id를 키로 하는 Value가 존재하지 않으면 insert한다. 59 | * map이지만 this[id]로 존재를 찾지 않고, lower_bound함수를 이용하는 것이 인상깊었다. 더 빠르게 구현되어 있나? 60 | * 61 | * @param id 62 | * @param v 63 | */ 64 | void BowVector::addIfNotExist(WordId id, WordValue v) 65 | { 66 | BowVector::iterator vit = this->lower_bound(id); 67 | 68 | if(vit == this->end() || (this->key_comp()(id, vit->first))) 69 | { 70 | this->insert(vit, BowVector::value_type(id, v)); 71 | } 72 | } 73 | 74 | // -------------------------------------------------------------------------- 75 | 76 | /** 77 | * @brief norm_type에 맞게 BowVector 전체를 normalize시킨다. 78 | * 79 | * @param norm_type 80 | */ 81 | void BowVector::normalize(LNorm norm_type) 82 | { 83 | double norm = 0.0; 84 | BowVector::iterator it; 85 | 86 | if(norm_type == DBoW2::L1) 87 | { 88 | for(it = begin(); it != end(); ++it) 89 | norm += fabs(it->second); 90 | } 91 | else 92 | { 93 | for(it = begin(); it != end(); ++it) 94 | norm += it->second * it->second; 95 | norm = sqrt(norm); 96 | } 97 | 98 | if(norm > 0.0) 99 | { 100 | for(it = begin(); it != end(); ++it) 101 | it->second /= norm; 102 | } 103 | } 104 | 105 | // -------------------------------------------------------------------------- 106 | 107 | std::ostream& operator<< (std::ostream &out, const BowVector &v) 108 | { 109 | BowVector::const_iterator vit; 110 | unsigned int i = 0; 111 | const unsigned int N = v.size(); 112 | for(vit = v.begin(); vit != v.end(); ++vit, ++i) 113 | { 114 | out << "<" << vit->first << ", " << vit->second << ">"; 115 | 116 | if(i < N-1) out << ", "; 117 | } 118 | return out; 119 | } 120 | 121 | // -------------------------------------------------------------------------- 122 | 123 | void BowVector::saveM(const std::string &filename, size_t W) const 124 | { 125 | std::fstream f(filename.c_str(), std::ios::out); 126 | 127 | WordId last = 0; 128 | BowVector::const_iterator bit; 129 | for(bit = this->begin(); bit != this->end(); ++bit) 130 | { 131 | for(; last < bit->first; ++last) 132 | { 133 | f << "0 "; 134 | } 135 | f << bit->second << " "; 136 | 137 | last = bit->first + 1; 138 | } 139 | for(; last < (WordId)W; ++last) 140 | f << "0 "; 141 | 142 | f.close(); 143 | } 144 | 145 | // -------------------------------------------------------------------------- 146 | 147 | } // namespace DBoW2 148 | 149 | -------------------------------------------------------------------------------- /CreateVoca/src/FSuperpoint.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * File: FSuperpoint.cpp 3 | * Date: September 2020 4 | * Author: Chanwoo Lee 5 | * Description: functions for SUPERPOINT descriptors 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "FSuperpoint.h" 17 | 18 | using namespace std; 19 | 20 | namespace DBoW2 { 21 | 22 | // -------------------------------------------------------------------------- 23 | const int FSUPERPOINT::L = 256; 24 | 25 | void FSUPERPOINT::meanValue(const std::vector &descriptors, 26 | FSUPERPOINT::TDescriptor &mean) 27 | { 28 | if(descriptors.empty()) 29 | { 30 | return; 31 | } 32 | else if(descriptors.size() == 1) 33 | { 34 | mean = descriptors[0]->clone(); 35 | } 36 | else 37 | { 38 | size_t len = descriptors.size(); 39 | std::vector::const_iterator iter; 40 | for(iter = descriptors.begin(); iter != descriptors.end(); ++iter) 41 | { 42 | mean += *(*iter); 43 | } 44 | 45 | mean /= float(len); 46 | } 47 | } 48 | 49 | // -------------------------------------------------------------------------- 50 | 51 | double FSUPERPOINT::distance(const FSUPERPOINT::TDescriptor &a, 52 | const FSUPERPOINT::TDescriptor &b) 53 | { 54 | if(a.empty() || b.empty()) 55 | { 56 | std::cout << "ERROR: Mat is empty!\n"; 57 | exit(1); 58 | } 59 | 60 | TDescriptor dist = cv::abs(a - b); 61 | return double(cv::sum(dist)[0]) / FSUPERPOINT::L; 62 | } 63 | 64 | // -------------------------------------------------------------------------- 65 | 66 | std::string FSUPERPOINT::toString(const FSUPERPOINT::TDescriptor &a) 67 | { 68 | stringstream ss; 69 | const float *ptr = a.ptr(0); 70 | for(int i=0; i(0); 83 | 84 | stringstream ss(s); 85 | for(int i = 0; i < FSUPERPOINT::L; ++i, ++ptr) 86 | { 87 | float n; 88 | ss >> n; 89 | 90 | if(!ss.fail()) 91 | *ptr = (float)n; 92 | } 93 | 94 | } 95 | 96 | // -------------------------------------------------------------------------- 97 | 98 | void FSUPERPOINT::toMat32F(const std::vector &descriptors, 99 | cv::Mat &mat) 100 | { 101 | if(descriptors.empty()) 102 | { 103 | mat.release(); 104 | return; 105 | } 106 | const size_t N = descriptors.size(); 107 | mat.create(N, FSUPERPOINT::L, CV_32F); 108 | cv::vconcat(descriptors, mat); 109 | } 110 | 111 | // -------------------------------------------------------------------------- 112 | 113 | void FSUPERPOINT::toMat32F(const cv::Mat &descriptors, cv::Mat &mat) 114 | { 115 | descriptors.convertTo(mat, CV_32F); 116 | } 117 | 118 | // -------------------------------------------------------------------------- 119 | 120 | void FSUPERPOINT::toMat8U(const std::vector &descriptors, cv::Mat &mat) 121 | { 122 | mat.create(descriptors.size(), FSUPERPOINT::L, CV_8U); 123 | mat.setTo(1); 124 | } 125 | 126 | // -------------------------------------------------------------------------- 127 | 128 | } // namespace DBoW2 129 | 130 | -------------------------------------------------------------------------------- /CreateVoca/src/FeatureVector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * File: FeatureVector.cpp 3 | * Date: November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: feature vector 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #include "FeatureVector.h" 11 | #include 12 | #include 13 | #include 14 | 15 | namespace DBoW2 { 16 | 17 | // --------------------------------------------------------------------------- 18 | 19 | FeatureVector::FeatureVector(void) 20 | { 21 | } 22 | 23 | // --------------------------------------------------------------------------- 24 | 25 | FeatureVector::~FeatureVector(void) 26 | { 27 | } 28 | 29 | // --------------------------------------------------------------------------- 30 | 31 | /** 32 | * @brief FeatureVector는 std::map구조를 가지고 있다. id를 키로 가지는 value가 없으면 추가한다. 33 | * 만약 존재하는 키가 있으면, i_feature를 push_back으로 집어넣는다. 34 | * 35 | * @param id 36 | * @param i_feature 37 | */ 38 | void FeatureVector::addFeature(NodeId id, unsigned int i_feature) 39 | { 40 | FeatureVector::iterator vit = this->lower_bound(id); 41 | 42 | if(vit != this->end() && vit->first == id) 43 | { 44 | vit->second.push_back(i_feature); 45 | } 46 | else 47 | { 48 | vit = this->insert(vit, FeatureVector::value_type(id, 49 | std::vector() )); 50 | vit->second.push_back(i_feature); 51 | } 52 | } 53 | 54 | // --------------------------------------------------------------------------- 55 | 56 | std::ostream& operator<<(std::ostream &out, 57 | const FeatureVector &v) 58 | { 59 | if(!v.empty()) 60 | { 61 | FeatureVector::const_iterator vit = v.begin(); 62 | 63 | const std::vector* f = &vit->second; 64 | 65 | out << "<" << vit->first << ": ["; 66 | if(!f->empty()) out << (*f)[0]; 67 | for(unsigned int i = 1; i < f->size(); ++i) 68 | { 69 | out << ", " << (*f)[i]; 70 | } 71 | out << "]>"; 72 | 73 | for(++vit; vit != v.end(); ++vit) 74 | { 75 | f = &vit->second; 76 | 77 | out << ", <" << vit->first << ": ["; 78 | if(!f->empty()) out << (*f)[0]; 79 | for(unsigned int i = 1; i < f->size(); ++i) 80 | { 81 | out << ", " << (*f)[i]; 82 | } 83 | out << "]>"; 84 | } 85 | } 86 | 87 | return out; 88 | } 89 | 90 | // --------------------------------------------------------------------------- 91 | 92 | } // namespace DBoW2 93 | -------------------------------------------------------------------------------- /CreateVoca/src/QueryResults.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * File: QueryResults.cpp 3 | * Date: March, November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: structure to store results of database queries 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #include 11 | #include 12 | #include "QueryResults.h" 13 | 14 | using namespace std; 15 | 16 | namespace DBoW2 17 | { 18 | 19 | // --------------------------------------------------------------------------- 20 | 21 | ostream & operator<<(ostream& os, const Result& ret ) 22 | { 23 | os << ""; 24 | return os; 25 | } 26 | 27 | // --------------------------------------------------------------------------- 28 | 29 | ostream & operator<<(ostream& os, const QueryResults& ret ) 30 | { 31 | if(ret.size() == 1) 32 | os << "1 result:" << endl; 33 | else 34 | os << ret.size() << " results:" << endl; 35 | 36 | QueryResults::const_iterator rit; 37 | for(rit = ret.begin(); rit != ret.end(); ++rit) 38 | { 39 | os << *rit; 40 | if(rit + 1 != ret.end()) os << endl; 41 | } 42 | return os; 43 | } 44 | 45 | // --------------------------------------------------------------------------- 46 | 47 | void QueryResults::saveM(const std::string &filename) const 48 | { 49 | fstream f(filename.c_str(), ios::out); 50 | 51 | QueryResults::const_iterator qit; 52 | for(qit = begin(); qit != end(); ++qit) 53 | { 54 | f << qit->Id << " " << qit->Score << endl; 55 | } 56 | 57 | f.close(); 58 | } 59 | 60 | // --------------------------------------------------------------------------- 61 | 62 | } // namespace DBoW2 63 | 64 | -------------------------------------------------------------------------------- /CreateVoca/src/ScoringObject.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * File: ScoringObject.cpp 3 | * Date: November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: functions to compute bow scores 6 | * License: see the LICENSE.txt file 7 | * 8 | */ 9 | 10 | #include 11 | #include "TemplatedVocabulary.h" 12 | #include "BowVector.h" 13 | 14 | using namespace DBoW2; 15 | 16 | // If you change the type of WordValue, make sure you change also the 17 | // epsilon value (this is needed by the KL method) 18 | const double GeneralScoring::LOG_EPS = log(DBL_EPSILON); // FLT_EPSILON 19 | 20 | // --------------------------------------------------------------------------- 21 | // --------------------------------------------------------------------------- 22 | 23 | double L1Scoring::score(const BowVector &v1, const BowVector &v2) const 24 | { 25 | BowVector::const_iterator v1_it, v2_it; 26 | const BowVector::const_iterator v1_end = v1.end(); 27 | const BowVector::const_iterator v2_end = v2.end(); 28 | 29 | v1_it = v1.begin(); 30 | v2_it = v2.begin(); 31 | 32 | double score = 0; 33 | 34 | while(v1_it != v1_end && v2_it != v2_end) 35 | { 36 | const WordValue& vi = v1_it->second; 37 | const WordValue& wi = v2_it->second; 38 | 39 | if(v1_it->first == v2_it->first) 40 | { 41 | score += fabs(vi - wi) - fabs(vi) - fabs(wi); 42 | 43 | // move v1 and v2 forward 44 | ++v1_it; 45 | ++v2_it; 46 | } 47 | else if(v1_it->first < v2_it->first) 48 | { 49 | // move v1 forward 50 | v1_it = v1.lower_bound(v2_it->first); 51 | // v1_it = (first element >= v2_it.id) 52 | } 53 | else 54 | { 55 | // move v2 forward 56 | v2_it = v2.lower_bound(v1_it->first); 57 | // v2_it = (first element >= v1_it.id) 58 | } 59 | } 60 | 61 | // ||v - w||_{L1} = 2 + Sum(|v_i - w_i| - |v_i| - |w_i|) 62 | // for all i | v_i != 0 and w_i != 0 63 | // (Nister, 2006) 64 | // scaled_||v - w||_{L1} = 1 - 0.5 * ||v - w||_{L1} 65 | score = -score/2.0; 66 | 67 | return score; // [0..1] 68 | } 69 | 70 | // --------------------------------------------------------------------------- 71 | // --------------------------------------------------------------------------- 72 | 73 | double L2Scoring::score(const BowVector &v1, const BowVector &v2) const 74 | { 75 | BowVector::const_iterator v1_it, v2_it; 76 | const BowVector::const_iterator v1_end = v1.end(); 77 | const BowVector::const_iterator v2_end = v2.end(); 78 | 79 | v1_it = v1.begin(); 80 | v2_it = v2.begin(); 81 | 82 | double score = 0; 83 | 84 | while(v1_it != v1_end && v2_it != v2_end) 85 | { 86 | const WordValue& vi = v1_it->second; 87 | const WordValue& wi = v2_it->second; 88 | 89 | if(v1_it->first == v2_it->first) 90 | { 91 | score += vi * wi; 92 | 93 | // move v1 and v2 forward 94 | ++v1_it; 95 | ++v2_it; 96 | } 97 | else if(v1_it->first < v2_it->first) 98 | { 99 | // move v1 forward 100 | v1_it = v1.lower_bound(v2_it->first); 101 | // v1_it = (first element >= v2_it.id) 102 | } 103 | else 104 | { 105 | // move v2 forward 106 | v2_it = v2.lower_bound(v1_it->first); 107 | // v2_it = (first element >= v1_it.id) 108 | } 109 | } 110 | 111 | // ||v - w||_{L2} = sqrt( 2 - 2 * Sum(v_i * w_i) ) 112 | // for all i | v_i != 0 and w_i != 0 ) 113 | // (Nister, 2006) 114 | if(score >= 1) // rounding errors 115 | score = 1.0; 116 | else 117 | score = 1.0 - sqrt(1.0 - score); // [0..1] 118 | 119 | return score; 120 | } 121 | 122 | // --------------------------------------------------------------------------- 123 | // --------------------------------------------------------------------------- 124 | 125 | double ChiSquareScoring::score(const BowVector &v1, const BowVector &v2) 126 | const 127 | { 128 | BowVector::const_iterator v1_it, v2_it; 129 | const BowVector::const_iterator v1_end = v1.end(); 130 | const BowVector::const_iterator v2_end = v2.end(); 131 | 132 | v1_it = v1.begin(); 133 | v2_it = v2.begin(); 134 | 135 | double score = 0; 136 | 137 | // all the items are taken into account 138 | 139 | while(v1_it != v1_end && v2_it != v2_end) 140 | { 141 | const WordValue& vi = v1_it->second; 142 | const WordValue& wi = v2_it->second; 143 | 144 | if(v1_it->first == v2_it->first) 145 | { 146 | // (v-w)^2/(v+w) - v - w = -4 vw/(v+w) 147 | // we move the -4 out 148 | if(vi + wi != 0.0) score += vi * wi / (vi + wi); 149 | 150 | // move v1 and v2 forward 151 | ++v1_it; 152 | ++v2_it; 153 | } 154 | else if(v1_it->first < v2_it->first) 155 | { 156 | // move v1 forward 157 | v1_it = v1.lower_bound(v2_it->first); 158 | } 159 | else 160 | { 161 | // move v2 forward 162 | v2_it = v2.lower_bound(v1_it->first); 163 | } 164 | } 165 | 166 | // this takes the -4 into account 167 | score = 2. * score; // [0..1] 168 | 169 | return score; 170 | } 171 | 172 | // --------------------------------------------------------------------------- 173 | // --------------------------------------------------------------------------- 174 | 175 | double KLScoring::score(const BowVector &v1, const BowVector &v2) const 176 | { 177 | BowVector::const_iterator v1_it, v2_it; 178 | const BowVector::const_iterator v1_end = v1.end(); 179 | const BowVector::const_iterator v2_end = v2.end(); 180 | 181 | v1_it = v1.begin(); 182 | v2_it = v2.begin(); 183 | 184 | double score = 0; 185 | 186 | // all the items or v are taken into account 187 | 188 | while(v1_it != v1_end && v2_it != v2_end) 189 | { 190 | const WordValue& vi = v1_it->second; 191 | const WordValue& wi = v2_it->second; 192 | 193 | if(v1_it->first == v2_it->first) 194 | { 195 | if(vi != 0 && wi != 0) score += vi * log(vi/wi); 196 | 197 | // move v1 and v2 forward 198 | ++v1_it; 199 | ++v2_it; 200 | } 201 | else if(v1_it->first < v2_it->first) 202 | { 203 | // move v1 forward 204 | score += vi * (log(vi) - LOG_EPS); 205 | ++v1_it; 206 | } 207 | else 208 | { 209 | // move v2_it forward, do not add any score 210 | v2_it = v2.lower_bound(v1_it->first); 211 | // v2_it = (first element >= v1_it.id) 212 | } 213 | } 214 | 215 | // sum rest of items of v 216 | for(; v1_it != v1_end; ++v1_it) 217 | if(v1_it->second != 0) 218 | score += v1_it->second * (log(v1_it->second) - LOG_EPS); 219 | 220 | return score; // cannot be scaled 221 | } 222 | 223 | // --------------------------------------------------------------------------- 224 | // --------------------------------------------------------------------------- 225 | 226 | double BhattacharyyaScoring::score(const BowVector &v1, 227 | const BowVector &v2) const 228 | { 229 | BowVector::const_iterator v1_it, v2_it; 230 | const BowVector::const_iterator v1_end = v1.end(); 231 | const BowVector::const_iterator v2_end = v2.end(); 232 | 233 | v1_it = v1.begin(); 234 | v2_it = v2.begin(); 235 | 236 | double score = 0; 237 | 238 | while(v1_it != v1_end && v2_it != v2_end) 239 | { 240 | const WordValue& vi = v1_it->second; 241 | const WordValue& wi = v2_it->second; 242 | 243 | if(v1_it->first == v2_it->first) 244 | { 245 | score += sqrt(vi * wi); 246 | 247 | // move v1 and v2 forward 248 | ++v1_it; 249 | ++v2_it; 250 | } 251 | else if(v1_it->first < v2_it->first) 252 | { 253 | // move v1 forward 254 | v1_it = v1.lower_bound(v2_it->first); 255 | // v1_it = (first element >= v2_it.id) 256 | } 257 | else 258 | { 259 | // move v2 forward 260 | v2_it = v2.lower_bound(v1_it->first); 261 | // v2_it = (first element >= v1_it.id) 262 | } 263 | } 264 | 265 | return score; // already scaled 266 | } 267 | 268 | // --------------------------------------------------------------------------- 269 | // --------------------------------------------------------------------------- 270 | 271 | double DotProductScoring::score(const BowVector &v1, 272 | const BowVector &v2) const 273 | { 274 | BowVector::const_iterator v1_it, v2_it; 275 | const BowVector::const_iterator v1_end = v1.end(); 276 | const BowVector::const_iterator v2_end = v2.end(); 277 | 278 | v1_it = v1.begin(); 279 | v2_it = v2.begin(); 280 | 281 | double score = 0; 282 | 283 | while(v1_it != v1_end && v2_it != v2_end) 284 | { 285 | const WordValue& vi = v1_it->second; 286 | const WordValue& wi = v2_it->second; 287 | 288 | if(v1_it->first == v2_it->first) 289 | { 290 | score += vi * wi; 291 | 292 | // move v1 and v2 forward 293 | ++v1_it; 294 | ++v2_it; 295 | } 296 | else if(v1_it->first < v2_it->first) 297 | { 298 | // move v1 forward 299 | v1_it = v1.lower_bound(v2_it->first); 300 | // v1_it = (first element >= v2_it.id) 301 | } 302 | else 303 | { 304 | // move v2 forward 305 | v2_it = v2.lower_bound(v1_it->first); 306 | // v2_it = (first element >= v1_it.id) 307 | } 308 | } 309 | 310 | return score; // cannot scale 311 | } 312 | 313 | // --------------------------------------------------------------------------- 314 | // --------------------------------------------------------------------------- 315 | 316 | -------------------------------------------------------------------------------- /Dataset/icl_snippet/250.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/250.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/254.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/254.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/258.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/258.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/262.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/262.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/266.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/266.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/270.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/270.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/274.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/274.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/278.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/278.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/282.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/282.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/286.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/286.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/290.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/290.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/294.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/294.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/298.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/298.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/302.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/302.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/306.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/306.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/310.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/314.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/314.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/318.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/318.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/322.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/322.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/326.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/326.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/330.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/330.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/334.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/334.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/338.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/338.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/342.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/342.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/346.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/346.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/350.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/350.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/354.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/354.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/358.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/358.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/362.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/362.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/366.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/366.png -------------------------------------------------------------------------------- /Dataset/icl_snippet/magicleap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/icl_snippet/magicleap.png -------------------------------------------------------------------------------- /Dataset/magicleap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/magicleap.png -------------------------------------------------------------------------------- /Dataset/nyu_snippet.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Dataset/nyu_snippet.mp4 -------------------------------------------------------------------------------- /INSTALL_LibTorch.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | wget -O LibTorch.zip https://download.pytorch.org/libtorch/cu102/libtorch-cxx11-abi-shared-with-deps-1.6.0.zip 4 | sudo unzip LibTorch.zip -d /usr/local 5 | 6 | -------------------------------------------------------------------------------- /INSTALL_OpenCV.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | cd ~ 4 | 5 | echo "Download eigen 3.3.8 version zip file..." 6 | wget -O eigen3.zip https://gitlab.com/libeigen/eigen/-/archive/3.3.8/eigen-3.3.8.zip 7 | 8 | # INSTALL EIGEN3 9 | echo "Install Eigen3..." 10 | unzip eigen3.zip 11 | cd eigen-3.3.8 12 | mkdir build && cd build 13 | cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local 14 | sudo make install && cd ../.. 15 | 16 | 17 | echo "Download 3.4.11 version OpenCV and OpenCV Contrib..." 18 | wget -O opencv.zip https://github.com/opencv/opencv/archive/3.4.11.zip 19 | wget -O opencv_contrib.zip https://github.com/opencv/opencv_contrib/archive/3.4.11.zip 20 | 21 | # INSTALL OpenCV 22 | echo "Install OpenCV with some options..." 23 | sudo unzip opencv.zip -d /usr/local 24 | sudo unzip opencv_contrib.zip -d /usr/local 25 | sudo apt install git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev 26 | 27 | cd /usr/local/opencv-3.4.11 28 | sudo mkdir build 29 | cd build 30 | 31 | sudo cmake -DBUILD_opencv_world -DCMAKE_CONFIGURATION_TYPE=Release -DOPENCV_ENABLE_NONFREE -DOPENCV_EXTRA_MODULES_PATH=/usr/local/opencv_contrib-3.4.11/modules -DCMAKE_INSTALL_PREFIX=/usr/local .. 32 | sudo make -j$(nproc) 33 | sudo make install 34 | 35 | # Remove .zip files. 36 | # cd ~ && rm eigen3.zip opencv.zip opencv_contrib.zip 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | SuperPoint feature extractor with cpp/c++ Implementation 2 | ====================================== 3 | 4 | > Hello, this is my old project. 5 | > But, for some people that have a interest in applying superpoint featuer extractor to own project, I've written this README file. 6 | > If this project was of little help to your project, please strike the star button. ***Enjoy!!*** 7 | 8 | 9 | 10 | ## Prerequisite 11 | 12 | In this project, following packages are used. Make sure that the right version of libraries is installed and linked. 13 | 14 | 1. Nvidia-driver & Cuda Toolkit 10.2 with cuDNN 7.6.5 15 | > About Cuda toolkit installation, there are so many guides. Follow it! \ 16 | > **First** : sudo apt-get install nvidia-driver-440 (It is normal for a higher version of the secondary nvidia library to be installed. Just check that 'nvidia-driver-440' is installed.) \ 17 | > **Second** : Follow instructions in this link (https://developer.nvidia.com/cuda-10.2-download-archive) 18 | 19 | 2. OpenCV (C++) 3.4.11 version 20 | ``` shell 21 | # Installation script 22 | chmod +x INSTALL_OpenCV.sh 23 | ./INSTALL_OpenCV.sh 24 | ``` 25 | 26 | 3. LibTorch 1.6.0 version (with GPU | Cuda Toolkit 10.2, cuDNN 7.6.5) 27 | > If only CPU can be used, install cpu-version LibTorch. Some code change about tensor device should be required. 28 | ```shell 29 | # Installation script 30 | chmod +x INSTALL_LibTorch.sh 31 | ./INSTALL_LibTorch.sh 32 | ``` 33 | 34 | ## BUILD 35 | 36 | ```shell 37 | cmake -B build -S . 38 | cmake --build build -t SuperPoint 39 | ``` 40 | 41 | ## RUN 42 | - You can use your webcam for fancy test. It'll be automatically detected in normal case. 43 | ```shell 44 | # argument is frequncy 45 | ./bin/SuperPoint 100 46 | ``` 47 | 48 | ### BibTeX Citation 49 | ``` 50 | @inproceedings{detone18superpoint, 51 | author = {Daniel DeTone and 52 | Tomasz Malisiewicz and 53 | Andrew Rabinovich}, 54 | title = {SuperPoint: Self-Supervised Interest Point Detection and Description}, 55 | booktitle = {CVPR Deep Learning for Visual SLAM Workshop}, 56 | year = {2018}, 57 | url = {http://arxiv.org/abs/1712.07629} 58 | } 59 | ``` 60 | -------------------------------------------------------------------------------- /Weights/superpoint.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChanWoo25/SuperPoint2CPP/f4083d1added529a79740482b20603f5dcb6a8f8/Weights/superpoint.pt -------------------------------------------------------------------------------- /gitpush.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # $# 는 인자의 개수 4 | # $1 은 첫번째 인자를 문자열로 5 | # $2 는 2번째 인자.... 6 | 7 | # 8 | # if [ ... ]; then 9 | # ... 10 | # else 11 | # ... 12 | # fi 13 | 14 | if [ $# -eq 0 ]; then 15 | echo "Please type some message to commit." 16 | exit 17 | fi 18 | 19 | git add . && git commit -m "$@" && git push -u origin master -------------------------------------------------------------------------------- /include/SPDetector.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SPDETECTOR_HPP 2 | #define SPDETECTOR_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace SuperPointSLAM 12 | { 13 | 14 | 15 | 16 | #define EPSILON 1e-19 17 | 18 | class SPDetector 19 | { 20 | public: 21 | 22 | // Default Constuctor. No Use. 23 | SPDetector(); 24 | /** 25 | * @brief Construct a new SPDetector::SPDetector object. 26 | * When Initialize SPDetector, (1) First we initialize 27 | * SuperPoint Class with weight_dir and use_cuda arguments. 28 | * (2) and Move to device(cpu or gpu) we'll use. 29 | * (3) Make the model eveluation mode, too. 30 | * 31 | * @param _weight_dir the PATH that contains pretrained weight. 32 | * @param _use_cuda whether the model operates in cpu or gpu. 33 | */ 34 | SPDetector(std::string _weight_dir, bool _use_cuda); 35 | 36 | ~SPDetector(){} 37 | 38 | /** 39 | * @brief Detect input image's Keypoints and Compute Descriptor. 40 | * 41 | * @param img Input image. We use img's deep copy object. 42 | * @return cv::Mat 43 | */ 44 | void detect(cv::InputArray _image, std::vector& _keypoints, 45 | cv::Mat &_descriptors); 46 | 47 | int n_keypoints; 48 | 49 | private: 50 | std::shared_ptr model; // Superpoint model object 51 | c10::TensorOptions tensor_opts; // Contains necessary info for creating proper at::Tensor 52 | c10::DeviceType mDeviceType; // If our device can use the GPU, it has 'kCUDA', otherwise it has 'kCPU'. 53 | c10::Device mDevice; // c10::Device corresponding to mDeviceType. 54 | 55 | 56 | torch::Tensor mProb; // Superpoint Output Probability Tensor 57 | torch::Tensor mDesc; // Superpoint Output Descriptor Tensor 58 | 59 | /** 60 | * kpts is [H, W] size Tensor. 61 | * Its elements has 1 if there is a featrue, and 0 otherwise. 62 | * After this function, kpts Tensor can guarantee that 63 | * no overlapping feature points exist in each 3x3 patch. 64 | * 65 | * This function is executed on the CPU because it requires direct access 66 | * to the tensor elements. Therefore, in some cases, (Specifically, if 67 | * detect() is done on the GPU,) performance can be greatly impaired 68 | * because of the data travel time You can turn this option on and off 69 | * using the 'nms' member variable. 70 | */ 71 | void SemiNMS(at::Tensor& kpts); 72 | 73 | // SemiNMS() on/off flag. 74 | bool nms = true; 75 | 76 | // Interest Point Threshold 77 | float mConfThres=0.125; 78 | }; 79 | 80 | } 81 | 82 | 83 | #endif -------------------------------------------------------------------------------- /include/SuperPoint.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SUPERPOINT_HPP 2 | #define SUPERPOINT_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace SuperPointSLAM 14 | { 15 | 16 | using namespace torch; 17 | using namespace nn; 18 | 19 | /** 20 | * @brief 순수 SuperPoint 클래스 21 | */ 22 | class SuperPoint : public Module { 23 | public: 24 | // SuperPoint Model Constructor 25 | SuperPoint(); 26 | 27 | /** 28 | * Display some information 29 | * 1. Cuda Availability 30 | * 2. GPU number 31 | * 3. cudnn availability. 32 | */ 33 | void display(); 34 | 35 | // Display Module and Submodule's detail informations. 36 | // (1)Whether it is trainable 37 | // (2)module's name(ex. Conv2D or Linear etc.). 38 | void display(std::shared_ptr net); 39 | 40 | /** 41 | * @brief Forward propagation 42 | * @param x input Tensor. 43 | * @param Prob Output Probabities Tensor 44 | * @param Desc Output Descriptors Tensor 45 | * @details Return probabilities and descriptors using Prob, Desc. 46 | * - all Arguments are by reference to speed up. 47 | * - When input x's demension is [B, H, W], (B = Batch_size) 48 | * - Output dimension is as follows. 49 | * - Prob: [B, H, W] 50 | * - Desc: [B, 256, H/8, W/8] 51 | */ 52 | void forward(torch::Tensor& x, torch::Tensor& Prob, torch::Tensor& Desc); 53 | 54 | protected: 55 | //SHARED ENCODER 56 | Conv2d conv1a{nullptr}; 57 | Conv2d conv1b{nullptr}; 58 | Conv2d conv2a{nullptr}; 59 | Conv2d conv2b{nullptr}; 60 | Conv2d conv3a{nullptr}; 61 | Conv2d conv3b{nullptr}; 62 | Conv2d conv4a{nullptr}; 63 | Conv2d conv4b{nullptr}; 64 | 65 | //DETECTOR 66 | Conv2d convPa{nullptr}; 67 | Conv2d convPb{nullptr}; 68 | 69 | //DESCRIPTOR 70 | Conv2d convDa{nullptr}; 71 | Conv2d convDb{nullptr}; 72 | 73 | const int c1 = 64; 74 | const int c2 = 64; 75 | const int c3 = 128; 76 | const int c4 = 128; 77 | const int c5 = 256; 78 | const int d1 = 256; 79 | 80 | const float EPSILON = 1e-19; 81 | }; 82 | 83 | } 84 | 85 | #endif -------------------------------------------------------------------------------- /include/Tools.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TOOLS_HPP 2 | #define TOOLS_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class VideoStreamer{ 13 | private: 14 | enum class input_device{ 15 | IS_CAMERA=0, 16 | IS_VIDEO_FILE=1 17 | }; 18 | input_device img_source; 19 | //--- INITIALIZE VIDEOCAPTURE 20 | cv::VideoCapture cap; 21 | // open the default camera using default API 22 | // cap.open(0); 23 | // OR advance usage: select any API backend 24 | int deviceID = 0; // 0 = open default camera 25 | int apiID = cv::CAP_ANY; // 0 = autodetect default API 26 | int MAX_FRAME_NUM = 1000000; 27 | int current_frame_num = 0; 28 | cv::Size *pImgSize = NULL; 29 | 30 | public: 31 | VideoStreamer(){ 32 | cap.open(deviceID, apiID); 33 | if (!cap.isOpened()) { 34 | std::cerr << "ERROR! Unable to open camera\n"; 35 | std::exit(1); 36 | } 37 | img_source = input_device::IS_CAMERA; 38 | } 39 | ~VideoStreamer() 40 | { 41 | // When everything done, release the video capture object 42 | cap.release(); 43 | } 44 | VideoStreamer(int cam_id):img_source(input_device::IS_CAMERA) 45 | { 46 | deviceID = cam_id; 47 | cap.open(deviceID, apiID); 48 | if (!cap.isOpened()) { 49 | std::cerr << "ERROR! Unable to open camera\n"; 50 | std::exit(1); 51 | } 52 | 53 | cv::Mat test_grab; 54 | while(!cap.read(test_grab)); 55 | pImgSize = new cv::Size(test_grab.size()); 56 | } 57 | VideoStreamer(const cv::String& filename):img_source(input_device::IS_VIDEO_FILE) 58 | { 59 | cap.open(filename, apiID); 60 | if (!cap.isOpened()) { 61 | std::cerr << "ERROR! Unable to open camera\n"; 62 | std::exit(1); 63 | } 64 | 65 | // cv::Mat test_grab; 66 | // while(!cap.read(test_grab)); 67 | // image_size = test_grab.size(); 68 | // W_scale = (float)image_size.width / (float)input_size.width; 69 | // H_scale = (float)image_size.height / (float)input_size.height; 70 | } 71 | 72 | float H_scale=1.0, W_scale=1.0; 73 | cv::Mat img, input; 74 | cv::Mat read_image(const cv::String& path); 75 | // Read a image as grayscale and resize to img_size. 76 | 77 | bool next_frame(); 78 | void setImageSize(const cv::Size &_size){ 79 | pImgSize = new cv::Size(_size); 80 | } 81 | }; 82 | 83 | std::string cv_type2str(int type); 84 | 85 | #endif -------------------------------------------------------------------------------- /include/zed.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ZED_H 2 | #define ZED_H 3 | 4 | #include 5 | namespace ZED 6 | { 7 | 8 | void zed_serial_num(); 9 | 10 | } 11 | 12 | #endif -------------------------------------------------------------------------------- /main_CreateVoca.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * File: Demo.cpp 3 | * Date: November 2011 4 | * Author: Dorian Galvez-Lopez 5 | * Description: demo application of DBoW2 6 | * License: see the LICENSE.txt file 7 | */ 8 | 9 | #include 10 | #include 11 | 12 | // OpenCV 13 | #include 14 | // #include 15 | // #include 16 | #include 17 | #include 18 | #include 19 | 20 | using namespace DBoW2; 21 | using namespace std; 22 | using namespace SuperPointSLAM; 23 | 24 | /** You need to modify the path below that corresponds to your dataset and weight path. **/ 25 | const std::string weight_dir = "./Weights/superpoint.pt"; 26 | /***************************************************************************/ 27 | 28 | void SuperpointVocCreation(const vector > &features); 29 | void TestDatabase(const vector > &features); 30 | 31 | static int N_IMG = 10; 32 | 33 | void wait() 34 | { 35 | cout << endl << "Press enter to continue" << endl; 36 | getchar(); 37 | } 38 | 39 | void test() 40 | { 41 | 42 | float a[] = {1, 2, 3, 4, 5, 6}; 43 | float b[] = {4, 5, 6}; 44 | cv::Mat A(3, 2, CV_32F, a); 45 | cv::Mat B(3, 1, CV_32F, b); 46 | std::cout << A << std::endl; 47 | std::cout << A.row(0) << std::endl; 48 | std::cout << A.row(1) << std::endl; 49 | std::cout << A.col(0) << std::endl; 50 | std::cout << A.col(0) << std::endl; 51 | std::cout << A.size() << std::endl; 52 | double c = cv::sum(A)[0]; 53 | double d = A.at(0, 0); 54 | 55 | std::cout << c << std::endl; 56 | std::cout << std::sqrt(c) << std::endl; 57 | 58 | // std::cout << d << std::endl; 59 | // std::cout << A.at(0) << std::endl; 60 | // std::cout << B.at(0) << std::endl; 61 | } 62 | 63 | /***************************************************************************/ 64 | 65 | int main(int argc, char* argv[]) 66 | { 67 | cv::String DATA_PATH = "/home/leecw/Dataset/place365gray/%6d.png"; 68 | if(argc == 2) 69 | { 70 | DATA_PATH = cv::String(argv[1]); 71 | cout << DATA_PATH << endl; 72 | } 73 | 74 | vector< vector > features; 75 | VideoStreamer vs(DATA_PATH); 76 | 77 | 78 | /** Superpoint Detector **/ 79 | SPDetector SPF(weight_dir, torch::cuda::is_available()); 80 | std::cout << "VC created, SPDetector Constructed.\n"; 81 | 82 | long long n_features = 0; 83 | int t = N_IMG; 84 | 85 | long long cnt = 0; 86 | while(vs.next_frame()){ 87 | 88 | features.push_back(vector()); 89 | features[cnt].resize(0); 90 | 91 | /* Feature extraction */ 92 | cv::Mat descriptors; // [N_kpts, 256] Size format:[W, H] 93 | std::vector keypoints; 94 | SPF.detect(vs.input, keypoints, descriptors); 95 | 96 | // Insert descriptors to "featrues". 97 | int len = keypoints.size(); 98 | for(unsigned i = 0; i < len; i++) 99 | features[cnt].push_back(descriptors.row(i)); 100 | 101 | /* Count */ 102 | n_features += features[cnt].size(); cnt++; 103 | cout << descriptors.size().height << ' '; 104 | if(cnt%10 == 0) cout << endl; 105 | } 106 | 107 | N_IMG = cnt; 108 | std::cout << "\nFrom " << N_IMG << " images ... "; 109 | std::cout << "\nAll features extracted. [ Total: " << n_features << " ]\n"; 110 | 111 | SuperpointVocCreation(features); 112 | 113 | // // wait(); 114 | 115 | // TestDatabase(features); 116 | 117 | return 0; 118 | } 119 | 120 | // ---------------------------------------------------------------------------- 121 | 122 | /** 123 | * @brief create the vocabulary 124 | * 125 | * @param features loadfeature()를 통해 얻는 features 이중 벡터를 이용. 126 | */ 127 | void SuperpointVocCreation(const vector> &features) 128 | { 129 | // branching factor and depth levels 130 | const int k = 10; 131 | const int L = 6; 132 | const WeightingType weight = TF_IDF; 133 | const ScoringType scoring = L1_NORM; 134 | 135 | SuperpointVocabulary voc(k, L, weight, scoring); 136 | 137 | cout << "Creating a Big " << k << "^" << L << " SuperPoint Vocabulary..." << endl; 138 | /* Vocabulary 생성 함수 */ 139 | voc.create(features); 140 | cout << "... done!" << endl; 141 | 142 | // cout을 이용한 vocabulary 정보 출력 가능. 143 | cout << "Vocabulary information: " << endl 144 | << voc << endl << endl; 145 | 146 | // lets do something with this vocabulary 147 | // voc를 클래스로 하여 feature정보를 BoWVector Type으로 변환하여 scoring 가능. 148 | // 기억을 되살리자면, BoWVector란, Vocabulary에 들어있는 word의 히스토그램을 얻고 149 | // 분별력을 더하기 위해 TF-IDF reweighting을 하여 얻은 벡터이다. 150 | // cout << "Matching images against themselves (0 low, 1 high): " << endl; 151 | 152 | // BowVector v1, v2; 153 | // for(int i = 0; i < N_IMG; i++) 154 | // { 155 | // voc.transform(features[i], v1); 156 | // for(int j = 0; j < N_IMG; j++) 157 | // { 158 | // voc.transform(features[j], v2); 159 | 160 | // double score = voc.score(v1, v2); 161 | // if(score >= 0.3) 162 | // cout << "Image " << i << " vs Image " << j << ": " << score << endl; 163 | // } 164 | // } 165 | 166 | // save the vocabulary to disk 167 | cout << endl << "Saving vocabulary..." << endl; 168 | voc.saveToTextFile("SPVoc1_Iter800_Img12262_Thres_0_0625.txt"); 169 | cout << "Done" << endl; 170 | } 171 | 172 | // ---------------------------------------------------------------------------- 173 | 174 | 175 | void TestDatabase(const vector > &features) 176 | { 177 | cout << "Creating a small database..." << endl; 178 | 179 | // Load the vocabulary from disk 180 | SuperpointVocabulary voc("SP_voc_v2.yml.gz"); 181 | 182 | 183 | SuperpointDatabase db(voc, true, 0); 184 | // false = do not use direct index 185 | // (so ignore the last param) 186 | // The direct index is useful if we want to retrieve the features that 187 | // belong to some vocabulary node. 188 | // db creates a copy of the vocabulary, 189 | // we may get rid of "voc" now 190 | 191 | // add images to the database 192 | for(int i = 0; i < N_IMG; i++) 193 | { 194 | db.add(features[i]); 195 | } 196 | 197 | cout << "... done!" << endl; 198 | 199 | cout << "Database information: " << endl << db << endl; 200 | 201 | // and query the database 202 | cout << "Querying the database: " << endl; 203 | 204 | // Vocabulary에 image에 대해 query. return type = QueryResults. 205 | QueryResults ret; 206 | for(int i = 0; i < N_IMG; i+=40) 207 | { 208 | db.query(features[i], ret, 10); 209 | 210 | // ret[0] is always the same image in this case, because we added it to the 211 | // database. ret[1] is the second best match. 212 | 213 | // QueryResults 타입의 변수도 Cout으로 출력을 지원. 214 | cout << "\n[Searching for Image " << i << ". " << ret << "]\n"; 215 | } 216 | 217 | cout << endl; 218 | 219 | // we can save the database. The created file includes the vocabulary 220 | // and the entries added 221 | cout << "Saving database..." << endl; 222 | db.save("SP_db_v2.yml.gz"); 223 | cout << "... done!" << endl; 224 | 225 | // once saved, we can load it again 226 | cout << "Retrieving database once again..." << endl; 227 | SuperpointDatabase db2("SP_db_v2.yml.gz"); 228 | cout << "... done! This is: " << endl << db2 << endl; 229 | } 230 | 231 | // ---------------------------------------------------------------------------- 232 | 233 | 234 | -------------------------------------------------------------------------------- /main_SuperPoint.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | using namespace SuperPointSLAM; 5 | 6 | /** You need to modify the path below that corresponds to your dataset and weight path. **/ 7 | const std::string weight_dir = "./Weights/superpoint.pt"; 8 | void test(); 9 | 10 | 11 | int main(const int argc, char* argv[]) 12 | { 13 | /** From the main argument, Retrieve waiting period to control Displaying.**/ 14 | int ms; 15 | if(argc == 2) 16 | { 17 | char* a = argv[1]; 18 | ms = std::atoi(a); 19 | } 20 | else ms = 100; 21 | std::cout << "Frame rate is " << ms << "ms.\n"; 22 | 23 | 24 | /** Initialize VideoSteamer and SuperPoint Object **/ 25 | // VideoStreamer vs("../Dataset/nyu_snippet.mp4"); 26 | VideoStreamer vs(0); 27 | // VideoStreamer vs(kitti_dir); 28 | // vs.setImageSize(cv::Size(720, 240)); 29 | 30 | 31 | /** Superpoint Detector **/ 32 | SPDetector SPF(weight_dir, torch::cuda::is_available()); 33 | std::cout << "VC created, SPDetector Constructed.\n"; 34 | 35 | // Test input/output file 36 | // std::ifstream inputFile("input.txt", std::ios::in); 37 | // std::ofstream outputFile("output.txt", std::ios::out | std::ios::app); 38 | // int test_nms_dist_thres; 39 | // float test_conf_thres; 40 | // inputFile >> test_nms_dist_thres >> test_conf_thres; 41 | 42 | 43 | cv::namedWindow("superpoint", cv::WINDOW_AUTOSIZE); 44 | long long idx=0; 45 | while(++idx){ 46 | // Capture frame-by-frame 47 | // Image's size is [640 x 480] 48 | if(!vs.next_frame()) { std::cout << "main -- Video End\n"; break; } 49 | 50 | std::vector Keypoints; 51 | cv::Mat Descriptors; 52 | 53 | auto start = std::chrono::system_clock::now(); 54 | SPF.detect(vs.input, Keypoints, Descriptors); 55 | auto end = std::chrono::system_clock::now(); 56 | std::chrono::milliseconds mill = std::chrono::duration_cast(end - start); 57 | 58 | /* Logging */ 59 | std::cout << idx << "th ProcessTime: " << mill.count() << "ms\n"; 60 | std::cout << "Keypoint num: " << Keypoints.size() << std::endl; 61 | 62 | 63 | /* Visualization */ 64 | float x_scale(vs.W_scale), y_scale(vs.H_scale); 65 | auto kpt_iter = Keypoints.begin(); 66 | for(; kpt_iter != Keypoints.end(); kpt_iter++) 67 | { 68 | float X(kpt_iter->pt.x), Y(kpt_iter->pt.y); 69 | double conf(kpt_iter->response); 70 | cv::circle(vs.img, cv::Point(int(X*x_scale), int(Y*y_scale)), 1, cv::Scalar(0, 0, (255 * conf * 10)), 2); 71 | } 72 | 73 | /* Display the resulting frame */ 74 | cv::imshow( "superpoint", vs.img ); 75 | 76 | // Press ESC on keyboard to exit 77 | char c = (char)cv::waitKey(ms); 78 | if(c==27){ break; } 79 | } 80 | 81 | 82 | // Closes all the frames 83 | cv::destroyAllWindows(); 84 | } -------------------------------------------------------------------------------- /runSuperPoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cmake -B build -S . 3 | cmake --build build -t SuperPoint 4 | ./bin/SuperPoint 100 -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### LIBRARY - Super 4 | add_library(Super SHARED 5 | SuperPoint.cpp 6 | SPDetector.cpp 7 | #SPExtractor.cpp 8 | Tools.cpp 9 | ) 10 | target_compile_features(Super PUBLIC cxx_std_14) 11 | target_link_libraries(Super PUBLIC 12 | ${TORCH_LIBRARIES} 13 | ${OpenCV_LIBS} 14 | ) 15 | set_target_properties(Super PROPERTIES 16 | LIBRARY_OUTPUT_DIRECTORY "${CMAKE_HOME_DIRECTORY}/lib" 17 | ) 18 | target_include_directories(Super PUBLIC "../include") -------------------------------------------------------------------------------- /src/SPDetector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright (c) 2009, Willow Garage, Inc. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following 15 | * disclaimer in the documentation and/or other materials provided 16 | * with the distribution. 17 | * * Neither the name of the Willow Garage nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 | * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | * POSSIBILITY OF SUCH DAMAGE. 33 | * 34 | */ 35 | 36 | #include 37 | 38 | namespace SuperPointSLAM 39 | { 40 | 41 | SPDetector::SPDetector(std::string _weight_dir, bool _use_cuda) 42 | : mDeviceType((_use_cuda) ? c10::kCUDA : c10::kCPU), 43 | mDevice(c10::Device(mDeviceType)) 44 | { 45 | /* SuperPoint model loading */ 46 | model = std::make_shared(); 47 | torch::load(model, _weight_dir); 48 | 49 | /* This option should be done exactly as below */ 50 | tensor_opts = c10::TensorOptions() 51 | .dtype(torch::kFloat32) 52 | .layout(c10::kStrided) 53 | .requires_grad(false); 54 | if (_use_cuda) 55 | model->to(mDevice); 56 | model->eval(); 57 | } 58 | 59 | void SPDetector::detect(cv::InputArray _image, std::vector& _keypoints, 60 | cv::Mat &_descriptors) 61 | { 62 | cv::Mat img = _image.getMat(); 63 | at::Tensor x = torch::from_blob((void*)img.clone().data, \ 64 | {1, 1, img.rows, img.cols}, \ 65 | tensor_opts).to(mDevice); 66 | 67 | // To avoid Error caused by division by zero. 68 | // "EPSILON" is mostly used for this purpose. 69 | x = (x + EPSILON) / 255.0; 70 | 71 | model->forward(x, mProb, mDesc); 72 | mProb = mProb.squeeze(0); 73 | 74 | /* Return a "CUDA bool type Tensor" 75 | * 1 if there is a featrue, and 0 otherwise */ 76 | at::Tensor kpts = (mProb > mConfThres); 77 | 78 | /* Remove potential redundent features. */ 79 | if(nms) 80 | { // Default=true 81 | SemiNMS(kpts); 82 | } 83 | 84 | /* Prepare grid_sampler() */ 85 | kpts = at::nonzero(kpts); // [N, 2] (y, x) 86 | at::Tensor fkpts = kpts.to(kFloat); 87 | at::Tensor grid = torch::zeros({1, 1, kpts.size(0), 2}).to(mDevice); 88 | // grid.print(); // [CUDAFloatType [1, 1, 225, 2]] 89 | 90 | // mProb size(1): W - cols - 320, size(0): H - rows - 240 91 | 92 | /** Get each Keypoints' descriptor. **/ 93 | grid[0][0].slice(1, 0, 1) = (2.0 * (fkpts.slice(1, 1, 2) / mProb.size(1))) - 1; // x 94 | grid[0][0].slice(1, 1, 2) = (2.0 * (fkpts.slice(1, 0, 1) / mProb.size(0))) - 1; // y 95 | mDesc = at::grid_sampler(mDesc, grid, 0, 0, false); // [1, 256, 1, n_keypoints] 96 | mDesc = mDesc.squeeze(0).squeeze(1); // [256, n_keypoints] 97 | 98 | /** Normalize 1-Dimension with 2-Norm. **/ 99 | at::Tensor dn = at::norm(mDesc, 2, 1); // [CUDAFloatType [256]] 100 | mDesc = at::div((mDesc + EPSILON), unsqueeze(dn, 1)); 101 | //mDesc = mDesc.div(unsqueeze(dn, 1)); // [256, n_keypoints] <- unsqueeezed dn[CUDAFloatType [256, 1]] 102 | mDesc = mDesc.transpose(0, 1).contiguous(); // [CUDAFloatType [N, 256]] 103 | 104 | // After processing, back to CPU only descriptor 105 | if (mDeviceType == c10::kCUDA) 106 | mDesc = mDesc.to(kCPU); 107 | 108 | /** Convert descriptor From at::Tensor To cv::Mat **/ 109 | cv::Size desc_size(mDesc.size(1), mDesc.size(0)); 110 | n_keypoints = mDesc.size(0); 111 | 112 | // [256, N], CV_32F 113 | _descriptors.create(n_keypoints, 256, CV_32FC1); 114 | memcpy((void*)_descriptors.data, mDesc.data_ptr(), sizeof(float) * mDesc.numel()); 115 | // descriptors = cv::Mat(desc_size, CV_32FC1, mDesc.data_ptr()); 116 | // std::cout << _descriptors.row(0) << std::endl; 117 | 118 | /* Convert Keypoint 119 | * From torch::Tensor kpts(=keypoints) 120 | * To cv::KeyPoint keypoints_no_nms */ 121 | _keypoints.clear(); 122 | _keypoints.reserve(n_keypoints); 123 | for (int i = 0; i < n_keypoints; i++) 124 | { 125 | float x = kpts[i][1].item(), y = kpts[i][0].item(); 126 | float conf = mProb[kpts[i][0]][kpts[i][1]].item(); 127 | _keypoints.push_back(cv::KeyPoint(cv::Point((int)x, (int)y), 1.0, 0.0, conf)); 128 | } 129 | 130 | mProb.reset(); 131 | mDesc.reset(); 132 | } 133 | 134 | void SPDetector::SemiNMS(at::Tensor& kpts) 135 | { 136 | if (mDeviceType == c10::kCUDA) 137 | kpts = kpts.to(kCPU); 138 | // std::cout << kpts.scalar_type() << sizeof(kpts.scalar_type()) << std::endl; 139 | // NMS alternative 140 | int rowlen = kpts.size(0); 141 | int collen = kpts.size(1); 142 | 143 | //auto accessor = kpts.accessor(); 144 | auto pT1 = kpts.data_ptr(); 145 | auto pT2 = pT1 + collen; 146 | // auto pT3 = pT2 + collen; 147 | 148 | for(int i = 0; i < rowlen; i++) 149 | { 150 | for(int j = 0 ; j < collen; j++) 151 | { 152 | if(*pT1 && (i < rowlen-1) && (j < collen-1)) 153 | { 154 | *(pT1 + 1) = 0; // *(pT1 + 2) = 0; 155 | *pT2 = 0; *(pT2 + 1) = 0; // *(pT2 + 2) = 0; 156 | //*pT3 = 0; *(pT3 + 1) = 0; *(pT3 + 2) = 0; 157 | } 158 | pT1++; 159 | pT2++; 160 | // pT3++; 161 | } 162 | } 163 | 164 | if (mDeviceType == c10::kCUDA) 165 | kpts = kpts.to(kCUDA); 166 | } 167 | 168 | } //END namespace SuperPointSLAM -------------------------------------------------------------------------------- /src/SuperPoint.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace SuperPointSLAM 4 | { 5 | 6 | SuperPoint::SuperPoint() 7 | { 8 | /* 9 | A Module is registered as a submodule to another Module 10 | by calling register_module(), typically from within a parent 11 | module’s constructor. 12 | */ 13 | 14 | //SHARED ENCODER 15 | conv1a = register_module("conv1a", Conv2d(Conv2dOptions(1, c1, 3).stride(1).padding(1))); 16 | conv1b = register_module("conv1b", Conv2d(Conv2dOptions(c1, c1, 3).stride(1).padding(1))); 17 | 18 | conv2a = register_module("conv2a", Conv2d(Conv2dOptions(c1, c2, 3).stride(1).padding(1))); 19 | conv2b = register_module("conv2b", Conv2d(Conv2dOptions(c2, c2, 3).stride(1).padding(1))); 20 | 21 | conv3a = register_module("conv3a", Conv2d(Conv2dOptions(c2, c3, 3).stride(1).padding(1))); 22 | conv3b = register_module("conv3b", Conv2d(Conv2dOptions(c3, c3, 3).stride(1).padding(1))); 23 | 24 | conv4a = register_module("conv4a", Conv2d(Conv2dOptions(c3, c4, 3).stride(1).padding(1))); 25 | conv4b = register_module("conv4b", Conv2d(Conv2dOptions(c4, c4, 3).stride(1).padding(1))); 26 | 27 | //DETECTOR 28 | convPa = register_module("convPa", Conv2d(Conv2dOptions(c4, c5, 3).stride(1).padding(1))); 29 | convPb = register_module("convPb", Conv2d(Conv2dOptions(c5, 65, 1).stride(1).padding(0))); 30 | 31 | //DESCRIPTOR 32 | convDa = register_module("convDa", Conv2d(Conv2dOptions(c4, c5, 3).stride(1).padding(1))); 33 | convDb = register_module("convDb", Conv2d(Conv2dOptions(c5, d1, 1).stride(1).padding(0))); 34 | } 35 | 36 | void SuperPoint::forward(torch::Tensor& x, torch::Tensor& Prob, torch::Tensor& Desc) 37 | { 38 | //SHARED ENCODER 39 | x = relu(conv1a->forward(x)); 40 | x = relu(conv1b->forward(x)); 41 | x = max_pool2d(x, 2, 2); 42 | 43 | x = relu(conv2a->forward(x)); 44 | x = relu(conv2b->forward(x)); 45 | x = max_pool2d(x, 2, 2); 46 | 47 | x = relu(conv3a->forward(x)); 48 | x = relu(conv3b->forward(x)); 49 | x = max_pool2d(x, 2, 2); 50 | 51 | x = relu(conv4a->forward(x)); 52 | x = relu(conv4b->forward(x)); 53 | 54 | //DETECTOR 55 | auto cPa = relu(convPa->forward(x)); 56 | auto semi = convPb->forward(cPa); // [B, 65, H/8, W/8] 57 | 58 | //DESCRIPTOR 59 | auto cDa = relu(convDa->forward(x)); 60 | auto desc = convDb->forward(cDa); // [B, 256, H/8, W/8] 61 | auto dn = norm(desc, 2, 1); 62 | desc = at::div((desc + EPSILON), unsqueeze(dn, 1)); 63 | 64 | //DETECTOR - POST PROCESS 65 | semi = softmax(semi, 1); // 65개 채널에서 [H/8 * W/8] 개 원소으 총합 1이 되도록 regression. 66 | semi = semi.slice(1, 0, 64); // remove rest_bin 67 | semi = semi.permute({0, 2, 3, 1}); // [B, H/8, W/8, 64] 68 | 69 | int Hc = semi.size(1); 70 | int Wc = semi.size(2); 71 | semi = semi.contiguous().view({-1, Hc, Wc, 8, 8}); 72 | semi = semi.permute({0, 1, 3, 2, 4}); 73 | semi = semi.contiguous().view({-1, Hc * 8, Wc * 8}); // [B, H, W] 74 | 75 | //Return Tensor 76 | Prob = semi; // [B, H, W] 77 | Desc = desc; // [B, 256, H/8, W/8] 78 | } 79 | 80 | } // Namespace NAMU_TEST END 81 | -------------------------------------------------------------------------------- /src/Tools.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | cv::Mat VideoStreamer::read_image(const cv::String& path) 4 | { 5 | cv::Mat gray_img = cv::imread(path, cv::IMREAD_GRAYSCALE); 6 | // resampling using pixel area relation 7 | // resize(input, output, Size, scale_factor_x, scale_factor_y, interpolation_method) 8 | if(pImgSize != NULL) 9 | cv::resize(gray_img, gray_img, (*pImgSize), 0, 0, cv::INTER_AREA); 10 | 11 | if(gray_img.empty()){ 12 | std::cerr << "Error reading image.\n"; 13 | exit('2'); 14 | } 15 | gray_img.convertTo(gray_img, CV_32F); 16 | return gray_img/255.; 17 | } 18 | 19 | bool VideoStreamer::next_frame() 20 | { 21 | if(current_frame_num >= MAX_FRAME_NUM) return false; 22 | if(img_source == input_device::IS_CAMERA) 23 | { 24 | if(!cap.read(img)) 25 | { 26 | std::cout << "No Image.\n"; 27 | return false; 28 | } 29 | input = img.clone(); 30 | 31 | if(pImgSize != NULL) 32 | cv::resize(input, input, (*pImgSize), 1., 1., cv::INTER_AREA); 33 | 34 | cv::cvtColor(input, input, cv::COLOR_RGB2GRAY); 35 | input.convertTo(input, CV_32F); 36 | } 37 | else if(img_source == input_device::IS_VIDEO_FILE) 38 | { 39 | /* Read next image. Return false if failed.*/ 40 | if(!cap.read(img)) 41 | { 42 | std::cout << "No Image.\n"; 43 | return false; 44 | } 45 | 46 | input = img.clone(); 47 | cv::cvtColor(input, input, cv::COLOR_RGB2GRAY); 48 | 49 | /* If the size is fixed, adjust it. */ 50 | if(pImgSize != NULL) 51 | { 52 | W_scale = (float)img.size().width / (float)pImgSize->width; 53 | H_scale = (float)img.size().height / (float)pImgSize->height; 54 | cv::resize(input, input, (*pImgSize), 1., 1., cv::INTER_AREA); 55 | } 56 | 57 | input.convertTo(input, CV_32F); 58 | } 59 | else 60 | { 61 | std::cerr << "There is no source of image frames!\n"; 62 | exit(2); 63 | } 64 | current_frame_num++; 65 | return true; 66 | } 67 | 68 | std::string cv_type2str(int type) { 69 | std::string r; 70 | 71 | uchar depth = type & CV_MAT_DEPTH_MASK; 72 | uchar chans = 1 + (type >> CV_CN_SHIFT); 73 | 74 | switch ( depth ) { 75 | case CV_8U: r = "8U"; break; 76 | case CV_8S: r = "8S"; break; 77 | case CV_16U: r = "16U"; break; 78 | case CV_16S: r = "16S"; break; 79 | case CV_32S: r = "32S"; break; 80 | case CV_32F: r = "32F"; break; 81 | case CV_64F: r = "64F"; break; 82 | default: r = "User"; break; 83 | } 84 | 85 | r += "C"; 86 | r += (chans+'0'); 87 | 88 | return r; 89 | } -------------------------------------------------------------------------------- /zed/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | option(LINK_SHARED_ZED "Link with the ZED SDK shared executable" ON) 4 | 5 | if (NOT LINK_SHARED_ZED AND MSVC) 6 | message(FATAL_ERROR "LINK_SHARED_ZED OFF : ZED SDK static libraries not available on Windows") 7 | endif() 8 | 9 | # if(COMMAND cmake_policy) 10 | # cmake_policy(SET CMP0003 OLD) 11 | # cmake_policy(SET CMP0015 OLD) 12 | # endif(COMMAND cmake_policy) 13 | 14 | # link_directories(${ZED_LIBRARY_DIR}) 15 | # link_directories(${CUDA_LIBRARY_DIRS}) 16 | 17 | if (LINK_SHARED_ZED) 18 | SET(ZED_LIBS ${ZED_LIBRARIES} ${CUDA_CUDA_LIBRARY} ${CUDA_CUDART_LIBRARY}) 19 | else() 20 | SET(ZED_LIBS ${ZED_STATIC_LIBRARIES} ${CUDA_CUDA_LIBRARY} ${CUDA_LIBRARY}) 21 | endif() 22 | 23 | ADD_LIBRARY(libzed zed.cpp ../include/zed.hpp) 24 | TARGET_LINK_LIBRARIES(libzed PUBLIC ${ZED_LIBS} ${OpenCV_LIBS}) 25 | set_target_properties(libzed PROPERTIES 26 | CXX_STANDARD 14 27 | CXX_EXTENSIONS OFF 28 | CXX_STANDARD_REQUIRED ON 29 | LIBRARY_OUTPUT_DIRECTORY "${CMAKE_HOME_DIRECTORY}/lib" 30 | ) 31 | target_include_directories(libzed 32 | PUBLIC 33 | "${CMAKE_HOME_DIRECTORY}/include" 34 | ${CUDA_INCLUDE_DIRS} 35 | 36 | ) 37 | -------------------------------------------------------------------------------- /zed/zed.cpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////// 2 | // 3 | // Copyright (c) 2020, STEREOLABS. 4 | // 5 | // All rights reserved. 6 | // 7 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 8 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 9 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 10 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 11 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 12 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 13 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 14 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 15 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 16 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 17 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 18 | // 19 | /////////////////////////////////////////////////////////////////////////// 20 | 21 | #include 22 | 23 | namespace ZED 24 | { 25 | 26 | void zed_serial_num() 27 | { 28 | // Create a ZED camera object 29 | sl::Camera zed; 30 | 31 | // Open the camera 32 | sl::ERROR_CODE err = zed.open(); 33 | if (err != sl::ERROR_CODE::SUCCESS) { 34 | std::cout << "Error " << err << ", exit program.\n"; 35 | exit(1); 36 | } 37 | 38 | // Get camera information (ZED serial number) 39 | auto camera_infos = zed.getCameraInformation(); 40 | printf("Hello! This is my serial number: %d\n", camera_infos.serial_number); 41 | 42 | // Close the camera 43 | zed.close(); 44 | } 45 | 46 | } //ZED END 47 | --------------------------------------------------------------------------------