├── .gitattributes ├── CMakeLists.txt ├── README.md ├── engines └── model.txt ├── image ├── 523539369cec4410e5a28336208fc94.jpg ├── akaze-image.png ├── akaze-video.png ├── akaze_example.gif ├── deep-learning_example.gif ├── superpoint-superglue-image.png └── superpoint-superglue-video.png ├── include ├── akaze_match.h ├── data_body.h ├── local_match.h ├── logging.h ├── match.h └── tools.h └── src ├── akaze_match.cpp ├── example.cpp ├── local_match.cpp └── match.cpp /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | project(IR) 3 | add_definitions(-std=c++11) 4 | 5 | option(CUDA_USE_STATIC_CUDA_RUNTIME OFF) 6 | set(CMAKE_CXX_STANDARD 11) 7 | set(CMAKE_BUILD_TYPE Release) 8 | 9 | include_directories(${PROJECT_SOURCE_DIR}/include) 10 | # link_directories(${PROJECT_SOURCE_DIR}/lib) 11 | 12 | 13 | # cuda 14 | find_package(CUDA REQUIRED) 15 | include_directories(/usr/local/cuda-10.2/include) 16 | link_directories(/usr/local/cuda-10.2/lib64) 17 | 18 | # tensorrt 19 | include_directories(/usr/local/TensorRT-7.2.2.3/include) 20 | link_directories(/usr/local/TensorRT-7.2.2.3/lib) 21 | 22 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Ofast -Wfatal-errors -D_MWAITXINTRIN_H_INCLUDED") 23 | 24 | find_package(OpenCV) 25 | include_directories(${OpenCV_INCLUDE_DIRS}) 26 | 27 | set(SRC 28 | src/local_match.cpp 29 | src/akaze_match.cpp 30 | src/match.cpp 31 | src/example.cpp 32 | ) 33 | 34 | add_executable(IR ${SRC}) 35 | 36 | target_link_libraries(IR nvinfer) 37 | target_link_libraries(IR nvinfer_plugin) 38 | target_link_libraries(IR cudart) 39 | target_link_libraries(IR ${OpenCV_LIBS}) 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # feature detection and matching algorithm models 2 | 3 | 4 | 5 | ## Introduction 6 | 7 | 🚀🚀This warehouse mainly uses C++ to compare traditional image feature detection and matching, and deep learning feature detection and matching algorithm models. Deep learning includes superpoint-superglue, and traditional algorithms include AKAZE, SURF, ORB, etc. 8 | 9 | 1. akaze feature point detection and matching display. 10 | 11 | ![akaze-image](./image/akaze_example.gif) 12 | 13 | 2. superpoint-superpoint feature point detection and matching display. 14 | 15 | ![akaze-video](./image/deep-learning_example.gif) 16 | 17 | ## Dependencies 18 | 19 | All operating environments, please strictly follow the given configuration，the configuration is as follows： 20 | 21 | OpenCV >= 3.4 22 | 23 | CUDA >=10.2 24 | 25 | CUDNN>=8.02 26 | 27 | TensorRT>=7.2.3 28 | 29 | ## How to Run 30 | 31 | 1. build. 32 | 33 | ``` 34 | cd feature-detection-matching-algorithm/ 35 | mkdir build 36 | cd build 37 | cmake .. 38 | make 39 | ``` 40 | 41 | 2. run camera. 42 | 43 | deep learning algorithms. 44 | 45 | ``` 46 | ./IR --deeplearning --camera 0 47 | ``` 48 | 49 | traditional algorithms. 50 | 51 | ``` 52 | ./IR --traditional --camera 0 53 | ``` 54 | 55 | 3. run image-pair. 56 | 57 | deep learning algorithms. 58 | 59 | ``` 60 | ./IR --deeplearning --image-pair xx01.jpg xx02.jpg 61 | ``` 62 | 63 | traditional algorithms. 64 | 65 | ``` 66 | ./IR --traditional --image-pair xx01.jpg xx02.jpg 67 | ``` 68 | 69 | ## Models TRT 70 | https://pan.baidu.com/s/1CoK_KuC42BFD-mtO-BBhHg 71 | Code：cb7x 72 | 73 | ## TODO 74 | 75 | - [ ] Optimizing post-processing using custom TensorRT layer or Cublass. 76 | - [ ] Model conversion script. 77 | - [ ] support for FP16/INT8. 78 | 79 | ## Discussion 80 | 81 | WeChat ID: sigma1573 82 | 83 | Welcome to *add* WeChat(note: unit + name) and join the group discussion！ 84 | 85 | For more details, please refer to zhihu: https://zhuanlan.zhihu.com/p/518877309 86 | 87 | ## SuperPoint 88 | 89 | Superpoint pretrained models are from [magicleap/SuperPointPretrainedNetwork.](https://github.com/magicleap/SuperPointPretrainedNetwork) 90 | 91 | ## SuperGlue 92 | 93 | SuperGlue pretrained models are from [magicleap/SuperGluePretrainedNetwork.](https://github.com/magicleap/SuperGluePretrainedNetwork) 94 | 95 | ## Reference 96 | 97 | ``` 98 | @inproceedings{sarlin20superglue, 99 | author = {Paul-Edouard Sarlin and 100 | Daniel DeTone and 101 | Tomasz Malisiewicz and 102 | Andrew Rabinovich}, 103 | title = {{SuperGlue}: Learning Feature Matching with Graph Neural Networks}, 104 | booktitle = {CVPR}, 105 | year = {2020}, 106 | url = {https://arxiv.org/abs/1911.11763} 107 | } 108 | ``` 109 | 110 | -------------------------------------------------------------------------------- /engines/model.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/engines/model.txt -------------------------------------------------------------------------------- /image/523539369cec4410e5a28336208fc94.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/523539369cec4410e5a28336208fc94.jpg -------------------------------------------------------------------------------- /image/akaze-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/akaze-image.png -------------------------------------------------------------------------------- /image/akaze-video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/akaze-video.png -------------------------------------------------------------------------------- /image/akaze_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/akaze_example.gif -------------------------------------------------------------------------------- /image/deep-learning_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/deep-learning_example.gif -------------------------------------------------------------------------------- /image/superpoint-superglue-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/superpoint-superglue-image.png -------------------------------------------------------------------------------- /image/superpoint-superglue-video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/image/superpoint-superglue-video.png -------------------------------------------------------------------------------- /include/akaze_match.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #ifndef _AKAZE_MATCH_H_ 8 | #define _AKAZE_MATCH_H_ 9 | #include"opencv2/opencv.hpp" 10 | #include"data_body.h" 11 | 12 | 13 | class akaze_match 14 | { 15 | public: 16 | akaze_match(float inlier_threshold, float nn_match_ratio); 17 | ~akaze_match(); 18 | size_t akaze_forward(cv::Mat & image0, cv::Mat & image1, std::vector &vdm); 19 | private: 20 | float inlier_threshold = 2.5f; // Distance threshold to identify inliers with homography check 21 | float nn_match_ratio = 0.8f; // Nearest neighbor matching ratio 22 | }; 23 | 24 | 25 | #endif // !_AKAZE_MATCH_H_ 26 | 27 | 28 | -------------------------------------------------------------------------------- /include/data_body.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #ifndef _DATA_BODY_H_ 8 | #define _DATA_BODY_H_ 9 | #include 10 | #include 11 | 12 | struct lm_params 13 | { 14 | int32_t batch_size; 15 | int32_t dla_core; 16 | 17 | bool int8; 18 | bool fp16; 19 | bool seria; 20 | 21 | std::vector data_dirs; 22 | std::vector input_names; 23 | std::vector output_names; 24 | }; 25 | 26 | struct point_lm_params : public lm_params 27 | { 28 | std::string point_weight_file; 29 | int input_w; 30 | int input_h; 31 | float scores_thresh; 32 | int border; 33 | }; 34 | 35 | struct match_lm_params : public lm_params 36 | { 37 | std::string match_weight_file; 38 | float match_threshold; 39 | }; 40 | 41 | struct infer_deleter 42 | { 43 | template 44 | void operator()(T*obj) const 45 | { 46 | if (obj) 47 | { 48 | obj->destroy(); 49 | } 50 | } 51 | }; 52 | 53 | struct data 54 | { 55 | float x; 56 | float y; 57 | float s; 58 | }; 59 | 60 | struct data_point 61 | { 62 | float desc_w; 63 | float desc_h; 64 | int keypoint_size; 65 | float * descriptors; 66 | float * scores; 67 | float * keypoints; 68 | 69 | std::vector vdt; 70 | int status_code; // normal 1, error 0 71 | }; 72 | 73 | struct maxv_indices 74 | { 75 | float max_value; 76 | int indices; 77 | }; 78 | 79 | struct data_match 80 | { 81 | float valid_keypoint_x0; 82 | float valid_keypoint_y0; 83 | float valid_keypoint_x1; 84 | float valid_keypoint_y1; 85 | float msscores0; 86 | }; 87 | 88 | struct data_image 89 | { 90 | cv::Mat srcimg0; 91 | cv::Mat srcimg1; 92 | cv::Mat srcimg0_gray; 93 | cv::Mat srcimg1_gray; 94 | cv::Mat resized_srcimg0; 95 | cv::Mat resized_srcimg1; 96 | cv::Mat warpP0; 97 | cv::Mat warpP1; 98 | cv::Mat out_gray; 99 | cv::Mat out_rgb; 100 | cv::Mat out_match; 101 | cv::Mat H; 102 | 103 | }; 104 | 105 | #endif // !_DATA_BODY_H_ 106 | 107 | -------------------------------------------------------------------------------- /include/local_match.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #ifndef _LOCAL_MATCH_H_ 8 | #define _LOCAL_MATCH_H_ 9 | #include 10 | #include 11 | #include 12 | #include"cuda_runtime_api.h" 13 | #include 14 | #include 15 | #include"opencv2/opencv.hpp" 16 | #include"logging.h" 17 | #include"data_body.h" 18 | #include"tools.h" 19 | 20 | class point_lm_trt 21 | { 22 | public: 23 | point_lm_trt(); 24 | ~point_lm_trt(); 25 | bool initial_point_model(); 26 | size_t forward(cv::Mat& srcimg, data_point& dp); 27 | int get_input_h(); 28 | int get_input_w(); 29 | 30 | private: 31 | template 32 | using _unique_ptr = std::unique_ptr; 33 | std::shared_ptr< nvinfer1::ICudaEngine> _engine_ptr; 34 | std::shared_ptr< nvinfer1::IExecutionContext> _context_ptr; 35 | point_lm_params _point_lm_params; 36 | 37 | 38 | private: 39 | bool build_model(); 40 | float * imnormalize(cv::Mat & img); 41 | }; 42 | 43 | class match_lm_trt 44 | { 45 | public: 46 | match_lm_trt(); 47 | ~match_lm_trt(); 48 | bool initial_match_model(); 49 | size_t forward(data_point & dp0, data_point & dp1, std::vector &vdm); 50 | 51 | private: 52 | template 53 | using _unique_ptr = std::unique_ptr; 54 | std::shared_ptr< nvinfer1::ICudaEngine> _engine_ptr; 55 | std::shared_ptr< nvinfer1::IExecutionContext> _context_ptr; 56 | match_lm_params _match_lm_params; 57 | 58 | private: 59 | bool build_model(); 60 | float * log_optimal_transport(float * scores, float bin_score, 61 | int iters, int scores_output_h, int scores_output_w); 62 | int get_match_keypoint(data_point & dp0, data_point & dp1, 63 | float * lopt_scores, int scores_output_h, int scores_output_w, std::vector &vdm); 64 | void prepare_data(data_point & dp, float * arr1, float * arr2); 65 | 66 | }; 67 | 68 | #endif // !_LOCAL_MATCH_H_ 69 | -------------------------------------------------------------------------------- /include/logging.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Broad-sky/feature-detection-matching-algorithm/a393606862760e42ab88cec92d9cc3ee180e5e5a/include/logging.h -------------------------------------------------------------------------------- /include/match.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #ifndef _MATCH_H_ 8 | #define _MATCH_H_ 9 | #include"local_match.h" 10 | #include"akaze_match.h" 11 | //#include"surf_match.h" 12 | #include"tools.h" 13 | 14 | class IrInterface 15 | { 16 | public: 17 | static IrInterface * createIrInterface(); 18 | virtual void initial_model() = 0; 19 | virtual int dplearning_method_forward(std::vector &vdi, std::vector&vdm) = 0; 20 | virtual int traditional_method_forward(std::vector &vdi, std::vector&vdm, std::string md_name) = 0; 21 | }; 22 | 23 | 24 | class image_registration : public IrInterface 25 | { 26 | public: 27 | image_registration(); 28 | ~image_registration(); 29 | 30 | void initial_model(); 31 | 32 | int dplearning_method_forward(std::vector &vdi, std::vector & vdm); 33 | int traditional_method_forward(std::vector &vdi, std::vector&vdm, std::string md_name); 34 | 35 | 36 | private: 37 | point_lm_trt * _plt; 38 | match_lm_trt * _mlt; 39 | 40 | }; 41 | 42 | #endif // !_MATCH_H_ 43 | -------------------------------------------------------------------------------- /include/tools.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #ifndef _TOOLS_H 8 | #define _TOOLS_H 9 | #include"opencv2/opencv.hpp" 10 | #include"data_body.h" 11 | 12 | namespace lm_tools { 13 | static inline float sigmoid(float x) { 14 | return static_cast(1.f / (1.f + exp(-x))); 15 | } 16 | 17 | static inline bool within_bounds_2d(int64_t h, int64_t w, int64_t H, int64_t W) 18 | { 19 | return h >= 0 && h < H && w >= 0 && w < W; 20 | } 21 | 22 | static void print_data_1dim(float*arr, int length) 23 | { 24 | for (int i = 0; i < length; i++) 25 | { 26 | std::cout << arr[i] << ","; 27 | } 28 | std::cout << "\n"; 29 | } 30 | 31 | static void print_data_2dim(float*arr, int h, int w) 32 | { 33 | for (int i = 0; i < h; i++) 34 | { 35 | for (int j = 0; j < w; j++) 36 | { 37 | std::cout << arr[i*w + j] << ","; 38 | if ((j+1) % w == 0) 39 | { 40 | std::cout << "\n"; 41 | } 42 | } 43 | } 44 | } 45 | 46 | 47 | static void make_matching_plot_fast(std::vector &vdi, std::vector & vdm) 48 | { 49 | 50 | int H0 = vdi[0].resized_srcimg0.rows; 51 | int W0 = vdi[0].resized_srcimg0.cols; 52 | int H1 = vdi[0].resized_srcimg1.rows; 53 | int W1 = vdi[0].resized_srcimg1.cols; 54 | int margin = 30; 55 | int H = std::max(H0, H1); 56 | int W = W0 + W1 + margin; 57 | 58 | cv::Mat out = cv::Mat(H, W, CV_8UC1); 59 | out = cv::Scalar(255); 60 | 61 | cv::Rect rect0 = cv::Rect(0, 0, W0, H0); 62 | vdi[0].resized_srcimg0.copyTo(out(rect0)); 63 | cv::Rect rect1 = cv::Rect(W0 + margin, 0, W1, H1); 64 | vdi[0].resized_srcimg1.copyTo(out(rect1)); 65 | 66 | cv::Mat out_c3; 67 | cv::cvtColor(out, out_c3, cv::COLOR_GRAY2BGR); 68 | 69 | if (vdm.size()>=5) 70 | { 71 | for (auto & vdm_t : vdm) 72 | { 73 | if (false) 74 | { 75 | int valid_keypoint_x0 = vdm_t.valid_keypoint_x0 * double(vdi[0].srcimg0.cols) / double(vdi[0].resized_srcimg0.cols); 76 | int valid_keypoint_y0 = vdm_t.valid_keypoint_y0 * double(vdi[0].srcimg0.rows) / double(vdi[0].resized_srcimg0.rows); 77 | int valid_keypoint_x1 = vdm_t.valid_keypoint_x1 * double(vdi[0].srcimg1.cols) / double(vdi[0].resized_srcimg1.cols); 78 | int valid_keypoint_y1 = vdm_t.valid_keypoint_y1 * double(vdi[0].srcimg1.rows) / double(vdi[0].resized_srcimg1.rows); 79 | cv::circle(vdi[0].srcimg0, cv::Point(valid_keypoint_x0, valid_keypoint_y0), 3, cv::Scalar(255, 0, 0), 1, cv::LINE_AA); 80 | cv::circle(vdi[0].srcimg1, cv::Point(valid_keypoint_x1, valid_keypoint_y1), 5, cv::Scalar(255, 0, 0), 1, cv::LINE_AA); 81 | cv::circle(vdi[0].srcimg0_gray, cv::Point(valid_keypoint_x0, valid_keypoint_y0), 3, cv::Scalar(0, 0, 0), 1, cv::LINE_AA); 82 | cv::circle(vdi[0].srcimg1_gray, cv::Point(valid_keypoint_x1, valid_keypoint_y1), 5, cv::Scalar(0, 0, 0), 1, cv::LINE_AA); 83 | } 84 | 85 | int x0 = vdm_t.valid_keypoint_x0; 86 | int y0 = vdm_t.valid_keypoint_y0; 87 | int x1 = vdm_t.valid_keypoint_x1; 88 | int y1 = vdm_t.valid_keypoint_y1; 89 | if (vdm_t.msscores0>0.6) 90 | { 91 | cv::line(out_c3, cv::Point(x0, y0), cv::Point(x1 + margin + W0, y1), cv::Scalar(255, 0, 0), 1, cv::LINE_AA); 92 | cv::circle(out_c3, cv::Point(x0, y0), 2, cv::Scalar(0, 0, 255), -1, cv::LINE_AA); 93 | cv::circle(out_c3, cv::Point(x1 + margin + W0, y1), 2, cv::Scalar(0, 0, 255), -1, cv::LINE_AA); 94 | } 95 | } 96 | } 97 | vdi[0].out_match = out_c3; 98 | } 99 | 100 | static void image_color_convert(std::vector &vdi) 101 | { 102 | if (vdi[0].srcimg1.channels() == 3) 103 | { 104 | cv::cvtColor(vdi[0].srcimg0, vdi[0].srcimg0_gray, cv::COLOR_BGR2GRAY); 105 | cv::cvtColor(vdi[0].srcimg1, vdi[0].srcimg1_gray, cv::COLOR_BGR2GRAY); 106 | } 107 | if (vdi[0].srcimg1.channels() == 1) 108 | { 109 | vdi[0].srcimg0_gray = vdi[0].srcimg0; 110 | cv::cvtColor(vdi[0].srcimg0, vdi[0].srcimg0, cv::COLOR_GRAY2BGR); 111 | vdi[0].srcimg1_gray = vdi[0].srcimg1; 112 | cv::cvtColor(vdi[0].srcimg1, vdi[0].srcimg1, cv::COLOR_GRAY2BGR); 113 | } 114 | } 115 | 116 | 117 | } 118 | 119 | 120 | #endif // !_TOOLS_H 121 | -------------------------------------------------------------------------------- /src/akaze_match.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #include"akaze_match.h" 8 | 9 | 10 | akaze_match::akaze_match(float inlier_threshold, float nn_match_ratio) 11 | { 12 | inlier_threshold = inlier_threshold; 13 | nn_match_ratio = nn_match_ratio; 14 | } 15 | 16 | akaze_match::~akaze_match() 17 | { 18 | } 19 | 20 | size_t akaze_match::akaze_forward(cv::Mat & image0, cv::Mat & image1, std::vector &vdm) 21 | { 22 | std::vector kpts1, kpts2; 23 | cv::Mat desc1, desc2; 24 | cv::Ptr akaze = cv::AKAZE::create(); 25 | akaze->detectAndCompute(image0, cv::noArray(), kpts1, desc1); 26 | akaze->detectAndCompute(image1, cv::noArray(), kpts2, desc2); 27 | cv::BFMatcher matcher(cv::NORM_HAMMING); 28 | std::vector< std::vector > nn_matches; 29 | matcher.knnMatch(desc1, desc2, nn_matches, 2); 30 | std::vector matched1, matched2; 31 | for (size_t i = 0; i < nn_matches.size(); i++) { 32 | cv::DMatch first = nn_matches[i][0]; 33 | float dist1 = nn_matches[i][0].distance; 34 | float dist2 = nn_matches[i][1].distance; 35 | if (dist1 < nn_match_ratio * dist2) { 36 | matched1.push_back(kpts1[first.queryIdx]); 37 | matched2.push_back(kpts2[first.trainIdx]); 38 | } 39 | } 40 | std::vector good_matches; 41 | std::vector inliers1, inliers2; 42 | for (size_t i = 0; i < matched1.size(); i++) { 43 | data_match dm; 44 | if (true) { 45 | int new_i = static_cast(inliers1.size()); 46 | inliers1.push_back(matched1[i]); 47 | inliers2.push_back(matched2[i]); 48 | good_matches.push_back(cv::DMatch(new_i, new_i, 0)); 49 | dm.valid_keypoint_x0 = matched1[i].pt.x; 50 | dm.valid_keypoint_y0 = matched1[i].pt.y; 51 | dm.valid_keypoint_x1 = matched2[i].pt.x; 52 | dm.valid_keypoint_y1 = matched2[i].pt.y; 53 | dm.msscores0 = 1.0; 54 | vdm.push_back(dm); 55 | } 56 | } 57 | 58 | std::cout << "A-KAZE Matching Results" << std::endl; 59 | std::cout << "*******************************" << std::endl; 60 | std::cout << "# Keypoints 1: \t" << kpts1.size() << std::endl; 61 | std::cout << "# Keypoints 2: \t" << kpts2.size() << std::endl; 62 | std::cout << "# Matches: \t" << good_matches.size() << std::endl; 63 | std::cout << "# vdm size: \t" << vdm.size() << std::endl; 64 | std::cout << std::endl; 65 | 66 | return vdm.size(); 67 | } -------------------------------------------------------------------------------- /src/example.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/03/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #include"match.h" 8 | #include 9 | 10 | 11 | int main(int argc, char **argv) 12 | { 13 | if (argc > 3) 14 | { 15 | std::string argv1 = std::string(argv[1]); 16 | std::string argv2 = std::string(argv[2]); 17 | std::string argv3 = std::string(argv[3]); 18 | 19 | if (std::string(argv[1]) != "--deeplearning"&& (std::string(argv[1]) != "--traditional")) return 0; 20 | std::vector vdi; 21 | data_image di; 22 | cv::VideoCapture cap(0); 23 | if (!cap.isOpened()) 24 | { 25 | std::cout << "Failed to open camera." << std::endl; 26 | return -1; 27 | } 28 | if (argv2 == "--camera") 29 | { 30 | const char *ip = argv[3]; 31 | int index = std::stoi(argv3); 32 | if (cap.isOpened()) 33 | { 34 | cap>>di.srcimg0; 35 | } 36 | } 37 | if (argv2 == "--image-pair") 38 | { 39 | const char *image_path0 = argv[3]; // input image path 40 | const char *image_path1 = argv[4]; // save result image path 41 | di.srcimg0 = cv::imread(image_path0); 42 | di.srcimg1 = cv::imread(image_path1); 43 | if (di.srcimg0.empty() || di.srcimg0.empty()) 44 | { 45 | std::cerr << "image is null, please check image path \n"; 46 | return -1; 47 | } 48 | } 49 | vdi.emplace_back(di); 50 | cv::Mat out, out_gray; 51 | auto ir_ptr = std::unique_ptr(IrInterface::createIrInterface()); 52 | if (std::string(argv[1]) == "--deeplearning") ir_ptr->initial_model(); 53 | std::vector vdm; 54 | float cost_time; 55 | while (true) 56 | { 57 | if (argv2 == "--camera") 58 | { 59 | if (cap.isOpened()) 60 | { 61 | cap >> vdi[0].srcimg1; 62 | } 63 | else 64 | { 65 | std::cout << "open camera error!" << "\n"; 66 | break; 67 | } 68 | } 69 | lm_tools::image_color_convert(vdi); 70 | auto start = std::chrono::high_resolution_clock::now(); 71 | if (argv1 == "--deeplearning") 72 | { 73 | int ret = ir_ptr->dplearning_method_forward(vdi, vdm); 74 | if (ret >= 4) 75 | { 76 | std::cout << "deep learning method detect successed!" << "\n"; 77 | } 78 | 79 | } 80 | if (argv1 == "--traditional") 81 | { 82 | int ret = ir_ptr->traditional_method_forward(vdi, vdm, "akaze"); 83 | if (ret >= 4) 84 | { 85 | std::cout << "traditional method detect successed!" << "\n"; 86 | } 87 | } 88 | auto end = std::chrono::high_resolution_clock::now(); 89 | cost_time = std::chrono::duration(end-start).count(); 90 | std::cout<<"cost: "<arguments not right!" << std::endl; 113 | return -1; 114 | } 115 | } -------------------------------------------------------------------------------- /src/local_match.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #include"local_match.h" 8 | 9 | 10 | #ifdef __cplusplus 11 | #define PUT_IN_REGISTER 12 | #else 13 | #define PUT_IN_REGISTER register 14 | #endif 15 | 16 | #define CV_VERSION_ID CVAUX_STR(CV_MAJOR_VERSION) CVAUX_STR(CV_MINOR_VERSION) CVAUX_STR(CV_SUBMINOR_VERSION) 17 | #define CHECK(status) \ 18 | do\ 19 | {\ 20 | auto ret = (status);\ 21 | if (ret != 0)\ 22 | {\ 23 | std::cerr << "Cuda failure: " << ret << std::endl;\ 24 | abort();\ 25 | }\ 26 | } while (0) 27 | 28 | static Logger gLogger; 29 | 30 | 31 | 32 | bool point_lm_trt::build_model() 33 | { 34 | initLibNvInferPlugins(&gLogger.getTRTLogger(), ""); 35 | nvinfer1::IRuntime* runtime = nvinfer1::createInferRuntime(gLogger); 36 | if (!runtime) 37 | { 38 | return false; 39 | } 40 | 41 | char* model_deser_buffer{ nullptr }; 42 | const std::string engine_file_path(_point_lm_params.point_weight_file); 43 | std::ifstream ifs; 44 | int ser_length; 45 | ifs.open(engine_file_path.c_str(), std::ios::in | std::ios::binary); 46 | if (ifs.is_open()) 47 | { 48 | ifs.seekg(0, std::ios::end); 49 | ser_length = ifs.tellg(); 50 | ifs.seekg(0, std::ios::beg); 51 | model_deser_buffer = new char[ser_length]; 52 | ifs.read(model_deser_buffer, ser_length); 53 | ifs.close(); 54 | } 55 | else 56 | { 57 | return false; 58 | } 59 | 60 | _engine_ptr = std::shared_ptr(runtime->deserializeCudaEngine(model_deser_buffer, ser_length, nullptr), infer_deleter()); 61 | if (!_engine_ptr) 62 | { 63 | return false; 64 | } 65 | else 66 | { 67 | std::cout << "load engine successed!" << std::endl; 68 | } 69 | delete[] model_deser_buffer; 70 | 71 | _context_ptr = std::shared_ptr(_engine_ptr->createExecutionContext(), infer_deleter()); 72 | if (!_context_ptr) 73 | { 74 | return false; 75 | } 76 | return true; 77 | } 78 | 79 | bool point_lm_trt::initial_point_model() 80 | { 81 | bool ret = build_model(); 82 | if (!ret) 83 | { 84 | std::cout << "build point model is failed!" << std::endl; 85 | } 86 | return ret; 87 | } 88 | 89 | size_t point_lm_trt::forward(cv::Mat& srcimg, data_point& dp) 90 | { 91 | float * blob(nullptr); 92 | blob = imnormalize(srcimg); 93 | if (blob==nullptr) 94 | { 95 | std::cout << "imnormalize error! " << std::endl; 96 | dp.status_code = 0; 97 | return -1; 98 | } 99 | 100 | int batch_size = _engine_ptr->getMaxBatchSize(); 101 | int dummy_inputIndex = _engine_ptr->getBindingIndex(_point_lm_params.input_names[0].c_str()); 102 | assert(_engine_ptr->getBindingDataType(dummy_inputIndex) == nvinfer1::DataType::kFLOAT); 103 | auto dummy_input_dims = _engine_ptr->getBindingDimensions(dummy_inputIndex); 104 | int dummy_input_size = 1; 105 | for (int i = 0; i < dummy_input_dims.nbDims; i++) 106 | { 107 | dummy_input_size *= dummy_input_dims.d[i]; 108 | } 109 | const int scores_outputIndex = _engine_ptr->getBindingIndex(_point_lm_params.output_names[0].c_str()); 110 | assert(_engine_ptr->getBindingDataType(scores_outputIndex) == nvinfer1::DataType::kFLOAT); 111 | int scores_size = 1; 112 | auto scores_dims = _engine_ptr->getBindingDimensions(scores_outputIndex); 113 | for (int i = 0; i < scores_dims.nbDims; i++) 114 | { 115 | scores_size *= scores_dims.d[i]; 116 | } 117 | 118 | const int descriptors_outputIndex = _engine_ptr->getBindingIndex(_point_lm_params.output_names[1].c_str()); 119 | assert(_engine_ptr->getBindingDataType(descriptors_outputIndex) == nvinfer1::DataType::kFLOAT); 120 | int descriptors_size = 1; 121 | auto descriptors_dims = _engine_ptr->getBindingDimensions(descriptors_outputIndex); 122 | for (int i = 0; i < descriptors_dims.nbDims; i++) 123 | { 124 | descriptors_size *= descriptors_dims.d[i]; 125 | } 126 | float * scores_output = new float[scores_size]; 127 | float * descriptors_output = new float[descriptors_size]; 128 | 129 | void* buffers[3]; 130 | assert(1 * 1 * _point_lm_params.input_h * _point_lm_params.input_w == dummy_input_size); 131 | CHECK(cudaMalloc(&buffers[dummy_inputIndex], dummy_input_size * sizeof(float))); 132 | CHECK(cudaMalloc(&buffers[scores_outputIndex], scores_size * sizeof(float))); 133 | CHECK(cudaMalloc(&buffers[descriptors_outputIndex], descriptors_size * sizeof(float))); 134 | 135 | cudaStream_t stream; 136 | CHECK(cudaStreamCreate(&stream)); 137 | CHECK(cudaMemcpyAsync(buffers[dummy_inputIndex], blob, dummy_input_size * sizeof(float), cudaMemcpyHostToDevice, stream)); 138 | bool status = _context_ptr->enqueue(batch_size, buffers, stream, nullptr); 139 | 140 | delete[] blob; 141 | blob = nullptr; 142 | 143 | if (!status) 144 | { 145 | std::cout << "execute ifer error! " << std::endl; 146 | dp.status_code = 0; 147 | return -1; 148 | } 149 | CHECK(cudaMemcpyAsync(scores_output, buffers[scores_outputIndex], scores_size * sizeof(float), cudaMemcpyDeviceToHost, stream)); 150 | CHECK(cudaMemcpyAsync(descriptors_output, buffers[descriptors_outputIndex], descriptors_size * sizeof(float), cudaMemcpyDeviceToHost, stream)); 151 | 152 | cudaStreamDestroy(stream); 153 | CHECK(cudaFree(buffers[dummy_inputIndex])); 154 | CHECK(cudaFree(buffers[scores_outputIndex])); 155 | CHECK(cudaFree(buffers[descriptors_outputIndex])); 156 | int scores_h = _point_lm_params.input_h; 157 | int scores_w = _point_lm_params.input_w; 158 | int border = _point_lm_params.border; 159 | for (int y = 0; y < scores_h; y++) 160 | { 161 | for (int x = 0; x < scores_w; x++) 162 | { 163 | float score = scores_output[y*scores_w + x]; 164 | if (score>_point_lm_params.scores_thresh 165 | && x>=border && x<(scores_w-border)&&y>=border&&y<(scores_h-border)) 166 | { 167 | data dt; 168 | dt.x = x; 169 | dt.y = y; 170 | dt.s = score; 171 | dp.vdt.push_back(dt); 172 | } 173 | } 174 | } 175 | int vdt_size = dp.vdt.size(); 176 | dp.keypoint_size = vdt_size; 177 | 178 | int desc_fea_c = descriptors_dims.d[1]; 179 | int desc_fea_h = descriptors_dims.d[2]; 180 | int desc_fea_w = descriptors_dims.d[3]; 181 | 182 | float * desc_channel_sum_sqrts = new float[desc_fea_h*desc_fea_w]; 183 | for (int dfh = 0; dfh < desc_fea_h; dfh++) 184 | { 185 | for (int dfw = 0; dfw < desc_fea_w; dfw++) 186 | { 187 | float desc_channel_sum_temp = 0.f; 188 | for (int dfc = 0; dfc < desc_fea_c; dfc++) 189 | { 190 | desc_channel_sum_temp += descriptors_output[dfc*desc_fea_w*desc_fea_h + dfh*desc_fea_w + dfw]* 191 | descriptors_output[dfc*desc_fea_w*desc_fea_h + dfh*desc_fea_w + dfw]; 192 | } 193 | float desc_channel_sum_sqrt = std::sqrt(desc_channel_sum_temp); 194 | desc_channel_sum_sqrts[dfh*desc_fea_w + dfw] = desc_channel_sum_sqrt; 195 | } 196 | 197 | } 198 | 199 | for (int dfh = 0; dfh < desc_fea_h; dfh++) 200 | { 201 | for (int dfw = 0; dfw < desc_fea_w; dfw++) 202 | { 203 | for (int dfc = 0; dfc < desc_fea_c; dfc++) 204 | { 205 | descriptors_output[dfc*desc_fea_w*desc_fea_h + dfh*desc_fea_w + dfw] = 206 | descriptors_output[dfc*desc_fea_w*desc_fea_h + dfh*desc_fea_w + dfw] / desc_channel_sum_sqrts[dfh*desc_fea_w + dfw]; 207 | } 208 | } 209 | } 210 | int s = 8; 211 | float * descriptors_output_f = new float[desc_fea_c*vdt_size]; 212 | float * descriptors_output_sqrt = new float[vdt_size]; 213 | int count = 0; 214 | for (auto & _vdt : dp.vdt) 215 | { 216 | float ix = ((_vdt.x - s / 2 + 0.5) / (desc_fea_w*s - s / 2 - 0.5))*(desc_fea_w - 1); 217 | float iy = (_vdt.y - s / 2 + 0.5)/ (desc_fea_h*s - s / 2 - 0.5)*(desc_fea_h - 1); 218 | 219 | int ix_nw = std::floor(ix); 220 | int iy_nw = std::floor(iy); 221 | 222 | int ix_ne = ix_nw + 1; 223 | int iy_ne = iy_nw; 224 | 225 | int ix_sw = ix_nw; 226 | int iy_sw = iy_nw + 1; 227 | 228 | int ix_se = ix_nw + 1; 229 | int iy_se = iy_nw + 1; 230 | 231 | float nw = (ix_se - ix) * (iy_se - iy); 232 | float ne = (ix - ix_sw) * (iy_sw - iy); 233 | float sw = (ix_ne - ix) * (iy - iy_ne); 234 | float se = (ix - ix_nw) * (iy - iy_nw); 235 | 236 | float descriptors_channel_sum_l2 = 0.f; 237 | for (int dfc = 0; dfc < desc_fea_c; dfc++) 238 | { 239 | float res = 0.f; 240 | 241 | if (lm_tools::within_bounds_2d(iy_nw, ix_nw, desc_fea_h, desc_fea_w)) 242 | { 243 | res += descriptors_output[dfc*desc_fea_h*desc_fea_w + iy_nw*desc_fea_w + ix_nw] * nw; 244 | } 245 | if (lm_tools::within_bounds_2d(iy_ne, ix_ne, desc_fea_h, desc_fea_w)) 246 | { 247 | res += descriptors_output[dfc*desc_fea_h*desc_fea_w + iy_ne*desc_fea_w + ix_ne] * ne; 248 | } 249 | if (lm_tools::within_bounds_2d(iy_sw, ix_sw, desc_fea_h, desc_fea_w)) 250 | { 251 | res += descriptors_output[dfc*desc_fea_h*desc_fea_w + iy_sw*desc_fea_w + ix_sw] * sw; 252 | } 253 | if (lm_tools::within_bounds_2d(iy_se, ix_se, desc_fea_h, desc_fea_w)) 254 | { 255 | res += descriptors_output[dfc*desc_fea_h*desc_fea_w + iy_se*desc_fea_w + ix_se] * se; 256 | } 257 | descriptors_output_f[dfc*vdt_size + count] = res; 258 | descriptors_channel_sum_l2 += res*res; 259 | } 260 | descriptors_output_sqrt[count] = descriptors_channel_sum_l2; 261 | for (int64_t dfc = 0; dfc < desc_fea_c; dfc++) 262 | { 263 | descriptors_output_f[dfc*vdt_size + count] /= std::sqrt(descriptors_output_sqrt[count]); 264 | } 265 | count++; 266 | } 267 | 268 | delete[]scores_output; 269 | delete[]descriptors_output; 270 | delete[]descriptors_output_sqrt; 271 | scores_output = nullptr; 272 | descriptors_output = nullptr; 273 | descriptors_output_sqrt = nullptr; 274 | 275 | dp.descriptors = descriptors_output_f; 276 | dp.desc_h = desc_fea_c; 277 | dp.desc_w = vdt_size; 278 | dp.status_code = 1; 279 | return 1; 280 | } 281 | 282 | point_lm_trt::point_lm_trt() 283 | { 284 | cudaSetDevice(0); 285 | _point_lm_params.input_names.push_back("dummy_input"); 286 | _point_lm_params.output_names.push_back("scores"); 287 | _point_lm_params.output_names.push_back("descriptors"); 288 | _point_lm_params.dla_core = -1; 289 | _point_lm_params.int8 = false; 290 | _point_lm_params.fp16 = false; 291 | _point_lm_params.batch_size = 1; 292 | _point_lm_params.seria = false; 293 | _point_lm_params.point_weight_file = "../engines/point_model.bin"; 294 | _point_lm_params.input_h = 240; 295 | _point_lm_params.input_w = 320; 296 | _point_lm_params.scores_thresh = 0.01; 297 | _point_lm_params.border = 4; 298 | } 299 | 300 | point_lm_trt::~point_lm_trt() 301 | { 302 | } 303 | 304 | int point_lm_trt::get_input_h() 305 | { 306 | return _point_lm_params.input_h; 307 | } 308 | 309 | int point_lm_trt::get_input_w() 310 | { 311 | return _point_lm_params.input_w; 312 | } 313 | 314 | float * point_lm_trt::imnormalize(cv::Mat & img) 315 | { 316 | int img_h = img.rows; 317 | int img_w = img.cols; 318 | float * blob = new float[img_h*img_w]; 319 | for (int h = 0; h < img_h; h++) 320 | { 321 | for (int w = 0; w < img_w; w++) 322 | { 323 | blob[img_w*h + w] = ((float)img.at(h, w)) / 255.f; 324 | } 325 | } 326 | return blob; 327 | } 328 | 329 | bool match_lm_trt::build_model() 330 | { 331 | initLibNvInferPlugins(&gLogger.getTRTLogger(), ""); 332 | nvinfer1::IRuntime* runtime = nvinfer1::createInferRuntime(gLogger); 333 | if (!runtime) 334 | { 335 | return false; 336 | } 337 | 338 | char* model_deser_buffer{ nullptr }; 339 | const std::string engine_file_path(_match_lm_params.match_weight_file); 340 | std::ifstream ifs; 341 | int ser_length; 342 | ifs.open(engine_file_path.c_str(), std::ios::in | std::ios::binary); 343 | if (ifs.is_open()) 344 | { 345 | ifs.seekg(0, std::ios::end); 346 | ser_length = ifs.tellg(); 347 | ifs.seekg(0, std::ios::beg); 348 | model_deser_buffer = new char[ser_length]; 349 | ifs.read(model_deser_buffer, ser_length); 350 | ifs.close(); 351 | } 352 | else 353 | { 354 | return false; 355 | } 356 | 357 | _engine_ptr = std::shared_ptr(runtime->deserializeCudaEngine(model_deser_buffer, ser_length, nullptr), infer_deleter()); 358 | if (!_engine_ptr) 359 | { 360 | std::cout << "load engine failed!" << std::endl; 361 | return false; 362 | } 363 | delete[] model_deser_buffer; 364 | 365 | _context_ptr = std::shared_ptr(_engine_ptr->createExecutionContext(), infer_deleter()); 366 | if (!_context_ptr) 367 | { 368 | return false; 369 | } 370 | return true; 371 | } 372 | 373 | bool match_lm_trt::initial_match_model() 374 | { 375 | bool ret = build_model(); 376 | if (!ret) 377 | { 378 | std::cout << "build model is failed!" << std::endl; 379 | } 380 | return ret; 381 | } 382 | 383 | size_t match_lm_trt::forward(data_point & dp0, data_point & dp1, std::vector &vdm) 384 | { 385 | float * keypoint0 = new float[dp0.keypoint_size * 2]; 386 | float * scores0 = new float[dp0.keypoint_size]; 387 | float * descriptors0 = dp0.descriptors; 388 | float * keypoint1 = new float[dp1.keypoint_size * 2]; 389 | float * scores1 = new float[dp1.keypoint_size]; 390 | float * descriptors1 = dp1.descriptors; 391 | prepare_data(dp0, keypoint0, scores0); 392 | prepare_data(dp1, keypoint1, scores1); 393 | int batch_size = _engine_ptr->getMaxBatchSize(); 394 | int keypoints0_inputIndex = _engine_ptr->getBindingIndex(_match_lm_params.input_names[0].c_str()); 395 | assert(_engine_ptr->getBindingDataType(keypoints0_inputIndex) == nvinfer1::DataType::kFLOAT); 396 | int keypoints1_inputIndex = _engine_ptr->getBindingIndex(_match_lm_params.input_names[1].c_str()); 397 | assert(_engine_ptr->getBindingDataType(keypoints1_inputIndex) == nvinfer1::DataType::kFLOAT); 398 | int descriptors0_inputIndex = _engine_ptr->getBindingIndex(_match_lm_params.input_names[2].c_str()); 399 | assert(_engine_ptr->getBindingDataType(descriptors0_inputIndex) == nvinfer1::DataType::kFLOAT); 400 | int descriptors1_inputIndex = _engine_ptr->getBindingIndex(_match_lm_params.input_names[3].c_str()); 401 | assert(_engine_ptr->getBindingDataType(descriptors1_inputIndex) == nvinfer1::DataType::kFLOAT); 402 | int scores0_inputIndex = _engine_ptr->getBindingIndex(_match_lm_params.input_names[4].c_str()); 403 | assert(_engine_ptr->getBindingDataType(scores0_inputIndex) == nvinfer1::DataType::kFLOAT); 404 | int scores1_inputIndex = _engine_ptr->getBindingIndex(_match_lm_params.input_names[5].c_str()); 405 | assert(_engine_ptr->getBindingDataType(scores1_inputIndex) == nvinfer1::DataType::kFLOAT); 406 | int scores_outputIndex = _engine_ptr->getBindingIndex(_match_lm_params.output_names[0].c_str()); 407 | assert(_engine_ptr->getBindingDataType(scores_outputIndex) == nvinfer1::DataType::kFLOAT); 408 | void* buffers[7]; 409 | CHECK(cudaMalloc(&buffers[keypoints0_inputIndex], dp0.keypoint_size * 2 * sizeof(float))); 410 | CHECK(cudaMalloc(&buffers[keypoints1_inputIndex], dp1.keypoint_size * 2 * sizeof(float))); 411 | CHECK(cudaMalloc(&buffers[descriptors0_inputIndex], dp0.keypoint_size * 256 * sizeof(float))); 412 | CHECK(cudaMalloc(&buffers[descriptors1_inputIndex], dp1.keypoint_size * 256 * sizeof(float))); 413 | CHECK(cudaMalloc(&buffers[scores0_inputIndex], dp0.keypoint_size * sizeof(float))); 414 | CHECK(cudaMalloc(&buffers[scores1_inputIndex], dp1.keypoint_size * sizeof(float))); 415 | CHECK(cudaMalloc(&buffers[scores_outputIndex], dp0.keypoint_size * dp1.keypoint_size * sizeof(float))); 416 | 417 | cudaStream_t stream; 418 | CHECK(cudaStreamCreate(&stream)); 419 | CHECK(cudaMemcpyAsync(buffers[keypoints0_inputIndex], keypoint0, dp0.keypoint_size * 2 * sizeof(float), cudaMemcpyHostToDevice, stream)); 420 | CHECK(cudaMemcpyAsync(buffers[keypoints1_inputIndex], keypoint1, dp1.keypoint_size * 2 * sizeof(float), cudaMemcpyHostToDevice, stream)); 421 | CHECK(cudaMemcpyAsync(buffers[descriptors0_inputIndex], descriptors0, dp0.keypoint_size * 256 * sizeof(float), cudaMemcpyHostToDevice, stream)); 422 | CHECK(cudaMemcpyAsync(buffers[descriptors1_inputIndex], descriptors1, dp1.keypoint_size * 256 * sizeof(float), cudaMemcpyHostToDevice, stream)); 423 | CHECK(cudaMemcpyAsync(buffers[scores0_inputIndex], scores0, dp0.keypoint_size * sizeof(float), cudaMemcpyHostToDevice, stream)); 424 | CHECK(cudaMemcpyAsync(buffers[scores1_inputIndex], scores1, dp1.keypoint_size * sizeof(float), cudaMemcpyHostToDevice, stream)); 425 | _context_ptr->setBindingDimensions(keypoints0_inputIndex, nvinfer1::Dims2(dp0.keypoint_size, 2)); 426 | _context_ptr->setBindingDimensions(keypoints1_inputIndex, nvinfer1::Dims2(dp1.keypoint_size, 2)); 427 | 428 | _context_ptr->setBindingDimensions(descriptors0_inputIndex, nvinfer1::Dims2(256, dp0.keypoint_size)); 429 | _context_ptr->setBindingDimensions(descriptors1_inputIndex, nvinfer1::Dims2(256, dp1.keypoint_size)); 430 | 431 | nvinfer1::Dims ddi0, ddi1; 432 | ddi0.nbDims = 1; 433 | ddi0.d[0] = dp0.keypoint_size; 434 | ddi1.nbDims = 1; 435 | ddi1.d[0] = dp1.keypoint_size; 436 | 437 | _context_ptr->setBindingDimensions(scores0_inputIndex, ddi0); 438 | _context_ptr->setBindingDimensions(scores1_inputIndex, ddi1); 439 | bool status = _context_ptr->enqueue(batch_size, buffers, stream, nullptr); 440 | if (!status) 441 | { 442 | std::cout << "execute ifer error! " << std::endl; 443 | return 101; 444 | } 445 | float * scores_output = new float[dp0.keypoint_size*dp1.keypoint_size]; 446 | CHECK(cudaMemcpyAsync(scores_output, buffers[scores_outputIndex], dp0.keypoint_size*dp1.keypoint_size * sizeof(float), cudaMemcpyDeviceToHost, stream)); 447 | 448 | cudaStreamDestroy(stream); 449 | CHECK(cudaFree(buffers[keypoints0_inputIndex])); 450 | CHECK(cudaFree(buffers[keypoints1_inputIndex])); 451 | CHECK(cudaFree(buffers[descriptors0_inputIndex])); 452 | CHECK(cudaFree(buffers[descriptors1_inputIndex])); 453 | 454 | CHECK(cudaFree(buffers[scores0_inputIndex])); 455 | CHECK(cudaFree(buffers[scores1_inputIndex])); 456 | CHECK(cudaFree(buffers[scores_outputIndex])); 457 | 458 | int scores_output_h = dp0.keypoint_size; 459 | int scores_output_w = dp1.keypoint_size; 460 | float bin_score = 4.4124f; 461 | int iters = 20; 462 | float*lopt_scores{nullptr}; 463 | lopt_scores = log_optimal_transport(scores_output, bin_score, iters, scores_output_h, scores_output_w); 464 | delete[]scores_output; 465 | scores_output = nullptr; 466 | int ret_gmk = get_match_keypoint(dp0, dp1, lopt_scores, scores_output_h, scores_output_w, vdm); 467 | delete[] keypoint0; 468 | delete[] scores0; 469 | delete[] descriptors0; 470 | delete[] keypoint1; 471 | delete[] scores1; 472 | delete[] descriptors1; 473 | delete[] lopt_scores; 474 | keypoint0 = nullptr; 475 | scores0 = nullptr; 476 | descriptors0 = nullptr; 477 | keypoint1 = nullptr; 478 | scores1 = nullptr; 479 | descriptors1 = nullptr; 480 | lopt_scores = nullptr; 481 | return ret_gmk; 482 | } 483 | 484 | int match_lm_trt::get_match_keypoint(data_point & dp0, data_point & dp1, 485 | float * lopt_scores, int scores_output_h, int scores_output_w, std::vector &vdm) 486 | { 487 | std::vector vmi0, vmi1; 488 | for (int i = 0; i < scores_output_h + 1; i++) 489 | { 490 | float temp_max_value0 = lopt_scores[0]; 491 | maxv_indices mi0; 492 | for (int j = 0; j < scores_output_w + 1; j++) 493 | { 494 | if ((j + 1) % (scores_output_w + 1) != 0 && i != scores_output_h) 495 | { 496 | float current_value = lopt_scores[i*(scores_output_w + 1) + j]; 497 | if (temp_max_value0=10 && vmi1_size >= 10) 535 | { 536 | for (int i = 0; i < vmi0.size(); i++) 537 | { 538 | int vmi0_index = vmi0[i].indices; 539 | if (vmi1[vmi0_index].indices == i) 540 | { 541 | float temp_mscores0 = std::exp(vmi1[vmi0[i].indices].max_value); 542 | if (temp_mscores0>_match_lm_params.match_threshold) 543 | { 544 | data_match dm0; 545 | dm0.msscores0 = temp_mscores0; 546 | dm0.valid_keypoint_x0 = dp0.vdt[i].x; 547 | dm0.valid_keypoint_y0 = dp0.vdt[i].y; 548 | 549 | if (vmi0[i].indices>dp1.keypoint_size) 550 | { 551 | std::cout << "get valid keypoint need carefull" << std::endl; 552 | } 553 | dm0.valid_keypoint_x1 = dp1.vdt[vmi0[i].indices].x; 554 | dm0.valid_keypoint_y1 = dp1.vdt[vmi0[i].indices].y; 555 | vdm.emplace_back(dm0); 556 | } 557 | } 558 | } 559 | return vdm.size(); 560 | } 561 | else 562 | { 563 | return 0; 564 | } 565 | 566 | 567 | } 568 | 569 | 570 | float * match_lm_trt::log_optimal_transport(float * scores, float bin_score, int iters, 571 | int scores_output_h, int scores_output_w) 572 | { 573 | float norm = -std::log(scores_output_h + scores_output_w); 574 | int socres_new_size = scores_output_h*scores_output_w + scores_output_h + scores_output_w + 1; 575 | float * scores_new = new float[socres_new_size]; 576 | 577 | for (int i = 0; i < scores_output_h+1; i++) 578 | { 579 | for (int j = 0; j < scores_output_w+1; j++) 580 | { 581 | if ((j+1)% (scores_output_w+1)==0 || i==scores_output_h) 582 | { 583 | scores_new[i*(scores_output_w + 1)+j] = bin_score; 584 | } 585 | else 586 | { 587 | scores_new[i*(scores_output_w + 1) + j] = scores[i*scores_output_w + j]; 588 | } 589 | } 590 | } 591 | 592 | float * log_mu = new float[scores_output_h + 1]; 593 | float * log_nu = new float[scores_output_w + 1]; 594 | for (int i = 0; i < scores_output_h + 1; i++) 595 | { 596 | if (i==scores_output_h){ 597 | log_mu[i] = std::log(scores_output_w) + norm; 598 | } 599 | else{ 600 | log_mu[i] = norm; 601 | } 602 | } 603 | 604 | for (int i = 0; i < scores_output_w + 1; i++) 605 | { 606 | if (i == scores_output_w){ 607 | log_nu[i] = std::log(scores_output_h) + norm; 608 | } 609 | else{ 610 | log_nu[i] = norm; 611 | } 612 | } 613 | 614 | float * v = new float[scores_output_w + 1]; 615 | float * u = new float[scores_output_h + 1]; 616 | memset(v, 0.f, (scores_output_w + 1) * sizeof(float)); 617 | memset(u, 0.f, (scores_output_h + 1) * sizeof(float)); 618 | 619 | for (int iter = 0; iter < iters; iter++) 620 | { 621 | for (int i = 0; i < scores_output_h+1; i++) 622 | { 623 | float zv_sum_exp = 0.f; 624 | for (int j = 0; j < scores_output_w+1; j++) 625 | { 626 | zv_sum_exp += std::exp(scores_new[i*(scores_output_w+1) + j] + v[j]); 627 | } 628 | u[i] = log_mu[i] - std::log(zv_sum_exp); 629 | } 630 | for (int i = 0; i < scores_output_w + 1; i++) 631 | { 632 | float zu_sum_exp = 0.f; 633 | for (int j = 0; j < scores_output_h + 1; j++) 634 | { 635 | zu_sum_exp += std::exp(scores_new[j*(scores_output_w+1) + i] + u[j]); 636 | } 637 | v[i] = log_nu[i] - std::log(zu_sum_exp); 638 | } 639 | } 640 | 641 | for (int i = 0; i < scores_output_h + 1; i++) 642 | { 643 | for (int j = 0; j < scores_output_w + 1; j++) 644 | { 645 | scores_new[i*(scores_output_w+1) + j] = scores_new[i*(scores_output_w+1) + j] + u[i] + v[j]-norm; 646 | } 647 | } 648 | delete[] log_mu; 649 | delete[] log_nu; 650 | delete[] u; 651 | delete[] v; 652 | log_mu = nullptr; 653 | log_nu = nullptr; 654 | u = nullptr; 655 | v = nullptr; 656 | 657 | return scores_new; 658 | } 659 | 660 | void match_lm_trt::prepare_data(data_point & dp, float * arr1, float * arr2) 661 | { 662 | int count_k = 0; 663 | int count_s = 0; 664 | for (auto & dpv : dp.vdt) 665 | { 666 | arr1[count_k] = dpv.x; 667 | arr1[count_k + 1] = dpv.y; 668 | count_k += 2; 669 | 670 | arr2[count_s] = dpv.s; 671 | count_s++; 672 | } 673 | } 674 | 675 | 676 | match_lm_trt::match_lm_trt() 677 | { 678 | cudaSetDevice(0); 679 | _match_lm_params.input_names.push_back("keypoints0"); 680 | _match_lm_params.input_names.push_back("keypoints1"); 681 | _match_lm_params.input_names.push_back("descriptors0"); 682 | _match_lm_params.input_names.push_back("descriptors1"); 683 | _match_lm_params.input_names.push_back("scores0"); 684 | _match_lm_params.input_names.push_back("scores1"); 685 | 686 | _match_lm_params.output_names.push_back("scores"); 687 | _match_lm_params.dla_core = -1; 688 | _match_lm_params.int8 = false; 689 | _match_lm_params.fp16 = false; 690 | _match_lm_params.batch_size = 1; 691 | _match_lm_params.seria = false; 692 | _match_lm_params.match_weight_file = "../engines/match_model_indoor.bin"; 693 | _match_lm_params.match_threshold = 0.2f; 694 | 695 | } 696 | 697 | match_lm_trt::~match_lm_trt() 698 | { 699 | } 700 | -------------------------------------------------------------------------------- /src/match.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Time : 2022/01/01 3 | * Github: https://github.com/Broad-sky/feature-detection-matching-algorithm 4 | * Author: pingsheng shen 5 | */ 6 | 7 | #include"match.h" 8 | 9 | 10 | void image_registration::initial_model() 11 | { 12 | _plt = new point_lm_trt(); 13 | _plt->initial_point_model(); 14 | 15 | _mlt = new match_lm_trt(); 16 | _mlt->initial_match_model(); 17 | } 18 | 19 | 20 | image_registration::image_registration() 21 | { 22 | } 23 | 24 | image_registration::~image_registration() 25 | { 26 | delete _plt; 27 | delete _mlt; 28 | std::cout << "~image_registration()" << "\n"; 29 | } 30 | 31 | int image_registration::dplearning_method_forward(std::vector &vdi, std::vector & vdm) 32 | { 33 | int ret_mlt1 = 0; 34 | if (!vdi[0].srcimg0_gray.empty() && !vdi[0].srcimg1_gray.empty()) 35 | { 36 | data_point dp0, dp1; 37 | cv::resize(vdi[0].srcimg0_gray, vdi[0].resized_srcimg0, cv::Size(_plt->get_input_w(), _plt->get_input_h())); // INTER_LINEAR 38 | int ret_plt0 = _plt->forward(vdi[0].resized_srcimg0, dp0); 39 | 40 | cv::resize(vdi[0].srcimg1_gray, vdi[0].resized_srcimg1, cv::Size(_plt->get_input_w(), _plt->get_input_h())); 41 | int ret_plt1 = _plt->forward(vdi[0].resized_srcimg1, dp1); 42 | //assert(dp0.status_code == 1); 43 | //assert(dp1.status_code == 1); 44 | 45 | if (dp1.keypoint_size>=10) 46 | { 47 | ret_mlt1 = _mlt->forward(dp0, dp1, vdm); 48 | return ret_mlt1; 49 | } 50 | return -1; 51 | } 52 | else 53 | { 54 | std::cerr << "image0 or image1 is invalid!" << std::endl; 55 | return -1; 56 | } 57 | } 58 | 59 | 60 | int image_registration::traditional_method_forward(std::vector &vdi, std::vector&vdm, std::string md_name) 61 | { 62 | if (!vdi[0].srcimg0_gray.empty() && !vdi[0].srcimg1_gray.empty()) 63 | { 64 | int ret = 0; 65 | cv::resize(vdi[0].srcimg0_gray, vdi[0].resized_srcimg0, cv::Size(320, 240)); // INTER_LINEAR 66 | cv::resize(vdi[0].srcimg1_gray, vdi[0].resized_srcimg1, cv::Size(320, 240)); 67 | if (md_name=="akaze") 68 | { 69 | auto _am_ptr = std::unique_ptr(new akaze_match(4.5f, 0.8f)); 70 | ret = _am_ptr->akaze_forward(vdi[0].resized_srcimg0, vdi[0].resized_srcimg1, vdm); 71 | return ret; 72 | } 73 | else 74 | { 75 | std::cout << "traditional method akaze vdm size: " << vdm.size() << "\n"; 76 | return ret; 77 | } 78 | 79 | } 80 | else 81 | { 82 | std::cerr << "image0 or image1 is invalid!" << std::endl; 83 | return -1; 84 | } 85 | } 86 | 87 | IrInterface * IrInterface::createIrInterface() 88 | { 89 | return new image_registration(); 90 | } 91 | --------------------------------------------------------------------------------