├── .gitignore ├── LICENCE ├── README.md ├── adapters.py ├── assets ├── results.png └── sequences.png ├── const.py ├── custom_impl.py ├── eval_classic.py ├── eval_deep.py ├── evaluations.py ├── jupyter ├── 1iteration.ipynb ├── 2iteration.ipynb ├── Homography.ipynb ├── PatchVerificationGeneralizacija.ipynb └── Testing.ipynb ├── other_algorithms ├── __init__.py ├── asift.py ├── masks.py └── susan.py ├── utils.py └── visualizations.py /.gitignore: -------------------------------------------------------------------------------- 1 | results 2 | data/descriptors 3 | vlfeat 4 | bin/hb 5 | bin/hb_run.sh 6 | bin/*.log 7 | bin/*.txt 8 | bin/*.sh 9 | *~ 10 | *.zip 11 | descrs/** 12 | tmp/** 13 | hpatch-release/** 14 | results/** 15 | .#* 16 | *.pyc 17 | data/descriptors/** 18 | data/hpatches-release/** 19 | 20 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2016 Karel Lenc, Vasileios Balntas 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the 12 | distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 15 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 17 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 18 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 20 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Detector-descriptor benchmark 2 | 3 | This is a benchmark for evaluation of state-of-the-art detector and descriptor algorithms, a source code for the "On the Comparison of Classic and Deep Keypoint Detector and Descriptor Methods" paper published on [ISPA](https://www.isispa.org/) conference: https://arxiv.org/abs/2007.10000 [[1]](#refs). 4 | 5 | ### HPSequences dataset 6 | 7 | ![hpsequences](assets/sequences.png) 8 | 9 | Download dataset: [HPSequences dataset](http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz) [1.3GB] 10 | 11 | Place the directory in a convenient location. The folder `hpatches-sequences-release` contains all the 116 directories, 57 of which represent only photometric changes, whereas 59 represent only geometric deformations. Each sequence consists of one reference image and 5 target images representing the appropriate illumination or viewpoint changes. Alongisde every target image there is a homography connecting it to the reference image (stored in files `H_1_`). In case of an illumination change sequence, the homography is an identity mapping. 12 | 13 | The sequence folders are named with the following convention: 14 | 15 | * `i_X`: image sequences with illumination changes 16 | * `v_X`: image sequences with viewpoint changes 17 | 18 | ### Results 19 | 20 | ![results](assets/results.png) 21 | 22 | ### Remarks 23 | 24 | This benchmark is based on the HPatches evaluation tasks [[2]](#refs) and HPSequences dataset published along with it ([HPatches dataset repository](https://github.com/hpatches/hpatches-dataset)). Thanks to the authors for providing the dataset and the evaluation details. 25 | 26 | ### References 27 | 28 | 29 | [1] *On the Comparison of Classic and Deep Keypoint Detector and Descriptor Methods*, Kristijan Bartol*, David Bojanić*, Tomislav Pribanić, Tomislav Petković, Yago Diez Donoso, Joaquim Salvi Mas, ISPA 2019. 30 | *Authors contributed equally. 31 | 32 | [2] *HPatches: A benchmark and evaluation of handcrafted and learned local descriptors*, Vassileios Balntas*, Karel Lenc*, Andrea Vedaldi and Krystian Mikolajczyk, CVPR 2017. 33 | *Authors contributed equally. 34 | -------------------------------------------------------------------------------- /adapters.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from os import listdir 3 | from os.path import join, isfile 4 | 5 | from const import * 6 | 7 | DATASET_ROOT = '../../../hpatches-sequences-release/' 8 | KPZ_FILENAME = 'kp.npz' 9 | DESC_FILENAME = 'des.npz' 10 | 11 | 12 | def adapt_lfnet(): 13 | algo_name = 'lfnet' 14 | 15 | for folder in listdir(DATASET_ROOT): 16 | print('Processing directory: {}'.format(folder)) 17 | folder_path = join(DATASET_ROOT, folder) 18 | kp_path = join(folder_path, KPZ_FILENAME) 19 | desc_path = join(folder_path, DESC_FILENAME) 20 | results_dir = join(folder_path, 'out') 21 | 22 | kps = [0.] * 6 23 | descs = [0.] * 6 24 | 25 | for img_result_filename in filter(lambda f: 'ppm.npz' in f, listdir(results_dir)): 26 | # Take index from filename as filter() doesn't keep the order. 27 | img_idx = int(img_result_filename[0]) 28 | img_result_path = join(results_dir, img_result_filename) 29 | img_result = dict(np.load(img_result_path)) 30 | 31 | orig_img = cv2.imread(join(folder_path, '{}.ppm'.format(img_idx))) 32 | processed_img = cv2.imread(join(results_dir, '{}.ppm'.format(img_idx))) 33 | 34 | x_scale = float(orig_img.shape[1]) / float(processed_img.shape[1]) 35 | y_scale = float(orig_img.shape[0]) / float(processed_img.shape[0]) 36 | 37 | img_result['kpts'][:,0] *= x_scale 38 | img_result['kpts'][:,1] *= y_scale 39 | 40 | kps[img_idx - 1] = img_result['kpts'] 41 | descs[img_idx - 1] = img_result['descs'] 42 | 43 | kp = dict(np.load(kp_path, allow_pickle=True)) if isfile(kp_path) else dict() 44 | desc = dict(np.load(desc_path, allow_pickle=True)) if isfile(desc_path) else dict() 45 | kp[algo_name] = np.array(kps) 46 | desc[ALGO_TEMPLATE.format(algo_name, algo_name)] = np.array(descs) 47 | 48 | np.savez(kp_path, **kp) 49 | np.savez(desc_path, **desc) 50 | 51 | 52 | def adapt_superpoint(): 53 | algo_name = 'superpoint' 54 | x_dim = 320. 55 | y_dim = 240. 56 | 57 | for folder in listdir(DATASET_ROOT): 58 | print('Processing directory: {}'.format(folder)) 59 | folder_path = join(DATASET_ROOT, folder) 60 | kp_path = join(folder_path, KPZ_FILENAME) 61 | desc_path = join(folder_path, DESC_FILENAME) 62 | results_dir = join(folder_path, algo_name) 63 | 64 | kps = [0.] * 6 65 | descs = [0.] * 6 66 | 67 | for img_result_filename in filter(lambda f: 'ppm.npz' in f, listdir(results_dir)): 68 | # Take index from filename as filter() doesn't keep the order. 69 | img_idx = int(img_result_filename[0]) 70 | img_result_path = join(results_dir, img_result_filename) 71 | img_result = dict(np.load(img_result_path)) 72 | 73 | orig_img = cv2.imread(join(folder_path, '{}.ppm'.format(img_idx))) 74 | processed_img = cv2.imread(join(results_dir, '{}.ppm'.format(img_idx))) 75 | 76 | x_scale = float(orig_img.shape[1]) / x_dim 77 | y_scale = float(orig_img.shape[0]) / y_dim 78 | 79 | img_result['kpts'][:,0] *= x_scale 80 | img_result['kpts'][:,1] *= y_scale 81 | 82 | kps[img_idx - 1] = img_result['kpts'].T 83 | descs[img_idx - 1] = img_result['descs'].T 84 | 85 | kp = dict(np.load(kp_path, allow_pickle=True)) if isfile(kp_path) else dict() 86 | desc = dict(np.load(desc_path, allow_pickle=True)) if isfile(desc_path) else dict() 87 | kp[algo_name] = np.array(kps) 88 | desc[ALGO_TEMPLATE.format(algo_name, algo_name)] = np.array(descs) 89 | 90 | np.savez(kp_path, **kp) 91 | np.savez(desc_path, **desc) 92 | 93 | 94 | def adapt_d2net(): 95 | algo_name = 'd2net' 96 | x_dim = 320. 97 | y_dim = 240. 98 | 99 | dataset_root = '/home/kristijan/hpatches-sequences-release/i_ajuntament/hpatches-sequences-release' 100 | 101 | for folder in listdir(dataset_root): 102 | print('Processing directory: {}'.format(folder)) 103 | folder_path = join(dataset_root, folder) 104 | kp_path = join(folder_path, KPZ_FILENAME) 105 | desc_path = join(folder_path, DESC_FILENAME) 106 | # Results dir is folder dir for D2Net. 107 | results_dir = folder_path 108 | 109 | kps = [0.] * 6 110 | descs = [0.] * 6 111 | 112 | for img_result_filename in filter(lambda f: 'd2-net' in f, listdir(results_dir)): 113 | # Take index from filename as filter() doesn't keep the order. 114 | img_idx = int(img_result_filename[0]) 115 | img_result_path = join(results_dir, img_result_filename) 116 | img_result = dict(np.load(img_result_path)) 117 | 118 | #orig_img = cv2.imread(join(folder_path, '{}.ppm'.format(img_idx))) 119 | #processed_img = cv2.imread(join(results_dir, '{}.ppm'.format(img_idx))) 120 | 121 | #x_scale = float(orig_img.shape[1]) / x_dim 122 | #y_scale = float(orig_img.shape[0]) / y_dim 123 | 124 | #img_result['kpts'][:,0] *= x_scale 125 | #img_result['kpts'][:,1] *= y_scale 126 | 127 | print(img_result_path) 128 | 129 | kps[img_idx - 1] = img_result['keypoints'][:, :2] 130 | descs[img_idx - 1] = img_result['descriptors'][:, :2] 131 | 132 | kp = dict(np.load(kp_path, allow_pickle=True)) if isfile(kp_path) else dict() 133 | desc = dict(np.load(desc_path, allow_pickle=True)) if isfile(desc_path) else dict() 134 | kp[algo_name] = np.array(kps) 135 | desc[ALGO_TEMPLATE.format(algo_name, algo_name)] = np.array(descs) 136 | 137 | np.savez(kp_path, **kp) 138 | np.savez(desc_path, **desc) 139 | 140 | 141 | if __name__ == '__main__': 142 | import argparse 143 | 144 | parser_of_args = argparse.ArgumentParser(description='Select algorithm to adapt') 145 | 146 | parser_of_args.add_argument('--algorithm', type=str, 147 | help='name of the algorithm') 148 | 149 | args = parser_of_args.parse_args() 150 | 151 | result = locals()['adapt_{}'.format(args.algorithm)]() 152 | -------------------------------------------------------------------------------- /assets/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kristijanbartol/keypoint-algorithms-benchmark/d3bf05220fd697fececb293ba139ca5c563acc21/assets/results.png -------------------------------------------------------------------------------- /assets/sequences.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kristijanbartol/keypoint-algorithms-benchmark/d3bf05220fd697fececb293ba139ca5c563acc21/assets/sequences.png -------------------------------------------------------------------------------- /const.py: -------------------------------------------------------------------------------- 1 | ############## 2 | # ALGORITHMS # 3 | ############## 4 | SIFT = 'sift' 5 | ASIFT = 'asift' 6 | SURF = 'surf' 7 | BRIEF = 'brief' 8 | BRISK = 'brisk' 9 | ORB = 'orb' 10 | FAST = 'fast' 11 | HARRIS = 'harris' 12 | SHI_TOMASI = 'shi_tomasi' 13 | KAZE = 'kaze' 14 | AKAZE = 'akaze' 15 | MSER = 'mser' 16 | AGAST = 'agast' 17 | GFTT = 'gftt' 18 | CENSUREE = 'censure' 19 | ROOT_SIFT = 'root_sift' 20 | SUSAN = 'susan' 21 | 22 | # descriptor-only 23 | FREAK = 'freak' 24 | 25 | # deep 26 | LFNET = 'lfnet' 27 | SUPERPOINT = 'superpoint' 28 | D2NET = 'd2net' 29 | 30 | # Det+desc template 31 | ALGO_TEMPLATE = '{}_{}' 32 | 33 | #################### 34 | # EVALUATION TASKS # 35 | #################### 36 | 37 | VERIFICATION = 'Verification' 38 | MATCHING = 'Matching' 39 | RETRIEVAL = 'Retrieval' 40 | 41 | TASKS = [VERIFICATION, MATCHING, RETRIEVAL] 42 | 43 | ########################### 44 | # ALGORITHM DICTIONARIES # 45 | ########################## 46 | import cv2 47 | 48 | all_detectors = {'sift':cv2.xfeatures2d.SIFT_create, 49 | 'surf':cv2.xfeatures2d.SURF_create, 50 | 'orb':cv2.ORB_create, 51 | 'fast':cv2.FastFeatureDetector_create, 52 | 'brisk':cv2.BRISK_create, 53 | #'harris':HarrisMataHarris, 54 | #'shi_tomasi':ShiTomasi, 55 | 'kaze':cv2.KAZE_create, 56 | 'akaze':cv2.AKAZE_create, 57 | 'mser':cv2.MSER_create, 58 | 'agast':cv2.AgastFeatureDetector_create, 59 | 'gftt':cv2.GFTTDetector_create, 60 | #'censure':CensureClass, 61 | #'asift':ASIFTClass 62 | #'susan':SusanClass 63 | } 64 | 65 | all_descriptors = {'sift':cv2.xfeatures2d.SIFT_create, 66 | 'surf':cv2.xfeatures2d.SURF_create, 67 | 'orb':cv2.ORB_create, 68 | 'brisk':cv2.BRISK_create, 69 | 'freak':cv2.xfeatures2d.FREAK_create, 70 | 'kaze':cv2.KAZE_create, 71 | 'akaze':cv2.AKAZE_create, 72 | 'brief':cv2.xfeatures2d.BriefDescriptorExtractor_create, 73 | #'root_sift':RootSIFT 74 | } 75 | 76 | descriptor_distance = {'sift':cv2.NORM_L2, 77 | 'surf':cv2.NORM_L2, 78 | 'orb':cv2.NORM_HAMMING2, 79 | 'brisk':cv2.NORM_HAMMING2, 80 | 'freak':cv2.NORM_HAMMING2, 81 | 'kaze':cv2.NORM_L2, 82 | 'akaze':cv2.NORM_HAMMING2, 83 | 84 | 'brief':cv2.NORM_HAMMING2, 85 | 'root_sift':cv2.NORM_L2, 86 | 'lfnet':cv2.NORM_L2, 87 | 'superpoint':cv2.NORM_L2, 88 | 'd2net':cv2.NORM_L2} 89 | 90 | all_at_once_dict = {'sift':False, 91 | 'surf':False, 92 | 'orb':False, 93 | 'fast':False, 94 | 'brisk':False, 95 | 'harris':False, 96 | 'shi_tomasi':False, 97 | 'kaze':False, 98 | 'akaze':False, 99 | 'mser':False, 100 | 'agast':False, 101 | 'gftt':False, 102 | 'censure':False, 103 | 'asift':True, 104 | #'susan':False 105 | 106 | 107 | # deep 108 | 'lfnet':True, 109 | 'superpoint':True} 110 | 111 | ######### 112 | # PATHS # 113 | ######### 114 | 115 | RESULTS_DIR = 'results/' 116 | -------------------------------------------------------------------------------- /custom_impl.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import cv2 4 | import numpy as np 5 | import skimage.feature 6 | from other_algorithms.asift import affine_detect 7 | from other_algorithms.susan import susanCorner 8 | 9 | # DETECTORS 10 | class HarrisMataHarris(): 11 | ''' 12 | Wrapper for the OpenCV implementation of the Harris detector. 13 | ''' 14 | def __init__(self, blockSize=2, ksize=3, k=0.04, borderType=cv2.BORDER_DEFAULT): 15 | # self.src = src 16 | self.blockSize = blockSize 17 | self.ksize = ksize 18 | self.k = k 19 | self.borderType = borderType 20 | 21 | def detect(self, img, mask=None): 22 | ''' 23 | Keypoint detector method. The threshold is set to 0.01. 24 | The parameter mask is set to keep standardized inputs for the detect method in the openCv implementations. 25 | ''' 26 | keypoints = [] 27 | 28 | # already taking care of this in the implementation 29 | # if len(img.shape) == 3: 30 | # img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 31 | 32 | dst = cv2.cornerHarris(img,self.blockSize,self.ksize,self.k) 33 | indexes = np.where(dst>0.01*dst.max()) 34 | 35 | for i in range(len(indexes[0])): 36 | keypoints.append(cv2.KeyPoint(x=int(indexes[1][i]), y=int(indexes[0][i]), _size=0)) 37 | 38 | return keypoints 39 | 40 | 41 | class ShiTomasi(): 42 | ''' 43 | other parameters are left to the default OpenCv parameters 44 | ''' 45 | def __init__(self, maxCorners=1000000, qualityLevel=0.01,minDistance=10): 46 | self.maxCorners = maxCorners 47 | self.qualityLevel = qualityLevel 48 | self.minDistance = minDistance 49 | 50 | def detect(self, img, mask=None): 51 | keypoints = [] 52 | 53 | # Already taken care of in the implementation 54 | # if len(img.shape) == 3: 55 | # img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 56 | 57 | corners = cv2.goodFeaturesToTrack(img, 58 | self.maxCorners, 59 | self.qualityLevel, 60 | self.minDistance) 61 | corners = np.int0(corners) 62 | corners = corners[:,0,:] 63 | 64 | for i in range(corners.shape[0]): 65 | keypoints.append(cv2.KeyPoint(x=int(corners[i,0]), 66 | y=int(corners[i,1]), 67 | _size=0)) 68 | 69 | return keypoints 70 | 71 | 72 | class CensureClass(): 73 | 74 | def detect(self, img, mask=None): 75 | keypoints = [] 76 | 77 | censure = skimage.feature.CENSURE() 78 | censure.detect(img) 79 | 80 | for i in range(censure.keypoints.shape[0]): 81 | keypoints.append(cv2.KeyPoint(x=censure.keypoints[i,1], 82 | y=censure.keypoints[i,0], 83 | _size=0)) 84 | 85 | return keypoints 86 | 87 | 88 | class SusanClass(): 89 | 90 | def detect(self, img, mask=None): 91 | keypoints = [] 92 | 93 | kp = susanCorner(img) 94 | 95 | kp = np.where(kp>0) 96 | 97 | for i in range(len(kp[0])): 98 | keypoints.append(cv2.KeyPoint(x=kp[0][i], 99 | y=kp[1][i], 100 | _size=0)) 101 | 102 | return keypoints 103 | 104 | 105 | # DESCRIPTORS 106 | class RootSIFT: 107 | def __init__(self, eps=1e-7): 108 | self.eps = eps 109 | 110 | 111 | def compute(self, image, kp): 112 | sift = cv2.xfeatures2d.SIFT_create() 113 | kp, des = sift.compute(image, kp) 114 | 115 | if len(kp) == 0: 116 | return kp, np.array([]) 117 | 118 | # apply the Hellinger kernel by first L1-normalizing and taking the 119 | # square-root 120 | des /= (des.sum(axis=1, keepdims=True) + self.eps) 121 | des = np.sqrt(des) 122 | #descs /= (np.linalg.norm(descs, axis=1, ord=2) + eps) 123 | 124 | return kp, des 125 | 126 | 127 | 128 | # DETECTOR + DESCRIPTOR 129 | class ASIFTClass(): 130 | 131 | def detectAndCompute(self, img, mask=None): 132 | kp, des = affine_detect(cv2.xfeatures2d.SIFT_create(), img) 133 | 134 | return kp, des 135 | -------------------------------------------------------------------------------- /eval_classic.py: -------------------------------------------------------------------------------- 1 | 2 | # PIPELINE2 JE ZA POKRENUTI SKUPINU det+des IN BULK 3 | import glob 4 | import cv2 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import math 8 | import re 9 | import os 10 | from matplotlib.patches import ConnectionPatch 11 | from sklearn.metrics import average_precision_score 12 | from random import sample 13 | import itertools 14 | from pprint import pprint 15 | import json 16 | #from skimage.feature import CENSURE 17 | 18 | from custom_impl import * 19 | from utils import * 20 | from evaluations import patchVerification, imageMatching, patchRetrieval 21 | #from visualizations import taskEvaluation 22 | 23 | # project_root = '/home/davidboja/PycharmProjects/FER/hpatches-benchmark/python/ISPA' 24 | # dataset_path = project_root + '/hpatches-sequences-release/*' 25 | dataset_path = '/home/dbojanic/hpatches-sequences-release/*' 26 | 27 | n = 100 28 | nr_iterations = 5 29 | failed_combinations = [] 30 | FILE_EXTENSION = '_zver3_3.txt' 31 | 32 | det_des_combinations = list(itertools.product(list(all_detectors.keys()), 33 | list(all_descriptors.keys()) ) 34 | ) 35 | 36 | combinations_to_remove = [(ORB, SIFT),(ORB, SURF),(ORB, BRISK),(ORB, FREAK),(ORB, KAZE),(ORB,AKAZE),(ORB, BRIEF),(ORB, ROOT_SIFT), 37 | (SIFT,ORB),(SURF,ORB),(FAST,ORB),(BRISK,ORB),(HARRIS,ORB),(SHI_TOMASI,ORB),(KAZE,ORB),(AKAZE,ORB),(MSER,ORB),(AGAST,ORB),(GFTT,ORB),(CENSUREE,ORB),(ASIFT,ORB),(SUSAN,ORB), 38 | (KAZE, SIFT),(KAZE, SURF),(KAZE, ORB),(KAZE, BRISK),(KAZE, FREAK),(KAZE, BRIEF),(KAZE, ROOT_SIFT), 39 | (SIFT,KAZE),(SURF,KAZE),(ORB, KAZE),(FAST,KAZE),(BRISK,KAZE),(HARRIS,KAZE),(SHI_TOMASI,KAZE),(AKAZE,KAZE),(MSER,KAZE),(AGAST,KAZE),(GFTT,KAZE),(CENSUREE,KAZE),(ASIFT,KAZE),(SUSAN,KAZE), 40 | (AKAZE, SIFT),(AKAZE, SURF),(AKAZE, ORB),(AKAZE, BRISK),(AKAZE, FREAK),(AKAZE, KAZE),(AKAZE, BRIEF),(AKAZE, ROOT_SIFT), 41 | (SIFT,AKAZE),(SURF,AKAZE),(ORB, AKAZE),(FAST,AKAZE),(BRISK,AKAZE),(HARRIS,AKAZE),(SHI_TOMASI,AKAZE),(MSER,AKAZE),(AGAST,AKAZE),(GFTT,AKAZE),(CENSUREE,AKAZE),(ASIFT,AKAZE),(SUSAN,AKAZE), 42 | (SURF, ROOT_SIFT),(ORB, ROOT_SIFT),(FAST, ROOT_SIFT),(BRISK, ROOT_SIFT),(HARRIS, ROOT_SIFT),(SHI_TOMASI,ROOT_SIFT),(KAZE, ROOT_SIFT),(AKAZE, ROOT_SIFT),(MSER, ROOT_SIFT),(AGAST, ROOT_SIFT),(GFTT, ROOT_SIFT),(CENSUREE, ROOT_SIFT),(ASIFT, ROOT_SIFT),(SUSAN, ROOT_SIFT), 43 | (ASIFT,SURF),(ASIFT,ORB),(ASIFT,BRISK),(ASIFT,FREAK),(ASIFT,KAZE),(ASIFT,AKAZE),(ASIFT,BRIEF),(ASIFT,ROOT_SIFT) 44 | ] 45 | 46 | # combinations_to_remove2 = [(SIFT,SIFT),(SIFT,SURF),(SIFT,BRISK),(SIFT,FREAK),(SIFT,BRIEF),(SIFT,ROOT_SIFT), 47 | # (SURF,SIFT),(SURF,SURF),(SURF,BRISK),(SURF,FREAK),(SURF,BRIEF), 48 | # (ORB,ORB), 49 | # (FAST,SIFT),(FAST,SURF),(FAST,BRISK),(FAST,FREAK),(FAST,BRIEF), 50 | # (BRISK,SIFT),(BRISK,SURF),(BRISK,BRISK),(BRISK,FREAK),(BRISK,BRIEF), 51 | # (HARRIS,SIFT),(HARRIS,BRISK),(HARRIS,FREAK),(HARRIS,BRIEF), 52 | # (SHI_TOMASI,SIFT),(SHI_TOMASI,BRISK),(SHI_TOMASI,FREAK),(SHI_TOMASI,BRIEF), 53 | # (KAZE,KAZE), 54 | # (AKAZE,AKAZE), 55 | # (MSER,SIFT),(MSER,SURF),(MSER,BRISK),(MSER,FREAK),(MSER,BRIEF), 56 | # (AGAST,SIFT),(AGAST,SURF),(AGAST,BRISK),(AGAST,FREAK),(AGAST,BRIEF), 57 | # (GFTT,SIFT),(GFTT,SURF),(GFTT,BRISK),(GFTT,FREAK),(GFTT,BRIEF)] 58 | 59 | # combinations_to_remove += combinations_to_remove2 60 | 61 | 62 | combinations_to_remove = list(set(combinations_to_remove)) 63 | 64 | for el in combinations_to_remove: 65 | try: 66 | det_des_combinations.remove(el) 67 | except Exception as e: 68 | print('{} NOT IN DICT'.format(el)) 69 | 70 | for dett, dess in det_des_combinations: 71 | print('Computing combination {}:{}'.format(dett,dess)) 72 | # Set Variables 73 | detector_name = dett 74 | descriptor_name = dess 75 | 76 | # Obtain keypoints 77 | try: 78 | createKeypoints(detector_name, descriptor_name, dataset_path,all_at_once_dict[detector_name]) 79 | removeUncommonPoints(detector_name, descriptor_name, dataset_path) 80 | except Exception as e: 81 | print('ERRORRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR!') 82 | print(e) 83 | with open('failed_combinations.txt','a+') as file: 84 | file.write(detector_name + '_' + descriptor_name +'\n') 85 | failed_combinations.append(detector_name + '_' + descriptor_name) 86 | continue 87 | 88 | # Evaluations 89 | list_of_APs, list_of_APs_i, list_of_APs_v = patchVerification(detector_name, 90 | descriptor_name, 91 | n, 92 | dataset_path, 93 | nr_iterations) 94 | 95 | with open('pV' + FILE_EXTENSION,'a+') as file: 96 | file.write('{}_{}|{}|{}|{}'.format(detector_name, 97 | descriptor_name, 98 | str(list_of_APs), 99 | str(list_of_APs_i), 100 | str(list_of_APs_v)) + '\n') 101 | 102 | list_of_mAPs, list_of_mAPs_i, list_of_mAPs_v = imageMatching(detector_name, 103 | descriptor_name, 104 | n, 105 | dataset_path, 106 | nr_iterations) 107 | with open('iM' + FILE_EXTENSION,'a+') as file: 108 | file.write('{}_{}|{}|{}|{}'.format(detector_name, 109 | descriptor_name, 110 | str(list_of_mAPs), 111 | str(list_of_mAPs_i), 112 | str(list_of_mAPs_v)) + '\n') 113 | 114 | list_of_mAPs, list_of_mAPs_i, list_of_mAPs_v = patchRetrieval(detector_name, 115 | descriptor_name, 116 | n, 117 | dataset_path, 118 | nr_iterations) 119 | 120 | with open('pR' + FILE_EXTENSION,'a+') as file: 121 | file.write('{}_{}|{}|{}|{}'.format(detector_name, 122 | descriptor_name, 123 | str(list_of_mAPs), 124 | str(list_of_mAPs_i), 125 | str(list_of_mAPs_v)) + '\n') 126 | -------------------------------------------------------------------------------- /eval_deep.py: -------------------------------------------------------------------------------- 1 | 2 | # PIPELINE 3 JE ZA POKRENUTI SAMO JEDAN det+des 3 | import glob 4 | import cv2 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import math 8 | import re 9 | import os 10 | from matplotlib.patches import ConnectionPatch 11 | from sklearn.metrics import average_precision_score 12 | from random import sample 13 | import itertools 14 | from pprint import pprint 15 | import json 16 | 17 | from utils import * 18 | from evaluations import patchVerification, imageMatching, patchRetrieval 19 | from visualizations import taskEvaluation 20 | from const import * 21 | 22 | 23 | project_root = '/home/kristijan/PhD/hpatches-benchmark/python/ISPA' 24 | dataset_path = '/home/kristijan/PhD/hpatches-sequences-release/*' 25 | n = 100 26 | nr_iterations = 5 27 | pV_results = {} 28 | iM_results = {} 29 | pR_results = {} 30 | FILE_EXTENSION = '_LFNET_NEW.txt' 31 | 32 | 33 | detector_name = LFNET 34 | descriptor_name = LFNET 35 | 36 | #createKeypoints(detector_name, descriptor_name, dataset_path) 37 | #removeUncommonPoints(detector_name, descriptor_name, dataset_path) 38 | 39 | # Evaluations 40 | list_of_APs, list_of_APs_i, list_of_APs_v = patchVerification(detector_name, 41 | descriptor_name, 42 | n, 43 | dataset_path, 44 | nr_iterations) 45 | 46 | with open('pV' + FILE_EXTENSION, 'a+') as file: 47 | file.write('{}_{}|{}|{}|{}'.format(detector_name, 48 | descriptor_name, 49 | str(list_of_APs), 50 | str(list_of_APs_i), 51 | str(list_of_APs_v)) + '\n') 52 | 53 | list_of_mAPs, list_of_mAPs_i, list_of_mAPs_v = imageMatching(detector_name, 54 | descriptor_name, 55 | n, 56 | dataset_path, 57 | nr_iterations) 58 | with open('iM' + FILE_EXTENSION, 'a+') as file: 59 | file.write('{}_{}|{}|{}|{}'.format(detector_name, 60 | descriptor_name, 61 | str(list_of_mAPs), 62 | str(list_of_mAPs_i), 63 | str(list_of_mAPs_v)) + '\n') 64 | 65 | list_of_mAPs, list_of_mAPs_i, list_of_mAPs_v = patchRetrieval(detector_name, 66 | descriptor_name, 67 | n, 68 | dataset_path, 69 | nr_iterations) 70 | 71 | with open('pR' + FILE_EXTENSION, 'a+') as file: 72 | file.write('{}_{}|{}|{}|{}'.format(detector_name, 73 | descriptor_name, 74 | str(list_of_mAPs), 75 | str(list_of_mAPs_i), 76 | str(list_of_mAPs_v)) + '\n') 77 | -------------------------------------------------------------------------------- /evaluations.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import glob 4 | import cv2 5 | import numpy as np 6 | import math 7 | import re 8 | import os 9 | from sklearn.metrics import average_precision_score 10 | from random import sample 11 | 12 | from utils import getTransformations, read_keypoints, read_next_keypoints 13 | from const import * 14 | 15 | 16 | 17 | def patchVerification(detector_name, descriptor_name, n, dataset_path, nr_of_iterations=1): 18 | ''' 19 | Task 1: Patch Verification 20 | 21 | Return: list_of_APs - list of APs per iteration 22 | list_of_APs_i - list of APs per iteration for only illumination folders 23 | list_of_APs_v - list of APs per iteration for only viewpoint folders 24 | ''' 25 | 26 | transformations = getTransformations(dataset_path) 27 | folders = glob.glob(dataset_path) 28 | list_of_APs = [] 29 | list_of_APs_i = [] 30 | list_of_APs_v = [] 31 | 32 | for i in range(nr_of_iterations): 33 | y = [] 34 | y_i = [] 35 | y_v = [] 36 | s = [] 37 | s_i = [] 38 | s_v = [] 39 | 40 | for folder_id, folder in enumerate(folders): 41 | 42 | folder_name = folder.split('/')[-1] 43 | 44 | # get keypoints and descriptors from sequence in folder 45 | kp, des = read_keypoints(folder, detector_name, descriptor_name) 46 | 47 | # get keypoints from next folder 48 | kp_next, des_next = read_next_keypoints(detector_name, descriptor_name, 49 | folders, folder_id, 100) 50 | # next_folder = folders[(folder_id + 1) % len(folders)] 51 | # kp_next, des_next = read_keypoints(next_folder, 52 | # detector_name, 53 | # descriptor_name) 54 | 55 | # printout 56 | print('pV: Working on folder {}'.format(folder_name)) 57 | 58 | # check if ref image has no keypoints and skip evaluation of sequence 59 | if 0 == kp[0].shape[0]: 60 | print('Folder {} has 0 keypoints for ref image'.format(folder)) 61 | print('SKIPPING THIS FOLDER') 62 | continue 63 | 64 | # random keypoints from ref image 65 | nr_of_indexes = min(n, kp[0].shape[0]) 66 | random_keypoint_indexes = sample(range(kp[0].shape[0]), nr_of_indexes) 67 | 68 | # choose ref image 69 | x_kp = kp[0][random_keypoint_indexes,:] 70 | x_des = des[0][random_keypoint_indexes,:] 71 | 72 | 73 | # iterate over descriptors of sequence images 74 | for id1, dess in enumerate(des[1:]): 75 | 76 | # match ref image and sequence image 77 | bf = cv2.BFMatcher(descriptor_distance[descriptor_name]) 78 | matches = bf.match(x_des, 79 | dess) 80 | 81 | # get the indexes from the matching procedure above 82 | x_idx = [random_keypoint_indexes[m.queryIdx] for m in matches] 83 | x_crtano_idx = [m.trainIdx for m in matches] 84 | 85 | # measure s with which we rank the AP 86 | s += [m.distance for m in matches] 87 | if 'i_' in folder_name: 88 | s_i += [m.distance for m in matches] 89 | else: 90 | s_v += [m.distance for m in matches] 91 | 92 | # image every keypoint from ref image to sequence image 93 | # we get Hx points as columns, 3xn matrix (third row are 1 - 94 | # homogen coordinates) 95 | tr = transformations[folder_name][id1] 96 | points = np.c_[ kp[0][x_idx,:][:,[0,1]] , np.ones(len(x_idx))] 97 | imaged_points = np.dot(tr, points.T) 98 | imaged_points_normal = imaged_points/imaged_points[2,:] 99 | 100 | # compute distance from Hx and x' (matching keypoints from sequence image) 101 | dist = kp[id1+1][x_crtano_idx,:][:,[0,1]].T - imaged_points_normal[[0,1],:] 102 | distances = np.sqrt(np.sum((dist)**2,axis=0)) 103 | 104 | # find if x' is the closest keypoint to Hx and assign y 105 | for i in range(imaged_points_normal.shape[1]): 106 | # dist_ is the distance from point Hx and every keypoint of image id1+1 107 | diff = kp[id1+1][:,[0,1]].T - imaged_points_normal[[0,1],i].reshape(2,1) 108 | dist_ = np.sqrt(np.sum((diff)**2,axis=0)) 109 | if (dist_ < distances[i]).any(): 110 | y.append(-1) 111 | if 'i_' in folder_name: 112 | y_i.append(-1) 113 | else: 114 | y_v.append(-1) 115 | else: 116 | y.append(1) 117 | if 'i_' in folder_name: 118 | y_i.append(1) 119 | else: 120 | y_v.append(1) 121 | 122 | # iterate over descriptors of non-sequence images 123 | for id2, dess in enumerate(des_next[1:]): 124 | print('#############') 125 | print(dess) 126 | print('#############') 127 | if 0 == dess.shape[0]: 128 | print('Image has 0 keypoints') 129 | print('SKIPPING THIS IMAGE') 130 | continue 131 | 132 | bf = cv2.BFMatcher(descriptor_distance[descriptor_name]) 133 | matches = bf.match(x_des, 134 | dess) 135 | 136 | s += [m.distance for m in matches] 137 | y += [-1 for m in matches] 138 | 139 | if 'i_' in folder_name: 140 | s_i += [m.distance for m in matches] 141 | y_i += [-1 for m in matches] 142 | else: 143 | s_v += [m.distance for m in matches] 144 | y_v += [-1 for m in matches] 145 | 146 | 147 | 148 | s2 = [-s_ for s_ in s] 149 | AP = average_precision_score(y,s2) 150 | list_of_APs.append(AP) 151 | 152 | s_i = [-s_ for s_ in s_i] 153 | AP_i = average_precision_score(y_i,s_i) 154 | list_of_APs_i.append(AP_i) 155 | 156 | s_v = [-s_ for s_ in s_v] 157 | AP_v = average_precision_score(y_v,s_v) 158 | list_of_APs_v.append(AP_v) 159 | 160 | 161 | return list_of_APs, list_of_APs_i, list_of_APs_v 162 | 163 | 164 | def imageMatching(detector_name, descriptor_name, n, dataset_path, nr_of_iterations=1): 165 | ''' 166 | Task 2: Image Matching. 167 | + save results to imageMatching.txt 168 | 169 | Return: list_of_APs: list of average precision for every iteration 170 | mAP: mean of list_of_APs 171 | ''' 172 | 173 | transformations = getTransformations(dataset_path) 174 | folders = glob.glob(dataset_path) 175 | list_of_mAPs = [] 176 | list_of_mAPs_i = [] 177 | list_of_mAPs_v = [] 178 | 179 | for i in range(nr_of_iterations): 180 | 181 | list_of_APs = [] 182 | list_of_APs_i = [] 183 | list_of_APs_v = [] 184 | 185 | for folder in folders: 186 | 187 | folder_name = folder.split('/')[-1] 188 | print('iM: Working on folder {}'.format(folder_name)) 189 | 190 | # get keypoints and descriptors from sequence 191 | kp, des = read_keypoints(folder, detector_name, descriptor_name) 192 | 193 | # if an image has no keypoints skip that evaluating sequence 194 | if 0 == kp[0].shape[0]: 195 | print('Folder {} has 0 keypoints for ref image'.format(folder)) 196 | print('SKIPPING THIS FOLDER') 197 | continue 198 | 199 | # random keypoints from ref image 200 | nr_of_indexes = min(n,kp[0].shape[0]) 201 | random_keypoint_indexes = sample(range(kp[0].shape[0]), nr_of_indexes) 202 | #print('NR OF INDEXES: {}'.format(nr_of_indexes)) 203 | 204 | 205 | # define ref image 206 | x_kp = kp[0][random_keypoint_indexes,:] 207 | x_des = des[0][random_keypoint_indexes,:] 208 | 209 | 210 | for id1, dess in enumerate(des[1:]): 211 | y = [] 212 | s = [] 213 | 214 | # match ref image and sequence image 215 | bf = cv2.BFMatcher(descriptor_distance[descriptor_name]) 216 | matches = bf.match(x_des, 217 | dess) 218 | 219 | # get indexes of matches 220 | x_idx = [random_keypoint_indexes[m.queryIdx] for m in matches] 221 | x_crtano_idx = [m.trainIdx for m in matches] 222 | #print(x_crtano_idx) 223 | #print(kp[id1+1].shape) 224 | 225 | # measure s used for AP 226 | s += [m.distance for m in matches] 227 | 228 | # image every keypoint on ref image on 229 | # Hx are saved in columns, 3xn matrix (third row are all ones) 230 | tr = transformations[folder_name][id1] 231 | points = np.c_[ kp[0][x_idx,:][:,[0,1]] , np.ones(len(x_idx))] 232 | imaged_points = np.dot(tr, points.T) 233 | imaged_points_normal = imaged_points/imaged_points[2,:] 234 | 235 | # compute distance from Hx and x' 236 | dist = kp[id1+1][x_crtano_idx,:][:,[0,1]].T - imaged_points_normal[[0,1],:] 237 | distances = np.sqrt(np.sum((dist)**2,axis=0)) 238 | 239 | # find if x' is the closest keypoint to Hx and assign y 240 | for i in range(imaged_points_normal.shape[1]): 241 | # dist_ is the distance from point Hx and every keypoint of image id1+1 242 | diff = kp[id1+1][:,[0,1]].T - imaged_points_normal[[0,1],i].reshape(2,1) 243 | dist_ = np.sqrt(np.sum((diff)**2,axis=0)) 244 | # if there is any keypoint closer to Hx than our matching one, y=-1 245 | if (dist_ < distances[i]).any(): 246 | y.append(-1) 247 | else: 248 | y.append(1) 249 | 250 | # compute AP for pair of images if there is at least one positive match 251 | # if there is no positive match the AP will be nan so we cant compute mAP 252 | if 1 in y: 253 | s2 = [-s_ for s_ in s] 254 | AP = average_precision_score(y,s2) 255 | list_of_APs.append(AP) 256 | if 'i_' in folder_name: 257 | list_of_APs_i.append(AP) 258 | else: 259 | list_of_APs_v.append(AP) 260 | else: 261 | print('NO ONES IN Y!!!! {}/{}'.format(folder_name, id1)) 262 | 263 | if list_of_APs: 264 | list_of_mAPs.append(sum(list_of_APs) / len(list_of_APs)) 265 | list_of_mAPs_i.append(sum(list_of_APs_i) / len(list_of_APs_i)) 266 | list_of_mAPs_v.append(sum(list_of_APs_v) / len(list_of_APs_v)) 267 | else: 268 | list_of_mAPs.append(0) 269 | list_of_all_APs.append(list_of_APs) 270 | 271 | return list_of_mAPs, list_of_mAPs_i, list_of_mAPs_v 272 | 273 | 274 | def patchRetrieval(detector_name, descriptor_name, n, dataset_path, nr_of_iterations=1): 275 | ''' 276 | Task 3: Patch Retrieval. 277 | + save results to patchRetrieval.txt 278 | 279 | Return: list_of_APs: list of average precision for every iteration 280 | mAP: mean of list_of_APs 281 | ''' 282 | 283 | transformations = getTransformations(dataset_path) 284 | folders = glob.glob(dataset_path) 285 | list_of_mAPs = [] 286 | list_of_mAPs_i = [] 287 | list_of_mAPs_v = [] 288 | 289 | for i in range(nr_of_iterations): 290 | 291 | list_of_APs = [] 292 | list_of_APs_i = [] 293 | list_of_APs_v = [] 294 | 295 | for folder_id, folder in enumerate(folders): 296 | 297 | folder_name = folder.split('/')[-1] 298 | 299 | # get keypoints and descriptors from sequence in folder 300 | kp, des = read_keypoints(folder, detector_name, descriptor_name) 301 | 302 | # check if an image has no keypoints and skip that evaluating sequence 303 | if 0 == kp[0].shape[0]: 304 | print('Folder {} has 0 keypoints for ref image'.format(folder_name)) 305 | print('SKIPPING THIS FOLDER') 306 | continue 307 | 308 | # get keypoints from next folder 309 | kp_next, des_next = read_next_keypoints(detector_name, descriptor_name, folders, folder_id, 100) 310 | 311 | print('pR: Working on folders {}'.format(folder_name)) 312 | 313 | # random keypoints from ref image 314 | nr_of_indexes = min(n,kp[0].shape[0]) 315 | random_keypoint_indexes = sample(range(kp[0].shape[0]), nr_of_indexes) 316 | #print('NR OF INDEXES: {}'.format(nr_of_indexes)) 317 | 318 | # create dict which saves y and s for every keypoint separately 319 | y = {} 320 | s = {} 321 | for i in range(len(random_keypoint_indexes)): 322 | y[i] = [] 323 | s[i] = [] 324 | 325 | # choose ref image 326 | x_kp = kp[0][random_keypoint_indexes,:] 327 | x_des = des[0][random_keypoint_indexes,:] 328 | 329 | 330 | for id1, dess in enumerate(des[1:]): 331 | 332 | # image every keypoint from ref image onto sequence image 333 | # Hx are saved in columns, 3xn matrix (third row are all ones) 334 | tr = transformations[folder_name][id1] 335 | points = np.c_[ x_kp[:,[0,1]] , np.ones(x_kp.shape[0])] 336 | imaged_points = np.dot(tr, points.T) 337 | imaged_points_normal = imaged_points/imaged_points[2,:] 338 | 339 | 340 | # for every column in Hx, find its closest kp on the sequence image 341 | for i in range(imaged_points_normal.shape[1]): 342 | # computing distance between all kp and finding the minimal 343 | dist = kp[id1+1][:,[0,1]].T - imaged_points_normal[[0,1],i].reshape(2,1) 344 | distances = np.sqrt(np.sum((dist)**2,axis=0)) 345 | index_of_closest_kp = np.argmin(distances) 346 | #closest_keypoint = kp[id1+1][index_of_closest_kp,:] 347 | 348 | y[i].append(1) 349 | # find distance between descriptors and save as measure s 350 | index_in_orig_kp0 = random_keypoint_indexes[i] 351 | descriptors_distance = cv2.norm(dess[index_of_closest_kp,:], 352 | des[0][index_in_orig_kp0,:], 353 | descriptor_distance[descriptor_name]) 354 | s[i].append(descriptors_distance) 355 | 356 | 357 | # images out of the sequence 358 | for dess in des_next[1:]: 359 | 360 | if 0 == dess.shape[0]: 361 | print('Image has 0 keypoints') 362 | print('SKIPPING THIS IMAGE') 363 | continue 364 | 365 | # obtain random points from second image 366 | how_many = min(n,dess.shape[0]) 367 | random_keypoint_indexes_right_img = sample(range(dess.shape[0]), how_many) 368 | 369 | # match points 370 | bf = cv2.BFMatcher(descriptor_distance[descriptor_name]) 371 | matches = bf.match(x_des, 372 | dess[random_keypoint_indexes_right_img,:]) 373 | 374 | for m in matches: 375 | s[m.queryIdx].append(m.distance) 376 | y[m.queryIdx].append(-1) 377 | 378 | 379 | # after iterating over sequence, compute AP 380 | for i in range(len(y.keys())): 381 | s2 = [-s_ for s_ in s[i]] 382 | AP = average_precision_score(y[i],s2) 383 | list_of_APs.append(AP) 384 | 385 | if 'i_' in folder_name: 386 | list_of_APs_i.append(AP) 387 | else: 388 | list_of_APs_v.append(AP) 389 | 390 | list_of_mAPs.append(sum(list_of_APs) / len(list_of_APs)) 391 | list_of_mAPs_i.append(sum(list_of_APs_i) / len(list_of_APs_i)) 392 | list_of_mAPs_v.append(sum(list_of_APs_v) / len(list_of_APs_v)) 393 | 394 | return list_of_mAPs, list_of_mAPs_i, list_of_mAPs_v 395 | 396 | 397 | def write_result(l, f): 398 | s = '[' 399 | for e in l: 400 | s += str(e) + ', ' 401 | s += ']' 402 | f.write(s) 403 | 404 | 405 | if __name__ == '__main__': 406 | import argparse 407 | 408 | parser_of_args = argparse.ArgumentParser(description='Evaluate your algorithm on 3 tasks') 409 | parser_of_args.add_argument('detector_name', type=str, 410 | help='name of the detector') 411 | parser_of_args.add_argument('descriptor_name', type=str, 412 | help='name of descriptor') 413 | parser_of_args.add_argument('n', type=int, 414 | help='number of random keypoints to choose') 415 | parser_of_args.add_argument('dataset_path', type=str, 416 | help='path to hpatches dataset') 417 | parser_of_args.add_argument('nr_of_iterations', type=int, 418 | help='number of iterations for patch verification') 419 | 420 | args = parser_of_args.parse_args() 421 | 422 | # project_root = '/home/davidboja/PycharmProjects/FER/hpatches-benchmark/python/ISPA' 423 | # dataset_path = project_root + '/hpatches-sequences-release/*' 424 | 425 | list_of_APs_pV, list_of_APs_i_pV, list_of_APs_v_pV = patchVerification(args.detector_name, 426 | args.descriptor_name, 427 | args.n, 428 | args.dataset_path, 429 | args.nr_of_iterations) 430 | 431 | list_of_mAPs_iM, list_of_mAPs_i_iM, list_of_mAPs_v_iM = imageMatching(args.detector_name, 432 | args.descriptor_name, 433 | args.n, 434 | args.dataset_path, 435 | args.nr_of_iterations) 436 | 437 | list_of_mAPs_pR, list_of_mAPs_i_pR, list_of_mAPs_v_pR = patchRetrieval(args.detector_name, 438 | args.descriptor_name, 439 | args.n, 440 | args.dataset_path, 441 | args.nr_of_iterations) 442 | 443 | print(list_of_APs_pV) 444 | print(list_of_APs_i_pV) 445 | print(list_of_APs_v_pV) 446 | 447 | print(list_of_mAPs_iM) 448 | print(list_of_mAPs_i_iM) 449 | print(list_of_mAPs_v_iM) 450 | 451 | print(list_of_mAPs_pR) 452 | print(list_of_mAPs_i_pR) 453 | print(list_of_mAPs_v_pR) 454 | 455 | with open(os.path.join(RESULTS_DIR, 456 | ALGO_TEMPLATE.format(args.detector_name, args.descriptor_name)), 'w+') as f: 457 | f.write('Patch verification:\n') 458 | write_result(list_of_APs_pV, f) 459 | write_result(list_of_APs_i_pV, f) 460 | write_result(list_of_APs_v_pV, f) 461 | f.write('\nImage matching:\n') 462 | write_result(list_of_mAPs_iM, f) 463 | write_result(list_of_mAPs_i_iM, f) 464 | write_result(list_of_mAPs_v_iM, f) 465 | f.write('\nPatch retrieval:\n') 466 | write_result(list_of_mAPs_pR, f) 467 | write_result(list_of_mAPs_i_pR, f) 468 | write_result(list_of_mAPs_v_pR, f) 469 | 470 | # TODO: izbrojati broj keypointova za svaki algo i vidjeti postoji li korelacija -------------------------------------------------------------------------------- /other_algorithms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kristijanbartol/keypoint-algorithms-benchmark/d3bf05220fd697fececb293ba139ca5c563acc21/other_algorithms/__init__.py -------------------------------------------------------------------------------- /other_algorithms/asift.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Affine invariant feature-based image matching sample. 5 | 6 | This sample is similar to find_obj.py, but uses the affine transformation 7 | space sampling technique, called ASIFT [1]. While the original implementation 8 | is based on SIFT, you can try to use SURF or ORB detectors instead. Homography RANSAC 9 | is used to reject outliers. Threading is used for faster affine sampling. 10 | 11 | [1] http://www.ipol.im/pub/algo/my_affine_sift/ 12 | 13 | USAGE 14 | asift.py [--feature=[-flann]] [ ] 15 | 16 | --feature - Feature to use. Can be sift, surf, orb or brisk. Append '-flann' 17 | to feature name to use Flann-based matcher instead bruteforce. 18 | 19 | Press left mouse button on a feature point to see its matching point. 20 | ''' 21 | 22 | # Python 2/3 compatibility 23 | from __future__ import print_function 24 | 25 | import numpy as np 26 | import cv2 27 | 28 | # built-in modules 29 | #import itertools as it 30 | from multiprocessing.pool import ThreadPool 31 | 32 | # local modules 33 | # from common import Timer 34 | # from find_obj import init_feature, filter_matches, explore_match 35 | 36 | 37 | def affine_skew(tilt, phi, img, mask=None): 38 | ''' 39 | affine_skew(tilt, phi, img, mask=None) -> skew_img, skew_mask, Ai 40 | 41 | Ai - is an affine transform matrix from skew_img to img 42 | ''' 43 | h, w = img.shape[:2] 44 | if mask is None: 45 | mask = np.zeros((h, w), np.uint8) 46 | mask[:] = 255 47 | A = np.float32([[1, 0, 0], [0, 1, 0]]) 48 | if phi != 0.0: 49 | phi = np.deg2rad(phi) 50 | s, c = np.sin(phi), np.cos(phi) 51 | A = np.float32([[c,-s], [ s, c]]) 52 | corners = [[0, 0], [w, 0], [w, h], [0, h]] 53 | tcorners = np.int32( np.dot(corners, A.T) ) 54 | x, y, w, h = cv2.boundingRect(tcorners.reshape(1,-1,2)) 55 | A = np.hstack([A, [[-x], [-y]]]) 56 | img = cv2.warpAffine(img, A, (w, h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE) 57 | if tilt != 1.0: 58 | s = 0.8*np.sqrt(tilt*tilt-1) 59 | img = cv2.GaussianBlur(img, (0, 0), sigmaX=s, sigmaY=0.01) 60 | img = cv2.resize(img, (0, 0), fx=1.0/tilt, fy=1.0, interpolation=cv2.INTER_NEAREST) 61 | A[0] /= tilt 62 | if phi != 0.0 or tilt != 1.0: 63 | h, w = img.shape[:2] 64 | mask = cv2.warpAffine(mask, A, (w, h), flags=cv2.INTER_NEAREST) 65 | Ai = cv2.invertAffineTransform(A) 66 | return img, mask, Ai 67 | 68 | 69 | def affine_detect(detector, img, mask=None, pool=None): 70 | ''' 71 | affine_detect(detector, img, mask=None, pool=None) -> keypoints, descrs 72 | 73 | Apply a set of affine transormations to the image, detect keypoints and 74 | reproject them into initial image coordinates. 75 | See http://www.ipol.im/pub/algo/my_affine_sift/ for the details. 76 | 77 | ThreadPool object may be passed to speedup the computation. 78 | ''' 79 | params = [(1.0, 0.0)] 80 | for t in 2**(0.5*np.arange(1,6)): 81 | for phi in np.arange(0, 180, 72.0 / t): 82 | params.append((t, phi)) 83 | 84 | def f(p): 85 | t, phi = p 86 | timg, tmask, Ai = affine_skew(t, phi, img) 87 | keypoints, descrs = detector.detectAndCompute(timg, tmask) 88 | for kp in keypoints: 89 | x, y = kp.pt 90 | kp.pt = tuple( np.dot(Ai, (x, y, 1)) ) 91 | if descrs is None: 92 | descrs = [] 93 | return keypoints, descrs 94 | 95 | keypoints, descrs = [], [] 96 | if pool is None: 97 | ires = map(f, params) 98 | # else: 99 | # ires = pool.imap(f, params) 100 | 101 | for i, (k, d) in enumerate(ires): 102 | print('affine sampling: %d / %d\r' % (i+1, len(params)), end='') 103 | keypoints.extend(k) 104 | descrs.extend(d) 105 | 106 | return keypoints, np.array(descrs) 107 | 108 | if __name__ == '__main__': 109 | print(__doc__) 110 | 111 | # import sys, getopt 112 | # opts, args = getopt.getopt(sys.argv[1:], '', ['feature=']) 113 | # opts = dict(opts) 114 | # feature_name = opts.get('--feature', 'brisk-flann') 115 | # try: 116 | # fn1, fn2 = args 117 | # except: 118 | # fn1 = '../data/aero1.jpg' 119 | # fn2 = '../data/aero3.jpg' 120 | # 121 | # img1 = cv2.imread(fn1, 0) 122 | # img2 = cv2.imread(fn2, 0) 123 | # detector, matcher = init_feature(feature_name) 124 | # 125 | # if img1 is None: 126 | # print('Failed to load fn1:', fn1) 127 | # sys.exit(1) 128 | # 129 | # if img2 is None: 130 | # print('Failed to load fn2:', fn2) 131 | # sys.exit(1) 132 | # 133 | # if detector is None: 134 | # print('unknown feature:', feature_name) 135 | # sys.exit(1) 136 | # 137 | # print('using', feature_name) 138 | # 139 | # pool=ThreadPool(processes = cv2.getNumberOfCPUs()) 140 | # kp1, desc1 = affine_detect(detector, img1, pool=pool) 141 | # kp2, desc2 = affine_detect(detector, img2, pool=pool) 142 | # print('img1 - %d features, img2 - %d features' % (len(kp1), len(kp2))) 143 | 144 | # def match_and_draw(win): 145 | # with Timer('matching'): 146 | # raw_matches = matcher.knnMatch(desc1, trainDescriptors = desc2, k = 2) #2 147 | # p1, p2, kp_pairs = filter_matches(kp1, kp2, raw_matches) 148 | # if len(p1) >= 4: 149 | # H, status = cv2.findHomography(p1, p2, cv2.RANSAC, 5.0) 150 | # print('%d / %d inliers/matched' % (np.sum(status), len(status))) 151 | # # do not draw outliers (there will be a lot of them) 152 | # kp_pairs = [kpp for kpp, flag in zip(kp_pairs, status) if flag] 153 | # else: 154 | # H, status = None, None 155 | # print('%d matches found, not enough for homography estimation' % len(p1)) 156 | # 157 | # vis = explore_match(win, img1, img2, kp_pairs, None, H) 158 | # 159 | # 160 | # match_and_draw('affine find_obj') 161 | # cv2.waitKey() 162 | # cv2.destroyAllWindows() 163 | -------------------------------------------------------------------------------- /other_algorithms/masks.py: -------------------------------------------------------------------------------- 1 | """ 2 | masks.py 3 | 4 | Provides functions which create specialized footprints or masks which are 5 | subsequently fed as inputs to functions in scipy.ndimage.filters module. 6 | 7 | AUTHOR: Andrew L. Stachyra 8 | DATE: 5/17/2013 9 | """ 10 | 11 | import numpy as np 12 | from scipy.ndimage import filters 13 | import math 14 | 15 | def getDeriv(image, weights=[3./16, 10./16, 3./16], axes=[0], mode="reflect", 16 | cval=0.0): 17 | 18 | """ 19 | Calculates a first or second derivative on a multi-dimensional image 20 | array by the method of finite differences using a 3X3 stencil. 21 | Images in principle need not necessarily be 2D; i.e., 3D tomographic 22 | images should work also, however, "caveat emptor" as this latter 23 | functionality has not yet actually been tested fully. 24 | 25 | Parameters 26 | ---------- 27 | 28 | image: array_like 29 | 30 | Array containing grayscale image data. Only grayscale pixel 31 | values are supported--cannot handle 3-channel color. 32 | 33 | weights: array, optional 34 | 35 | 1D sequence of three numbers representing the type of finite 36 | differences derivative (Prewitt, Sobel, Scharr, etc.) to compute. 37 | Defaults to [3./16, 10./16, 3./16], i.e., Scharr type. It is 38 | recommended that this sequence should be normalized so that all 39 | components sum to 1. If not, the function will still return a 40 | result, however, cross derivative (dxdy type) results will be 41 | scaled incorrectly with respect to 1st derivatives and non-cross 42 | 2nd derivatives (e.g., dxdx, dydy). 43 | 44 | axes: scalar or array_like, optional 45 | 46 | Either a single value (1st derivative case) or two values (2nd 47 | derivative) indicating axes along which derivatives are to be 48 | taken. Examples: 49 | 50 | axes=0 1st derivative, x-axis 51 | axes=[0] Also indicates 1st derivative, x-axis 52 | axes=(1, 1) 2nd derivative, y-axis (i.e, dydy type) 53 | axes=[0, 2] 2nd derivative, x- and z-axes (i.e., dxdz, 54 | assuming a tomographic style image with 55 | three axes) 56 | 57 | mode: ('reflect', 'constant', 'nearest', 'mirror', 'wrap') 58 | 59 | Controls how edge pixels in the input image are treated. See 60 | scipy.ndimage.filters.correlate1d() for details. 61 | 62 | cval: scalar, optional 63 | 64 | Only meaningful if mode is set to 'constant'. See 65 | scipy.ndimage.filters.correlate1d() for details. 66 | 67 | Returns 68 | ------- 69 | 70 | output: ndarray 71 | 72 | An estimate of first or second partial derivative with respect 73 | to image brightness at each pixel. 74 | 75 | """ 76 | 77 | """Check and/or condition the input variables""" 78 | # Force treatment as float numpy array to avoid rounding errors later 79 | image = np.asarray(image, dtype=float) 80 | 81 | wmsg = 'weights input variable must be an array or list with ' + \ 82 | 'exactly three elements' 83 | try: 84 | nw = len(weights) # Fails if weights is not iterable type 85 | except: 86 | raise TypeError(wmsg) 87 | if nw != 3: # If weights is iterable, but still not correct length... 88 | raise ValueError(wmsg) 89 | 90 | """Set appropriate weights, and slightly reconfigure axes specification""" 91 | try: # Assume axes input value is iterable 92 | nx = len(axes) # Will raise a TypeError if axes is not iterable 93 | except TypeError: 94 | # First derivative 95 | wght = [-0.5, 0, 0.5] 96 | myaxes = [axes] # Force myaxes to be iterable list containing one item 97 | nx = 0 98 | 99 | # Skip the rest, if axes input value was scalar (i.e., not iterable) 100 | if nx == 0: 101 | pass 102 | # Alternative first derivative, if axes input is iterable 103 | elif nx == 1: 104 | wght = [-0.5, 0, 0.5] 105 | myaxes = axes 106 | elif nx == 2: 107 | # Second derivative, along same axis twice 108 | if axes[0] == axes[1]: 109 | wght = [1.0, -2.0, 1.0] 110 | myaxes = [axes[0]] 111 | # Second derivative, along orthogonal axes 112 | else: 113 | wght = [-0.5, 0, 0.5] 114 | myaxes = axes 115 | else: 116 | raise ValueError('Too many axes: 3rd derivatives and higher are ' + 117 | 'not yet supported') 118 | 119 | """Compute the derivative!!!""" 120 | for ii in myaxes: 121 | # Use fast compiled code from scipy.ndimage._nd_image.pyd 122 | output = filters.correlate1d(image, wght, ii, mode=mode, cval=cval) 123 | 124 | """Apply smoothing weights (Prewitt, Sobel, Scharr, or whatever the 125 | user has selected) to all remaining axes""" 126 | # Get a list of all other axes. For 2D images, this will produce either 127 | # a null list (in the dxdy case) or at most one other axis. For 3D 128 | # images (e.g., such as a tomographic image), there will be either one 129 | # or two other axes. 130 | otheraxes = [ii for ii in range(image.ndim) if ii not in myaxes] 131 | for ii in otheraxes: 132 | output = filters.correlate1d(output, weights, ii, mode=mode, cval=cval) 133 | 134 | return output 135 | 136 | def circFootprint(radius, method='Area', npoints=10, dtype=bool): 137 | 138 | """ 139 | Generates a circular footprint based on the input radius. Intended 140 | for use with usan() function. 141 | 142 | Parameters 143 | ---------- 144 | 145 | radius: scalar 146 | 147 | Radius of the circular footprint, in pixels. 148 | 149 | method: ('Center', 'center', 'Area', 'area'), optional 150 | 151 | Method by which to compute the footprint. If method=='Center' 152 | or method=='center', each pixel is tested for membership in 153 | the footprint based upon whether a single point at the center 154 | of the pixel falls within the radius. Depending upon the 155 | dtype selected, each pixel will assume either of two values: 156 | (True, False), (0, 1), or (0., 1.). If method=='Area' or 157 | method=='area', a square subgrid of size npoints X npoints 158 | is superimposed upon each pixel, and membership is determined 159 | by the total number of subgrid points (representing the fraction 160 | of pixel area) that falls within the radius. Depending upon the 161 | dtype selected, each pixel will assume values of either (True, 162 | False), (0, 1), or a sliding scale value between 0. and 1. 163 | 164 | npoints: scalar, optional 165 | 166 | Number of points to use in subgrid when method='Area' or 167 | method='area' is selected. See method input variable above 168 | for further discussion. 169 | 170 | dtype: data-type, optional 171 | 172 | Data type to use for output. Note that float is only really 173 | meaningful if method='Area' or method='area' is selected. If 174 | dtype is set to float but method='Center' or method='center', 175 | then the pixels in the footprint will be assigned floating 176 | point values of 0. or 1., but will be unable to assume any 177 | values in between. See method input variable above for further 178 | discussion. 179 | 180 | Returns 181 | ------- 182 | 183 | footprint: ndarray 184 | 185 | A square array defining a circular footprint. Values of zero 186 | indicate a pixel is not within the footprint radius, non-zero 187 | values indicate either absolute membership (bool or int) or 188 | degree of partial membership (float). 189 | 190 | """ 191 | 192 | # Determine whether each test pixel falls within the circular mask based 193 | # on whether the pixel's center falls within the radius 194 | if method == 'Center' or method == 'center': 195 | halfext = int(math.floor(radius)) 196 | ones = np.ones((2*halfext+1, 2*halfext+1), dtype=dtype) 197 | zeros = np.zeros((2*halfext+1, 2*halfext+1), dtype=dtype) 198 | # Make a square trial grid just large enough to contain radius 199 | v, h = np.ogrid[-halfext:(halfext+1), -halfext:(halfext+1)] 200 | # footprint consists of any pixel within the radius 201 | footprint = np.where(v**2 + h**2 <= radius**2, ones, zeros) 202 | # Determine each pixel's membership in circular mask based on total 203 | # percentage of pixel area that falls within the radius 204 | elif method == 'Area' or method == 'area': 205 | step = 1./npoints 206 | # Create a subgrid of size (npoints, npoints) within each pixel 207 | v, h = np.ogrid[(-0.5+step/2):(0.5+step/2):step, 208 | (-0.5+step/2):(0.5+step/2):step] 209 | halfext = int(math.ceil(radius-0.5)) 210 | fpfloat = np.zeros((2*halfext+1, 2*halfext+1), dtype=float) 211 | # Loop through each pixel in an implicit trial grid 212 | for ii in range(-halfext, (halfext+1)): 213 | for ij in range(-halfext, (halfext+1)): 214 | # Values of True signify points within the footprint radius 215 | subgrid = ((v-ii)**2 + (h-ij)**2 <= radius**2) 216 | # Total area of (ii,ij)'th pixel is proportional to total 217 | # fraction of True values in the subgrid 218 | fpfloat[(ii+halfext), 219 | (ij+halfext)] = float(sum(sum(subgrid)))/(npoints**2) 220 | ones = np.ones((2*halfext+1, 2*halfext+1), dtype=dtype) 221 | zeros = np.zeros((2*halfext+1, 2*halfext+1), dtype=dtype) 222 | # For dtypes that aren't capable of representing fpfloat properly, 223 | # create a footprint by rounding the values in fpfloat up or down... 224 | if dtype == bool or dtype == int: 225 | footprint = np.where(fpfloat >= 0.5, ones, zeros) 226 | # ...but otherwise, just use fpfloat directly 227 | else: 228 | footprint = fpfloat.astype(dtype) 229 | # If trial grid accidentally contains a one pixel wide perimeter band 230 | # which doesn't fall within the circular footprint, then trim it off 231 | if not footprint[0,halfext]: 232 | footprint = footprint[1:(2*halfext),1:(2*halfext)] 233 | else: 234 | raise ValueError('Method ' + str(method) + ' not supported') 235 | 236 | return footprint 237 | 238 | def usan(image, mode='Edge', radius=3.4, fptype=bool, t=25, gfrac=None, 239 | cgthresh=None, nshades=256): 240 | 241 | """ 242 | Calculates raw edge or corner response, based upon an algorithm 243 | described in: "SUSAN--A New Approach to Low Level Image Processing", 244 | S. M. Smith and J. M. Brady, Technical Report TR95SMSIc (1995), 245 | available at: 246 | http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.24.2763 . 247 | Alternatively, there is also a slightly abridged version of the 248 | same reference, with identical title and authors, available at 249 | International Journal of Computer Vision, 23(1), 45-78 (1997). 250 | 251 | Parameters 252 | ---------- 253 | 254 | image: array_like 255 | 256 | Array containing grayscale image data. Only grayscale pixel 257 | values are supported--cannot handle 3-channel color. 258 | 259 | mode: ('Edge', 'edge', 'EdgeDir', 'edgedir', 'Corner', 'corner'), 260 | optional 261 | 262 | Chooses usage mode. If mode=='Edge' or mode=='edge' or mode== 263 | 'Corner' or mode=='corner' then the output will be either an 264 | edge or corner response. If mode=='EdgeDir' or mode=='edgedir' 265 | then the output gives an estimate of the angle (in degrees) 266 | of the edge normal unit vector. In general, a value for the 267 | edge normal angle will be generated for all pixels, however it 268 | isn't usually meaningful unless the edge response is non-zero. 269 | 270 | radius: scalar, optional 271 | 272 | Circular footprint radius, in units of pixels; passed through 273 | as input to circFootprint(). Default is 3.4, as recommended 274 | in reference. 275 | 276 | fptype: data-type, optional 277 | 278 | Data type of circular footprint; passed to circFootprint(). 279 | Default is bool, since the version of the algorithm originally 280 | described in the reference did not define the algorithm behavior 281 | for any other type of footprint (we have trivially extended it 282 | to cover float type footprints as well, however there is not 283 | much observable different in the results, as it only affects the 284 | weighting of the pixels at the edge of the footprint, not the 285 | central region). 286 | 287 | t: scalar, optional 288 | 289 | Threshold value for color difference required to exclude/include 290 | a pixel in the USAN region described in the reference article. 291 | Default is 25 grayscale levels, as suggested by article authors, 292 | and assumes a grayscale image with a total of 256 distinct 293 | shades. As described in eq. 4, this is technically a "soft" 294 | threshold, rather than a hard cutoff, and moreover it's also 295 | bidirectional; i.e., a setting of t=25 actually means +/- 25. 296 | 297 | gfrac: scalar, optional 298 | 299 | Fraction of maximum number of USAN pixels to assign to the "g" 300 | parameter in eq. 3 of the reference article. Adhering to 301 | recommendation of article authors, it defaults to 0.75 if user 302 | specifies mode='Edge' or mode='edge', and 0.5 if user specifies 303 | mode='Corner' or mode='corner'. 304 | 305 | cgthesh: scaler, optional 306 | 307 | Threshold value of USAN center of gravity, in units of pixels, 308 | which will lead to a shift between one underlying set of 309 | approximations/assumptions vs. another. Defaults to 0.3*radius 310 | (i.e., 1.02 pixels, assuming a default USAN footprint radius of 311 | 3.4 pixels, which is consistent with author's nominal suggested 312 | setting of about 1 pixel) if mode='Edge' or mode= 'edge', and 313 | defaults to 0.45*radius if mode='Corner' or mode='corner' (the 314 | article authors did not suggest a default setting for corner 315 | mode, so this setting is based on trial-and-error). See 316 | reference article for more details. 317 | 318 | nshades: scalar, optional 319 | 320 | Total number of distinct integer grayscale levels available in 321 | image format. Defaults to 256 (i.e., 2**8), appropriate for an 322 | 8 bit image format such as jpeg. For image formats with higher 323 | grayscale color resolution, e.g. such as 12-bit or 16-bit, set 324 | nshades=4096 or nshades=65536 (i.e., 2**12 or 2**16). 325 | 326 | Returns 327 | ------- 328 | 329 | R or edgedir: ndarray 330 | 331 | Context sensitive value, depending on the value of the mode input 332 | variable. If mode='Edge' or mode='edge' or mode='Corner' or 333 | mode='corner' then the return array holds the edge or corner 334 | response as described by eq. 3 in the reference article. If 335 | mode='EdgeDir' or mode='edgedir' then the return array holds 336 | the raw estimated edge normal direction. In general this value 337 | will only be meaningful if the edge response of the same pixel is 338 | nonzero. 339 | 340 | """ 341 | 342 | """ Condition input variables """ 343 | # Make sure that methods of type ndarray are available for use later 344 | image = np.asarray(image) 345 | 346 | # Assign default values suggested in reference, if user doesn't override 347 | if gfrac is None: 348 | if mode == 'Edge' or mode == 'edge': 349 | gfrac = 0.75 350 | elif mode == 'Corner' or mode == 'corner': 351 | gfrac = 0.5 352 | 353 | # Assign default value based on reference for 'EdgeDir' mode, and based 354 | # on trial-and-error experimentation, for 'Corner' mode 355 | if cgthresh is None: 356 | if mode == 'EdgeDir' or mode == 'edgedir': 357 | cgthresh = 0.3 * radius 358 | elif mode == 'Corner' or mode == 'corner': 359 | cgthresh = 0.45 * radius 360 | 361 | """ Create a lookup table, as recommended in the reference """ 362 | idx = np.arange(nshades) 363 | explookup = np.zeros(len(idx)) 364 | for ii in range(len(idx)): 365 | # From eq. 4 in the reference 366 | explookup[ii] = math.exp(-(idx[ii].astype(float)/t)**6) 367 | 368 | """ Set up USAN circular footprint and several related variables """ 369 | # Get the circular USAN mask 370 | fp = circFootprint(radius, dtype=fptype) 371 | # For the dtype=bool case, the areawgt variable essentially turns into 372 | # a flattened array of ones, but for the dtype=float case, pixels near 373 | # the edge of the circular footprint will be reweighted according to what 374 | # fraction of their error falls within the footprint radius 375 | areawgt = fp[(fp != 0)] 376 | # Force fp to be all zeros and ones, to comport with expectations of 377 | # scipy.ndimage.filters.generic_filter() 378 | fp = fp.astype(bool).astype(int) 379 | # Define arrays consisting of horizontal and vertical offsets between 380 | # the nucleus and each of the surrounding pixels in the circular mask 381 | halfext = (fp.shape[1] - 1)/2 382 | xdiff, ydiff = np.mgrid[-halfext:(halfext+1), -halfext:(halfext+1)] 383 | xdiff = xdiff[(fp != 0)] 384 | ydiff = ydiff[(fp != 0)] 385 | 386 | """ Define a function which will be called iteratively upon every pixel 387 | in the image, using scipy.ndimage.filters.generic_filter() """ 388 | def filterfunc(maskout, areawgt, xdiff, ydiff, radius, mode, t, explookup, 389 | gfrac, cgthresh, npoints=10): 390 | 391 | """ 392 | Internal function to usan() which gets passed down through to 393 | scipy.ndimage.filters.generic_filters() to perform the actual 394 | work of filtering. Gets called once for each pixel in image. 395 | 396 | maskout: flattened ndarray 397 | 398 | Contains values from surrounding pixels which fell within 399 | footprint at each pixel. 400 | 401 | areawgt: flattened ndarray 402 | 403 | Same size as maskout; if fptype=float in usan() input, gives 404 | fraction of each pixel area which fell within the circular 405 | footprint radius. Otherwise, if fptype=bool or fptype=int, 406 | it's simply an array of ones. 407 | 408 | xdiff, ydiff: flattened ndarray 409 | 410 | (x, y) offset between each pixel and nucleus (center pixel 411 | of circular footprint). 412 | 413 | radius, mode, t, gfrac, cgthresh 414 | 415 | Straight pass-through of inputs to usan() function. 416 | 417 | explookup: ndarray 418 | 419 | 1D array containing lookup table for eq. 4 in reference 420 | article. Size should match nshades input argument to usan() 421 | function. 422 | 423 | npoints: scalar, optional 424 | 425 | Specifies number of subpixel lattice points to use per pixel 426 | when computing USAN contiguity if mode='Corner' or 427 | mode='corner'. 428 | 429 | Returns 430 | ------- 431 | 432 | R or edgedir: scalar 433 | 434 | Context sensitive value, depending on the value of the mode 435 | input variable. If mode='Edge' or mode='edge' or mode='Corner' 436 | or mode='corner' then the return value holds the edge or corner 437 | response as described by eq. 3 in the reference article. If 438 | mode='EdgeDir' or mode='edgedir' then the return value holds 439 | the raw estimated edge normal direction. In general this value 440 | will only be meaningful if the edge response of the same pixel 441 | is nonzero. 442 | 443 | """ 444 | 445 | """ Condition inputs and pre-compute expressions which are required 446 | in multiple locations below """ 447 | # Total number of pixels in mask 448 | ntot = len(maskout) - 1 449 | # Index and intensity of center pixel (i.e., the nucleus) 450 | ctridx = ntot//2 451 | nucleus = maskout[ctridx] 452 | # Delete data of center pixel in family of arrays with same dimension 453 | maskout = np.delete(maskout, ctridx) 454 | areawgt = np.delete(areawgt, ctridx) 455 | xdiff = np.delete(xdiff, ctridx) 456 | ydiff = np.delete(ydiff, ctridx) 457 | # Calculate color/grayscale shade difference between nucleus and all 458 | # other surrounding pixels in the footprint. Cast type back to int 459 | # in order to index lookup table--will definitely be handed off as 460 | # a float otherwise (see explanatory note below for reason behind 461 | # convoluted type casting flip-flop in the first place). 462 | graydiff = np.abs(maskout-nucleus*np.ones(len(maskout))).astype(int) 463 | # Calculate c as described in eq. 4 in the reference. 464 | c = explookup[graydiff] 465 | # Reduces to eq. 2 in reference, if areawgt values are all 1 466 | n = (areawgt * c).sum() 467 | # Total number of pixels in circular mask 468 | nmax = areawgt.sum().astype(float) 469 | 470 | """ Compute an appropriate response function for each usage mode """ 471 | if mode == 'Edge' or mode == 'edge': 472 | # Eq. 3 in reference 473 | R = gfrac*nmax - n 474 | if R<0: R=0. 475 | return R 476 | elif mode == 'EdgeDir' or mode == 'edgedir': 477 | denom = (areawgt * c).sum() 478 | # Usual case 479 | if denom: 480 | # Calculate center of gravity, using eq. 5 in reference 481 | xcg = ((xdiff * areawgt * c).sum()) / denom 482 | ycg = ((ydiff * areawgt * c).sum()) / denom 483 | # Divide-by-zero case, which can arise when a single noisy 484 | # pixels is surrounded by many others of dissimilar brightness 485 | else: 486 | xcg, ycg = 0, 0 487 | cgdist = math.sqrt(xcg**2 + ycg**2) 488 | # The so-called "inter-pixel" case mentioned in the reference 489 | if n >= (2*radius) and cgdist >= cgthresh: 490 | # Compute angle associated with edge normal direction unit 491 | # vector (i.e., the actual unit vector itself, had we needed 492 | # to calculate it explicitly in the code, would have been 493 | # (cos(edgedir), sin(edgedir))). Due to the way the USAN 494 | # concept is defined by the reference authors, the edge 495 | # direction is NOT a gradient pointing from lighter to darker 496 | # (or vice versa) regions as is commonly the case with other 497 | # types of edge finding algorithms. Rather, it always points 498 | # perpendicularly away from the edge, no matter which side of 499 | # the edge (lighter or darker) the pixel is on. Thus, as the 500 | # USAN circular mask moves across an edge, the edgedir as it 501 | # is defined in the inter-pixel case usually tends to flip 502 | # very suddenly by 180 degrees. For edgedir values falling 503 | # between 90 and 270 degrees, we will subtract 180 degrees 504 | # at a later stage of processing in order to map them onto 505 | # the interval -90 to +90 degrees, which is all that we have 506 | # available anyway for the intra-pixel case (see below). 507 | edgedir = math.atan2(ycg, xcg) * 180 / math.pi 508 | # The "intra-pixel" case; see reference for description 509 | else: 510 | xvar = (xdiff * xdiff * areawgt * c).sum() # Eq. 6 511 | yvar = (ydiff * ydiff * areawgt * c).sum() # Eq. 7 512 | xycovar = (xdiff * ydiff * areawgt * c).sum() # Eq. 8 513 | # Compute edge normal direction. The xvar and yvar quantities 514 | # are essentially weighted variances, and are therefore 515 | # positive definite. If the (x, y) content of the USAN 516 | # is positively covariant (i.e., lies along a positively 517 | # sloped line) then atan2(yvar, xvar)*180/pi gives an angle 518 | # parallel to the edge and between 0 and 90 degrees, while 519 | # (atan2(yvar, xvar)*180/pi - 90) gives the desired edge 520 | # normal direction (i.e., an angle perpendicular to the edge 521 | # and guaranteed to lie between -90 and 0 degrees). On the 522 | # other hand, if the USAN is negatively covariant, then 523 | # atan2(yvar, xvar)*180/pi instead gives an angle which is 524 | # mirror flipped about the y-axis compared to the true edge 525 | # line. I.e., say the true edge lies parallel to some angle 526 | # which we shall define as (90 + theta), with 527 | # 90 >= theta >= 0, (by definition therefore giving the edge 528 | # line a negative slope) then atan2(yvar, xvar)*pi/2 529 | # returns the value (90 - theta), which is mirror-flipped 530 | # about the y-axis relative to the true value of 531 | # (90 + theta). As a consequence of the way that we defined 532 | # the true edge parallel direction (90 + theta), theta itself 533 | # turns out to be just the edge normal direction that we had 534 | # wanted to find, and thus, solving for theta, we get 535 | # theta = -(atan2(yvar, xvar)*180/pi - 90). I.e., its's the 536 | # same as for the positive covariance case, except for a sign 537 | # flip. Note however that there is one key difference 538 | # between this case and the so-called the "inter-pixel" case 539 | # above: the angles in this "intra-pixel" case are 540 | # pre-constrained to fall only between -90 and +90 degrees, 541 | # whereas "inter-pixel" angles may fall anywhere from -180 542 | # to +180. 543 | edgedir = math.atan2(yvar, xvar) * 180 / math.pi - 90 544 | if xycovar < 0: 545 | edgedir = -edgedir 546 | return edgedir 547 | elif mode == 'Corner' or mode == 'corner': 548 | # Eq. 3, but with an alternative default setting (as compared to 549 | # 'Edge' mode) for the gfrac parameter 550 | R = gfrac*nmax - n 551 | if R<0: R=0. 552 | # Do false corner suppression, but only if there appears to be 553 | # a genuine non-zero corner response 554 | if R>0: 555 | denom = (areawgt * c).sum() 556 | # Usual case 557 | if denom: 558 | # Calculate center of gravity, using eq. 5 in reference 559 | xcg = ((xdiff * areawgt * c).sum()) / denom 560 | ycg = ((ydiff * areawgt * c).sum()) / denom 561 | # Divide-by-zero case, which can arise when a single noisy 562 | # pixels is surrounded by many others of dissimilar brightness 563 | else: 564 | xcg, ycg = 0, 0 565 | cgdist = math.sqrt(xcg**2 + ycg**2) 566 | # False corner check #1: CG is too close to nucleus 567 | if cgdist < cgthresh: 568 | R=0 569 | else: 570 | # CG vector direction 571 | theta = math.atan2(ycg.astype(float), xcg.astype(float)) 572 | # Calculate nearest pixel locations of a bunch of sub- 573 | # pixel-spaced points on a line along the CG direction 574 | for ii in np.arange(0, radius, 1./npoints): 575 | xtest = int(round(ii * math.cos(theta))) 576 | ytest = int(round(ii * math.sin(theta))) 577 | for ij in range(len(xdiff)): 578 | # Find corresponding index ij in footprint data 579 | if xtest == xdiff[ij] and ytest == ydiff[ij]: 580 | # False corner check #2: non-contiguous USAN 581 | if areawgt[ij] == 1 and graydiff[ij] > t: 582 | R=0 583 | break 584 | if R == 0: 585 | break 586 | return R 587 | else: 588 | raise ValueError('Mode ' + str(mode) + ' not recognized') 589 | 590 | """ Finally, perform the USAN filter operation! """ 591 | # Note that image must be cast as float in order to force 592 | # filters.generic_filter() to return the output type as float (which 593 | # is what we usually want). However, the image itself is intrinisically 594 | # type int, and the journal reference recommends using a lookup table to 595 | # compare the relatively limited number (nshades=256) of different 596 | # possible pixel grayscale color values against one another. This means 597 | # that as soon as program control drops down into the filterfunc() 598 | # (defined above), the color-related variables must be cast right back 599 | # to int again in order to be able to index the lookup table properly. 600 | # It's convoluted! 601 | extraarg = (areawgt, xdiff, ydiff, radius, mode, t, explookup, gfrac, 602 | cgthresh) 603 | return filters.generic_filter(image.astype(float), filterfunc, 604 | footprint=fp, extra_arguments=extraarg) 605 | -------------------------------------------------------------------------------- /other_algorithms/susan.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | from scipy.ndimage import filters 4 | from . import masks 5 | 6 | 7 | def susanCorner(image, radius=3.4, fptype=bool, t=25, gfrac=None, 8 | cgthresh=None, nshades=256): 9 | 10 | # Get raw corner response; many corners may consist of small clusters of 11 | # adjacent responsive pixels 12 | rawcorner = masks.usan(image, mode='Corner', radius=radius, 13 | fptype=fptype, t=t, gfrac=gfrac, 14 | cgthresh=cgthresh, nshades=nshades) 15 | 16 | # Find maximum corner response within circular USAN footprint (but force 17 | # footprint type to be bool in this case regardless of user-selected 18 | # input fptype, because float would make no sense in this context) 19 | fp = masks.circFootprint(radius=radius, dtype=bool) 20 | rawmax = filters.maximum_filter(rawcorner, footprint=fp) 21 | 22 | # True corners are those where response is both locally maximum as well 23 | # as non-zero 24 | corner = np.where(rawcorner == rawmax, rawcorner, 0) 25 | 26 | return corner 27 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | 2 | import glob 3 | import cv2 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import math 7 | import re 8 | import os 9 | import timeit 10 | from matplotlib.patches import ConnectionPatch 11 | from pprint import pprint 12 | from sklearn.metrics import average_precision_score 13 | from random import sample 14 | 15 | from const import * 16 | 17 | TIME_MEASURES_TXT = 'time_measures_zver_3.txt' 18 | 19 | 20 | def alreadyCompiledKeypoints(dataset_path): 21 | ''' 22 | Check which detectors and descriptors you already compiled. 23 | ''' 24 | 25 | if 'kp.npz' in os.listdir(dataset_path + '/v_london'): 26 | file = np.load(dataset_path + '/v_london/kp.npz') 27 | print('Compiled keypoints: {}'.format(file.files)) 28 | else: 29 | print('There are no keypoints already compiled in kp.npz') 30 | 31 | if 'des.npz' in os.listdir(dataset_path + '/v_london'): 32 | file = np.load(dataset_path + '/v_london/des.npz') 33 | print('Compiled descriptors: {}'.format(file.files)) 34 | else: 35 | print('There are no descriptors already compiled in des.npz') 36 | 37 | 38 | def deleteAllKeypoints(dataset_path): 39 | ''' 40 | Delete all kp.npz and des.npz in all folders. 41 | ''' 42 | folders = glob.glob(dataset_path) 43 | 44 | for folder in folders: 45 | desss = glob.glob(folder + '/des*') 46 | kpppp = glob.glob(folder + '/kp*') 47 | for d in desss: 48 | os.remove(d) 49 | for k in kpppp: 50 | os.remove(k) 51 | 52 | 53 | def createKeypoints(detector_name, descriptor_name, dataset_path, all_at_once=False): 54 | ''' 55 | For a given detector and descriptor pair, save keypoints 56 | and descriptors to kp.npz and des.npz in every folder for whole sequence 57 | ''' 58 | 59 | sequence_images = {} 60 | folders = glob.glob(dataset_path) 61 | 62 | for folder in folders: 63 | folder_name = folder.split('/')[-1] 64 | print('Working on folder {}'.format(folder_name)) 65 | 66 | # load sequence images in list 67 | sequence_images[folder_name] = glob.glob(folder + '/*.ppm') 68 | sequence_images[folder_name] = sorted(sequence_images[folder_name]) 69 | sequence_images[folder_name] = [cv2.imread(im) for im in sequence_images[folder_name]] 70 | 71 | # convert images to gray 72 | images_ = [ cv2.cvtColor(im ,cv2.COLOR_RGB2GRAY) for im in sequence_images[folder_name]] 73 | 74 | # create detector and descriptor instances 75 | detector = [ all_detectors[detector_name]() for im in sequence_images[folder_name]] 76 | descriptor = [ all_descriptors[descriptor_name]() for im in sequence_images[folder_name]] 77 | 78 | kp = [] 79 | des = [] 80 | 81 | for id1, algorithm in enumerate(zip(detector,descriptor)): 82 | 83 | if not all_at_once: 84 | start_time = timeit.default_timer() 85 | kp_ = algorithm[0].detect(images_[id1],None) 86 | 87 | kp_, des_ = algorithm[1].compute(images_[id1], kp_) 88 | end_time = timeit.default_timer() 89 | 90 | if (des_ == None).any(): 91 | des_ = np.array([]) 92 | 93 | with open(TIME_MEASURES_TXT,'a+') as file: 94 | file.write('{},{},{}\n'.format(detector_name, 95 | descriptor_name, 96 | end_time-start_time)) 97 | else: 98 | start_detector_and_descriptor = timeit.default_timer() 99 | kp_, des_ = det.detectAndCompute(images_[id1],None) 100 | end_detector_and_descriptor = timeit.default_timer() 101 | 102 | if (des_ == None).any(): 103 | des_ = np.array([]) 104 | 105 | with open(TIME_MEASURES_TXT,'a+') as file: 106 | file.write('{},{},{}\n'.format(detector_name, descriptor_name, 107 | end_detector_and_descriptor-start_detector_and_descriptor)) 108 | 109 | kp_np = np.array([(k.pt[0], k.pt[1], k.angle, k.size, k.response) for k in kp_]) 110 | 111 | kp.append(kp_np) 112 | des.append(des_) 113 | 114 | if 'kp.npz' in os.listdir(folder): 115 | file = np.load(folder + '/kp.npz') 116 | elements = dict(file) 117 | elements[detector_name] = kp 118 | np.savez(folder + '/kp.npz', **elements) 119 | 120 | else: 121 | np.savez(folder + '/kp.npz', **{detector_name:kp}) 122 | 123 | if 'des.npz' in os.listdir(folder): 124 | file = np.load(folder + '/des.npz') 125 | elements = dict(file) 126 | elements[detector_name + '_' + descriptor_name] = des 127 | np.savez(folder + '/des.npz', **elements) 128 | 129 | else: 130 | nm = detector_name + '_' + descriptor_name 131 | np.savez(folder + '/des.npz', **{nm:des}) 132 | 133 | 134 | def getTransformations(dataset_path): 135 | transformations = {} 136 | folders = glob.glob(dataset_path) 137 | 138 | def load_transform(tr): 139 | with open(tr) as file: 140 | s = file.read() 141 | nrs = re.split('\n| ',s)[:-1] 142 | nrs = [nr for nr in nrs if nr != ''] 143 | return np.array(nrs).reshape(3,3).astype(np.float) 144 | 145 | for folder in folders: 146 | folder_name = folder.split('/')[-1] 147 | transformations[folder_name] = glob.glob(folder + '/H*') 148 | transformations[folder_name] = sorted(transformations[folder_name]) 149 | transformations[folder_name] = [load_transform(tr) for tr in transformations[folder_name]] 150 | 151 | return transformations 152 | 153 | 154 | def removeUncommonPoints(detector_name, descriptor_name, dataset_path): 155 | ''' 156 | Remove keypoints from ref image that do not appear on sequence images 157 | when imaged with homography H. 158 | This function expects that keypoints already exist in kp.npz and des.npz 159 | for given detector and descriptor names. 160 | ''' 161 | transformations = getTransformations(dataset_path) 162 | folders = glob.glob(dataset_path) 163 | 164 | 165 | for folder in folders: 166 | folder_name = folder.split('/')[-1] 167 | print('#########################################') 168 | print('Working on folder {}'.format(folder_name)) 169 | 170 | 171 | kp_file = np.load(folder + '/kp.npz', allow_pickle=True) 172 | kp = kp_file[detector_name] 173 | kp = list(kp) 174 | 175 | des_file = np.load(folder + '/des.npz', allow_pickle=True) 176 | nm = detector_name + '_' + descriptor_name 177 | des = des_file[nm] 178 | des = list(des) 179 | 180 | indexes_to_remove = [] 181 | 182 | # remove keypoints from ref image that do not appear on sequence images 183 | remove = set() 184 | kp_ = kp[0].copy() 185 | 186 | # iterate over homographies H and image points onto sequence image 187 | for id2, tr in enumerate(transformations[folder_name]): 188 | # image points 189 | if kp_.shape[0] == 0: 190 | with open('missing_kp_on_ref','a+') as fl: 191 | fl.write('{}: algo with no kp on 1.ppm for folder {} and image {}.'.format(detector_name, 192 | folder_name, 193 | id2+2)) 194 | print('THERE ARE NO KEYPOINTS ON IMAGE 1.ppm IN FOLDER {}'.format(folder_name)) 195 | continue 196 | points = np.c_[ kp_[:,[0,1]] , np.ones(kp_.shape[0])] 197 | imaged_points = np.dot(tr, points.T) 198 | imaged_points_normal = imaged_points/imaged_points[2,:] 199 | 200 | # get bounds of image on which we are projecting 201 | image_size = cv2.imread(folder + '/' + str(id2+2) + '.ppm' ) 202 | image_size = image_size.shape 203 | 204 | # get indexes that are out of bounds on the sequence image 205 | x_indexes_out_of_bounds = np.where((imaged_points_normal[0,:] < 0) | 206 | (image_size[1] < imaged_points_normal[0,:]))[0] 207 | 208 | 209 | y_indexes_out_of_bounds = np.where((imaged_points_normal[1,:] < 0) | 210 | (image_size[0] < imaged_points_normal[1,:]))[0] 211 | 212 | # add the indexes to set 213 | remove = remove.union(x_indexes_out_of_bounds) 214 | remove = remove.union(y_indexes_out_of_bounds) 215 | 216 | # create a list from the set 217 | indexes_to_remove = list(remove) 218 | 219 | # delete the indexes 220 | print('Removing {} keypoints from image {}/1.ppm'.format(len(indexes_to_remove),folder_name)) 221 | print('old size: {}'.format(kp[0].shape)) 222 | kp[0] = np.delete(kp[0], indexes_to_remove, 0) 223 | print('new size: {}'.format(kp[0].shape)) 224 | 225 | des[0] = np.delete(des[0], indexes_to_remove, 0) 226 | 227 | # save the new keypoints to disk 228 | elements = dict(kp_file) 229 | elements[detector_name] = kp 230 | np.savez(folder + '/kp.npz', **elements) 231 | 232 | elements = dict(des_file) 233 | elements[nm] = des 234 | np.savez(folder + '/des.npz', **elements) 235 | 236 | 237 | def createMeTable(): 238 | ''' 239 | Loads execution times for detectors+descriptors and prints out how the 240 | table should look in latex. 241 | ''' 242 | import pandas as pd 243 | 244 | data = pd.read_csv('rezultati/time_measures_zver_1.txt', 245 | sep=",", 246 | header=None, 247 | names=['det','des','time'], 248 | index_col=None) 249 | data1 = pd.read_csv('rezultati/time_measures_lfnet_bartools.txt', 250 | sep=",", 251 | header=None, 252 | names=['det','des','time'], 253 | index_col=None) 254 | 255 | data2 = pd.read_csv('rezultati/time_measures_superpoint_bartools.txt', 256 | sep=",", 257 | header=None, 258 | names=['det','des','time'], 259 | index_col=None) 260 | data = data.append(data1, ignore_index=True) 261 | data = data.append(data2, ignore_index=True) 262 | 263 | data['time'] = data['time'].astype('float64') 264 | 265 | data2 = pd.DataFrame(columns=['det','des','time']) 266 | for name, df in data.groupby(['det','des']): 267 | me = np.mean(df['time']) 268 | data2 = data2.append(pd.Series([name[0].upper(),name[1].upper(),me], 269 | index=['det','des','time']), 270 | ignore_index=True) 271 | 272 | data2.sort_values(by=['time','det'], inplace=True) 273 | data2['time'] = data2['time'].round(3) 274 | 275 | data2.reset_index(inplace=True,drop=True) 276 | 277 | print(data2.head()) 278 | 279 | tt = min(data2.shape[0],10) 280 | data2 = data2.loc[:tt,:] 281 | 282 | print('Detector & ' + ' & '.join(data2['det']) + ' \\\\') 283 | print('\\hline') 284 | print('Descriptor & ' + ' & '.join(data2['des']) + ' \\\\') 285 | print('\\hline') 286 | print('Time & ' + ' & '.join(data2['time'].astype('str')) + ' \\\\') 287 | print('\\hline') 288 | 289 | 290 | def read_keypoints(folder, detector_name, descriptor_name): 291 | # Get keypoints for the sequence 292 | kp_all = np.load(folder + '/kp.npz', allow_pickle=True) 293 | kp = kp_all[detector_name] 294 | 295 | # Get descriptors for the sequence 296 | des_all = np.load(folder + '/des.npz', allow_pickle=True) 297 | nm = detector_name + '_' + descriptor_name 298 | des = des_all[nm] 299 | 300 | return list(kp), list(des) 301 | 302 | 303 | 304 | def read_next_keypoints(detector_name, descriptor_name, folders, folder_id, m=100): 305 | 306 | kp_all = [] 307 | des_all = [] 308 | 309 | random_images = sample(range(len(folders)), m) 310 | 311 | for ind in random_images: 312 | seq = (folder_id+ind) % len(folders) 313 | image_nr = ind%6 314 | 315 | kp = np.load(folders[seq] + '/kp.npz', allow_pickle=True) 316 | kp = list(kp[detector_name]) 317 | kp = kp[image_nr] 318 | kp_all.append(kp) 319 | 320 | des = np.load(folders[seq]+'/des.npz', allow_pickle=True) 321 | nm = detector_name + '_' + descriptor_name 322 | des = list(des[nm]) 323 | des = des[image_nr] 324 | des_all.append(des) 325 | 326 | return kp_all, des_all 327 | 328 | if __name__ == '__main__': 329 | import argparse 330 | 331 | parser_of_args = argparse.ArgumentParser(description='Create kp and des') 332 | parser_of_args.add_argument('detector_name', type=str, 333 | help='name of the detector') 334 | parser_of_args.add_argument('descriptor_name', type=str, 335 | help='name of descriptor') 336 | parser_of_args.add_argument('dataset_path', type=str, 337 | help='path to hpatches dataset') 338 | 339 | args = parser_of_args.parse_args() 340 | 341 | # project_root = '/home/davidboja/PycharmProjects/FER/hpatches-benchmark/python/ISPA' 342 | # dataset_path = project_root + '/hpatches-sequences-release/*' 343 | 344 | dataset_path = args.dataset_path + '/*' 345 | 346 | createKeypoints(args.detector_name, args.descriptor_name, dataset_path) 347 | removeUncommonPoints(args.detector_name, args.descriptor_name, dataset_path) 348 | -------------------------------------------------------------------------------- /visualizations.py: -------------------------------------------------------------------------------- 1 | 2 | import cv2 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import seaborn 6 | import math 7 | import glob 8 | import re 9 | import os 10 | from matplotlib.patches import ConnectionPatch 11 | from pprint import pprint 12 | from sklearn.metrics import average_precision_score 13 | from random import sample 14 | 15 | from const import * 16 | 17 | def processLine(line): 18 | name = line.split('|')[0] 19 | 20 | if name.split('_')[0] == name.split('_')[1]: 21 | name = name.split('_')[0] 22 | 23 | all = line.split('|')[1].split('[')[1].split(']')[0].split(',') 24 | all = [float(a_) for a_ in all] 25 | 26 | illumination = line.split('|')[2].split('[')[1].split(']')[0].split(',') 27 | illumination = [float(i_) for i_ in illumination] 28 | 29 | viewpoint = line.split('|')[3].split('[')[1].split(']')[0].split(',') 30 | viewpoint = [float(v_) for v_ in viewpoint] 31 | 32 | return name, all, illumination, viewpoint 33 | 34 | 35 | def visualizeKp(detector_name, folder_path): 36 | ''' 37 | Visualize keypoints of a detector on a sequence in a given folder. 38 | ''' 39 | 40 | sequence_images = glob.glob(folder_path + '/*.ppm') 41 | sequence_images = sorted(sequence_images) 42 | image_names = [im.split('/')[-1] for im in sequence_images] 43 | sequence_images = [cv2.imread(im) for im in sequence_images] 44 | 45 | kp = np.load(folder_path + '/kp.npz') 46 | kp = kp[detector_name] 47 | kp = list(kp) 48 | 49 | fig, ax = plt.subplots(nrows=3, ncols=2, figsize=(20,20)) 50 | for id1, axx in enumerate(ax.ravel()): 51 | 52 | gray = cv2.cvtColor(sequence_images[id1] ,cv2.COLOR_RGB2GRAY) 53 | # gray_with_kp = cv2.drawKeypoints(gray, 54 | # kp[id1], 55 | # cv2.DRAW_MATCHES_FLAGS_DEFAULT) 56 | 57 | # axx.imshow(gray_with_kp) 58 | 59 | axx.imshow(gray) 60 | axx.scatter(kp[id1][:,0],kp[id1][:,1], s=1, color='red') 61 | axx.set_title('{}'.format(image_names[id1])) 62 | plt.show() 63 | 64 | 65 | def topNrMatches(detector_name, descriptor_name, folder_path, nr_matches): 66 | ''' 67 | Draw grid of 5x2 images where the left one is always the ref image (1.ppm) 68 | and the right ones are 2-6.ppm. Draw lines between matching keypoints. 69 | ''' 70 | 71 | # load images 72 | sequence_images = glob.glob(folder_path + '/*.ppm') 73 | sequence_images = sorted(sequence_images) 74 | image_names = [im.split('/')[-1] for im in sequence_images] 75 | sequence_images = [cv2.imread(im) for im in sequence_images] 76 | 77 | # load detectors 78 | kp = np.load(folder_path + '/kp.npz') 79 | kp = kp[detector_name] 80 | kp = list(kp) 81 | 82 | # load descriptors 83 | des = np.load(folder_path + '/des.npz') 84 | nm = detector_name + '_' + descriptor_name 85 | des = des[nm] 86 | des = list(des) 87 | 88 | # compute matches 89 | matches_with_ref = [] 90 | 91 | for dess in des[1:]: 92 | bf = cv2.BFMatcher(cv2.NORM_L2) 93 | matches = bf.match(des[0], dess) 94 | matches_with_ref.append(sorted(matches, key = lambda x:x.distance)) 95 | 96 | fig, ax = plt.subplots(nrows=5, ncols=2, figsize=(30,30)) 97 | for i, axx in enumerate(ax.ravel()): 98 | if (i % 2) == 1: 99 | continue 100 | 101 | points_on_first_img = [] 102 | points_on_second_img = [] 103 | 104 | axx.imshow(sequence_images[0]) 105 | axx.set_title('{}'.format(image_names[0])) 106 | ax.ravel()[i+1].imshow(sequence_images[int(i/2)+1]) 107 | ax.ravel()[i+1].set_title('{}'.format(image_names[int(i/2)+1])) 108 | 109 | for m in matches_with_ref[int(i/2)][:nr_matches]: 110 | points_on_first_img.append(kp[0][m.queryIdx,[0,1]]) 111 | points_on_second_img.append(kp[int(i/2)+1][m.trainIdx,[0,1]]) 112 | 113 | axx.scatter([i[0] for i in points_on_first_img], 114 | [i[1] for i in points_on_first_img], 115 | s=20, 116 | c='red') 117 | 118 | ax.ravel()[i+1].scatter([i[0] for i in points_on_second_img], 119 | [i[1] for i in points_on_second_img], 120 | s=20, 121 | c='red') 122 | 123 | for j in range(len(points_on_first_img)): 124 | con = ConnectionPatch(xyA=(points_on_second_img[j][0],points_on_second_img[j][1]), 125 | xyB=(points_on_first_img[j][0],points_on_first_img[j][1]), 126 | coordsA="data", coordsB="data", 127 | axesA=ax.ravel()[i+1], axesB=axx, color="red") 128 | 129 | ax.ravel()[i+1].add_artist(con) 130 | 131 | 132 | plt.show() 133 | 134 | 135 | def processResultDict(dic): 136 | 137 | name = [] 138 | all = [] 139 | i_ = [] 140 | v_ = [] 141 | 142 | for alg,results in dic.items(): 143 | name.append(alg.upper()) 144 | 145 | mean_ = np.mean(results['a']) 146 | all.append(float('{0:.2f}'.format(mean_))) 147 | 148 | mean_ = np.mean(results['i']) 149 | i_.append(float('{0:.2f}'.format(mean_))) 150 | 151 | mean_ = np.mean(results['v']) 152 | v_.append(float('{0:.2f}'.format(mean_))) 153 | 154 | # error.append([float('{0:.2f}'.format(mean_ - np.min(results))), 155 | # float('{0:.2f}'.format(np.max(results) - mean_))]) 156 | 157 | return name, all, i_, v_ 158 | 159 | 160 | def taskEvaluation(dict_of_APs): 161 | ''' 162 | Returns a barchart with the keys of the dict_of_APs as names of algorithms 163 | and height as mAP from dict_of_APs values. Aditionally plot dots for min 164 | and max values on barchart. 165 | ''' 166 | names = {} 167 | all = {} 168 | i_ = {} 169 | v_ = {} 170 | labels = ['AP','mAP','mAP'] 171 | 172 | for i in range(3): 173 | names[i], all[i], i_[i], v_[i] = processResultDict(dict_of_APs[i]) 174 | 175 | pos = np.arange(len(names[0])) 176 | 177 | fig, ax1 = plt.subplots(ncols=3, figsize=(10,5)) 178 | fig.subplots_adjust(left=0.12, right=0.92, top=0.9, bottom=0.09, wspace=0.4) 179 | 180 | 181 | for i in range(3): 182 | ax1[i].barh( 183 | pos, 184 | all[i], 185 | align='center', 186 | height=0.5, 187 | tick_label=names[i] if i == 0 else ['' for _ in range(len(names[0]))], 188 | color=seaborn.color_palette("hls", len(names[0])),#seaborn.color_palette("Paired"),#seaborn.color_palette("Set2"), 189 | #xerr=np.array(err[i]).T, 190 | capsize=5., 191 | zorder=1) 192 | plt.xlim([0, 1]) 193 | ax2 = ax1[i].twinx() 194 | ax2.set_yticks(pos) 195 | ax2.set_ylim(ax1[i].get_ylim()) 196 | ax2.set_yticklabels(['{}%'.format(s) for s in all[i]]) 197 | 198 | ill = ax1[i].scatter(i_[i],pos, zorder=2, color='black', marker='x',s=15, linewidth=0.5) 199 | vie = ax1[i].scatter(v_[i],pos, zorder=2, color='black', marker='<', s=10, linewidth=0.8) 200 | 201 | ax1[i].set_title(TASKS[i]) 202 | ax1[i].set_xlabel(labels[i]) 203 | plt.figlegend((ill,vie), ('viewpoint','illumination'), loc = 'upper center', 204 | ncol=2, fontsize='small')#, labelspacing=0. ) 205 | #plt.show() 206 | #plt.title('') 207 | plt.savefig('graph.pdf', format='pdf') 208 | 209 | 210 | if __name__ == '__main__': 211 | import argparse 212 | import collections 213 | 214 | # parser_of_args = argparse.ArgumentParser(description='Visualize keypoints and graphs') 215 | # parser_of_args.add_argument('detector_name', type=str, 216 | # help='Name of the detector') 217 | # parser_of_args.add_argument('descriptor_name', type=str, 218 | # help='Name of the descriptor') 219 | # parser_of_args.add_argument('folder_path', type=str, 220 | # help='Path of folder') 221 | # parser_of_args.add_argument('nr_matches', type=int, 222 | # help='Number of matches to visualize') 223 | # 224 | # args = parser_of_args.parse_args() 225 | 226 | # visualizeKp(args.detector_name, args.folder_path) 227 | # topNrMatches(args.detector_name, args.descriptor_name, args.folder_path, args.nr_matches) 228 | 229 | # dict_of_APs = { 230 | # SIFT: np.random.rand(5), 231 | # SURF : np.random.rand(5), 232 | # BRISK : np.random.rand(5), 233 | # BRIEF : np.random.rand(5), 234 | # ORB : np.random.rand(5), 235 | # ALGO_TEMPLATE.format(SIFT, SURF) : np.random.rand(5), 236 | # ALGO_TEMPLATE.format(BRISK, BRIEF) : np.random.rand(5), 237 | # SHI_TOMASI : np.random.rand(5) 238 | # } 239 | 240 | pV = collections.OrderedDict() 241 | with open('rezultati/pV_zver3_1.txt','r') as f: 242 | for line in f: 243 | name, all, i_, v_ = processLine(line) 244 | pV[name] = {} 245 | pV[name]['a'] = all 246 | pV[name]['i'] = i_ 247 | pV[name]['v'] = v_ 248 | 249 | with open('rezultati/pV3_bartools.txt','r') as f: 250 | for line in f: 251 | name, all, i_, v_ = processLine(line) 252 | pV[name] = {} 253 | pV[name]['a'] = all 254 | pV[name]['i'] = i_ 255 | pV[name]['v'] = v_ 256 | 257 | iM = collections.OrderedDict() 258 | with open('rezultati/iM_zver3_1.txt','r') as f: 259 | for line in f: 260 | name, all, i_, v_ = processLine(line) 261 | iM[name] = {} 262 | iM[name]['a'] = all 263 | iM[name]['i'] = i_ 264 | iM[name]['v'] = v_ 265 | 266 | with open('rezultati/iM3_bartools.txt','r') as f: 267 | for line in f: 268 | name, all, i_, v_ = processLine(line) 269 | iM[name] = {} 270 | iM[name]['a'] = all 271 | iM[name]['i'] = i_ 272 | iM[name]['v'] = v_ 273 | 274 | pR = collections.OrderedDict() 275 | with open('rezultati/pR_zver3_1.txt','r') as f: 276 | for line in f: 277 | name, all, i_, v_ = processLine(line) 278 | pR[name] = {} 279 | pR[name]['a'] = all 280 | pR[name]['i'] = i_ 281 | pR[name]['v'] = v_ 282 | 283 | with open('rezultati/pR3_bartools.txt','r') as f: 284 | for line in f: 285 | name, all, i_, v_ = processLine(line) 286 | pR[name] = {} 287 | pR[name]['a'] = all 288 | pR[name]['i'] = i_ 289 | pR[name]['v'] = v_ 290 | 291 | dict_of_APs = {} 292 | dict_of_APs[0] = pV 293 | dict_of_APs[1] = iM 294 | dict_of_APs[2] = pR 295 | 296 | taskEvaluation(dict_of_APs) 297 | --------------------------------------------------------------------------------