├── LICENSE ├── PnP parse data.ipynb ├── README.md ├── create_E_submission.py ├── create_F_submission.py ├── create_H_submission.py ├── create_opencv_F_submission_example.py ├── create_opencv_homography_submission_example.py ├── download.sh ├── eval_E_submission.py ├── eval_F_submission.py ├── eval_H_submission.py ├── eval_all_test_E.py ├── eval_all_test_F.py ├── eval_all_test_H.py ├── hdf5reader.py ├── metrics.py ├── parse_EF_data.ipynb ├── parse_H_data.ipynb ├── tune_hyperparameters_and_create_test_E_submission.py ├── tune_hyperparameters_and_create_test_F_submission.py ├── tune_hyperparameters_and_create_test_H_submission.py ├── upgrade_E_submission.py └── utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## RANSAC 2020 tutorial starter pack (in progress) 2 | 3 | # Data for epipolar geometry training and validation 4 | 5 | Training and validation data you can download from http://cmp.felk.cvut.cz/~mishkdmy/CVPR-RANSAC-Tutorial-2020/RANSAC-Tutorial-Data-EF.tar 6 | It was obtained with RootSIFT features and mutual nearest neighbour matching. 7 | 12 scenes, 100k image pairs each, so 1.2M cases for training, 86 Gb. 8 | 9 | Validation-only set is available from [here](http://cmp.felk.cvut.cz/~mishkdmy/CVPR-RANSAC-Tutorial-2020/RANSAC-Tutorial-Data-ValOnly.tar). It is 2.1Gb. 10 | 11 | Correspondences, which are obtained without mutual nearest neighbor check are available from [here](http://ptak.felk.cvut.cz/personal/mishkdmy/CVPR2020-RANSAC-Tutorial/RANSAC-Tutorial-Data-uni.tar), 349 Gb. 12 | 13 | The data comes from train and validation set of the [CVPR IMW 2020 PhotoTourism challenge](https://vision.uvic.ca/image-matching-challenge/data/) 14 | 15 | 16 | Jupyter notebook, showing the format of the data and toy evaluation example is [here](parse_EF_data.ipynb). 17 | 18 | 19 | Your methods can use as an input: 20 | 21 | x, y, matching score for fundamental matrix case 22 | 23 | and x,y, matching score, calibration matrices K1, K2 for essential matrix case. 24 | 25 | The code for running OpenCV RANSACs evaluation is [here](create_opencv_F_submission_example.py) 26 | 27 | ```bash 28 | python -utt create_opencv_F_submission_example.py 29 | 30 | ``` 31 | 32 | 33 | The test data is here http://cmp.felk.cvut.cz/~mishkdmy/CVPR-RANSAC-Tutorial-2020/RANSAC-Tutorial-Data-EF-Test.tar 34 | 35 | [**PyTorch data loader for hdf5 files**](hdf5reader.py) 36 | 37 | 38 | # Data for training and validation of PnP methods 39 | 40 | Training and validation data you can download from http://cmp.felk.cvut.cz/~mishkdmy/CVPR-RANSAC-Tutorial-2020/RANSAC-Tutorial-Data-PnP.tar.gz (612Mb). 41 | The data is from [EPOS](http://cmp.felk.cvut.cz/epos/) datasset. 42 | 43 | The description of the dataset format and parser to read the data is in [this notebook](https://github.com/ducha-aiki/ransac-tutorial-2020-data/blob/master/PnP%20parse%20data.ipynb) 44 | 45 | 46 | 47 | # Data for hyperparameter tuning for homography. No training data 48 | 49 | Test(without GT) and validation data you can download from http://cmp.felk.cvut.cz/~mishkdmy/CVPR-RANSAC-Tutorial-2020/homography.tar.gz 50 | 51 | 52 | Jupyter notebook, showing the format of the data and toy evaluation example is [here](parse_H_data.ipynb). 53 | Your methods can use as an input: 54 | 55 | x, y, matching score 56 | 57 | The evaluation metric is mean averacy accuracy over set of thresholds. And the thresholded metric is MAE: mean absolute error on jointly visible part of the both images. 58 | For more details see [metrics.py](metrics.py) 59 | 60 | 61 | ## Example submission 62 | 63 | To tune hyperparameters on the validation set and create test set prediction with OpenCV RANSAC, run the following script [create_opencv_homography_submission_example.py](create_opencv_homography_submission_example.py) 64 | 65 | It will run hyper parameter search on the validation set and then creates two files: homography_opencv_EVD_submission.h5 and homography_opencv_HPatchesSeq_submission.h5 66 | Each of them has the same format as ground truth homography: h5 file with image pairs names as the key and [3x3] homography as an output. 67 | 68 | 69 | ```bash 70 | python -utt create_opencv_homography_submission_example.py 71 | > 72 | 73 | ``` 74 | 75 | 76 | # Data for 3D point cloud stitching 77 | 78 | 79 | ## Point clouds from ETHZ Photogrammetry and Remote Sensing Group 80 | 81 | Download dataset here: 82 | 83 | https://cloudstor.aarnet.edu.au/plus/s/229Wnoez2c35Cmw 84 | 85 | The data is organised as follows: 86 | - For each scene, there is a sequence of T lidar scans { keypoint_s[t].pcd }, where t=1,...,T. 87 | - Correspondences are available for only consecutive scans { corr_s[t]_s[t+1].txt }. 88 | 89 | In the zip package, there is a Matlab function ``pc_plotter(fname_pc1, fname_pc2, fname_corr)`` to plot pairs of point clouds and their 3D correspondences. The function requires file names for a pair of point clouds and their correspondences. As an example, for Arch, you could run in Matlab 90 | ``` 91 | pc_plotter('../arch/keypoint_s1.pcd', '../arch/keypoint_s2.pcd', '../arch/corr_s1_s2.txt') 92 | ``` 93 | to display the 3D correspondences. 94 | 95 | The inlier thresholds are ... (TBA) 96 | 97 | The ground truth transformation parameters are available here: 98 | 99 | https://cloudstor.aarnet.edu.au/plus/s/mDMT09jEk0tZTrL 100 | 101 | Use a text editor to open the files. 102 | 103 | This data was sourced from Photogrammetry and Remote Sensing Group at ETH Zurich: 104 | 105 | https://prs.igp.ethz.ch/research/completed_projects/automatic_registration_of_point_clouds.html 106 | 107 | If you use the data in any publication, please credit the original authors properly. 108 | 109 | ## Point clouds from Microsoft 7 Scenes data. 110 | 111 | Download dataset here: 112 | 113 | https://cloudstor.aarnet.edu.au/plus/s/9KQBYVFSjYn0PDH 114 | 115 | There are two instances in this dataset. 116 | 117 | In the zip package, there is a Matlab script `plot_7scenes.m` to plot the point clouds and their correspondences. After you run the script, the inlier threshold and ground truth transformation are available respectively in `inst.conf.th` and `GT`. 118 | 119 | This data was sourced from Microsoft RGB-D Dataset 7-Scenes: 120 | 121 | https://www.microsoft.com/en-us/research/project/rgb-d-dataset-7-scenes/ 122 | 123 | If you use the data in any publication, please credit the original authors properly. 124 | 125 | # Data for 3D object detection 126 | 127 | ## RGBD Object Dataset 128 | 129 | Download dataset here: 130 | 131 | https://cloudstor.aarnet.edu.au/plus/s/ko1F2tFQzAyG1I0 132 | 133 | There are two instances in this dataset. 134 | 135 | In the zip package, there is a Matlab script `plot_lai.m` to plot the point clouds and their correspondences. After you run the script, the inlier threshold and ground truth transformation are available respectively in `inst.conf.th` and `GT`. 136 | 137 | This data was sourced from RGB-D Object Dataset from University of Washington: 138 | 139 | https://rgbd-dataset.cs.washington.edu/ 140 | 141 | If you use the data in any publication, please credit the original authors properly. 142 | 143 | # Other cases 144 | 145 | TBA if any. 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /create_E_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | import pydegensac 12 | 13 | from skimage.measure import ransac as skransac 14 | from skimage.transform import EssentialMatrixTransform 15 | import multiprocessing 16 | 17 | from joblib import Parallel, delayed 18 | import sys 19 | try: 20 | from third_party.NM_Net_v2 import NMNET22 21 | import torch 22 | except Exception as e: 23 | print (e) 24 | #sys.exit(0) 25 | pass 26 | 27 | 28 | 29 | def get_single_result(ms, m, method, K1, K2, params): 30 | mask = ms <= params['match_th'] 31 | tentatives = m[mask] 32 | tentative_idxs = np.arange(len(mask))[mask] 33 | src_pts = normalize_keypoints(tentatives[:, :2], K1) 34 | dst_pts = normalize_keypoints(tentatives[:, 2:], K2) 35 | 36 | if tentatives.shape[0] <= 10: 37 | return np.eye(3), np.array([False] * len(mask)) 38 | if method == 'cv2e': 39 | E, mask_inl = cv2.findEssentialMat(src_pts, dst_pts, 40 | np.eye(3), cv2.RANSAC, 41 | threshold=params['inl_th'], 42 | prob=params['conf']) 43 | mask_inl = mask_inl.astype(bool).flatten() 44 | elif method == 'sklearn': 45 | try: 46 | #print(src_pts.shape, dst_pts.shape) 47 | E, mask_inl = skransac([src_pts, dst_pts], 48 | EssentialMatrixTransform, 49 | min_samples=8, 50 | residual_threshold=params['inl_th'], 51 | max_trials=params['maxiter'], 52 | stop_probability=params['conf']) 53 | mask_inl = mask_inl.astype(bool).flatten() 54 | E = E.params 55 | except Exception as e: 56 | print ("Fail!", e) 57 | return np.eye(3), np.array([False] * len(mask)) 58 | else: 59 | raise ValueError('Unknown method') 60 | 61 | final_inliers = np.array([False] * len(mask)) 62 | if E is not None: 63 | for i, x in enumerate(mask_inl): 64 | final_inliers[tentative_idxs[i]] = x 65 | return E, final_inliers 66 | 67 | def get_single_result_filtered(m, mask, method, K1, K2, params): 68 | tentatives = m[mask] 69 | tentative_idxs = np.arange(len(mask))[mask] 70 | src_pts = normalize_keypoints(tentatives[:, :2], K1) 71 | dst_pts = normalize_keypoints(tentatives[:, 2:], K2) 72 | 73 | if tentatives.shape[0] <= 10: 74 | return np.eye(3), np.array([False] * len(mask)) 75 | if method == 'cv2e': 76 | E, mask_inl = cv2.findEssentialMat(src_pts, dst_pts, 77 | np.eye(3), cv2.RANSAC, 78 | threshold=params['inl_th'], 79 | prob=params['conf']) 80 | mask_inl = mask_inl.astype(bool).flatten() 81 | elif method == 'sklearn': 82 | try: 83 | #print(src_pts.shape, dst_pts.shape) 84 | E, mask_inl = skransac([src_pts, dst_pts], 85 | EssentialMatrixTransform, 86 | min_samples=8, 87 | residual_threshold=params['inl_th'], 88 | max_trials=params['maxiter'], 89 | stop_probability=params['conf']) 90 | mask_inl = mask_inl.astype(bool).flatten() 91 | E = E.params 92 | except Exception as e: 93 | print ("Fail!", e) 94 | return np.eye(3), np.array([False] * len(mask)) 95 | else: 96 | raise ValueError('Unknown method') 97 | 98 | final_inliers = np.array([False] * len(mask)) 99 | if E is not None: 100 | for i, x in enumerate(mask_inl): 101 | final_inliers[tentative_idxs[i]] = x 102 | return E, final_inliers 103 | 104 | def get_single_result_nmnet(model,ms, m, method, params, K1, K2): 105 | E, mask = model.predict_E(m, K1, K2 ) 106 | return E, mask 107 | 108 | def create_E_submission(IN_DIR,seq, method, params = {}): 109 | out_model = {} 110 | inls = {} 111 | matches = load_h5(f'{IN_DIR}/{seq}/matches.h5') 112 | matches_scores = load_h5(f'{IN_DIR}/{seq}/match_conf.h5') 113 | K1_K2 = load_h5(f'{IN_DIR}/{seq}/K1_K2.h5') 114 | keys = [k for k in matches.keys()] 115 | if method == 'nmnet2': 116 | model = NMNET22('third_party/model.pth') 117 | results = [get_single_result_nmnet(model, matches_scores[k], matches[k], method, params, K1_K2[k][0][0], K1_K2[k][0][1]) for k in tqdm(keys) ] 118 | for i, k in enumerate(keys): 119 | v = results[i] 120 | out_model[k] = v[0] 121 | inls[k] = v[1] 122 | elif method == 'load_oanet': 123 | out_model = load_h5(f'oanet/essential_{args.split}/{seq}/E_weighted.h5') 124 | inls = load_h5(f'oanet/essential_{args.split}/{seq}/corr_th.h5') 125 | elif method == 'load_oanet_ransac': 126 | out_model = load_h5(f'oanet/essential_{args.split}/{seq}/E_post.h5') 127 | inls = load_h5(f'oanet/essential_{args.split}/{seq}/corr_post.h5') 128 | else: 129 | results = Parallel(n_jobs=num_cores)(delayed(get_single_result)(matches_scores[k], matches[k], method, K1_K2[k][0][0], K1_K2[k][0][1], params) for k in tqdm(keys)) 130 | for i, k in enumerate(keys): 131 | v = results[i] 132 | out_model[k] = v[0] 133 | inls[k] = v[1] 134 | return out_model, inls 135 | 136 | def evaluate_results(submission, split = 'val'): 137 | ang_errors = {} 138 | DIR = split 139 | seqs = os.listdir(DIR) 140 | for seq in seqs: 141 | matches = load_h5(f'{DIR}/{seq}/matches.h5') 142 | K1_K2 = load_h5(f'{DIR}/{seq}/K1_K2.h5') 143 | R = load_h5(f'{DIR}/{seq}/R.h5') 144 | T = load_h5(f'{DIR}/{seq}/T.h5') 145 | F_pred, inl_mask = submission[0][seq], submission[1][seq] 146 | ang_errors[seq] = {} 147 | for k, m in tqdm(matches.items()): 148 | if F_pred[k] is None: 149 | ang_errors[seq][k] = 3.14 150 | continue 151 | img_id1 = k.split('-')[0] 152 | img_id2 = k.split('-')[1] 153 | K1 = K1_K2[k][0][0] 154 | K2 = K1_K2[k][0][1] 155 | try: 156 | E_cv_from_F = get_E_from_F(F_pred[k], K1, K2) 157 | except: 158 | print ("Fail") 159 | E = np.eye(3) 160 | R1 = R[img_id1] 161 | R2 = R[img_id2] 162 | T1 = T[img_id1] 163 | T2 = T[img_id2] 164 | dR = np.dot(R2, R1.T) 165 | dT = T2 - np.dot(dR, T1) 166 | pts1 = m[inl_mask[k],:2] # coordinates in image 1 167 | pts2 = m[inl_mask[k],2:] # coordinates in image 2 168 | p1n = normalize_keypoints(pts1, K1) 169 | p2n = normalize_keypoints(pts2, K2) 170 | ang_errors[seq][k] = max(eval_essential_matrix(p1n, p2n, E_cv_from_F, dR, dT)) 171 | return ang_errors 172 | 173 | 174 | 175 | if __name__ == '__main__': 176 | parser = argparse.ArgumentParser() 177 | parser.add_argument( 178 | "--split", 179 | default='val', 180 | type=str, 181 | help='split to run on. Can be val or test') 182 | parser.add_argument( 183 | "--method", default='cv2e', type=str, 184 | help=' can be cv2f, pyransac, degensac, sklearn' ) 185 | parser.add_argument( 186 | "--inlier_th", 187 | default=0.75, 188 | type=float, 189 | help='inlier threshold. Default is 0.75') 190 | parser.add_argument( 191 | "--conf", 192 | default=0.999, 193 | type=float, 194 | help='confidence Default is 0.999') 195 | parser.add_argument( 196 | "--maxiter", 197 | default=100000, 198 | type=int, 199 | help='max iter Default is 100000') 200 | parser.add_argument( 201 | "--match_th", 202 | default=0.85, 203 | type=float, 204 | help='match filetring th. Default is 0.85') 205 | 206 | parser.add_argument( 207 | "--force", 208 | default=False, 209 | type=bool, 210 | help='Force recompute if exists') 211 | parser.add_argument( 212 | "--data_dir", 213 | default='f_data', 214 | type=str, 215 | help='path to the data') 216 | 217 | args = parser.parse_args() 218 | 219 | if args.split not in ['val', 'test']: 220 | raise ValueError('Unknown value for --split') 221 | 222 | if args.method.lower() not in ['cv2e', 'sklearn', 'nmnet2', 'oanet2', 'load_oanet', 'load_oanet_ransac']: 223 | raise ValueError('Unknown value for --method') 224 | NUM_RUNS = 1 225 | if args.split == 'test': 226 | NUM_RUNS = 3 227 | params = {"maxiter": args.maxiter, 228 | "inl_th": args.inlier_th, 229 | "conf": args.conf, 230 | "match_th": args.match_th 231 | } 232 | problem = 'e' 233 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 234 | IN_DIR = os.path.join(args.data_dir, args.split) 235 | if not os.path.isdir(OUT_DIR): 236 | os.makedirs(OUT_DIR) 237 | num_cores = int(len(os.sched_getaffinity(0)) * 0.6) 238 | for run in range(NUM_RUNS): 239 | seqs = os.listdir(IN_DIR) 240 | for seq in seqs: 241 | print (f'Working on {seq}') 242 | out_models_fname = os.path.join(OUT_DIR, f'submission_models_seq_{seq}_run_{run}.h5') 243 | out_inliers_fname = os.path.join(OUT_DIR, f'submission_inliers_seq_{seq}_run_{run}.h5') 244 | 245 | if os.path.isfile(out_models_fname) and not args.force: 246 | print (f"Submission file {out_models_fname} already exists, skipping") 247 | continue 248 | models, inlier_masks = create_E_submission(IN_DIR, seq, 249 | args.method, 250 | params) 251 | save_h5(models, out_models_fname) 252 | save_h5(inlier_masks, out_inliers_fname) 253 | print ('Done!') 254 | 255 | 256 | 257 | 258 | -------------------------------------------------------------------------------- /create_F_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | import pydegensac 12 | 13 | from skimage.measure import ransac as skransac 14 | from skimage.transform import FundamentalMatrixTransform 15 | import multiprocessing 16 | import sys 17 | from joblib import Parallel, delayed 18 | import PIL 19 | try: 20 | from third_party.NM_Net_v2 import NMNET22 21 | import torch 22 | import kornia.geometry as KG 23 | import torch.nn.functional as TF 24 | except Exception as e: 25 | print (e) 26 | #sys.exit(0) 27 | pass 28 | 29 | 30 | def kornia_find_fundamental_wdlt(points1: torch.Tensor, 31 | points2: torch.Tensor, 32 | weights: torch.Tensor, 33 | params) -> torch.Tensor: 34 | '''Function, which finds homography via iteratively-reweighted 35 | least squares ToDo: add citation''' 36 | F = KG.find_fundamental(points1, points2, weights) 37 | for i in range(params['maxiter']): 38 | error = KG.epipolar.metrics.symmetrical_epipolar_distance(points1, points2, F) 39 | error_norm = TF.normalize(1.0 / (error + 1e-5), dim=1, p=params['conf']) 40 | F = KG.find_fundamental(points1, points2, error_norm) 41 | error = KG.epipolar.metrics.symmetrical_epipolar_distance(points1, points2, F) 42 | mask = error <= params['inl_th'] 43 | return F.detach().cpu().numpy().reshape(3,3), mask.detach().cpu().numpy().reshape(-1) 44 | 45 | def norm_test_data(xs_initial, w1,h1,w2,h2): 46 | cx1 = (w1 - 1.0) * 0.5 47 | cy1 = (h1 - 1.0) * 0.5 48 | f1 = max(h1 - 1.0, w1 - 1.0) 49 | scale1 = 1.0 / f1 50 | 51 | T1 = np.zeros((3, 3,)) 52 | T1[0, 0], T1[1, 1], T1[2, 2] = scale1, scale1, 1 53 | T1[0, 2], T1[1, 2] = -scale1 * cx1, -scale1 * cy1 54 | 55 | cx2 = (w2 - 1.0) * 0.5 56 | cy2 = (h2 - 1.0) * 0.5 57 | f2 = max(h2 - 1.0, w2 - 1.0) 58 | scale2 = 1.0 / f2 59 | 60 | T2 = np.zeros((3, 3,)) 61 | T2[0, 0], T2[1, 1], T2[2, 2] = scale2, scale2, 1 62 | T2[0, 2], T2[1, 2] = -scale2 * cx2, -scale2 * cy2 63 | 64 | kp1 = (xs_initial[:, :2] - np.asarray([cx1, cy1])) / np.asarray([f1, f1]) 65 | kp2 = (xs_initial[:, 2:] - np.asarray([cx2, cy2])) / np.asarray([f2, f2]) 66 | 67 | xs = np.concatenate([kp1, kp2], axis=-1) 68 | return xs, T1, T2 69 | 70 | def get_single_result(ms, m, method, params, w1 = None, h1 = None, w2 = None, h2 = None): 71 | mask = ms <= params['match_th'] 72 | tentatives = m[mask] 73 | tentative_idxs = np.arange(len(mask))[mask] 74 | src_pts = tentatives[:, :2] 75 | dst_pts = tentatives[:, 2:] 76 | if tentatives.shape[0] <= 10: 77 | return np.eye(3), np.array([False] * len(mask)) 78 | if method == 'cv2f': 79 | F, mask_inl = cv2.findFundamentalMat(src_pts, dst_pts, 80 | cv2.RANSAC, 81 | params['inl_th'], 82 | confidence=params['conf']) 83 | if method == 'kornia': 84 | #mask = ms <= params['match_th'] 85 | #tentatives = m[mask] 86 | #tentative_idxs = np.arange(len(mask))[mask] 87 | src_pts = m[:, :2] 88 | dst_pts = m[:, 2:] 89 | pts1 = torch.from_numpy(src_pts).view(1, -1, 2) 90 | pts2 = torch.from_numpy(dst_pts).view(1, -1, 2) 91 | weights = torch.from_numpy(1.0-ms).view(1, -1).pow(params['match_th']) 92 | weights = TF.normalize(weights, dim=1) 93 | F, mask_inl = kornia_find_fundamental_wdlt(pts1.float(), pts2.float(), weights.float(), params) 94 | elif method == 'cv2eimg': 95 | tent_norm, T1, T2 = norm_test_data(tentatives, w1,h1,w2,h2) 96 | #print (T1) 97 | #K1 = compute_T_with_imagesize(w1,h1) 98 | #K2 = compute_T_with_imagesize(w2,h2) 99 | #print (K1, K2) 100 | #src_pts = normalize_keypoints(src_pts, K1) 101 | #dst_pts = normalize_keypoints(dst_pts, K2) 102 | #print (src_pts) 103 | E, mask_inl = cv2.findEssentialMat(tent_norm[:, :2], tent_norm[:, 2:], 104 | np.eye(3), cv2.RANSAC, 105 | threshold=params['inl_th'], 106 | prob=params['conf']) 107 | F = np.matmul(np.matmul(T2.T, E), T1) 108 | elif method == 'pyransac': 109 | F, mask_inl = pydegensac.findFundamentalMatrix(src_pts, dst_pts, 110 | params['inl_th'], 111 | conf=params['conf'], 112 | max_iters = params['maxiter'], 113 | enable_degeneracy_check=False) 114 | elif method == 'degensac': 115 | F, mask_inl = pydegensac.findFundamentalMatrix(src_pts, dst_pts, 116 | params['inl_th'], 117 | conf=params['conf'], 118 | max_iters = params['maxiter'], 119 | enable_degeneracy_check=True) 120 | elif method == 'sklearn': 121 | try: 122 | #print(src_pts.shape, dst_pts.shape) 123 | F, mask_inl = skransac([src_pts, dst_pts], 124 | FundamentalMatrixTransform, 125 | min_samples=8, 126 | residual_threshold=params['inl_th'], 127 | max_trials=params['maxiter'], 128 | stop_probability=params['conf']) 129 | mask_inl = mask_inl.astype(bool).flatten() 130 | F = F.params 131 | except Exception as e: 132 | print ("Fail!", e) 133 | return np.eye(3), np.array([False] * len(mask)) 134 | else: 135 | raise ValueError('Unknown method') 136 | 137 | final_inliers = np.array([False] * len(mask)) 138 | if F is not None: 139 | for i, x in enumerate(mask_inl): 140 | final_inliers[tentative_idxs[i]] = x 141 | return F, final_inliers 142 | 143 | def get_single_result_nmnet(model,ms, m, method, params, w1, h1, w2, h2): 144 | with torch.no_grad(): 145 | F, mask = model.predict_F(m, w1, h1, w2, h2) 146 | return F, mask 147 | 148 | 149 | def create_F_submission(IN_DIR,seq, method, params = {}): 150 | out_model = {} 151 | inls = {} 152 | matches = load_h5(f'{IN_DIR}/{seq}/matches.h5') 153 | matches_scores = load_h5(f'{IN_DIR}/{seq}/match_conf.h5') 154 | keys = [k for k in matches.keys()] 155 | if method == 'nmnet2': 156 | model = NMNET22('third_party/model.pth') 157 | img_names = set() 158 | for k in keys: 159 | k1,k2 = k.split('-') 160 | img_names.add(k1) 161 | img_names.add(k2) 162 | img_names = list(img_names) 163 | wh = {} 164 | for fname in img_names: 165 | img = PIL.Image.open(f'{IN_DIR}/{seq}/images/{fname}.jpg') 166 | w,h = img.size 167 | wh[fname] = (w,h) 168 | results = [get_single_result_nmnet(model, matches_scores[k], matches[k], method, params, *(wh[k.split('-')[0]]), *(wh[k.split('-')[1]])) for k in tqdm(keys) ] 169 | for i, k in enumerate(keys): 170 | v = results[i] 171 | out_model[k] = v[0] 172 | inls[k] = v[1] 173 | elif method == 'cv2eimg': 174 | img_names = set() 175 | for k in keys: 176 | k1,k2 = k.split('-') 177 | img_names.add(k1) 178 | img_names.add(k2) 179 | img_names = list(img_names) 180 | wh = {} 181 | for fname in img_names: 182 | img = PIL.Image.open(f'{IN_DIR}/{seq}/images/{fname}.jpg') 183 | w,h = img.size 184 | wh[fname] = (w,h) 185 | results = Parallel(n_jobs=num_cores)(delayed(get_single_result)(matches_scores[k], matches[k], method, params, *(wh[k.split('-')[0]]), *(wh[k.split('-')[1]]) ) for k in tqdm(keys)) 186 | for i, k in enumerate(keys): 187 | v = results[i] 188 | out_model[k] = v[0] 189 | inls[k] = v[1] 190 | elif method == 'load_dfe': 191 | out_model = load_h5(f'4Dmytro/F_dfe_{seq}_submission.h5') 192 | inls = load_h5(f'4Dmytro/inls_dfe_{seq}_submission.h5') 193 | elif method == 'load_oanet': 194 | out_model = load_h5(f'oanet/fundamental_{args.split}/{seq}/F_weighted.h5') 195 | inls = load_h5(f'oanet/fundamental_{args.split}/{seq}/corr_th.h5') 196 | elif method == 'load_oanet_degensac': 197 | out_model = load_h5(f'oanet/fundamental_{args.split}/{seq}/F_post.h5') 198 | inls = load_h5(f'oanet/fundamental_{args.split}/{seq}/corr_post.h5') 199 | else: 200 | results = Parallel(n_jobs=num_cores)(delayed(get_single_result)(matches_scores[k], matches[k], method, params) for k in tqdm(keys)) 201 | for i, k in enumerate(keys): 202 | v = results[i] 203 | out_model[k] = v[0] 204 | inls[k] = v[1] 205 | return out_model, inls 206 | 207 | def evaluate_results(submission, split = 'val'): 208 | ang_errors = {} 209 | DIR = split 210 | seqs = os.listdir(DIR) 211 | for seq in seqs: 212 | matches = load_h5(f'{DIR}/{seq}/matches.h5') 213 | K1_K2 = load_h5(f'{DIR}/{seq}/K1_K2.h5') 214 | R = load_h5(f'{DIR}/{seq}/R.h5') 215 | T = load_h5(f'{DIR}/{seq}/T.h5') 216 | F_pred, inl_mask = submission[0][seq], submission[1][seq] 217 | ang_errors[seq] = {} 218 | for k, m in tqdm(matches.items()): 219 | if F_pred[k] is None: 220 | ang_errors[seq][k] = 3.14 221 | continue 222 | img_id1 = k.split('-')[0] 223 | img_id2 = k.split('-')[1] 224 | K1 = K1_K2[k][0][0] 225 | K2 = K1_K2[k][0][1] 226 | try: 227 | E_cv_from_F = get_E_from_F(F_pred[k], K1, K2) 228 | except: 229 | print ("Fail") 230 | E = np.eye(3) 231 | R1 = R[img_id1] 232 | R2 = R[img_id2] 233 | T1 = T[img_id1] 234 | T2 = T[img_id2] 235 | dR = np.dot(R2, R1.T) 236 | dT = T2 - np.dot(dR, T1) 237 | pts1 = m[inl_mask[k],:2] # coordinates in image 1 238 | pts2 = m[inl_mask[k],2:] # coordinates in image 2 239 | p1n = normalize_keypoints(pts1, K1) 240 | p2n = normalize_keypoints(pts2, K2) 241 | ang_errors[seq][k] = max(eval_essential_matrix(p1n, p2n, E_cv_from_F, dR, dT)) 242 | return ang_errors 243 | 244 | 245 | def grid_search_hypers_opencv(INL_THs = [0.75, 1.0, 1.5, 2.0, 3.0, 4.0], 246 | MATCH_THs = [0.75, 0.8, 0.85, 0.9, 0.95]): 247 | res = {} 248 | for inl_th in INL_THs: 249 | for match_th in MATCH_THs: 250 | key = f'{inl_th}_{match_th}' 251 | print (f"inlier_th = {inl_th}, snn_ration = {match_th}") 252 | cv2_results = create_F_submission_cv2(split = 'val', 253 | inlier_th = inl_th, 254 | match_th = match_th) 255 | MAEs = evaluate_results(cv2_results, 'val') 256 | mAA = calc_mAA_FE(MAEs) 257 | final = 0 258 | for k,v in mAA.items(): 259 | final+= v / float(len(mAA)) 260 | print (f'Validation mAA = {final}') 261 | res[key] = final 262 | max_MAA = 0 263 | inl_good = 0 264 | match_good = 0 265 | for k, v in res.items(): 266 | if max_MAA < v: 267 | max_MAA = v 268 | pars = k.split('_') 269 | match_good = float(pars[1]) 270 | inl_good = float(pars[0]) 271 | return inl_good, match_good, max_MAA 272 | 273 | if __name__ == '__main__': 274 | supported_methods = ['kornia', 'cv2f','cv2eimg','load_oanet', 'load_oanet_degensac', 'pyransac', 'degensac', 'sklearn', 'load_dfe', 'nmnet2'] 275 | parser = argparse.ArgumentParser() 276 | parser.add_argument( 277 | "--split", 278 | default='val', 279 | type=str, 280 | help='split to run on. Can be val or test') 281 | parser.add_argument( 282 | "--method", default='cv2F', type=str, 283 | help=f' can be {supported_methods}' ) 284 | parser.add_argument( 285 | "--inlier_th", 286 | default=0.75, 287 | type=float, 288 | help='inlier threshold. Default is 0.75') 289 | parser.add_argument( 290 | "--conf", 291 | default=0.999, 292 | type=float, 293 | help='confidence Default is 0.999') 294 | parser.add_argument( 295 | "--maxiter", 296 | default=100000, 297 | type=int, 298 | help='max iter Default is 100000') 299 | parser.add_argument( 300 | "--match_th", 301 | default=0.85, 302 | type=float, 303 | help='match filetring th. Default is 0.85') 304 | 305 | parser.add_argument( 306 | "--force", 307 | default=False, 308 | type=bool, 309 | help='Force recompute if exists') 310 | parser.add_argument( 311 | "--data_dir", 312 | default='f_data', 313 | type=str, 314 | help='path to the data') 315 | 316 | args = parser.parse_args() 317 | 318 | if args.split not in ['val', 'test']: 319 | raise ValueError('Unknown value for --split') 320 | 321 | if args.method.lower() not in supported_methods: 322 | raise ValueError(f'Unknown value {args.method.lower()} for --method') 323 | NUM_RUNS = 1 324 | if args.split == 'test': 325 | NUM_RUNS = 3 326 | params = {"maxiter": args.maxiter, 327 | "inl_th": args.inlier_th, 328 | "conf": args.conf, 329 | "match_th": args.match_th 330 | } 331 | problem = 'f' 332 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 333 | IN_DIR = os.path.join(args.data_dir, args.split) 334 | if not os.path.isdir(OUT_DIR): 335 | os.makedirs(OUT_DIR) 336 | num_cores = int(len(os.sched_getaffinity(0)) * 0.9) 337 | for run in range(NUM_RUNS): 338 | seqs = os.listdir(IN_DIR) 339 | for seq in seqs: 340 | print (f'Working on {seq}') 341 | out_models_fname = os.path.join(OUT_DIR, f'submission_models_seq_{seq}_run_{run}.h5') 342 | out_inliers_fname = os.path.join(OUT_DIR, f'submission_inliers_seq_{seq}_run_{run}.h5') 343 | 344 | if os.path.isfile(out_models_fname) and not args.force: 345 | print (f"Submission file {out_models_fname} already exists, skipping") 346 | continue 347 | models, inlier_masks = create_F_submission(IN_DIR, seq, 348 | args.method, 349 | params) 350 | save_h5(models, out_models_fname) 351 | save_h5(inlier_masks, out_inliers_fname) 352 | print ('Done!') 353 | -------------------------------------------------------------------------------- /create_H_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | import pydegensac 12 | 13 | from skimage.measure import ransac as skransac 14 | from skimage.transform import ProjectiveTransform 15 | import multiprocessing 16 | import sys 17 | from joblib import Parallel, delayed 18 | import PIL 19 | 20 | 21 | def get_single_result(ms, m, method, params): 22 | mask = (ms <= params['match_th']).reshape(-1) 23 | tentatives = m[mask] 24 | tentative_idxs = np.arange(len(mask))[mask] 25 | src_pts = tentatives[:, :2] 26 | dst_pts = tentatives[:, 2:] 27 | if tentatives.shape[0] <= 5: 28 | return np.eye(3), np.array([False] * len(mask)) 29 | if method == 'cv2h': 30 | H, mask_inl = cv2.findHomography(src_pts, dst_pts, 31 | cv2.RANSAC, 32 | params['inl_th'], 33 | maxIters=params['maxiter'], 34 | confidence=params['conf']) 35 | elif method == 'pyransac': 36 | H, mask_inl = pydegensac.findHomography(src_pts, dst_pts, 37 | params['inl_th'], 38 | conf=params['conf'], 39 | max_iters = params['maxiter']) 40 | elif method == 'sklearn': 41 | try: 42 | #print(src_pts.shape, dst_pts.shape) 43 | H, mask_inl = skransac([src_pts, dst_pts], 44 | ProjectiveTransform, 45 | min_samples=4, 46 | residual_threshold=params['inl_th'], 47 | max_trials=params['maxiter'], 48 | stop_probability=params['conf']) 49 | mask_inl = mask_inl.astype(bool).flatten() 50 | H = H.params 51 | except Exception as e: 52 | print ("Fail!", e) 53 | return np.eye(3), np.array([False] * len(mask)) 54 | else: 55 | raise ValueError('Unknown method') 56 | 57 | final_inliers = np.array([False] * len(mask)) 58 | if H is not None: 59 | for i, x in enumerate(mask_inl): 60 | final_inliers[tentative_idxs[i]] = x 61 | return H, final_inliers 62 | 63 | 64 | def create_H_submission(IN_DIR, seq, method, params = {}): 65 | out_model = {} 66 | inls = {} 67 | matches = load_h5(f'{IN_DIR}/matches.h5') 68 | matches_scores = load_h5(f'{IN_DIR}/match_conf.h5') 69 | keys = [k for k in matches.keys()] 70 | results = Parallel(n_jobs=min(num_cores,len(keys)))(delayed(get_single_result)(matches_scores[k], matches[k], method, params) for k in tqdm(keys)) 71 | for i, k in enumerate(keys): 72 | v = results[i] 73 | out_model[k] = v[0] 74 | inls[k] = v[1] 75 | return out_model, inls 76 | 77 | 78 | 79 | if __name__ == '__main__': 80 | supported_methods = [ 'cv2h', 'pyransac', 'sklearn', 'cv2lmeds'] 81 | parser = argparse.ArgumentParser() 82 | parser.add_argument( 83 | "--split", 84 | default='val', 85 | type=str, 86 | help='split to run on. Can be val or test') 87 | parser.add_argument( 88 | "--method", default='cv2h', type=str, 89 | help=f'can be {supported_methods}' ) 90 | parser.add_argument( 91 | "--inlier_th", 92 | default=0.75, 93 | type=float, 94 | help='inlier threshold. Default is 0.75') 95 | parser.add_argument( 96 | "--conf", 97 | default=0.999, 98 | type=float, 99 | help='confidence Default is 0.999') 100 | parser.add_argument( 101 | "--maxiter", 102 | default=100000, 103 | type=int, 104 | help='max iter Default is 100000') 105 | parser.add_argument( 106 | "--match_th", 107 | default=0.85, 108 | type=float, 109 | help='match filetring th. Default is 0.85') 110 | 111 | parser.add_argument( 112 | "--force", 113 | default=False, 114 | type=bool, 115 | help='Force recompute if exists') 116 | parser.add_argument( 117 | "--data_dir", 118 | default='h_data', 119 | type=str, 120 | help='path to the data') 121 | 122 | args = parser.parse_args() 123 | 124 | if args.split not in ['val', 'test']: 125 | raise ValueError('Unknown value for --split') 126 | 127 | if args.method.lower() not in supported_methods: 128 | raise ValueError(f'Unknown value {args.method.lower()} for --method') 129 | NUM_RUNS = 1 130 | if args.split == 'test': 131 | NUM_RUNS = 3 132 | params = {"maxiter": args.maxiter, 133 | "inl_th": args.inlier_th, 134 | "conf": args.conf, 135 | "match_th": args.match_th 136 | } 137 | problem = 'h' 138 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 139 | IN_DIR = args.data_dir 140 | if not os.path.isdir(OUT_DIR): 141 | os.makedirs(OUT_DIR) 142 | num_cores = int(len(os.sched_getaffinity(0)) * 0.5) 143 | for run in range(NUM_RUNS): 144 | seqs = os.listdir(IN_DIR) 145 | for seq in seqs: 146 | IN_DIR_CURRENT = os.path.join(IN_DIR, seq, args.split) 147 | print (f'Working on {seq}') 148 | out_models_fname = os.path.join(OUT_DIR, f'submission_models_seq_{seq}_run_{run}.h5') 149 | out_inliers_fname = os.path.join(OUT_DIR, f'submission_inliers_seq_{seq}_run_{run}.h5') 150 | 151 | if os.path.isfile(out_models_fname) and not args.force: 152 | print (f"Submission file {out_models_fname} already exists, skipping") 153 | continue 154 | models, inlier_masks = create_H_submission(IN_DIR_CURRENT, seq, 155 | args.method, 156 | params) 157 | save_h5(models, out_models_fname) 158 | save_h5(inlier_masks, out_inliers_fname) 159 | print ('Done!') 160 | -------------------------------------------------------------------------------- /create_opencv_F_submission_example.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | 11 | 12 | 13 | def create_F_submission_cv2(split = 'val', inlier_th = 1.0, match_th = 0.8): 14 | DIR = split 15 | seqs = os.listdir(DIR) 16 | out_model = {} 17 | inls = {} 18 | for seq in seqs: 19 | matches = load_h5(f'{DIR}/{seq}/matches.h5') 20 | matches_scores = load_h5(f'{DIR}/{seq}/match_conf.h5') 21 | out_model[seq] = {} 22 | inls[seq] = {} 23 | for k, m in tqdm(matches.items()): 24 | ms = matches_scores[k].reshape(-1) 25 | mask = ms <= match_th 26 | tentatives = m[mask] 27 | tentative_idxs = np.arange(len(mask))[mask] 28 | src_pts = tentatives[:,:2] 29 | dst_pts = tentatives[:,2:] 30 | F, mask_inl = cv2.findFundamentalMat(src_pts, dst_pts, cv2.RANSAC, 31 | inlier_th, confidence=0.9999) 32 | out_model[seq][k] = F 33 | final_inliers = np.array([False] * len(mask)) 34 | if F is not None: 35 | for i, x in enumerate(mask_inl): 36 | final_inliers[tentative_idxs[i]] = x 37 | inls[seq][k] = final_inliers 38 | return out_model, inls 39 | 40 | def evaluate_results(submission, split = 'val'): 41 | ang_errors = {} 42 | DIR = split 43 | seqs = os.listdir(DIR) 44 | for seq in seqs: 45 | matches = load_h5(f'{DIR}/{seq}/matches.h5') 46 | K1_K2 = load_h5(f'{DIR}/{seq}/K1_K2.h5') 47 | R = load_h5(f'{DIR}/{seq}/R.h5') 48 | T = load_h5(f'{DIR}/{seq}/T.h5') 49 | F_pred, inl_mask = submission[0][seq], submission[1][seq] 50 | ang_errors[seq] = {} 51 | for k, m in tqdm(matches.items()): 52 | if F_pred[k] is None: 53 | ang_errors[seq][k] = 3.14 54 | continue 55 | img_id1 = k.split('-')[0] 56 | img_id2 = k.split('-')[1] 57 | K1 = K1_K2[k][0][0] 58 | K2 = K1_K2[k][0][1] 59 | try: 60 | E_cv_from_F = get_E_from_F(F_pred[k], K1, K2) 61 | except: 62 | print ("Fail") 63 | E = np.eye(3) 64 | R1 = R[img_id1] 65 | R2 = R[img_id2] 66 | T1 = T[img_id1] 67 | T2 = T[img_id2] 68 | dR = np.dot(R2, R1.T) 69 | dT = T2 - np.dot(dR, T1) 70 | pts1 = m[inl_mask[k],:2] # coordinates in image 1 71 | pts2 = m[inl_mask[k],2:] # coordinates in image 2 72 | p1n = normalize_keypoints(pts1, K1) 73 | p2n = normalize_keypoints(pts2, K2) 74 | ang_errors[seq][k] = max(eval_essential_matrix(p1n, p2n, E_cv_from_F, dR, dT)) 75 | return ang_errors 76 | 77 | 78 | def grid_search_hypers_opencv(INL_THs = [0.75, 1.0, 1.5, 2.0, 3.0, 4.0], 79 | MATCH_THs = [0.75, 0.8, 0.85, 0.9, 0.95]): 80 | res = {} 81 | for inl_th in INL_THs: 82 | for match_th in MATCH_THs: 83 | key = f'{inl_th}_{match_th}' 84 | print (f"inlier_th = {inl_th}, snn_ration = {match_th}") 85 | cv2_results = create_F_submission_cv2(split = 'val', 86 | inlier_th = inl_th, 87 | match_th = match_th) 88 | MAEs = evaluate_results(cv2_results, 'val') 89 | mAA = calc_mAA_FE(MAEs) 90 | final = 0 91 | for k,v in mAA.items(): 92 | final+= v / float(len(mAA)) 93 | print (f'Validation mAA = {final}') 94 | res[key] = final 95 | max_MAA = 0 96 | inl_good = 0 97 | match_good = 0 98 | for k, v in res.items(): 99 | if max_MAA < v: 100 | max_MAA = v 101 | pars = k.split('_') 102 | match_good = float(pars[1]) 103 | inl_good = float(pars[0]) 104 | return inl_good, match_good, max_MAA 105 | 106 | if __name__ == '__main__': 107 | # Search for the best hyperparameters on the validation set 108 | print ("Searching hypers") 109 | inl_good, match_good, max_MAA = grid_search_hypers_opencv() 110 | print (f"The best hyperparameters for OpenCV H RANSAC are") 111 | print (f"inlier_th = {inl_good}, snn_ration = {match_good}. Validation mAA = {max_MAA}") 112 | print ("Creating submission") 113 | cv2_test_submission = create_F_submission_cv2(split = 'test', inlier_th = inl_good, match_th = match_good) 114 | for ds_name, models in cv2_test_submission.items(): 115 | save_h5(models, f'F_opencv_{ds_name}_submission.h5') 116 | print (f"Saved to F_opencv_{ds_name}_submission.h5") 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /create_opencv_homography_submission_example.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import h5py 4 | import cv2 5 | from utils import * 6 | from metrics import * 7 | from tqdm import tqdm 8 | 9 | def create_cv2_submission(split = 'val', inlier_th = 3.0, match_th = 0.85, n_iter=100000): 10 | DIR = 'homography' 11 | out_model = {} 12 | for ds in ['EVD', 'HPatchesSeq']: 13 | out_model[ds] = {} 14 | matches = load_h5(f'{DIR}/{ds}/{split}/matches.h5') 15 | matches_scores = load_h5(f'{DIR}/{ds}/{split}/match_conf.h5') 16 | for k, m in tqdm(matches.items()): 17 | ms = matches_scores[k].reshape(-1) 18 | mask = ms <= match_th 19 | tentatives = m[mask] 20 | src_pts = tentatives[:,:2] 21 | dst_pts = tentatives[:,2:] 22 | H, mask_inl = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 23 | inlier_th, maxIters=n_iter, confidence=0.9999) 24 | out_model[ds][k] = H 25 | return out_model 26 | 27 | 28 | def evaluate_results(results_dict, split='val'): 29 | DIR = 'homography' 30 | MAEs = {} 31 | for ds in ['EVD', 'HPatchesSeq']: 32 | Hgt_dict = load_h5(f'{DIR}/{ds}/{split}/Hgt.h5') 33 | models = results_dict[ds] 34 | MAEs[ds] = {} 35 | for k, H_est in tqdm(models.items()): 36 | H_gt = Hgt_dict[k] 37 | img1, img2 = get_h_imgpair(k, ds, split) 38 | MAE = get_visible_part_mean_absolute_reprojection_error(img1, img2, H_gt, H_est) 39 | MAEs[ds][k] = MAE 40 | return MAEs 41 | 42 | 43 | def grid_search_hypers_opencv(INL_THs = [0.75, 1.0, 1.5, 2.0, 3.0, 4.0], 44 | MATCH_THs = [0.75, 0.8, 0.85, 0.9, 0.95]): 45 | res = {} 46 | for inl_th in INL_THs: 47 | for match_th in MATCH_THs: 48 | key = f'{inl_th}_{match_th}' 49 | print (f"inlier_th = {inl_th}, snn_ration = {match_th}") 50 | cv2_results = create_cv2_submission(split = 'val', 51 | inlier_th = inl_th, 52 | match_th = match_th, 53 | n_iter=50000) 54 | MAEs = evaluate_results(cv2_results) 55 | mAA = calc_mAA(MAEs) 56 | final = (mAA['EVD'] + mAA['HPatchesSeq'])/2.0 57 | print (f'Validation mAA = {final}') 58 | res[key] = final 59 | max_MAA = 0 60 | inl_good = 0 61 | match_good = 0 62 | for k, v in res.items(): 63 | if max_MAA < v: 64 | max_MAA = v 65 | pars = k.split('_') 66 | match_good = float(pars[1]) 67 | inl_good = float(pars[0]) 68 | return inl_good, match_good, max_MAA 69 | 70 | 71 | if __name__ == '__main__': 72 | # Search for the best hyperparameters on the validation set 73 | print ("Searching hypers") 74 | inl_good, match_good, max_MAA = grid_search_hypers_opencv() 75 | print (f"The best hyperparameters for OpenCV H RANSAC are") 76 | print (f"inlier_th = {inl_good}, snn_ration = {match_good}. Validation mAA = {max_MAA}") 77 | print ("Creating submission") 78 | cv2_test_submission = create_cv2_submission(split = 'test', inlier_th = inl_good, match_th = match_good, 79 | n_iter=50000) 80 | for ds_name, models in cv2_test_submission.items(): 81 | save_h5(models, f'homography_opencv_{ds_name}_submission.h5') 82 | print (f"Saved to homography_opencv_{ds_name}_submission.h5") 83 | -------------------------------------------------------------------------------- /download.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ducha-aiki/ransac-tutorial-2020-data/52810309d8341d538e24a13577c44ae2b4a5ec77/download.sh -------------------------------------------------------------------------------- /eval_E_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | 12 | 13 | 14 | 15 | 16 | def evaluate_results(IN_DIR, seq, models, inliers): 17 | ang_errors = {} 18 | matches = load_h5(f'{IN_DIR}/{seq}/matches.h5') 19 | K1_K2 = load_h5(f'{IN_DIR}/{seq}/K1_K2.h5') 20 | R = load_h5(f'{IN_DIR}/{seq}/R.h5') 21 | T = load_h5(f'{IN_DIR}/{seq}/T.h5') 22 | E_pred, inl_mask = models, inliers 23 | for k, m in tqdm(matches.items()): 24 | if E_pred[k] is None: 25 | ang_errors[k] = 3.14 26 | continue 27 | img_id1 = k.split('-')[0] 28 | img_id2 = k.split('-')[1] 29 | K1 = K1_K2[k][0][0] 30 | K2 = K1_K2[k][0][1] 31 | E = E_pred[k].astype(np.float64) 32 | R1 = R[img_id1] 33 | R2 = R[img_id2] 34 | T1 = T[img_id1] 35 | T2 = T[img_id2] 36 | dR = np.dot(R2, R1.T) 37 | dT = T2 - np.dot(dR, T1) 38 | if args.method.lower() == 'load_oanet': #They provided not the mask, but actual correspondences 39 | pts1 = inl_mask[k][:, :2] # coordinates in image 1 40 | pts2 = inl_mask[k][:, 2:] # coordinates in image 2 41 | #print (pts1.shape) 42 | elif args.method.lower() == 'load_oanet_ransac': #They provided not the mask, but actual correspondences 43 | pts1 = inl_mask[k][:, :2] # coordinates in image 1 44 | pts2 = inl_mask[k][:, 2:] # coordinates in image 2 45 | else: 46 | pts1 = m[inl_mask[k].astype(bool),:2] # coordinates in image 1 47 | pts2 = m[inl_mask[k].astype(bool),2:] # coordinates in image 2 48 | p1n = normalize_keypoints(pts1, K1).astype(np.float64) 49 | p2n = normalize_keypoints(pts2, K2).astype(np.float64) 50 | ang_errors[k] = max(eval_essential_matrix(p1n, p2n, E.reshape(3,3), dR, dT)) 51 | return ang_errors 52 | 53 | 54 | 55 | if __name__ == '__main__': 56 | parser = argparse.ArgumentParser() 57 | parser.add_argument( 58 | "--split", 59 | default='val', 60 | type=str, 61 | help='split to run on. Can be val or test') 62 | parser.add_argument( 63 | "--method", default='cv2F', type=str, 64 | help=' can be cv2f, pyransac, degensac, sklearn' ) 65 | parser.add_argument( 66 | "--inlier_th", 67 | default=0.75, 68 | type=float, 69 | help='inlier threshold. Default is 0.75') 70 | parser.add_argument( 71 | "--conf", 72 | default=0.999, 73 | type=float, 74 | help='confidence Default is 0.999') 75 | parser.add_argument( 76 | "--maxiter", 77 | default=100000, 78 | type=int, 79 | help='max iter Default is 100000') 80 | parser.add_argument( 81 | "--match_th", 82 | default=0.85, 83 | type=float, 84 | help='match filetring th. Default is 0.85') 85 | 86 | parser.add_argument( 87 | "--force", 88 | default=False, 89 | type=bool, 90 | help='Force recompute if exists') 91 | parser.add_argument( 92 | "--upgraded", 93 | default=False, 94 | type=bool, 95 | help='for processing 5pt-postprocessed after deep methods') 96 | parser.add_argument( 97 | "--data_dir", 98 | default='f_data', 99 | type=str, 100 | help='path to the data') 101 | 102 | args = parser.parse_args() 103 | 104 | if args.split not in ['val', 'test']: 105 | raise ValueError('Unknown value for --split') 106 | 107 | if args.method.lower() not in ['cv2e', 'nmnet2','load_dfe', 'sklearn', 'cne', 'acne', 'load_oanet', 'load_oanet_ransac']: 108 | raise ValueError('Unknown value for --method') 109 | NUM_RUNS = 1 110 | if args.split == 'test': 111 | NUM_RUNS = 3 112 | if args.method.lower() in ['nmnet2', 'cne', 'acne', 'load_oanet']: 113 | NUM_RUNS=1 114 | params = {"maxiter": args.maxiter, 115 | "inl_th": args.inlier_th, 116 | "conf": args.conf, 117 | "match_th": args.match_th 118 | } 119 | problem = 'e' 120 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 121 | IN_DIR = os.path.join(args.data_dir, args.split) 122 | if not os.path.isdir(OUT_DIR): 123 | os.makedirs(OUT_DIR) 124 | num_cores = int(len(os.sched_getaffinity(0)) * 0.9) 125 | all_maas = [] 126 | suf = '' 127 | if args.upgraded: 128 | suf='_upgraded' 129 | for run in range(NUM_RUNS): 130 | seqs = os.listdir(IN_DIR) 131 | for seq in seqs: 132 | print (f'Working on {seq}') 133 | in_models_fname = os.path.join(OUT_DIR, f'submission{suf}_models_seq_{seq}_run_{run}.h5') 134 | in_inliers_fname = os.path.join(OUT_DIR, f'submission{suf}_inliers_seq_{seq}_run_{run}.h5') 135 | out_errors_fname = os.path.join(OUT_DIR, f'errors{suf}_seq_{seq}_run_{run}.h5') 136 | out_maa_fname = os.path.join(OUT_DIR, f'maa{suf}_seq_{seq}_run_{run}.h5') 137 | if not os.path.isfile(in_models_fname) or not os.path.isfile(in_inliers_fname): 138 | print (f"Submission file {in_inliers_fname} is missing, cannot evaluate, skipping") 139 | continue 140 | models = load_h5(in_models_fname) 141 | inlier_masks = load_h5(in_inliers_fname) 142 | if os.path.isfile(out_errors_fname) and not args.force: 143 | print (f"Submission file {in_inliers_fname} exists, read it") 144 | error = load_h5(out_errors_fname) 145 | else: 146 | error = evaluate_results(IN_DIR, seq, models, inlier_masks) 147 | save_h5(error, out_errors_fname) 148 | mAA = calc_mAA_FE({seq: error}) 149 | print (f" mAA {seq} = {mAA[seq]:.5f}") 150 | save_h5({"mAA": mAA[seq]}, out_maa_fname) 151 | all_maas.append(mAA[seq]) 152 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa{suf}_FINAL.h5') 153 | final_mAA = (np.array(all_maas)).mean() 154 | print (f" mAA total = {final_mAA:.5f}") 155 | save_h5({"mAA": final_mAA}, out_maa_final_fname) 156 | print ('Done!') 157 | 158 | 159 | 160 | -------------------------------------------------------------------------------- /eval_F_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | 12 | 13 | 14 | 15 | 16 | def evaluate_results(IN_DIR, seq, models, inliers): 17 | ang_errors = {} 18 | matches = load_h5(f'{IN_DIR}/{seq}/matches.h5') 19 | K1_K2 = load_h5(f'{IN_DIR}/{seq}/K1_K2.h5') 20 | R = load_h5(f'{IN_DIR}/{seq}/R.h5') 21 | T = load_h5(f'{IN_DIR}/{seq}/T.h5') 22 | F_pred, inl_mask = models, inliers 23 | for k, m in tqdm(matches.items()): 24 | if F_pred[k] is None: 25 | ang_errors[k] = 3.14 26 | continue 27 | img_id1 = k.split('-')[0] 28 | img_id2 = k.split('-')[1] 29 | K1 = K1_K2[k][0][0] 30 | K2 = K1_K2[k][0][1] 31 | try: 32 | E_cv_from_F = get_E_from_F(F_pred[k], K1, K2) 33 | except: 34 | print ("Fail") 35 | E = np.eye(3) 36 | R1 = R[img_id1] 37 | R2 = R[img_id2] 38 | T1 = T[img_id1] 39 | T2 = T[img_id2] 40 | dR = np.dot(R2, R1.T) 41 | dT = T2 - np.dot(dR, T1) 42 | if args.method.lower() == 'load_oanet': #They provided not the mask, but actual correspondences 43 | pts1 = inl_mask[k][:, :2] # coordinates in image 1 44 | pts2 = inl_mask[k][:, 2:] # coordinates in image 2 45 | elif args.method.lower() == 'load_oanet_degensac': #They provided not the mask, but actual correspondences 46 | pts1 = inl_mask[k][:, :2] # coordinates in image 1 47 | pts2 = inl_mask[k][:, 2:] # coordinates in image 2 48 | else: 49 | pts1 = m[inl_mask[k],:2] # coordinates in image 1 50 | pts2 = m[inl_mask[k],2:] # coordinates in image 2 51 | p1n = normalize_keypoints(pts1, K1) 52 | p2n = normalize_keypoints(pts2, K2) 53 | ang_errors[k] = max(eval_essential_matrix(p1n, p2n, E_cv_from_F, dR, dT)) 54 | return ang_errors 55 | 56 | 57 | 58 | if __name__ == '__main__': 59 | parser = argparse.ArgumentParser() 60 | parser.add_argument( 61 | "--split", 62 | default='val', 63 | type=str, 64 | help='split to run on. Can be val or test') 65 | parser.add_argument( 66 | "--method", default='cv2F', type=str, 67 | help=' can be cv2f, pyransac, degensac, sklearn' ) 68 | parser.add_argument( 69 | "--inlier_th", 70 | default=0.75, 71 | type=float, 72 | help='inlier threshold. Default is 0.75') 73 | parser.add_argument( 74 | "--conf", 75 | default=0.999, 76 | type=float, 77 | help='confidence Default is 0.999') 78 | parser.add_argument( 79 | "--maxiter", 80 | default=100000, 81 | type=int, 82 | help='max iter Default is 100000') 83 | parser.add_argument( 84 | "--match_th", 85 | default=0.85, 86 | type=float, 87 | help='match filetring th. Default is 0.85') 88 | 89 | parser.add_argument( 90 | "--force", 91 | default=False, 92 | type=bool, 93 | help='Force recompute if exists') 94 | parser.add_argument( 95 | "--data_dir", 96 | default='f_data', 97 | type=str, 98 | help='path to the data') 99 | 100 | args = parser.parse_args() 101 | 102 | if args.split not in ['val', 'test']: 103 | raise ValueError('Unknown value for --split') 104 | 105 | if args.method.lower() not in ['cv2f', 'kornia', 'cv2eimg','load_oanet','load_oanet_degensac', 'pyransac', 'load_dfe', 'nmnet2', 'degensac', 'sklearn', 'cne', 'acne']: 106 | raise ValueError('Unknown value for --method') 107 | NUM_RUNS = 1 108 | if args.split == 'test': 109 | NUM_RUNS = 3 110 | if args.method.lower() in ['load_oanet','load_oanet_degensac', 'load_dfe', 'nmnet2', 'cne', 'acne']: 111 | NUM_RUNS=1 112 | params = {"maxiter": args.maxiter, 113 | "inl_th": args.inlier_th, 114 | "conf": args.conf, 115 | "match_th": args.match_th 116 | } 117 | problem = 'f' 118 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 119 | IN_DIR = os.path.join(args.data_dir, args.split) 120 | if not os.path.isdir(OUT_DIR): 121 | os.makedirs(OUT_DIR) 122 | num_cores = int(len(os.sched_getaffinity(0)) * 0.9) 123 | all_maas = [] 124 | for run in range(NUM_RUNS): 125 | seqs = os.listdir(IN_DIR) 126 | for seq in seqs: 127 | print (f'Working on {seq}') 128 | in_models_fname = os.path.join(OUT_DIR, f'submission_models_seq_{seq}_run_{run}.h5') 129 | in_inliers_fname = os.path.join(OUT_DIR, f'submission_inliers_seq_{seq}_run_{run}.h5') 130 | out_errors_fname = os.path.join(OUT_DIR, f'errors_seq_{seq}_run_{run}.h5') 131 | out_maa_fname = os.path.join(OUT_DIR, f'maa_seq_{seq}_run_{run}.h5') 132 | if not os.path.isfile(in_models_fname) or not os.path.isfile(in_inliers_fname): 133 | print (f"Submission file {in_inliers_fname} is missing, cannot evaluate, skipping") 134 | continue 135 | models = load_h5(in_models_fname) 136 | inlier_masks = load_h5(in_inliers_fname) 137 | if os.path.isfile(out_errors_fname) and not args.force: 138 | print (f"Submission file {in_inliers_fname} exists, read it") 139 | error = load_h5(out_errors_fname) 140 | else: 141 | error = evaluate_results(IN_DIR, seq, models, inlier_masks) 142 | save_h5(error, out_errors_fname) 143 | mAA = calc_mAA_FE({seq: error}) 144 | print (f" mAA {seq} = {mAA[seq]:.5f}") 145 | save_h5({"mAA": mAA[seq]}, out_maa_fname) 146 | all_maas.append(mAA[seq]) 147 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 148 | final_mAA = (np.array(all_maas)).mean() 149 | print (f" mAA total = {final_mAA:.5f}") 150 | save_h5({"mAA": final_mAA}, out_maa_final_fname) 151 | print ('Done!') 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /eval_H_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | import multiprocessing 12 | import sys 13 | from joblib import Parallel, delayed 14 | import PIL 15 | 16 | 17 | def eval_single_result(k, H_gt, H, IN_DIR): 18 | img1, img2 = get_h_imgpair2(k, IN_DIR) 19 | return get_visible_part_mean_absolute_reprojection_error(img1, img2, H_gt, H) 20 | 21 | def evaluate_results(IN_DIR, seq, models, inliers): 22 | MAEs = {} 23 | Hgt_dict = load_h5(f'{IN_DIR}/Hgt.h5') 24 | H_pred, inl_mask = models, inliers 25 | keys = sorted([k for k in H_pred.keys()]) 26 | num_cores = int(len(os.sched_getaffinity(0)) * 0.5) 27 | results = Parallel(n_jobs=min(num_cores,len(keys)))(delayed(eval_single_result)(k, Hgt_dict[k], H_pred[k], IN_DIR) for k in tqdm(keys)) 28 | for i, k in enumerate(keys): 29 | MAEs[k] = v = results[i] 30 | return MAEs 31 | 32 | 33 | 34 | if __name__ == '__main__': 35 | supported_methods = [ 'cv2h', 'pyransac', 'sklearn', 'cv2lmeds'] 36 | parser = argparse.ArgumentParser() 37 | parser.add_argument( 38 | "--split", 39 | default='val', 40 | type=str, 41 | help='split to run on. Can be val or test') 42 | parser.add_argument( 43 | "--method", default='cv2h', type=str, 44 | help=f'can be {supported_methods}' ) 45 | parser.add_argument( 46 | "--inlier_th", 47 | default=0.75, 48 | type=float, 49 | help='inlier threshold. Default is 0.75') 50 | parser.add_argument( 51 | "--conf", 52 | default=0.999, 53 | type=float, 54 | help='confidence Default is 0.999') 55 | parser.add_argument( 56 | "--maxiter", 57 | default=100000, 58 | type=int, 59 | help='max iter Default is 100000') 60 | parser.add_argument( 61 | "--match_th", 62 | default=0.85, 63 | type=float, 64 | help='match filetring th. Default is 0.85') 65 | 66 | parser.add_argument( 67 | "--force", 68 | default=False, 69 | type=bool, 70 | help='Force recompute if exists') 71 | parser.add_argument( 72 | "--data_dir", 73 | default='h_data', 74 | type=str, 75 | help='path to the data') 76 | 77 | args = parser.parse_args() 78 | 79 | if args.split not in ['val', 'test']: 80 | raise ValueError('Unknown value for --split') 81 | 82 | if args.method.lower() not in supported_methods: 83 | raise ValueError(f'Unknown value {args.method.lower()} for --method') 84 | NUM_RUNS = 1 85 | if args.split == 'test': 86 | NUM_RUNS = 3 87 | params = {"maxiter": args.maxiter, 88 | "inl_th": args.inlier_th, 89 | "conf": args.conf, 90 | "match_th": args.match_th 91 | } 92 | problem = 'h' 93 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 94 | IN_DIR = args.data_dir 95 | if not os.path.isdir(OUT_DIR): 96 | os.makedirs(OUT_DIR) 97 | num_cores = int(len(os.sched_getaffinity(0)) * 0.5) 98 | all_maas = [] 99 | all_maas_per_seq = {} 100 | seqs = os.listdir(IN_DIR) 101 | for seq in seqs: 102 | all_maas_per_seq[seq] = [] 103 | for run in range(NUM_RUNS): 104 | IN_DIR_CURRENT = os.path.join(IN_DIR, seq, args.split) 105 | print (f'Working on {seq}') 106 | in_models_fname = os.path.join(OUT_DIR, f'submission_models_seq_{seq}_run_{run}.h5') 107 | in_inliers_fname = os.path.join(OUT_DIR, f'submission_inliers_seq_{seq}_run_{run}.h5') 108 | out_errors_fname = os.path.join(OUT_DIR, f'errors_seq_{seq}_run_{run}.h5') 109 | out_maa_fname = os.path.join(OUT_DIR, f'maa_seq_{seq}_run_{run}.h5') 110 | if not os.path.isfile(in_models_fname) or not os.path.isfile(in_inliers_fname): 111 | print (f"Submission file {in_inliers_fname} is missing, cannot evaluate, skipping") 112 | continue 113 | models = load_h5(in_models_fname) 114 | inlier_masks = load_h5(in_inliers_fname) 115 | if os.path.isfile(out_errors_fname) and not args.force: 116 | print (f"Submission file {in_inliers_fname} exists, read it") 117 | error = load_h5(out_errors_fname) 118 | else: 119 | error = evaluate_results(IN_DIR_CURRENT, seq, models, inlier_masks) 120 | save_h5(error, out_errors_fname) 121 | mAA = calc_mAA({seq: error}) 122 | print (f" mAA {seq} = {mAA[seq]:.5f}") 123 | save_h5({"mAA": mAA[seq]}, out_maa_fname) 124 | all_maas.append(mAA[seq]) 125 | all_maas_per_seq[seq].append(mAA[seq]) 126 | print (OUT_DIR) 127 | for k, v in all_maas_per_seq.items(): 128 | print (k, np.array(v).mean()) 129 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 130 | final_mAA = (np.array(all_maas)).mean() 131 | print (f" mAA total = {final_mAA:.5f}") 132 | save_h5({"mAA": final_mAA}, out_maa_final_fname) 133 | print ('Done!') 134 | 135 | 136 | 137 | -------------------------------------------------------------------------------- /eval_all_test_E.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import cv2 4 | from utils import * 5 | from tqdm import tqdm 6 | import os 7 | from metrics import * 8 | 9 | import argparse 10 | 11 | 12 | if __name__ == '__main__': 13 | # Search for the best hyperparameters on the validation set 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | "--data_dir", 17 | default='f_data', 18 | type=str, 19 | help='path to the data') 20 | args = parser.parse_args() 21 | BASE = f'results/test/e/' 22 | res = {} 23 | for method in os.listdir(BASE): 24 | B1 = f'{BASE}/{method}/' 25 | for hypers in os.listdir(B1): 26 | key = f'{method}-{hypers}' 27 | print (f"Evaluating {key}") 28 | parts = hypers.split('_') 29 | conf = float(parts[1].split('-')[-1]) 30 | inl_th = float(parts[3].split('-')[-1]) 31 | match_th = float(parts[5].split('-')[-1]) 32 | maxiters = int(parts[6].split('-')[-1]) 33 | run_str_eval = f'python eval_E_submission.py --data_dir {args.data_dir} --conf {conf} --match_th {match_th} --inlier_th {inl_th} --method {method} --split test --maxiter {maxiters}' 34 | os.system(run_str_eval) 35 | params = {"maxiter": maxiters, 36 | "inl_th": inl_th, 37 | "conf": conf, 38 | "match_th": match_th 39 | } 40 | OUT_DIR = get_output_dir('e', 'test', method, params) 41 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 42 | final_res = load_h5(out_maa_final_fname) 43 | res[key] = final_res['mAA'] 44 | print (final_res['mAA']) 45 | -------------------------------------------------------------------------------- /eval_all_test_F.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import cv2 4 | from utils import * 5 | from tqdm import tqdm 6 | import os 7 | from metrics import * 8 | 9 | import argparse 10 | 11 | 12 | if __name__ == '__main__': 13 | # Search for the best hyperparameters on the validation set 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | "--data_dir", 17 | default='f_data', 18 | type=str, 19 | help='path to the data') 20 | args = parser.parse_args() 21 | BASE = f'results/test/f/' 22 | res = {} 23 | for method in os.listdir(BASE): 24 | B1 = f'{BASE}/{method}/' 25 | for hypers in os.listdir(B1): 26 | key = f'{method}-{hypers}' 27 | print (f"Evaluating {key}") 28 | parts = hypers.split('_') 29 | conf = float(parts[1].split('-')[-1]) 30 | inl_th = float(parts[3].split('-')[-1]) 31 | match_th = float(parts[5].split('-')[-1]) 32 | maxiters = int(parts[6].split('-')[-1]) 33 | run_str_eval = f'python eval_F_submission.py --data_dir {args.data_dir} --conf {conf} --match_th {match_th} --inlier_th {inl_th} --method {method} --split test --maxiter {maxiters}' 34 | os.system(run_str_eval) 35 | params = {"maxiter": maxiters, 36 | "inl_th": inl_th, 37 | "conf": conf, 38 | "match_th": match_th 39 | } 40 | OUT_DIR = get_output_dir('f', 'test', method, params) 41 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 42 | final_res = load_h5(out_maa_final_fname) 43 | res[key] = final_res['mAA'] 44 | print (final_res['mAA']) 45 | -------------------------------------------------------------------------------- /eval_all_test_H.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import cv2 4 | from utils import * 5 | from tqdm import tqdm 6 | import os 7 | from metrics import * 8 | 9 | import argparse 10 | 11 | 12 | if __name__ == '__main__': 13 | # Search for the best hyperparameters on the validation set 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | "--data_dir", 17 | default='f_data', 18 | type=str, 19 | help='path to the data') 20 | args = parser.parse_args() 21 | BASE = f'results/test/h/' 22 | res = {} 23 | for method in os.listdir(BASE): 24 | B1 = f'{BASE}/{method}/' 25 | if 'sklearn' not in method: 26 | continue 27 | for hypers in os.listdir(B1): 28 | key = f'{method}-{hypers}' 29 | print (f"Evaluating {key}") 30 | parts = hypers.split('_') 31 | conf = float(parts[1].split('-')[-1]) 32 | inl_th = float(parts[3].split('-')[-1]) 33 | match_th = float(parts[5].split('-')[-1]) 34 | maxiters = int(parts[6].split('-')[-1]) 35 | run_str_eval = f'python eval_H_submission.py --force True --data_dir {args.data_dir} --conf {conf} --match_th {match_th} --inlier_th {inl_th} --method {method} --split test --maxiter {maxiters}' 36 | os.system(run_str_eval) 37 | params = {"maxiter": maxiters, 38 | "inl_th": inl_th, 39 | "conf": conf, 40 | "match_th": match_th 41 | } 42 | OUT_DIR = get_output_dir('h', 'test', method, params) 43 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 44 | final_res = load_h5(out_maa_final_fname) 45 | res[key] = final_res['mAA'] 46 | print (final_res['mAA']) 47 | -------------------------------------------------------------------------------- /hdf5reader.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import torch 3 | from torch.utils.data import Dataset 4 | 5 | 6 | # torch.multiprocessing.set_start_method("spawn") 7 | 8 | 9 | class H5DataReader: 10 | def __init__(self, path): 11 | 12 | ## If this crashes, try swmr=False 13 | self.mapping = { 14 | "F": h5py.File(f"{path}/Fgt.h5", "r", libver="latest", swmr=True), 15 | "matches": h5py.File(f"{path}/matches.h5", "r", libver="latest", swmr=True), 16 | "confidence": h5py.File( 17 | f"{path}/match_conf.h5", "r", libver="latest", swmr=True 18 | ), 19 | } 20 | 21 | self.path = path 22 | self.keys = self.__get_h5_keys(self.mapping["F"]) 23 | self.num_keys = len(self.keys) 24 | 25 | def __get_h5_keys(self, file): 26 | return [key for key in file.keys()] 27 | 28 | def __load_h5_key(self, mapping, key): 29 | out = {k: v[key][()] for k, v in mapping.items()} 30 | 31 | return out 32 | 33 | def __getitem__(self, idx): 34 | return self.__load_h5_key(self.mapping, self.keys[idx]), self.keys[idx] 35 | 36 | def __len__(self): 37 | return len(self.keys) 38 | 39 | 40 | class DummyH5Dataset(Dataset): 41 | def __init__(self, path): 42 | self.path = path 43 | self.reader = None 44 | 45 | def __getitem__(self, idx): 46 | ## This is important to make hdf5 work with multiprocessing 47 | ## Opening the file in the constructor will lead to crashes 48 | if self.reader is None: 49 | self.reader = H5DataReader(self.path) 50 | 51 | data, name = self.reader.__getitem__(idx) 52 | 53 | ## Do something 54 | 55 | def __len__(self): 56 | return len(H5DataReader(self.path)) 57 | -------------------------------------------------------------------------------- /metrics.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | 5 | def get_visible_part_mean_absolute_reprojection_error(img1, img2, H_gt, H): 6 | '''We reproject the image 1 mask to image2 and back to get the visible part mask. 7 | Then we average the reprojection absolute error over that area''' 8 | h,w = img1.shape[:2] 9 | mask1 = np.ones((h,w)) 10 | mask1in2 = cv2.warpPerspective(mask1, H_gt, img2.shape[:2][::-1]) 11 | mask1inback = cv2.warpPerspective(mask1in2, np.linalg.inv(H_gt), img1.shape[:2][::-1]) > 0 12 | xi = np.arange(w) 13 | yi = np.arange(h) 14 | xg, yg = np.meshgrid(xi,yi) 15 | coords = np.concatenate([xg.reshape(*xg.shape,1), yg.reshape(*yg.shape,1)], axis=-1) 16 | shape_orig = coords.shape 17 | xy_rep_gt = cv2.perspectiveTransform(coords.reshape(-1, 1,2).astype(np.float32), H_gt.astype(np.float32)).squeeze(1) 18 | xy_rep_estimated = cv2.perspectiveTransform(coords.reshape(-1, 1,2).astype(np.float32), 19 | H.astype(np.float32)).squeeze(1) 20 | #error = np.abs(xy_rep_gt-xy_rep_estimated).sum(axis=1).reshape(xg.shape) * mask1inback 21 | error = np.sqrt(((xy_rep_gt-xy_rep_estimated)**2).sum(axis=1)).reshape(xg.shape) * mask1inback 22 | mean_error = error.sum() / mask1inback.sum() 23 | return mean_error 24 | 25 | 26 | def calc_mAA(MAEs, ths = np.logspace(np.log2(1.0), np.log2(20), 10, base=2.0)): 27 | res = {} 28 | for ds_name, MAEs_cur in MAEs.items(): 29 | cur_results = [] 30 | for k, MAE in MAEs_cur.items(): 31 | acc = [] 32 | for th in ths: 33 | A = (MAE <= th).astype(np.float32).mean() 34 | acc.append(A) 35 | cur_results.append(np.array(acc).mean()) 36 | res[ds_name] = np.array(cur_results).mean() 37 | return res 38 | 39 | def calc_mAA_FE(ang_errors, ths = np.deg2rad(np.linspace(1.0, 10., 10))): 40 | res = {} 41 | for ds_name, MAEs_cur in ang_errors.items(): 42 | cur_results = [] 43 | for k, MAE in MAEs_cur.items(): 44 | acc = [] 45 | for th in ths: 46 | A = (MAE <= th).astype(np.float32).mean() 47 | acc.append(A) 48 | cur_results.append(np.array(acc).mean()) 49 | res[ds_name] = np.array(cur_results).mean() 50 | return res 51 | -------------------------------------------------------------------------------- /tune_hyperparameters_and_create_test_E_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | 11 | import argparse 12 | 13 | 14 | if __name__ == '__main__': 15 | # Search for the best hyperparameters on the validation set 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument( 18 | "--method", default='cv2F', type=str, 19 | help=' can be cv2f, pyransac, degensac, sklearn' ) 20 | parser.add_argument( 21 | "--data_dir", 22 | default='f_data', 23 | type=str, 24 | help='path to the data') 25 | parser.add_argument( 26 | "--conf", 27 | default=0.999, 28 | type=float, 29 | help='confidence Default is 0.999') 30 | parser.add_argument( 31 | "--maxiter", 32 | default=100000, 33 | type=int, 34 | help='max iter Default is 100000') 35 | 36 | args = parser.parse_args() 37 | 38 | print (f"Searching hypers for {args.method}, conf={args.conf}, maxIters={args.maxiter}") 39 | inl_ths = [0.001, 0.0001, 0.0002, 0.00005, 0.0004, 0.000025, 0.00008]#, 1.5, 2.0] 40 | match_ths = [0.75, 0.8, 0.85] 41 | res = {} 42 | for m_th in match_ths: 43 | for inl_th in inl_ths: 44 | key = f'{inl_th}_{m_th}' 45 | print (f'inlier threshold = {inl_th}, match threshold={m_th}') 46 | run_str = f'python -utt create_E_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {m_th} --inlier_th {inl_th} --method {args.method} --split val --maxiter {args.maxiter}' 47 | os.system(run_str) 48 | run_str_eval = f'python -utt eval_E_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {m_th} --inlier_th {inl_th} --method {args.method} --split val --maxiter {args.maxiter}' 49 | os.system(run_str_eval) 50 | params = {"maxiter": args.maxiter, 51 | "inl_th": inl_th, 52 | "conf": args.conf, 53 | "match_th": m_th 54 | } 55 | OUT_DIR = get_output_dir('e', 'val', args.method, params) 56 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 57 | final_res = load_h5(out_maa_final_fname) 58 | res[key] = final_res['mAA'] 59 | max_MAA = 0 60 | inl_good = 0 61 | match_good = 0 62 | for k, v in res.items(): 63 | if max_MAA < v: 64 | max_MAA = v 65 | pars = k.split('_') 66 | match_good = float(pars[1]) 67 | inl_good = float(pars[0]) 68 | print (f"The best hyperparameters for {args.method}, conf={args.conf}, maxIters={args.maxiter} are") 69 | print (f"inlier_th = {inl_good}, snn_ration = {match_good}. Validation mAA = {max_MAA}") 70 | print ("Creating submission") 71 | run_str = f'python -utt create_E_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {match_good} --inlier_th {inl_good} --method {args.method} --split test --maxiter {args.maxiter}' 72 | os.system(run_str) 73 | print ('Done!') 74 | -------------------------------------------------------------------------------- /tune_hyperparameters_and_create_test_F_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | 11 | import argparse 12 | 13 | 14 | if __name__ == '__main__': 15 | # Search for the best hyperparameters on the validation set 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument( 18 | "--method", default='cv2F', type=str, 19 | help=' can be cv2f, pyransac, degensac, sklearn' ) 20 | parser.add_argument( 21 | "--data_dir", 22 | default='f_data', 23 | type=str, 24 | help='path to the data') 25 | parser.add_argument( 26 | "--conf", 27 | default=0.999, 28 | type=float, 29 | help='confidence Default is 0.999') 30 | parser.add_argument( 31 | "--maxiter", 32 | default=100000, 33 | type=int, 34 | help='max iter Default is 100000') 35 | 36 | args = parser.parse_args() 37 | 38 | print (f"Searching hypers for {args.method}, conf={args.conf}, maxIters={args.maxiter}") 39 | inl_ths = [0.1, 0.2, 0.25, 0.5, 0.75, 1.0]#, 1.5, 2.0] 40 | match_ths = [0.75, 0.8, 0.85] 41 | res = {} 42 | for m_th in match_ths: 43 | for inl_th in inl_ths: 44 | key = f'{inl_th}_{m_th}' 45 | print (f'inlier threshold = {inl_th}, match threshold={m_th}') 46 | run_str = f'python -utt create_F_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {m_th} --inlier_th {inl_th} --method {args.method} --split val --maxiter {args.maxiter}' 47 | os.system(run_str) 48 | run_str_eval = f'python -utt eval_F_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {m_th} --inlier_th {inl_th} --method {args.method} --split val --maxiter {args.maxiter}' 49 | os.system(run_str_eval) 50 | params = {"maxiter": args.maxiter, 51 | "inl_th": inl_th, 52 | "conf": args.conf, 53 | "match_th": m_th 54 | } 55 | OUT_DIR = get_output_dir('f', 'val', args.method, params) 56 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 57 | final_res = load_h5(out_maa_final_fname) 58 | res[key] = final_res['mAA'] 59 | max_MAA = 0 60 | inl_good = 0 61 | match_good = 0 62 | for k, v in res.items(): 63 | if max_MAA < v: 64 | max_MAA = v 65 | pars = k.split('_') 66 | match_good = float(pars[1]) 67 | inl_good = float(pars[0]) 68 | print (f"The best hyperparameters for {args.method}, conf={args.conf}, maxIters={args.maxiter} are") 69 | print (f"inlier_th = {inl_good}, snn_ration = {match_good}. Validation mAA = {max_MAA}") 70 | print ("Creating submission") 71 | run_str = f'python -utt create_F_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {match_good} --inlier_th {inl_good} --method {args.method} --split test --maxiter {args.maxiter}' 72 | os.system(run_str) 73 | print ('Done!') 74 | -------------------------------------------------------------------------------- /tune_hyperparameters_and_create_test_H_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | 11 | import argparse 12 | 13 | 14 | if __name__ == '__main__': 15 | # Search for the best hyperparameters on the validation set 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument( 18 | "--method", default='cv2h', type=str, 19 | help=' can be cv2f, pyransac, degensac, sklearn' ) 20 | parser.add_argument( 21 | "--data_dir", 22 | default='h_data', 23 | type=str, 24 | help='path to the data') 25 | parser.add_argument( 26 | "--conf", 27 | default=0.999, 28 | type=float, 29 | help='confidence Default is 0.999') 30 | parser.add_argument( 31 | "--maxiter", 32 | default=100000, 33 | type=int, 34 | help='max iter Default is 100000') 35 | 36 | args = parser.parse_args() 37 | 38 | print (f"Searching hypers for {args.method}, conf={args.conf}, maxIters={args.maxiter}") 39 | inl_ths = [0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0] 40 | match_ths = [0.7, 0.75, 0.8, 0.85, 0.9] 41 | res = {} 42 | for m_th in match_ths: 43 | for inl_th in inl_ths: 44 | key = f'{inl_th}_{m_th}' 45 | print (f'inlier threshold = {inl_th}, match threshold={m_th}') 46 | run_str = f'python -utt create_H_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {m_th} --inlier_th {inl_th} --method {args.method} --split val --maxiter {args.maxiter}' 47 | os.system(run_str) 48 | run_str_eval = f'python -utt eval_H_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {m_th} --inlier_th {inl_th} --method {args.method} --split val --maxiter {args.maxiter}' 49 | os.system(run_str_eval) 50 | params = {"maxiter": args.maxiter, 51 | "inl_th": inl_th, 52 | "conf": args.conf, 53 | "match_th": m_th 54 | } 55 | OUT_DIR = get_output_dir('h', 'val', args.method, params) 56 | out_maa_final_fname = os.path.join(OUT_DIR, f'maa_FINAL.h5') 57 | final_res = load_h5(out_maa_final_fname) 58 | res[key] = final_res['mAA'] 59 | max_MAA = 0 60 | inl_good = 0 61 | match_good = 0 62 | for k, v in res.items(): 63 | if max_MAA < v: 64 | max_MAA = v 65 | pars = k.split('_') 66 | match_good = float(pars[1]) 67 | inl_good = float(pars[0]) 68 | print (f"The best hyperparameters for {args.method}, conf={args.conf}, maxIters={args.maxiter} are") 69 | print (f"inlier_th = {inl_good}, snn_ration = {match_good}. Validation mAA = {max_MAA}") 70 | print ("Creating submission") 71 | run_str = f'python -utt create_E_submission.py --data_dir {args.data_dir} --conf {args.conf} --match_th {match_good} --inlier_th {inl_good} --method {args.method} --split test --maxiter {args.maxiter}' 72 | os.system(run_str) 73 | print ('Done!') 74 | -------------------------------------------------------------------------------- /upgrade_E_submission.py: -------------------------------------------------------------------------------- 1 | # select the data 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import h5py 5 | import cv2 6 | from utils import * 7 | from tqdm import tqdm 8 | import os 9 | from metrics import * 10 | import argparse 11 | import pydegensac 12 | 13 | from skimage.measure import ransac as skransac 14 | from skimage.transform import EssentialMatrixTransform 15 | import multiprocessing 16 | 17 | from joblib import Parallel, delayed 18 | import sys 19 | 20 | 21 | 22 | 23 | def get_single_result(ms, m, method, K1, K2, params): 24 | if args.method.lower() == 'load_oanet': #They provided not the mask, but actual correspondences 25 | tentatives = ms 26 | src_pts = tentatives[:, :2] 27 | dst_pts = tentatives[:, 2:] 28 | if tentatives.shape[0] <= 10: 29 | return np.eye(3), tentatives 30 | else: 31 | final_inliers = np.array([False] * len(mask)) 32 | mask = ms 33 | tentatives = m[mask] 34 | tentative_idxs = np.arange(len(mask))[mask] 35 | if tentatives.shape[0] <= 10: 36 | return np.eye(3), np.array([False] * len(mask)) 37 | src_pts = normalize_keypoints(tentatives[:, :2], K1) 38 | dst_pts = normalize_keypoints(tentatives[:, 2:], K2) 39 | E, mask_inl = cv2.findEssentialMat(src_pts, dst_pts, 40 | np.eye(3), cv2.RANSAC, 41 | threshold=0.0002, 42 | prob=0.999) 43 | mask_inl = mask_inl.astype(bool).flatten() 44 | #print (mask_inl.sum()) 45 | if args.method.lower() == 'load_oanet': #They provided not the mask, but actual correspondences 46 | final_inliers = tentatives[mask_inl] 47 | else: 48 | if E is not None: 49 | for i, x in enumerate(mask_inl): 50 | final_inliers[tentative_idxs[i]] = x 51 | return E, final_inliers 52 | 53 | 54 | def upgrade_E_submission(IN_DIR, inliers, seq, method, params = {}): 55 | out_model = {} 56 | inls = {} 57 | matches = load_h5(f'{IN_DIR}/{seq}/matches.h5') 58 | #matches_scores = load_h5(f'{IN_DIR}/{seq}/match_conf.h5') 59 | K1_K2 = load_h5(f'{IN_DIR}/{seq}/K1_K2.h5') 60 | keys = [k for k in matches.keys()] 61 | 62 | results = Parallel(n_jobs=num_cores)(delayed(get_single_result)(inliers[k], matches[k], method, K1_K2[k][0][0], K1_K2[k][0][1], params) for k in tqdm(keys)) 63 | for i, k in enumerate(keys): 64 | v = results[i] 65 | out_model[k] = v[0] 66 | inls[k] = v[1] 67 | return out_model, inls 68 | 69 | def evaluate_results(submission, split = 'val'): 70 | ang_errors = {} 71 | DIR = split 72 | seqs = os.listdir(DIR) 73 | for seq in seqs: 74 | matches = load_h5(f'{DIR}/{seq}/matches.h5') 75 | K1_K2 = load_h5(f'{DIR}/{seq}/K1_K2.h5') 76 | R = load_h5(f'{DIR}/{seq}/R.h5') 77 | T = load_h5(f'{DIR}/{seq}/T.h5') 78 | F_pred, inl_mask = submission[0][seq], submission[1][seq] 79 | ang_errors[seq] = {} 80 | for k, m in tqdm(matches.items()): 81 | if F_pred[k] is None: 82 | ang_errors[seq][k] = 3.14 83 | continue 84 | img_id1 = k.split('-')[0] 85 | img_id2 = k.split('-')[1] 86 | K1 = K1_K2[k][0][0] 87 | K2 = K1_K2[k][0][1] 88 | try: 89 | E_cv_from_F = get_E_from_F(F_pred[k], K1, K2) 90 | except: 91 | print ("Fail") 92 | E = np.eye(3) 93 | R1 = R[img_id1] 94 | R2 = R[img_id2] 95 | T1 = T[img_id1] 96 | T2 = T[img_id2] 97 | dR = np.dot(R2, R1.T) 98 | dT = T2 - np.dot(dR, T1) 99 | pts1 = m[inl_mask[k],:2] # coordinates in image 1 100 | pts2 = m[inl_mask[k],2:] # coordinates in image 2 101 | p1n = normalize_keypoints(pts1, K1) 102 | p2n = normalize_keypoints(pts2, K2) 103 | ang_errors[seq][k] = max(eval_essential_matrix(p1n, p2n, E_cv_from_F, dR, dT)) 104 | return ang_errors 105 | 106 | 107 | 108 | if __name__ == '__main__': 109 | parser = argparse.ArgumentParser() 110 | parser.add_argument( 111 | "--split", 112 | default='val', 113 | type=str, 114 | help='split to run on. Can be val or test') 115 | parser.add_argument( 116 | "--method", default='cv2e', type=str, 117 | help=' can be cv2f, pyransac, degensac, sklearn' ) 118 | parser.add_argument( 119 | "--inlier_th", 120 | default=0.75, 121 | type=float, 122 | help='inlier threshold. Default is 0.75') 123 | parser.add_argument( 124 | "--conf", 125 | default=0.999, 126 | type=float, 127 | help='confidence Default is 0.999') 128 | parser.add_argument( 129 | "--maxiter", 130 | default=100000, 131 | type=int, 132 | help='max iter Default is 100000') 133 | parser.add_argument( 134 | "--match_th", 135 | default=0.85, 136 | type=float, 137 | help='match filetring th. Default is 0.85') 138 | 139 | parser.add_argument( 140 | "--force", 141 | default=False, 142 | type=bool, 143 | help='Force recompute if exists') 144 | parser.add_argument( 145 | "--data_dir", 146 | default='f_data', 147 | type=str, 148 | help='path to the data') 149 | 150 | args = parser.parse_args() 151 | 152 | if args.split not in ['val', 'test']: 153 | raise ValueError('Unknown value for --split') 154 | 155 | if args.method.lower() not in ['cne', 'acne','load_dfe', 'cv2e', 'sklearn', 'nmnet2', 'oanet2', 'load_oanet', 'load_oanet_ransac']: 156 | raise ValueError('Unknown value for --method') 157 | NUM_RUNS = 1 158 | if args.split == 'test': 159 | NUM_RUNS = 1 160 | params = {"maxiter": args.maxiter, 161 | "inl_th": args.inlier_th, 162 | "conf": args.conf, 163 | "match_th": args.match_th 164 | } 165 | problem = 'e' 166 | OUT_DIR = get_output_dir(problem, args.split, args.method, params) 167 | IN_DIR = os.path.join(args.data_dir, args.split) 168 | if not os.path.isdir(OUT_DIR): 169 | os.makedirs(OUT_DIR) 170 | num_cores = int(len(os.sched_getaffinity(0)) * 0.9) 171 | for run in range(NUM_RUNS): 172 | seqs = sorted(os.listdir(IN_DIR)) 173 | for seq in seqs: 174 | print (f'Working on {seq}') 175 | in_models_fname = os.path.join(OUT_DIR, f'submission_models_seq_{seq}_run_{run}.h5') 176 | in_inliers_fname = os.path.join(OUT_DIR, f'submission_inliers_seq_{seq}_run_{run}.h5') 177 | if args.method.lower() == 'load_dfe': 178 | in_inliers_fname = in_inliers_fname.replace('/e/','/f/') 179 | out_models_fname = os.path.join(OUT_DIR, f'submission_upgraded_models_seq_{seq}_run_{run}.h5') 180 | out_inliers_fname = os.path.join(OUT_DIR, f'submission_upgraded_inliers_seq_{seq}_run_{run}.h5') 181 | if os.path.isfile(out_models_fname) and not args.force: 182 | print (f"Submission file {out_models_fname} already exists, skipping") 183 | continue 184 | inliers = load_h5(in_inliers_fname) 185 | models, inlier_masks = upgrade_E_submission(IN_DIR, inliers, seq, 186 | args.method, 187 | params) 188 | save_h5(models, out_models_fname) 189 | save_h5(inlier_masks, out_inliers_fname) 190 | print ('Done!') 191 | 192 | 193 | 194 | 195 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC, University of Victoria, Czech Technical University 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | #The most of this code is taked from https://github.com/vcg-uvic/image-matching-benchmark 17 | import numpy as np 18 | import matplotlib.pyplot as plt 19 | import h5py 20 | import cv2 21 | from copy import deepcopy 22 | import os 23 | 24 | 25 | def load_h5(filename): 26 | '''Loads dictionary from hdf5 file''' 27 | dict_to_load = {} 28 | try: 29 | with h5py.File(filename, 'r') as f: 30 | keys = [key for key in f.keys()] 31 | for key in keys: 32 | dict_to_load[key] = f[key][()] 33 | except: 34 | print('Cannot find file {}'.format(filename)) 35 | return dict_to_load 36 | 37 | def save_h5(dict_to_save, filename): 38 | '''Saves dictionary to HDF5 file''' 39 | 40 | with h5py.File(filename, 'w') as f: 41 | for key in dict_to_save: 42 | #if 'img' in key: 43 | # continue 44 | #print (key) 45 | f.create_dataset(key, data=dict_to_save[key]) 46 | 47 | def normalize_keypoints(keypoints, K): 48 | '''Normalize keypoints using the calibration data.''' 49 | 50 | C_x = K[0, 2] 51 | C_y = K[1, 2] 52 | f_x = K[0, 0] 53 | f_y = K[1, 1] 54 | keypoints = (keypoints - np.array([[C_x, C_y]])) / np.array([[f_x, f_y]]) 55 | 56 | return keypoints 57 | 58 | def get_E_from_F(F, K1, K2): 59 | return np.matmul(np.matmul(K2.T, F), K1) 60 | 61 | def quaternion_from_matrix(matrix, isprecise=False): 62 | '''Return quaternion from rotation matrix. 63 | If isprecise is True, the input matrix is assumed to be a precise rotation 64 | matrix and a faster algorithm is used. 65 | >>> q = quaternion_from_matrix(numpy.identity(4), True) 66 | >>> numpy.allclose(q, [1, 0, 0, 0]) 67 | True 68 | >>> q = quaternion_from_matrix(numpy.diag([1, -1, -1, 1])) 69 | >>> numpy.allclose(q, [0, 1, 0, 0]) or numpy.allclose(q, [0, -1, 0, 0]) 70 | True 71 | >>> R = rotation_matrix(0.123, (1, 2, 3)) 72 | >>> q = quaternion_from_matrix(R, True) 73 | >>> numpy.allclose(q, [0.9981095, 0.0164262, 0.0328524, 0.0492786]) 74 | True 75 | >>> R = [[-0.545, 0.797, 0.260, 0], [0.733, 0.603, -0.313, 0], 76 | ... [-0.407, 0.021, -0.913, 0], [0, 0, 0, 1]] 77 | >>> q = quaternion_from_matrix(R) 78 | >>> numpy.allclose(q, [0.19069, 0.43736, 0.87485, -0.083611]) 79 | True 80 | >>> R = [[0.395, 0.362, 0.843, 0], [-0.626, 0.796, -0.056, 0], 81 | ... [-0.677, -0.498, 0.529, 0], [0, 0, 0, 1]] 82 | >>> q = quaternion_from_matrix(R) 83 | >>> numpy.allclose(q, [0.82336615, -0.13610694, 0.46344705, -0.29792603]) 84 | True 85 | >>> R = random_rotation_matrix() 86 | >>> q = quaternion_from_matrix(R) 87 | >>> is_same_transform(R, quaternion_matrix(q)) 88 | True 89 | >>> R = euler_matrix(0.0, 0.0, numpy.pi/2.0) 90 | >>> numpy.allclose(quaternion_from_matrix(R, isprecise=False), 91 | ... quaternion_from_matrix(R, isprecise=True)) 92 | True 93 | ''' 94 | 95 | M = np.array(matrix, dtype=np.float64, copy=False)[:4, :4] 96 | if isprecise: 97 | q = np.empty((4, )) 98 | t = np.trace(M) 99 | if t > M[3, 3]: 100 | q[0] = t 101 | q[3] = M[1, 0] - M[0, 1] 102 | q[2] = M[0, 2] - M[2, 0] 103 | q[1] = M[2, 1] - M[1, 2] 104 | else: 105 | i, j, k = 1, 2, 3 106 | if M[1, 1] > M[0, 0]: 107 | i, j, k = 2, 3, 1 108 | if M[2, 2] > M[i, i]: 109 | i, j, k = 3, 1, 2 110 | t = M[i, i] - (M[j, j] + M[k, k]) + M[3, 3] 111 | q[i] = t 112 | q[j] = M[i, j] + M[j, i] 113 | q[k] = M[k, i] + M[i, k] 114 | q[3] = M[k, j] - M[j, k] 115 | q *= 0.5 / math.sqrt(t * M[3, 3]) 116 | else: 117 | m00 = M[0, 0] 118 | m01 = M[0, 1] 119 | m02 = M[0, 2] 120 | m10 = M[1, 0] 121 | m11 = M[1, 1] 122 | m12 = M[1, 2] 123 | m20 = M[2, 0] 124 | m21 = M[2, 1] 125 | m22 = M[2, 2] 126 | 127 | # symmetric matrix K 128 | K = np.array([[m00 - m11 - m22, 0.0, 0.0, 0.0], 129 | [m01 + m10, m11 - m00 - m22, 0.0, 0.0], 130 | [m02 + m20, m12 + m21, m22 - m00 - m11, 0.0], 131 | [m21 - m12, m02 - m20, m10 - m01, m00 + m11 + m22]]) 132 | K /= 3.0 133 | 134 | # quaternion is eigenvector of K that corresponds to largest eigenvalue 135 | w, V = np.linalg.eigh(K) 136 | q = V[[3, 0, 1, 2], np.argmax(w)] 137 | 138 | if q[0] < 0.0: 139 | np.negative(q, q) 140 | 141 | return q 142 | 143 | def evaluate_R_t(R_gt, t_gt, R, t, q_gt=None): 144 | t = t.flatten() 145 | t_gt = t_gt.flatten() 146 | 147 | eps = 1e-15 148 | 149 | if q_gt is None: 150 | q_gt = quaternion_from_matrix(R_gt) 151 | q = quaternion_from_matrix(R) 152 | q = q / (np.linalg.norm(q) + eps) 153 | q_gt = q_gt / (np.linalg.norm(q_gt) + eps) 154 | loss_q = np.maximum(eps, (1.0 - np.sum(q * q_gt)**2)) 155 | err_q = np.arccos(1 - 2 * loss_q) 156 | 157 | t = t / (np.linalg.norm(t) + eps) 158 | t_gt = t_gt / (np.linalg.norm(t_gt) + eps) 159 | loss_t = np.maximum(eps, (1.0 - np.sum(t * t_gt)**2)) 160 | err_t = np.arccos(np.sqrt(1 - loss_t)) 161 | 162 | if np.sum(np.isnan(err_q)) or np.sum(np.isnan(err_t)): 163 | # This should never happen! Debug here 164 | print(R_gt, t_gt, R, t, q_gt) 165 | import IPython 166 | IPython.embed() 167 | 168 | return err_q, err_t 169 | 170 | def eval_essential_matrix(p1n, p2n, E, dR, dt): 171 | if len(p1n) != len(p2n): 172 | raise RuntimeError('Size mismatch in the keypoint lists') 173 | 174 | if p1n.shape[0] < 5: 175 | return np.pi, np.pi / 2 176 | 177 | if E.size > 0: 178 | _, R, t, _ = cv2.recoverPose(E, p1n, p2n) 179 | try: 180 | err_q, err_t = evaluate_R_t(dR, dt, R, t) 181 | except: 182 | err_q = np.pi 183 | err_t = np.pi / 2 184 | 185 | else: 186 | err_q = np.pi 187 | err_t = np.pi / 2 188 | 189 | return err_q, err_t 190 | 191 | 192 | def decolorize(img): 193 | return cv2.cvtColor(cv2.cvtColor(img,cv2.COLOR_RGB2GRAY), cv2.COLOR_GRAY2RGB) 194 | 195 | def drawlines(img1,img2,lines,pts1,pts2): 196 | ''' img1 - image on which we draw the epilines for the points in img2 197 | lines - corresponding epilines ''' 198 | r,c,ch = img1.shape 199 | img1o = deepcopy(img1) 200 | img2o = deepcopy(img2) 201 | for r,pt1,pt2 in zip(lines,pts1,pts2): 202 | color = tuple(np.random.randint(0,255,3).tolist()) 203 | x0,y0 = map(int, [0, -r[2]/r[1] ]) 204 | x1,y1 = map(int, [c, -(r[2]+r[0]*c)/r[1] ]) 205 | img1o = cv2.line(img1o, (x0,y0), (x1,y1), color,1) 206 | img1o = cv2.circle(img1o,tuple(pt1.squeeze().astype(np.int32)),5,color,-1) 207 | img2o = cv2.circle(img2o,tuple(pt2.squeeze().astype(np.int32)),5,color,-1) 208 | return img1o,img2o 209 | 210 | 211 | def draw_everything(img1, img2, pts1_good, pts2_good, F_gt): 212 | lines1gt = cv2.computeCorrespondEpilines(pts2_good.reshape(-1,1,2), 2, F_gt) 213 | lines1gt = lines1gt.reshape(-1,3) 214 | img5gt,img6gt = drawlines(img1,img2,lines1gt,pts1_good,pts2_good) 215 | # Find epilines corresponding to points in left image (first image) and 216 | # drawing its lines on right image 217 | lines2gt = cv2.computeCorrespondEpilines(pts1_good.reshape(-1,1,2), 1,F_gt) 218 | lines2gt = lines2gt.reshape(-1,3) 219 | img3gt,img4gt = drawlines(img2,img1,lines2gt,pts2_good,pts1_good) 220 | plt.figure(figsize = (12,12)) 221 | plt.subplot(121) 222 | plt.imshow(img5gt) 223 | plt.subplot(122) 224 | plt.imshow(img3gt) 225 | return 226 | 227 | 228 | def get_h_imgpair(key, dataset, split = 'val'): 229 | DIR = 'homography' 230 | if dataset == 'EVD': 231 | img1_fname = f'{DIR}/{dataset}/{split}/imgs/1/' + key.split('-')[0] + '.png' 232 | img2_fname = f'{DIR}/{dataset}/{split}/imgs/2/' + key.split('-')[0] + '.png' 233 | elif dataset == 'HPatchesSeq': 234 | img1_fname = f'{DIR}/{dataset}/{split}/imgs/{key[:-4]}/1.ppm' 235 | img2_fname = f'{DIR}/{dataset}/{split}/imgs/{key[:-4]}/{key[-1]}.ppm' 236 | else: 237 | raise ValueError ('Unknown dataset, try EVD or HPatchesSeq') 238 | img1 = cv2.cvtColor(cv2.imread(img1_fname), cv2.COLOR_BGR2RGB) 239 | img2 = cv2.cvtColor(cv2.imread(img2_fname), cv2.COLOR_BGR2RGB) 240 | return img1, img2 241 | 242 | def get_h_imgpair2(key, DIR): 243 | if 'EVD' in DIR: 244 | img1_fname = f'{DIR}/imgs/1/' + key.split('-')[0] + '.png' 245 | img2_fname = f'{DIR}/imgs/2/' + key.split('-')[0] + '.png' 246 | elif 'HPatchesSeq' in DIR: 247 | img1_fname = f'{DIR}/imgs/{key[:-4]}/1.ppm' 248 | img2_fname = f'{DIR}/imgs/{key[:-4]}/{key[-1]}.ppm' 249 | else: 250 | raise ValueError ('Unknown dataset, try EVD or HPatchesSeq') 251 | img1 = cv2.cvtColor(cv2.imread(img1_fname), cv2.COLOR_BGR2RGB) 252 | img2 = cv2.cvtColor(cv2.imread(img2_fname), cv2.COLOR_BGR2RGB) 253 | return img1, img2 254 | 255 | def get_output_dir(problem: str, split: str, method: str, params: dict): 256 | problem = problem.lower() 257 | if problem not in ['e', 'f', 'h', 'pnp']: 258 | raise ValueError(f'{problem} is unknown problem. Try e, f, h, or pnp') 259 | param_string = '' 260 | sorted_keys = sorted([str(x) for x in params.keys()]) 261 | for k in sorted_keys: 262 | param_string += f'_{k}-{str(params[k])}' 263 | return os.path.join('results', split, problem, method, param_string) 264 | 265 | 266 | 267 | # from ACNe code 268 | def compute_T_with_imagesize(w, h, f=None, ratio=1.0): 269 | cx = (w - 1.0) * 0.5 270 | cy = (h - 1.0) * 0.5 271 | mean = np.array([cx, cy]) 272 | if f is not None: 273 | f = f 274 | else: 275 | f = max(w - 1.0, h - 1.0) * ratio 276 | 277 | scale = 1.0 / f 278 | T = np.zeros((3, 3,)) 279 | T[0, 0], T[1, 1], T[2, 2] = scale, scale, 1 280 | T[0, 2], T[1, 2] = -scale * mean[0], -scale * mean[1] 281 | 282 | return T.copy() 283 | --------------------------------------------------------------------------------