├── .gitignore ├── HSequences_bench ├── HPatches_images.txt ├── splits.json └── tools │ ├── HSequences_reader.py │ ├── aux_tools.py │ ├── geometry_tools.py │ ├── matching_tools.py │ ├── opencv_matcher.py │ └── repeatability_tools.py ├── README.md ├── extract_multiscale_features.py ├── hsequences_bench.py ├── keyNet ├── aux │ ├── desc_aux_function.py │ ├── logger.py │ └── tools.py ├── config.py ├── config_hpatches.py ├── datasets │ ├── dataset_utils.py │ └── pytorch_dataset.py ├── loss │ └── score_loss_function.py ├── model │ ├── hardnet_pytorch.py │ └── keynet_architecture.py └── pretrained_nets │ ├── HardNet++.pth │ └── keyNet.pt ├── train.py └── train_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *__pycache__* 2 | logs/ 3 | .ipynb_checkpoints/ 4 | 5 | hpatches-sequences-release/ 6 | extracted_features/ 7 | HSequences_bench/results/ 8 | samples_for_matching/ 9 | 10 | datasets/synth 11 | keyNet/tfrecords 12 | keyNet/weights/* 13 | keyNet/data/* 14 | 15 | 0.sh 16 | 1.sh -------------------------------------------------------------------------------- /HSequences_bench/HPatches_images.txt: -------------------------------------------------------------------------------- 1 | hpatches-sequences-release/v_churchill/1.ppm 2 | hpatches-sequences-release/v_churchill/2.ppm 3 | hpatches-sequences-release/v_churchill/3.ppm 4 | hpatches-sequences-release/v_churchill/4.ppm 5 | hpatches-sequences-release/v_churchill/5.ppm 6 | hpatches-sequences-release/v_churchill/6.ppm 7 | hpatches-sequences-release/v_dogman/1.ppm 8 | hpatches-sequences-release/v_dogman/2.ppm 9 | hpatches-sequences-release/v_dogman/3.ppm 10 | hpatches-sequences-release/v_dogman/4.ppm 11 | hpatches-sequences-release/v_dogman/5.ppm 12 | hpatches-sequences-release/v_dogman/6.ppm 13 | hpatches-sequences-release/v_maskedman/1.ppm 14 | hpatches-sequences-release/v_maskedman/2.ppm 15 | hpatches-sequences-release/v_maskedman/3.ppm 16 | hpatches-sequences-release/v_maskedman/4.ppm 17 | hpatches-sequences-release/v_maskedman/5.ppm 18 | hpatches-sequences-release/v_maskedman/6.ppm 19 | hpatches-sequences-release/v_wall/1.ppm 20 | hpatches-sequences-release/v_wall/2.ppm 21 | hpatches-sequences-release/v_wall/3.ppm 22 | hpatches-sequences-release/v_wall/4.ppm 23 | hpatches-sequences-release/v_wall/5.ppm 24 | hpatches-sequences-release/v_wall/6.ppm 25 | hpatches-sequences-release/v_pomegranate/1.ppm 26 | hpatches-sequences-release/v_pomegranate/2.ppm 27 | hpatches-sequences-release/v_pomegranate/3.ppm 28 | hpatches-sequences-release/v_pomegranate/4.ppm 29 | hpatches-sequences-release/v_pomegranate/5.ppm 30 | hpatches-sequences-release/v_pomegranate/6.ppm 31 | hpatches-sequences-release/v_samples/1.ppm 32 | hpatches-sequences-release/v_samples/2.ppm 33 | hpatches-sequences-release/v_samples/3.ppm 34 | hpatches-sequences-release/v_samples/4.ppm 35 | hpatches-sequences-release/v_samples/5.ppm 36 | hpatches-sequences-release/v_samples/6.ppm 37 | hpatches-sequences-release/v_beyus/1.ppm 38 | hpatches-sequences-release/v_beyus/2.ppm 39 | hpatches-sequences-release/v_beyus/3.ppm 40 | hpatches-sequences-release/v_beyus/4.ppm 41 | hpatches-sequences-release/v_beyus/5.ppm 42 | hpatches-sequences-release/v_beyus/6.ppm 43 | hpatches-sequences-release/v_wormhole/1.ppm 44 | hpatches-sequences-release/v_wormhole/2.ppm 45 | hpatches-sequences-release/v_wormhole/3.ppm 46 | hpatches-sequences-release/v_wormhole/4.ppm 47 | hpatches-sequences-release/v_wormhole/5.ppm 48 | hpatches-sequences-release/v_wormhole/6.ppm 49 | hpatches-sequences-release/v_bird/1.ppm 50 | hpatches-sequences-release/v_bird/2.ppm 51 | hpatches-sequences-release/v_bird/3.ppm 52 | hpatches-sequences-release/v_bird/4.ppm 53 | hpatches-sequences-release/v_bird/5.ppm 54 | hpatches-sequences-release/v_bird/6.ppm 55 | hpatches-sequences-release/v_weapons/1.ppm 56 | hpatches-sequences-release/v_weapons/2.ppm 57 | hpatches-sequences-release/v_weapons/3.ppm 58 | hpatches-sequences-release/v_weapons/4.ppm 59 | hpatches-sequences-release/v_weapons/5.ppm 60 | hpatches-sequences-release/v_weapons/6.ppm 61 | hpatches-sequences-release/v_vitro/1.ppm 62 | hpatches-sequences-release/v_vitro/2.ppm 63 | hpatches-sequences-release/v_vitro/3.ppm 64 | hpatches-sequences-release/v_vitro/4.ppm 65 | hpatches-sequences-release/v_vitro/5.ppm 66 | hpatches-sequences-release/v_vitro/6.ppm 67 | hpatches-sequences-release/v_posters/1.ppm 68 | hpatches-sequences-release/v_posters/2.ppm 69 | hpatches-sequences-release/v_posters/3.ppm 70 | hpatches-sequences-release/v_posters/4.ppm 71 | hpatches-sequences-release/v_posters/5.ppm 72 | hpatches-sequences-release/v_posters/6.ppm 73 | hpatches-sequences-release/v_cartooncity/1.ppm 74 | hpatches-sequences-release/v_cartooncity/2.ppm 75 | hpatches-sequences-release/v_cartooncity/3.ppm 76 | hpatches-sequences-release/v_cartooncity/4.ppm 77 | hpatches-sequences-release/v_cartooncity/5.ppm 78 | hpatches-sequences-release/v_cartooncity/6.ppm 79 | hpatches-sequences-release/v_birdwoman/1.ppm 80 | hpatches-sequences-release/v_birdwoman/2.ppm 81 | hpatches-sequences-release/v_birdwoman/3.ppm 82 | hpatches-sequences-release/v_birdwoman/4.ppm 83 | hpatches-sequences-release/v_birdwoman/5.ppm 84 | hpatches-sequences-release/v_birdwoman/6.ppm 85 | hpatches-sequences-release/v_bark/1.ppm 86 | hpatches-sequences-release/v_bark/2.ppm 87 | hpatches-sequences-release/v_bark/3.ppm 88 | hpatches-sequences-release/v_bark/4.ppm 89 | hpatches-sequences-release/v_bark/5.ppm 90 | hpatches-sequences-release/v_bark/6.ppm 91 | hpatches-sequences-release/v_colors/1.ppm 92 | hpatches-sequences-release/v_colors/2.ppm 93 | hpatches-sequences-release/v_colors/3.ppm 94 | hpatches-sequences-release/v_colors/4.ppm 95 | hpatches-sequences-release/v_colors/5.ppm 96 | hpatches-sequences-release/v_colors/6.ppm 97 | hpatches-sequences-release/v_strand/1.ppm 98 | hpatches-sequences-release/v_strand/2.ppm 99 | hpatches-sequences-release/v_strand/3.ppm 100 | hpatches-sequences-release/v_strand/4.ppm 101 | hpatches-sequences-release/v_strand/5.ppm 102 | hpatches-sequences-release/v_strand/6.ppm 103 | hpatches-sequences-release/v_eastsouth/1.ppm 104 | hpatches-sequences-release/v_eastsouth/2.ppm 105 | hpatches-sequences-release/v_eastsouth/3.ppm 106 | hpatches-sequences-release/v_eastsouth/4.ppm 107 | hpatches-sequences-release/v_eastsouth/5.ppm 108 | hpatches-sequences-release/v_eastsouth/6.ppm 109 | hpatches-sequences-release/v_coffeehouse/1.ppm 110 | hpatches-sequences-release/v_coffeehouse/2.ppm 111 | hpatches-sequences-release/v_coffeehouse/3.ppm 112 | hpatches-sequences-release/v_coffeehouse/4.ppm 113 | hpatches-sequences-release/v_coffeehouse/5.ppm 114 | hpatches-sequences-release/v_coffeehouse/6.ppm 115 | hpatches-sequences-release/v_abstract/1.ppm 116 | hpatches-sequences-release/v_abstract/2.ppm 117 | hpatches-sequences-release/v_abstract/3.ppm 118 | hpatches-sequences-release/v_abstract/4.ppm 119 | hpatches-sequences-release/v_abstract/5.ppm 120 | hpatches-sequences-release/v_abstract/6.ppm 121 | hpatches-sequences-release/v_blueprint/1.ppm 122 | hpatches-sequences-release/v_blueprint/2.ppm 123 | hpatches-sequences-release/v_blueprint/3.ppm 124 | hpatches-sequences-release/v_blueprint/4.ppm 125 | hpatches-sequences-release/v_blueprint/5.ppm 126 | hpatches-sequences-release/v_blueprint/6.ppm 127 | hpatches-sequences-release/v_artisans/1.ppm 128 | hpatches-sequences-release/v_artisans/2.ppm 129 | hpatches-sequences-release/v_artisans/3.ppm 130 | hpatches-sequences-release/v_artisans/4.ppm 131 | hpatches-sequences-release/v_artisans/5.ppm 132 | hpatches-sequences-release/v_artisans/6.ppm 133 | hpatches-sequences-release/v_dirtywall/1.ppm 134 | hpatches-sequences-release/v_dirtywall/2.ppm 135 | hpatches-sequences-release/v_dirtywall/3.ppm 136 | hpatches-sequences-release/v_dirtywall/4.ppm 137 | hpatches-sequences-release/v_dirtywall/5.ppm 138 | hpatches-sequences-release/v_dirtywall/6.ppm 139 | hpatches-sequences-release/v_yard/1.ppm 140 | hpatches-sequences-release/v_yard/2.ppm 141 | hpatches-sequences-release/v_yard/3.ppm 142 | hpatches-sequences-release/v_yard/4.ppm 143 | hpatches-sequences-release/v_yard/5.ppm 144 | hpatches-sequences-release/v_yard/6.ppm 145 | hpatches-sequences-release/v_there/1.ppm 146 | hpatches-sequences-release/v_there/2.ppm 147 | hpatches-sequences-release/v_there/3.ppm 148 | hpatches-sequences-release/v_there/4.ppm 149 | hpatches-sequences-release/v_there/5.ppm 150 | hpatches-sequences-release/v_there/6.ppm 151 | hpatches-sequences-release/v_tabletop/1.ppm 152 | hpatches-sequences-release/v_tabletop/2.ppm 153 | hpatches-sequences-release/v_tabletop/3.ppm 154 | hpatches-sequences-release/v_tabletop/4.ppm 155 | hpatches-sequences-release/v_tabletop/5.ppm 156 | hpatches-sequences-release/v_tabletop/6.ppm 157 | hpatches-sequences-release/v_busstop/1.ppm 158 | hpatches-sequences-release/v_busstop/2.ppm 159 | hpatches-sequences-release/v_busstop/3.ppm 160 | hpatches-sequences-release/v_busstop/4.ppm 161 | hpatches-sequences-release/v_busstop/5.ppm 162 | hpatches-sequences-release/v_busstop/6.ppm 163 | hpatches-sequences-release/v_adam/1.ppm 164 | hpatches-sequences-release/v_adam/2.ppm 165 | hpatches-sequences-release/v_adam/3.ppm 166 | hpatches-sequences-release/v_adam/4.ppm 167 | hpatches-sequences-release/v_adam/5.ppm 168 | hpatches-sequences-release/v_adam/6.ppm 169 | hpatches-sequences-release/v_fest/1.ppm 170 | hpatches-sequences-release/v_fest/2.ppm 171 | hpatches-sequences-release/v_fest/3.ppm 172 | hpatches-sequences-release/v_fest/4.ppm 173 | hpatches-sequences-release/v_fest/5.ppm 174 | hpatches-sequences-release/v_fest/6.ppm 175 | hpatches-sequences-release/v_azzola/1.ppm 176 | hpatches-sequences-release/v_azzola/2.ppm 177 | hpatches-sequences-release/v_azzola/3.ppm 178 | hpatches-sequences-release/v_azzola/4.ppm 179 | hpatches-sequences-release/v_azzola/5.ppm 180 | hpatches-sequences-release/v_azzola/6.ppm 181 | hpatches-sequences-release/v_boat/1.ppm 182 | hpatches-sequences-release/v_boat/2.ppm 183 | hpatches-sequences-release/v_boat/3.ppm 184 | hpatches-sequences-release/v_boat/4.ppm 185 | hpatches-sequences-release/v_boat/5.ppm 186 | hpatches-sequences-release/v_boat/6.ppm 187 | hpatches-sequences-release/v_war/1.ppm 188 | hpatches-sequences-release/v_war/2.ppm 189 | hpatches-sequences-release/v_war/3.ppm 190 | hpatches-sequences-release/v_war/4.ppm 191 | hpatches-sequences-release/v_war/5.ppm 192 | hpatches-sequences-release/v_war/6.ppm 193 | hpatches-sequences-release/v_talent/1.ppm 194 | hpatches-sequences-release/v_talent/2.ppm 195 | hpatches-sequences-release/v_talent/3.ppm 196 | hpatches-sequences-release/v_talent/4.ppm 197 | hpatches-sequences-release/v_talent/5.ppm 198 | hpatches-sequences-release/v_talent/6.ppm 199 | hpatches-sequences-release/v_charing/1.ppm 200 | hpatches-sequences-release/v_charing/2.ppm 201 | hpatches-sequences-release/v_charing/3.ppm 202 | hpatches-sequences-release/v_charing/4.ppm 203 | hpatches-sequences-release/v_charing/5.ppm 204 | hpatches-sequences-release/v_charing/6.ppm 205 | hpatches-sequences-release/v_man/1.ppm 206 | hpatches-sequences-release/v_man/2.ppm 207 | hpatches-sequences-release/v_man/3.ppm 208 | hpatches-sequences-release/v_man/4.ppm 209 | hpatches-sequences-release/v_man/5.ppm 210 | hpatches-sequences-release/v_man/6.ppm 211 | hpatches-sequences-release/v_laptop/1.ppm 212 | hpatches-sequences-release/v_laptop/2.ppm 213 | hpatches-sequences-release/v_laptop/3.ppm 214 | hpatches-sequences-release/v_laptop/4.ppm 215 | hpatches-sequences-release/v_laptop/5.ppm 216 | hpatches-sequences-release/v_laptop/6.ppm 217 | hpatches-sequences-release/v_bees/1.ppm 218 | hpatches-sequences-release/v_bees/2.ppm 219 | hpatches-sequences-release/v_bees/3.ppm 220 | hpatches-sequences-release/v_bees/4.ppm 221 | hpatches-sequences-release/v_bees/5.ppm 222 | hpatches-sequences-release/v_bees/6.ppm 223 | hpatches-sequences-release/v_grace/1.ppm 224 | hpatches-sequences-release/v_grace/2.ppm 225 | hpatches-sequences-release/v_grace/3.ppm 226 | hpatches-sequences-release/v_grace/4.ppm 227 | hpatches-sequences-release/v_grace/5.ppm 228 | hpatches-sequences-release/v_grace/6.ppm 229 | hpatches-sequences-release/v_graffiti/1.ppm 230 | hpatches-sequences-release/v_graffiti/2.ppm 231 | hpatches-sequences-release/v_graffiti/3.ppm 232 | hpatches-sequences-release/v_graffiti/4.ppm 233 | hpatches-sequences-release/v_graffiti/5.ppm 234 | hpatches-sequences-release/v_graffiti/6.ppm 235 | hpatches-sequences-release/v_wounded/1.ppm 236 | hpatches-sequences-release/v_wounded/2.ppm 237 | hpatches-sequences-release/v_wounded/3.ppm 238 | hpatches-sequences-release/v_wounded/4.ppm 239 | hpatches-sequences-release/v_wounded/5.ppm 240 | hpatches-sequences-release/v_wounded/6.ppm 241 | hpatches-sequences-release/v_feast/1.ppm 242 | hpatches-sequences-release/v_feast/2.ppm 243 | hpatches-sequences-release/v_feast/3.ppm 244 | hpatches-sequences-release/v_feast/4.ppm 245 | hpatches-sequences-release/v_feast/5.ppm 246 | hpatches-sequences-release/v_feast/6.ppm 247 | hpatches-sequences-release/v_apprentices/1.ppm 248 | hpatches-sequences-release/v_apprentices/2.ppm 249 | hpatches-sequences-release/v_apprentices/3.ppm 250 | hpatches-sequences-release/v_apprentices/4.ppm 251 | hpatches-sequences-release/v_apprentices/5.ppm 252 | hpatches-sequences-release/v_apprentices/6.ppm 253 | hpatches-sequences-release/v_home/1.ppm 254 | hpatches-sequences-release/v_home/2.ppm 255 | hpatches-sequences-release/v_home/3.ppm 256 | hpatches-sequences-release/v_home/4.ppm 257 | hpatches-sequences-release/v_home/5.ppm 258 | hpatches-sequences-release/v_home/6.ppm 259 | hpatches-sequences-release/v_astronautis/1.ppm 260 | hpatches-sequences-release/v_astronautis/2.ppm 261 | hpatches-sequences-release/v_astronautis/3.ppm 262 | hpatches-sequences-release/v_astronautis/4.ppm 263 | hpatches-sequences-release/v_astronautis/5.ppm 264 | hpatches-sequences-release/v_astronautis/6.ppm 265 | hpatches-sequences-release/v_soldiers/1.ppm 266 | hpatches-sequences-release/v_soldiers/2.ppm 267 | hpatches-sequences-release/v_soldiers/3.ppm 268 | hpatches-sequences-release/v_soldiers/4.ppm 269 | hpatches-sequences-release/v_soldiers/5.ppm 270 | hpatches-sequences-release/v_soldiers/6.ppm 271 | hpatches-sequences-release/v_machines/1.ppm 272 | hpatches-sequences-release/v_machines/2.ppm 273 | hpatches-sequences-release/v_machines/3.ppm 274 | hpatches-sequences-release/v_machines/4.ppm 275 | hpatches-sequences-release/v_machines/5.ppm 276 | hpatches-sequences-release/v_machines/6.ppm 277 | hpatches-sequences-release/v_gardens/1.ppm 278 | hpatches-sequences-release/v_gardens/2.ppm 279 | hpatches-sequences-release/v_gardens/3.ppm 280 | hpatches-sequences-release/v_gardens/4.ppm 281 | hpatches-sequences-release/v_gardens/5.ppm 282 | hpatches-sequences-release/v_gardens/6.ppm 283 | hpatches-sequences-release/v_bricks/1.ppm 284 | hpatches-sequences-release/v_bricks/2.ppm 285 | hpatches-sequences-release/v_bricks/3.ppm 286 | hpatches-sequences-release/v_bricks/4.ppm 287 | hpatches-sequences-release/v_bricks/5.ppm 288 | hpatches-sequences-release/v_bricks/6.ppm 289 | hpatches-sequences-release/v_tempera/1.ppm 290 | hpatches-sequences-release/v_tempera/2.ppm 291 | hpatches-sequences-release/v_tempera/3.ppm 292 | hpatches-sequences-release/v_tempera/4.ppm 293 | hpatches-sequences-release/v_tempera/5.ppm 294 | hpatches-sequences-release/v_tempera/6.ppm 295 | hpatches-sequences-release/v_bip/1.ppm 296 | hpatches-sequences-release/v_bip/2.ppm 297 | hpatches-sequences-release/v_bip/3.ppm 298 | hpatches-sequences-release/v_bip/4.ppm 299 | hpatches-sequences-release/v_bip/5.ppm 300 | hpatches-sequences-release/v_bip/6.ppm 301 | hpatches-sequences-release/v_courses/1.ppm 302 | hpatches-sequences-release/v_courses/2.ppm 303 | hpatches-sequences-release/v_courses/3.ppm 304 | hpatches-sequences-release/v_courses/4.ppm 305 | hpatches-sequences-release/v_courses/5.ppm 306 | hpatches-sequences-release/v_courses/6.ppm 307 | hpatches-sequences-release/v_sunseason/1.ppm 308 | hpatches-sequences-release/v_sunseason/2.ppm 309 | hpatches-sequences-release/v_sunseason/3.ppm 310 | hpatches-sequences-release/v_sunseason/4.ppm 311 | hpatches-sequences-release/v_sunseason/5.ppm 312 | hpatches-sequences-release/v_sunseason/6.ppm 313 | hpatches-sequences-release/v_yuri/1.ppm 314 | hpatches-sequences-release/v_yuri/2.ppm 315 | hpatches-sequences-release/v_yuri/3.ppm 316 | hpatches-sequences-release/v_yuri/4.ppm 317 | hpatches-sequences-release/v_yuri/5.ppm 318 | hpatches-sequences-release/v_yuri/6.ppm 319 | hpatches-sequences-release/v_calder/1.ppm 320 | hpatches-sequences-release/v_calder/2.ppm 321 | hpatches-sequences-release/v_calder/3.ppm 322 | hpatches-sequences-release/v_calder/4.ppm 323 | hpatches-sequences-release/v_calder/5.ppm 324 | hpatches-sequences-release/v_calder/6.ppm 325 | hpatches-sequences-release/v_woman/1.ppm 326 | hpatches-sequences-release/v_woman/2.ppm 327 | hpatches-sequences-release/v_woman/3.ppm 328 | hpatches-sequences-release/v_woman/4.ppm 329 | hpatches-sequences-release/v_woman/5.ppm 330 | hpatches-sequences-release/v_woman/6.ppm 331 | hpatches-sequences-release/v_london/1.ppm 332 | hpatches-sequences-release/v_london/2.ppm 333 | hpatches-sequences-release/v_london/3.ppm 334 | hpatches-sequences-release/v_london/4.ppm 335 | hpatches-sequences-release/v_london/5.ppm 336 | hpatches-sequences-release/v_london/6.ppm 337 | hpatches-sequences-release/v_wapping/1.ppm 338 | hpatches-sequences-release/v_wapping/2.ppm 339 | hpatches-sequences-release/v_wapping/3.ppm 340 | hpatches-sequences-release/v_wapping/4.ppm 341 | hpatches-sequences-release/v_wapping/5.ppm 342 | hpatches-sequences-release/v_wapping/6.ppm 343 | hpatches-sequences-release/v_underground/1.ppm 344 | hpatches-sequences-release/v_underground/2.ppm 345 | hpatches-sequences-release/v_underground/3.ppm 346 | hpatches-sequences-release/v_underground/4.ppm 347 | hpatches-sequences-release/v_underground/5.ppm 348 | hpatches-sequences-release/v_underground/6.ppm 349 | hpatches-sequences-release/v_circus/1.ppm 350 | hpatches-sequences-release/v_circus/2.ppm 351 | hpatches-sequences-release/v_circus/3.ppm 352 | hpatches-sequences-release/v_circus/4.ppm 353 | hpatches-sequences-release/v_circus/5.ppm 354 | hpatches-sequences-release/v_circus/6.ppm 355 | hpatches-sequences-release/i_ajuntament/1.ppm 356 | hpatches-sequences-release/i_ajuntament/2.ppm 357 | hpatches-sequences-release/i_ajuntament/3.ppm 358 | hpatches-sequences-release/i_ajuntament/4.ppm 359 | hpatches-sequences-release/i_ajuntament/5.ppm 360 | hpatches-sequences-release/i_ajuntament/6.ppm 361 | hpatches-sequences-release/i_parking/1.ppm 362 | hpatches-sequences-release/i_parking/2.ppm 363 | hpatches-sequences-release/i_parking/3.ppm 364 | hpatches-sequences-release/i_parking/4.ppm 365 | hpatches-sequences-release/i_parking/5.ppm 366 | hpatches-sequences-release/i_parking/6.ppm 367 | hpatches-sequences-release/i_kions/1.ppm 368 | hpatches-sequences-release/i_kions/2.ppm 369 | hpatches-sequences-release/i_kions/3.ppm 370 | hpatches-sequences-release/i_kions/4.ppm 371 | hpatches-sequences-release/i_kions/5.ppm 372 | hpatches-sequences-release/i_kions/6.ppm 373 | hpatches-sequences-release/i_fog/1.ppm 374 | hpatches-sequences-release/i_fog/2.ppm 375 | hpatches-sequences-release/i_fog/3.ppm 376 | hpatches-sequences-release/i_fog/4.ppm 377 | hpatches-sequences-release/i_fog/5.ppm 378 | hpatches-sequences-release/i_fog/6.ppm 379 | hpatches-sequences-release/i_brooklyn/1.ppm 380 | hpatches-sequences-release/i_brooklyn/2.ppm 381 | hpatches-sequences-release/i_brooklyn/3.ppm 382 | hpatches-sequences-release/i_brooklyn/4.ppm 383 | hpatches-sequences-release/i_brooklyn/5.ppm 384 | hpatches-sequences-release/i_brooklyn/6.ppm 385 | hpatches-sequences-release/i_ski/1.ppm 386 | hpatches-sequences-release/i_ski/2.ppm 387 | hpatches-sequences-release/i_ski/3.ppm 388 | hpatches-sequences-release/i_ski/4.ppm 389 | hpatches-sequences-release/i_ski/5.ppm 390 | hpatches-sequences-release/i_ski/6.ppm 391 | hpatches-sequences-release/i_gonnenberg/1.ppm 392 | hpatches-sequences-release/i_gonnenberg/2.ppm 393 | hpatches-sequences-release/i_gonnenberg/3.ppm 394 | hpatches-sequences-release/i_gonnenberg/4.ppm 395 | hpatches-sequences-release/i_gonnenberg/5.ppm 396 | hpatches-sequences-release/i_gonnenberg/6.ppm 397 | hpatches-sequences-release/i_yellowtent/1.ppm 398 | hpatches-sequences-release/i_yellowtent/2.ppm 399 | hpatches-sequences-release/i_yellowtent/3.ppm 400 | hpatches-sequences-release/i_yellowtent/4.ppm 401 | hpatches-sequences-release/i_yellowtent/5.ppm 402 | hpatches-sequences-release/i_yellowtent/6.ppm 403 | hpatches-sequences-release/i_lionday/1.ppm 404 | hpatches-sequences-release/i_lionday/2.ppm 405 | hpatches-sequences-release/i_lionday/3.ppm 406 | hpatches-sequences-release/i_lionday/4.ppm 407 | hpatches-sequences-release/i_lionday/5.ppm 408 | hpatches-sequences-release/i_lionday/6.ppm 409 | hpatches-sequences-release/i_tools/1.ppm 410 | hpatches-sequences-release/i_tools/2.ppm 411 | hpatches-sequences-release/i_tools/3.ppm 412 | hpatches-sequences-release/i_tools/4.ppm 413 | hpatches-sequences-release/i_tools/5.ppm 414 | hpatches-sequences-release/i_tools/6.ppm 415 | hpatches-sequences-release/i_books/1.ppm 416 | hpatches-sequences-release/i_books/2.ppm 417 | hpatches-sequences-release/i_books/3.ppm 418 | hpatches-sequences-release/i_books/4.ppm 419 | hpatches-sequences-release/i_books/5.ppm 420 | hpatches-sequences-release/i_books/6.ppm 421 | hpatches-sequences-release/i_salon/1.ppm 422 | hpatches-sequences-release/i_salon/2.ppm 423 | hpatches-sequences-release/i_salon/3.ppm 424 | hpatches-sequences-release/i_salon/4.ppm 425 | hpatches-sequences-release/i_salon/5.ppm 426 | hpatches-sequences-release/i_salon/6.ppm 427 | hpatches-sequences-release/i_leuven/1.ppm 428 | hpatches-sequences-release/i_leuven/2.ppm 429 | hpatches-sequences-release/i_leuven/3.ppm 430 | hpatches-sequences-release/i_leuven/4.ppm 431 | hpatches-sequences-release/i_leuven/5.ppm 432 | hpatches-sequences-release/i_leuven/6.ppm 433 | hpatches-sequences-release/i_contruction/1.ppm 434 | hpatches-sequences-release/i_contruction/2.ppm 435 | hpatches-sequences-release/i_contruction/3.ppm 436 | hpatches-sequences-release/i_contruction/4.ppm 437 | hpatches-sequences-release/i_contruction/5.ppm 438 | hpatches-sequences-release/i_contruction/6.ppm 439 | hpatches-sequences-release/i_santuario/1.ppm 440 | hpatches-sequences-release/i_santuario/2.ppm 441 | hpatches-sequences-release/i_santuario/3.ppm 442 | hpatches-sequences-release/i_santuario/4.ppm 443 | hpatches-sequences-release/i_santuario/5.ppm 444 | hpatches-sequences-release/i_santuario/6.ppm 445 | hpatches-sequences-release/i_zion/1.ppm 446 | hpatches-sequences-release/i_zion/2.ppm 447 | hpatches-sequences-release/i_zion/3.ppm 448 | hpatches-sequences-release/i_zion/4.ppm 449 | hpatches-sequences-release/i_zion/5.ppm 450 | hpatches-sequences-release/i_zion/6.ppm 451 | hpatches-sequences-release/i_nijmegen/1.ppm 452 | hpatches-sequences-release/i_nijmegen/2.ppm 453 | hpatches-sequences-release/i_nijmegen/3.ppm 454 | hpatches-sequences-release/i_nijmegen/4.ppm 455 | hpatches-sequences-release/i_nijmegen/5.ppm 456 | hpatches-sequences-release/i_nijmegen/6.ppm 457 | hpatches-sequences-release/i_autannes/1.ppm 458 | hpatches-sequences-release/i_autannes/2.ppm 459 | hpatches-sequences-release/i_autannes/3.ppm 460 | hpatches-sequences-release/i_autannes/4.ppm 461 | hpatches-sequences-release/i_autannes/5.ppm 462 | hpatches-sequences-release/i_autannes/6.ppm 463 | hpatches-sequences-release/i_nuts/1.ppm 464 | hpatches-sequences-release/i_nuts/2.ppm 465 | hpatches-sequences-release/i_nuts/3.ppm 466 | hpatches-sequences-release/i_nuts/4.ppm 467 | hpatches-sequences-release/i_nuts/5.ppm 468 | hpatches-sequences-release/i_nuts/6.ppm 469 | hpatches-sequences-release/i_crownday/1.ppm 470 | hpatches-sequences-release/i_crownday/2.ppm 471 | hpatches-sequences-release/i_crownday/3.ppm 472 | hpatches-sequences-release/i_crownday/4.ppm 473 | hpatches-sequences-release/i_crownday/5.ppm 474 | hpatches-sequences-release/i_crownday/6.ppm 475 | hpatches-sequences-release/i_pinard/1.ppm 476 | hpatches-sequences-release/i_pinard/2.ppm 477 | hpatches-sequences-release/i_pinard/3.ppm 478 | hpatches-sequences-release/i_pinard/4.ppm 479 | hpatches-sequences-release/i_pinard/5.ppm 480 | hpatches-sequences-release/i_pinard/6.ppm 481 | hpatches-sequences-release/i_londonbridge/1.ppm 482 | hpatches-sequences-release/i_londonbridge/2.ppm 483 | hpatches-sequences-release/i_londonbridge/3.ppm 484 | hpatches-sequences-release/i_londonbridge/4.ppm 485 | hpatches-sequences-release/i_londonbridge/5.ppm 486 | hpatches-sequences-release/i_londonbridge/6.ppm 487 | hpatches-sequences-release/i_miniature/1.ppm 488 | hpatches-sequences-release/i_miniature/2.ppm 489 | hpatches-sequences-release/i_miniature/3.ppm 490 | hpatches-sequences-release/i_miniature/4.ppm 491 | hpatches-sequences-release/i_miniature/5.ppm 492 | hpatches-sequences-release/i_miniature/6.ppm 493 | hpatches-sequences-release/i_resort/1.ppm 494 | hpatches-sequences-release/i_resort/2.ppm 495 | hpatches-sequences-release/i_resort/3.ppm 496 | hpatches-sequences-release/i_resort/4.ppm 497 | hpatches-sequences-release/i_resort/5.ppm 498 | hpatches-sequences-release/i_resort/6.ppm 499 | hpatches-sequences-release/i_fenis/1.ppm 500 | hpatches-sequences-release/i_fenis/2.ppm 501 | hpatches-sequences-release/i_fenis/3.ppm 502 | hpatches-sequences-release/i_fenis/4.ppm 503 | hpatches-sequences-release/i_fenis/5.ppm 504 | hpatches-sequences-release/i_fenis/6.ppm 505 | hpatches-sequences-release/i_dome/1.ppm 506 | hpatches-sequences-release/i_dome/2.ppm 507 | hpatches-sequences-release/i_dome/3.ppm 508 | hpatches-sequences-release/i_dome/4.ppm 509 | hpatches-sequences-release/i_dome/5.ppm 510 | hpatches-sequences-release/i_dome/6.ppm 511 | hpatches-sequences-release/i_kurhaus/1.ppm 512 | hpatches-sequences-release/i_kurhaus/2.ppm 513 | hpatches-sequences-release/i_kurhaus/3.ppm 514 | hpatches-sequences-release/i_kurhaus/4.ppm 515 | hpatches-sequences-release/i_kurhaus/5.ppm 516 | hpatches-sequences-release/i_kurhaus/6.ppm 517 | hpatches-sequences-release/i_toy/1.ppm 518 | hpatches-sequences-release/i_toy/2.ppm 519 | hpatches-sequences-release/i_toy/3.ppm 520 | hpatches-sequences-release/i_toy/4.ppm 521 | hpatches-sequences-release/i_toy/5.ppm 522 | hpatches-sequences-release/i_toy/6.ppm 523 | hpatches-sequences-release/i_veggies/1.ppm 524 | hpatches-sequences-release/i_veggies/2.ppm 525 | hpatches-sequences-release/i_veggies/3.ppm 526 | hpatches-sequences-release/i_veggies/4.ppm 527 | hpatches-sequences-release/i_veggies/5.ppm 528 | hpatches-sequences-release/i_veggies/6.ppm 529 | hpatches-sequences-release/i_objects/1.ppm 530 | hpatches-sequences-release/i_objects/2.ppm 531 | hpatches-sequences-release/i_objects/3.ppm 532 | hpatches-sequences-release/i_objects/4.ppm 533 | hpatches-sequences-release/i_objects/5.ppm 534 | hpatches-sequences-release/i_objects/6.ppm 535 | hpatches-sequences-release/i_village/1.ppm 536 | hpatches-sequences-release/i_village/2.ppm 537 | hpatches-sequences-release/i_village/3.ppm 538 | hpatches-sequences-release/i_village/4.ppm 539 | hpatches-sequences-release/i_village/5.ppm 540 | hpatches-sequences-release/i_village/6.ppm 541 | hpatches-sequences-release/i_crownnight/1.ppm 542 | hpatches-sequences-release/i_crownnight/2.ppm 543 | hpatches-sequences-release/i_crownnight/3.ppm 544 | hpatches-sequences-release/i_crownnight/4.ppm 545 | hpatches-sequences-release/i_crownnight/5.ppm 546 | hpatches-sequences-release/i_crownnight/6.ppm 547 | hpatches-sequences-release/i_fruits/1.ppm 548 | hpatches-sequences-release/i_fruits/2.ppm 549 | hpatches-sequences-release/i_fruits/3.ppm 550 | hpatches-sequences-release/i_fruits/4.ppm 551 | hpatches-sequences-release/i_fruits/5.ppm 552 | hpatches-sequences-release/i_fruits/6.ppm 553 | hpatches-sequences-release/i_castle/1.ppm 554 | hpatches-sequences-release/i_castle/2.ppm 555 | hpatches-sequences-release/i_castle/3.ppm 556 | hpatches-sequences-release/i_castle/4.ppm 557 | hpatches-sequences-release/i_castle/5.ppm 558 | hpatches-sequences-release/i_castle/6.ppm 559 | hpatches-sequences-release/i_partyfood/1.ppm 560 | hpatches-sequences-release/i_partyfood/2.ppm 561 | hpatches-sequences-release/i_partyfood/3.ppm 562 | hpatches-sequences-release/i_partyfood/4.ppm 563 | hpatches-sequences-release/i_partyfood/5.ppm 564 | hpatches-sequences-release/i_partyfood/6.ppm 565 | hpatches-sequences-release/i_steps/1.ppm 566 | hpatches-sequences-release/i_steps/2.ppm 567 | hpatches-sequences-release/i_steps/3.ppm 568 | hpatches-sequences-release/i_steps/4.ppm 569 | hpatches-sequences-release/i_steps/5.ppm 570 | hpatches-sequences-release/i_steps/6.ppm 571 | hpatches-sequences-release/i_school/1.ppm 572 | hpatches-sequences-release/i_school/2.ppm 573 | hpatches-sequences-release/i_school/3.ppm 574 | hpatches-sequences-release/i_school/4.ppm 575 | hpatches-sequences-release/i_school/5.ppm 576 | hpatches-sequences-release/i_school/6.ppm 577 | hpatches-sequences-release/i_ktirio/1.ppm 578 | hpatches-sequences-release/i_ktirio/2.ppm 579 | hpatches-sequences-release/i_ktirio/3.ppm 580 | hpatches-sequences-release/i_ktirio/4.ppm 581 | hpatches-sequences-release/i_ktirio/5.ppm 582 | hpatches-sequences-release/i_ktirio/6.ppm 583 | hpatches-sequences-release/i_chestnuts/1.ppm 584 | hpatches-sequences-release/i_chestnuts/2.ppm 585 | hpatches-sequences-release/i_chestnuts/3.ppm 586 | hpatches-sequences-release/i_chestnuts/4.ppm 587 | hpatches-sequences-release/i_chestnuts/5.ppm 588 | hpatches-sequences-release/i_chestnuts/6.ppm 589 | hpatches-sequences-release/i_indiana/1.ppm 590 | hpatches-sequences-release/i_indiana/2.ppm 591 | hpatches-sequences-release/i_indiana/3.ppm 592 | hpatches-sequences-release/i_indiana/4.ppm 593 | hpatches-sequences-release/i_indiana/5.ppm 594 | hpatches-sequences-release/i_indiana/6.ppm 595 | hpatches-sequences-release/i_nescafe/1.ppm 596 | hpatches-sequences-release/i_nescafe/2.ppm 597 | hpatches-sequences-release/i_nescafe/3.ppm 598 | hpatches-sequences-release/i_nescafe/4.ppm 599 | hpatches-sequences-release/i_nescafe/5.ppm 600 | hpatches-sequences-release/i_nescafe/6.ppm 601 | hpatches-sequences-release/i_porta/1.ppm 602 | hpatches-sequences-release/i_porta/2.ppm 603 | hpatches-sequences-release/i_porta/3.ppm 604 | hpatches-sequences-release/i_porta/4.ppm 605 | hpatches-sequences-release/i_porta/5.ppm 606 | hpatches-sequences-release/i_porta/6.ppm 607 | hpatches-sequences-release/i_duda/1.ppm 608 | hpatches-sequences-release/i_duda/2.ppm 609 | hpatches-sequences-release/i_duda/3.ppm 610 | hpatches-sequences-release/i_duda/4.ppm 611 | hpatches-sequences-release/i_duda/5.ppm 612 | hpatches-sequences-release/i_duda/6.ppm 613 | hpatches-sequences-release/i_lionnight/1.ppm 614 | hpatches-sequences-release/i_lionnight/2.ppm 615 | hpatches-sequences-release/i_lionnight/3.ppm 616 | hpatches-sequences-release/i_lionnight/4.ppm 617 | hpatches-sequences-release/i_lionnight/5.ppm 618 | hpatches-sequences-release/i_lionnight/6.ppm 619 | hpatches-sequences-release/i_melon/1.ppm 620 | hpatches-sequences-release/i_melon/2.ppm 621 | hpatches-sequences-release/i_melon/3.ppm 622 | hpatches-sequences-release/i_melon/4.ppm 623 | hpatches-sequences-release/i_melon/5.ppm 624 | hpatches-sequences-release/i_melon/6.ppm 625 | hpatches-sequences-release/i_bridger/1.ppm 626 | hpatches-sequences-release/i_bridger/2.ppm 627 | hpatches-sequences-release/i_bridger/3.ppm 628 | hpatches-sequences-release/i_bridger/4.ppm 629 | hpatches-sequences-release/i_bridger/5.ppm 630 | hpatches-sequences-release/i_bridger/6.ppm 631 | hpatches-sequences-release/i_greentea/1.ppm 632 | hpatches-sequences-release/i_greentea/2.ppm 633 | hpatches-sequences-release/i_greentea/3.ppm 634 | hpatches-sequences-release/i_greentea/4.ppm 635 | hpatches-sequences-release/i_greentea/5.ppm 636 | hpatches-sequences-release/i_greentea/6.ppm 637 | hpatches-sequences-release/i_greenhouse/1.ppm 638 | hpatches-sequences-release/i_greenhouse/2.ppm 639 | hpatches-sequences-release/i_greenhouse/3.ppm 640 | hpatches-sequences-release/i_greenhouse/4.ppm 641 | hpatches-sequences-release/i_greenhouse/5.ppm 642 | hpatches-sequences-release/i_greenhouse/6.ppm 643 | hpatches-sequences-release/i_pencils/1.ppm 644 | hpatches-sequences-release/i_pencils/2.ppm 645 | hpatches-sequences-release/i_pencils/3.ppm 646 | hpatches-sequences-release/i_pencils/4.ppm 647 | hpatches-sequences-release/i_pencils/5.ppm 648 | hpatches-sequences-release/i_pencils/6.ppm 649 | hpatches-sequences-release/i_pool/1.ppm 650 | hpatches-sequences-release/i_pool/2.ppm 651 | hpatches-sequences-release/i_pool/3.ppm 652 | hpatches-sequences-release/i_pool/4.ppm 653 | hpatches-sequences-release/i_pool/5.ppm 654 | hpatches-sequences-release/i_pool/6.ppm 655 | hpatches-sequences-release/i_bologna/1.ppm 656 | hpatches-sequences-release/i_bologna/2.ppm 657 | hpatches-sequences-release/i_bologna/3.ppm 658 | hpatches-sequences-release/i_bologna/4.ppm 659 | hpatches-sequences-release/i_bologna/5.ppm 660 | hpatches-sequences-release/i_bologna/6.ppm 661 | hpatches-sequences-release/i_table/1.ppm 662 | hpatches-sequences-release/i_table/2.ppm 663 | hpatches-sequences-release/i_table/3.ppm 664 | hpatches-sequences-release/i_table/4.ppm 665 | hpatches-sequences-release/i_table/5.ppm 666 | hpatches-sequences-release/i_table/6.ppm 667 | hpatches-sequences-release/i_smurf/1.ppm 668 | hpatches-sequences-release/i_smurf/2.ppm 669 | hpatches-sequences-release/i_smurf/3.ppm 670 | hpatches-sequences-release/i_smurf/4.ppm 671 | hpatches-sequences-release/i_smurf/5.ppm 672 | hpatches-sequences-release/i_smurf/6.ppm 673 | hpatches-sequences-release/i_troulos/1.ppm 674 | hpatches-sequences-release/i_troulos/2.ppm 675 | hpatches-sequences-release/i_troulos/3.ppm 676 | hpatches-sequences-release/i_troulos/4.ppm 677 | hpatches-sequences-release/i_troulos/5.ppm 678 | hpatches-sequences-release/i_troulos/6.ppm 679 | hpatches-sequences-release/i_boutique/1.ppm 680 | hpatches-sequences-release/i_boutique/2.ppm 681 | hpatches-sequences-release/i_boutique/3.ppm 682 | hpatches-sequences-release/i_boutique/4.ppm 683 | hpatches-sequences-release/i_boutique/5.ppm 684 | hpatches-sequences-release/i_boutique/6.ppm 685 | hpatches-sequences-release/i_dc/1.ppm 686 | hpatches-sequences-release/i_dc/2.ppm 687 | hpatches-sequences-release/i_dc/3.ppm 688 | hpatches-sequences-release/i_dc/4.ppm 689 | hpatches-sequences-release/i_dc/5.ppm 690 | hpatches-sequences-release/i_dc/6.ppm 691 | hpatches-sequences-release/i_whitebuilding/1.ppm 692 | hpatches-sequences-release/i_whitebuilding/2.ppm 693 | hpatches-sequences-release/i_whitebuilding/3.ppm 694 | hpatches-sequences-release/i_whitebuilding/4.ppm 695 | hpatches-sequences-release/i_whitebuilding/5.ppm 696 | hpatches-sequences-release/i_whitebuilding/6.ppm 697 | -------------------------------------------------------------------------------- /HSequences_bench/splits.json: -------------------------------------------------------------------------------- 1 | {"a": {"test": ["i_ajuntament", "i_resort", "i_table", "i_troulos", "i_bologna", "i_lionnight", "i_porta", "i_zion", "i_brooklyn", "i_fruits", "i_books", "i_bridger", "i_whitebuilding", 2 | "i_kurhaus", "i_salon", "i_autannes", "i_tools", "i_santuario", "i_fog", "i_nijmegen", "v_courses", "v_coffeehouse", "v_abstract", "v_feast", "v_woman", "v_talent", "v_tabletop", 3 | "v_bees", "v_strand", "v_fest", "v_yard", "v_underground", "v_azzola", "v_eastsouth", "v_yuri", "v_soldiers", "v_man", "v_pomegranate", "v_birdwoman", "v_busstop"], 4 | "train": ["v_there", "i_yellowtent", "i_boutique", "v_wapping", "i_leuven", "i_school", "i_crownnight", "v_artisans", "v_colors", "i_ski", "v_circus", "v_tempera", 5 | "v_london", "v_war", "i_parking", "v_bark", "v_charing", "i_indiana", "v_weapons", "v_wormhole", "v_maskedman", "v_dirtywall", "v_wall", "v_vitro", "i_nuts", 6 | "i_londonbridge", "i_pool", "i_pinard", "i_greentea", "v_calder", "i_lionday", "i_crownday", "i_kions", "v_posters", "i_dome", "v_machines", "v_laptop", "v_boat", 7 | "v_churchill", "i_pencils", "v_beyus", "v_sunseason", "v_samples", "v_cartooncity", "v_gardens", "v_bip", "v_home", "i_veggies", "i_nescafe", "v_wounded", "i_toy", 8 | "v_dogman", "i_duda", "i_contruction", "v_graffiti", "i_gonnenberg", "v_astronautis", "i_ktirio", "i_castle", "i_greenhouse", "i_fenis", "i_partyfood", "v_adam", 9 | "v_apprentices", "v_blueprint", "i_smurf", "i_objects", "v_bird", "i_melon", "v_grace", "i_miniature", "v_bricks", "i_chestnuts", "i_village", "i_steps", "i_dc"], "name": "a"}, 10 | 11 | "c": {"test": ["i_ski", "i_table", "i_troulos", "i_melon", "i_tools", "i_kions", "i_londonbridge", "i_nijmegen", "i_boutique", "i_parking", "i_steps", "i_fog", "i_leuven", "i_dc", 12 | "i_partyfood", "i_pool", "i_castle", "i_bologna", "i_smurf", "i_crownnight", "v_azzola", "v_tempera", "v_machines", "v_coffeehouse", "v_graffiti", "v_artisans", "v_maskedman", 13 | "v_talent", "v_bees", "v_dirtywall", "v_blueprint", "v_war", "v_adam", "v_pomegranate", "v_busstop", "v_weapons", "v_gardens", "v_feast", "v_man", "v_wounded"], 14 | "train": ["v_there", "i_yellowtent", "i_whitebuilding", "v_wapping", "v_laptop", "i_school", "v_calder", "i_duda", "v_circus", "i_porta", "v_home", "i_lionnight", "i_chestnuts", 15 | "v_abstract", "v_soldiers", "i_contruction", "v_charing", "i_indiana", "v_strand", "v_fest", "v_yuri", "v_wormhole", "v_eastsouth", "i_autannes", "v_colors", "v_wall", "v_vitro", 16 | "i_nuts", "i_pinard", "v_tabletop", "i_brooklyn", "i_lionday", "i_crownday", "v_bip", "v_posters", "v_underground", "i_dome", "v_grace", "i_ajuntament", "v_cartooncity", "v_boat", 17 | "v_churchill", "i_pencils", "v_beyus", "v_sunseason", "v_samples", "i_kurhaus", "i_santuario", "i_resort", "i_zion", "i_veggies", "i_nescafe", "i_toy", "v_dogman", "i_books", 18 | "v_courses", "v_birdwoman", "v_yard", "i_salon", "i_gonnenberg", "v_astronautis", "i_ktirio", "i_bridger", "i_greenhouse", "i_fenis", "v_woman", "v_bricks", "v_apprentices", 19 | "i_greentea", "i_objects", "v_bird", "v_london", "i_fruits", "i_miniature", "i_village", "v_bark"], "name": "c"}, 20 | 21 | "b": {"test": ["i_fruits", "i_melon", "i_castle", "i_resort", "i_chestnuts", "i_kions", "i_kurhaus", "i_autannes", "i_duda", "i_partyfood", "i_ski", "i_dome", "i_greenhouse", 22 | "i_pencils", "i_porta", "i_lionday", "i_school", "i_bridger", "i_village", "i_fog", "v_astronautis", "v_bip", "v_charing", "v_woman", "v_feast", "v_yard", "v_churchill", 23 | "v_graffiti", "v_london", "v_sunseason", "v_posters", "v_bees", "v_apprentices", "v_birdwoman", "v_colors", "v_laptop", "v_there", "v_adam", "v_underground", "v_war"], 24 | "train": ["v_wormhole", "i_yellowtent", "i_boutique", "v_wapping", "i_leuven", "i_pinard", "i_crownnight", "v_artisans", "i_toy", "v_circus", "v_tempera", "i_lionnight", 25 | "i_parking", "v_soldiers", "i_contruction", "i_whitebuilding", "i_indiana", "v_azzola", "v_weapons", "v_fest", "v_yuri", "v_dirtywall", "v_eastsouth", "v_man", 26 | "v_wall", "v_vitro", "i_nuts", "i_londonbridge", "i_pool", "v_tabletop", "i_greentea", "i_brooklyn", "i_smurf", "v_cartooncity", "v_wounded", "v_calder", "v_coffeehouse", 27 | "v_grace", "v_machines", "i_tools", "v_boat", "v_beyus", "v_strand", "i_santuario", "i_crownday", "v_bark", "i_veggies", "i_nescafe", "v_maskedman", "v_abstract", "v_talent", 28 | "i_books", "i_table", "v_courses", "i_nijmegen", "i_salon", "i_gonnenberg", "v_samples", "i_ktirio", "v_gardens", "i_zion", "v_pomegranate", "i_fenis", "v_home", "i_ajuntament", 29 | "i_objects", "v_bird", "v_dogman", "i_troulos", "i_miniature", "v_bricks", "i_bologna", "v_busstop", "v_blueprint", "i_steps", "i_dc"], "name": "b"}, 30 | 31 | "illum_test": {"test": ["i_crownnight", "i_table", "i_objects", "i_nescafe", "i_nijmegen", "i_whitebuilding", "i_porta", "i_santuario", "i_dc", "i_castle", "i_steps", 32 | "i_contruction", "i_melon", "i_miniature", "i_troulos", "i_veggies", "i_zion", "i_gonnenberg", "i_autannes", "i_boutique", "i_fruits", "i_pool", "i_fog", "i_fenis", 33 | "i_ajuntament", "i_partyfood", "i_kurhaus", "i_school", "i_chestnuts", "i_smurf", "i_indiana", "i_pinard", "i_lionnight", "i_kions", "i_ski", "i_greenhouse", "i_ktirio", 34 | "i_tools", "i_toy", "i_bridger", "i_lionday", "i_brooklyn", "i_londonbridge", "i_greentea", "i_leuven", "i_nuts", "i_resort", "i_bologna", "i_duda", "i_dome", "i_pencils", 35 | "i_books", "i_parking", "i_salon"], "name": "illum_test"}, 36 | 37 | "illum": {"test": ["i_crownnight", "i_table", "i_objects", "i_nescafe", "i_nijmegen", "i_whitebuilding", 38 | "i_porta", "i_santuario", "i_dc", "i_castle", "i_steps", "i_contruction", "i_melon", "i_yellowtent", "i_miniature", "i_troulos", "i_veggies", "i_zion", "i_gonnenberg", 39 | "i_autannes", "i_boutique", "i_fruits", "i_pool", "i_fog", "i_fenis", "i_village", "i_ajuntament", "i_partyfood", "i_kurhaus", "i_school", "i_chestnuts", "i_smurf", 40 | "i_indiana", "i_pinard", "i_lionnight", "i_kions", "i_ski", "i_greenhouse", "i_ktirio", "i_tools", "i_toy", "i_bridger", "i_lionday", "i_brooklyn", "i_crownday", 41 | "i_londonbridge", "i_greentea", "i_leuven", "i_nuts", "i_resort", "i_bologna", "i_duda", "i_dome", "i_pencils", "i_books", "i_parking", "i_salon"], "name": "illum"}, 42 | 43 | "full": {"test": ["i_crownnight", "i_table", "i_objects", "i_nescafe", "i_nijmegen", "i_whitebuilding", "i_porta", "i_santuario", "i_dc", "i_castle", "i_steps", 44 | "i_contruction", "i_melon", "i_yellowtent", "i_miniature", "i_troulos", "i_veggies", "i_zion", "i_gonnenberg", "i_autannes", "i_boutique", "i_fruits", "i_pool", 45 | "i_fog", "i_fenis", "i_village", "i_ajuntament", "i_partyfood", "i_kurhaus", "i_school", "i_chestnuts", "i_smurf", "i_indiana", "i_pinard", "i_lionnight", "i_kions", 46 | "i_ski", "i_greenhouse", "i_ktirio", "i_tools", "i_toy", "i_bridger", "i_lionday", "i_brooklyn", "i_crownday", "i_londonbridge", "i_greentea", "i_leuven", "i_nuts", 47 | "i_resort", "i_bologna", "i_duda", "i_dome", "i_pencils", "i_books", "i_parking", "i_salon", "v_circus", "v_charing", "v_colors", "v_astronautis", "v_maskedman", 48 | "v_talent", "v_london", "v_underground", "v_coffeehouse", "v_calder", "v_grace", "v_yard", "v_dogman", "v_laptop", "v_eastsouth", "v_boat", "v_strand", "v_busstop", 49 | "v_artisans", "v_machines", "v_soldiers", "v_home", "v_wapping", "v_wounded", "v_weapons", "v_adam", "v_there", "v_vitro", "v_cartooncity", "v_abstract", "v_dirtywall", 50 | "v_beyus", "v_apprentices", "v_sunseason", "v_wall", "v_war", "v_bricks", "v_fest", "v_churchill", "v_blueprint", "v_tempera", "v_samples", "v_man", "v_bees", 51 | "v_pomegranate", "v_bip", "v_feast", "v_azzola", "v_woman", "v_yuri", "v_posters", "v_bird", "v_graffiti", "v_bark", "v_wormhole", "v_tabletop", "v_courses", "v_birdwoman", "v_gardens"], 52 | "name": "full"}, 53 | 54 | "view_test": {"test": ["v_circus", "v_charing", "v_colors", "v_astronautis", "v_maskedman", "v_talent", "v_london", "v_underground", "v_coffeehouse", "v_calder", 55 | "v_grace", "v_yard", "v_dogman", "v_laptop", "v_boat", "v_strand", "v_busstop", "v_machines", "v_soldiers", "v_home", "v_wapping", "v_wounded", "v_weapons", "v_adam", "v_there", 56 | "v_vitro", "v_cartooncity", "v_abstract", "v_dirtywall", "v_beyus", "v_apprentices", "v_sunseason", "v_wall", "v_war", "v_bricks", "v_fest", "v_churchill", "v_blueprint", 57 | "v_tempera", "v_samples", "v_man", "v_bees", "v_pomegranate", "v_bip", "v_feast", "v_azzola", "v_woman", "v_yuri", "v_posters", "v_bird", "v_graffiti", "v_bark", "v_wormhole", "v_tabletop", "v_courses", "v_gardens"], 58 | "name": "view_test"}, 59 | 60 | "view": {"test": ["v_circus", "v_charing", "v_colors", "v_astronautis", "v_maskedman", "v_talent", "v_london", "v_underground", "v_coffeehouse", "v_calder", "v_grace", 61 | "v_yard", "v_dogman", "v_laptop", "v_eastsouth", "v_boat", "v_strand", "v_busstop", "v_artisans", "v_machines", "v_soldiers", "v_home", "v_wapping", "v_wounded", 62 | "v_weapons", "v_adam", "v_there", "v_vitro", "v_cartooncity", "v_abstract", "v_dirtywall", "v_beyus", "v_apprentices", "v_sunseason", "v_wall", "v_war", "v_bricks", "v_fest", 63 | "v_churchill", "v_blueprint", "v_tempera", "v_samples", "v_man", "v_bees", "v_pomegranate", "v_bip", "v_feast", "v_azzola", "v_woman", "v_yuri", "v_posters", "v_bird", 64 | "v_graffiti", "v_bark", "v_wormhole", "v_tabletop", "v_courses", "v_birdwoman", "v_gardens"], "name": "view"}, 65 | 66 | "debug_view": {"test": ["v_wormhole", "v_tabletop", "v_courses", "v_birdwoman", "v_gardens"], "name": "debug_view"}, 67 | 68 | "debug_illum": {"test": ["i_ajuntament", "i_resort", "i_table", "i_troulos", "i_bologna"], "name": "debug_illum"}, 69 | "debug": {"test": ["v_wormhole", "v_tabletop", "v_courses", "v_birdwoman", "v_gardens", "i_ajuntament", "i_resort", "i_table", "i_troulos", "i_bologna"], "name": "debug"} 70 | } 71 | -------------------------------------------------------------------------------- /HSequences_bench/tools/HSequences_reader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import numpy as np 4 | from skimage import io 5 | 6 | 7 | class HSequences_dataset(object): 8 | 9 | def __init__(self, dataset_path, split, split_path): 10 | 11 | self.dataset_path = dataset_path 12 | self.split = split 13 | 14 | self.splits = json.load(open(split_path)) 15 | self.sequences = self.splits[self.split]['test'] 16 | 17 | self.count = 0 18 | 19 | def read_image(self, path): 20 | im = io.imread(path, as_gray=True) 21 | return im.reshape(im.shape[0], im.shape[1], 1) 22 | 23 | def read_homography(self, h_name): 24 | 25 | h = np.zeros((3, 3)) 26 | h_file = open(h_name, 'r') 27 | 28 | # Prepare Homography 29 | for j in range(3): 30 | line = h_file.readline() 31 | line = str.split(line); 32 | for i in range(3): 33 | h[j, i] = float(line[i]) 34 | 35 | inv_h = np.linalg.inv(h) 36 | inv_h = inv_h / inv_h[2, 2] 37 | 38 | return h, inv_h 39 | 40 | def get_sequence(self, folder_id): 41 | 42 | images_dst = [] 43 | h_src_2_dst = [] 44 | h_dst_2_src = [] 45 | 46 | sequence_path = os.path.join(self.dataset_path, self.sequences[folder_id]) 47 | name_image_src_path = sequence_path + '/1.ppm' 48 | 49 | im_src = self.read_image(name_image_src_path) 50 | im_src = im_src.astype(float) / im_src.max() 51 | 52 | for i in range(5): 53 | 54 | name_image_dst_path = sequence_path + '/' + str(i+2) + '.ppm' 55 | 56 | dst = self.read_image(name_image_dst_path) 57 | dst = dst.astype(float) / dst.max() 58 | 59 | images_dst.append(dst) 60 | 61 | homography_path = sequence_path + '/H_1_'+str(i+2) 62 | src_2_dst, dst_2_src = self.read_homography(homography_path) 63 | h_src_2_dst.append(src_2_dst) 64 | h_dst_2_src.append(dst_2_src) 65 | 66 | images_dst = np.asarray(images_dst) 67 | h_src_2_dst = np.asarray(h_src_2_dst) 68 | h_dst_2_src = np.asarray(h_dst_2_src) 69 | 70 | return {'im_src': im_src, 'images_dst': images_dst, 'h_src_2_dst': h_src_2_dst, 'h_dst_2_src': h_dst_2_src, 71 | 'sequence_name': self.sequences[folder_id]} 72 | 73 | def extract_hsequences(self): 74 | 75 | for idx_sequence in range(len(self.sequences)): 76 | 77 | yield self.get_sequence(idx_sequence) 78 | -------------------------------------------------------------------------------- /HSequences_bench/tools/aux_tools.py: -------------------------------------------------------------------------------- 1 | from os import path, mkdir 2 | import numpy as np 3 | import cv2 4 | 5 | def convert_opencv_matches_to_numpy(matches): 6 | """Returns a np.ndarray array with points indices correspondences 7 | with the shape of Nx2 which each N feature is a vector containing 8 | the keypoints id [id_ref, id_dst]. 9 | """ 10 | assert isinstance(matches, list), type(matches) 11 | correspondences = [] 12 | for match in matches: 13 | assert isinstance(match, cv2.DMatch), type(match) 14 | correspondences.append([match.queryIdx, match.trainIdx]) 15 | return np.asarray(correspondences) 16 | 17 | 18 | def create_results(): 19 | return { 20 | 'num_features': [], 21 | 'rep_single_scale': [], 22 | 'rep_multi_scale': [], 23 | 'num_points_single_scale': [], 24 | 'num_points_multi_scale': [], 25 | 'error_overlap_single_scale': [], 26 | 'error_overlap_multi_scale': [], 27 | 'mma': [], 28 | 'mma_corr': [], 29 | 'num_matches': [], 30 | 'num_mutual_corresp': [], 31 | 'avg_mma': [], 32 | 'num_matches': [], 33 | } 34 | 35 | 36 | def create_overlapping_results(detector_name, overlap): 37 | 38 | results = create_results() 39 | results['detector'] = detector_name 40 | results['overlap'] = overlap 41 | return results 42 | 43 | 44 | def check_directory(dir): 45 | if not path.isdir(dir): 46 | mkdir(dir) 47 | 48 | 49 | def convert_openCV_to_np(pts, dsc, order_coord): 50 | for idx, kp in enumerate(pts): 51 | if order_coord == 'xysr': 52 | kp_np = np.asarray([kp.pt[0], kp.pt[1], kp.size, kp.response, kp.angle]) 53 | else: 54 | kp_np = np.asarray([kp.pt[1], kp.pt[0], kp.size, kp.response, kp.angle]) 55 | 56 | dsc_np = np.asarray(dsc[idx], np.uint8).flatten() 57 | 58 | if idx == 0: 59 | kps_np = kp_np 60 | dscs_np = dsc_np 61 | else: 62 | kps_np = np.vstack([kps_np, kp_np]) 63 | dscs_np = np.vstack([dscs_np, dsc_np]) 64 | 65 | return kps_np, dscs_np -------------------------------------------------------------------------------- /HSequences_bench/tools/geometry_tools.py: -------------------------------------------------------------------------------- 1 | from cv2 import warpPerspective as applyH 2 | 3 | import numpy as np ## For homography applying. 4 | 5 | 6 | def remove_borders(image, borders): 7 | ## Input : [B, H, W, C] or [H, W, C] or [H, W] 8 | 9 | shape = image.shape 10 | new_im = np.zeros_like(image) 11 | # if len(shape) == 4: 12 | # shape = [shape[1], shape[2], shape[3]] 13 | # new_im[:, borders:shape[0]-borders, borders:shape[1]-borders, :] = image[:, borders:shape[0]-borders, borders:shape[1]-borders, :] 14 | # elif len(shape) == 3: 15 | if len(shape) == 3: 16 | new_im[borders:shape[0] - borders, borders:shape[1] - borders, :] = image[borders:shape[0] - borders, borders:shape[1] - borders, :] 17 | else: 18 | new_im[borders:shape[0] - borders, borders:shape[1] - borders] = image[borders:shape[0] - borders, borders:shape[1] - borders] 19 | return new_im 20 | 21 | 22 | def create_common_region_masks(h_dst_2_src, shape_src, shape_dst): 23 | 24 | # Create mask. Only take into account pixels in the two images 25 | inv_h = np.linalg.inv(h_dst_2_src) 26 | inv_h = inv_h / inv_h[2, 2] 27 | 28 | # Applies mask to destination. Where there is no 1, we can no find a point in source. 29 | ones_dst = np.ones((shape_dst[0], shape_dst[1])) 30 | ones_dst = remove_borders(ones_dst, borders=15) 31 | mask_src = applyH(ones_dst, h_dst_2_src, (shape_src[1], shape_src[0])) 32 | mask_src = np.where(mask_src >= 0.75, 1.0, 0.0) 33 | mask_src = remove_borders(mask_src, borders=15) 34 | 35 | ones_src = np.ones((shape_src[0], shape_src[1])) 36 | ones_src = remove_borders(ones_src, borders=15) 37 | mask_dst = applyH(ones_src, inv_h, (shape_dst[1], shape_dst[0])) 38 | mask_dst = np.where(mask_dst >= 0.75, 1.0, 0.0) 39 | mask_dst = remove_borders(mask_dst, borders=15) 40 | 41 | return mask_src, mask_dst 42 | 43 | 44 | def prepare_homography(hom): 45 | 46 | h = np.zeros((3, 3)) 47 | # Prepare Homography 48 | for j in range(3): 49 | for i in range(3): 50 | if j == 2 and i == 2: 51 | h[j, i] = 1. 52 | else: 53 | h[j, i] = hom[j * 3 + i] 54 | return h 55 | 56 | def getAff(x,y,H): 57 | h11 = H[0,0] 58 | h12 = H[0,1] 59 | h13 = H[0,2] 60 | h21 = H[1,0] 61 | h22 = H[1,1] 62 | h23 = H[1,2] 63 | h31 = H[2,0] 64 | h32 = H[2,1] 65 | h33 = H[2,2] 66 | fxdx = h11 / (h31 * x + h32 * y + h33) - (h11 * x + h12 * y + h13) * h31 / (h31 * x + h32 * y + h33) ** 2 67 | fxdy = h12 / (h31 * x + h32 * y + h33) - (h11 * x + h12 * y + h13) * h32 / (h31 * x + h32 * y + h33) ** 2 68 | 69 | fydx = h21 / (h31 * x + h32 * y + h33) - (h21 * x + h22 * y + h23) * h31 / (h31 * x + h32 * y + h33) ** 2 70 | fydy = h22 / (h31 * x + h32 * y + h33) - (h21 * x + h22 * y + h23) * h32 / (h31 * x + h32 * y + h33) ** 2 71 | 72 | Aff = [[fxdx, fxdy], [fydx, fydy]] 73 | 74 | return np.asarray(Aff) 75 | 76 | def apply_homography_to_points(points, h): 77 | new_points = [] 78 | 79 | for point in points: 80 | 81 | new_point = h.dot([point[0], point[1], 1.0]) 82 | 83 | tmp = point[2]**2+np.finfo(np.float32).eps 84 | 85 | Mi1 = [[1/tmp, 0], [0, 1/tmp]] 86 | Mi1_inv = np.linalg.inv(Mi1) 87 | Aff = getAff(point[0], point[1], h) 88 | 89 | BMB = np.linalg.inv(np.dot(Aff, np.dot(Mi1_inv, np.matrix.transpose(Aff)))) 90 | 91 | [e, _] = np.linalg.eig(BMB) 92 | new_radious = 1/((e[0] * e[1])**0.5)**0.5 93 | 94 | new_point = [new_point[0] / new_point[2], new_point[1] / new_point[2], new_radious, point[3]] 95 | new_points.append(new_point) 96 | 97 | return np.asarray(new_points) 98 | 99 | 100 | def find_index_higher_scores(map, num_points = 1000, threshold = -1): 101 | # Best n points 102 | if threshold == -1: 103 | 104 | flatten = map.flatten() 105 | order_array = np.sort(flatten) 106 | 107 | order_array = np.flip(order_array, axis=0) 108 | 109 | threshold = order_array[num_points-1] 110 | if threshold <= 0.0: 111 | indexes = np.argwhere(order_array > 0.0) 112 | if len(indexes) == 0: 113 | threshold = 0.0 114 | else: 115 | threshold = order_array[indexes[len(indexes)-1]] 116 | # elif threshold == 0.0: 117 | # threshold = order_array[np.nonzero(order_array)].min() 118 | 119 | indexes = np.argwhere(map >= threshold) 120 | 121 | return indexes[:num_points] 122 | 123 | 124 | def get_point_coordinates(map, scale_value=1., num_points=1000, threshold=-1, order_coord='xysr'): 125 | ## input numpy array score map : [H, W] 126 | indexes = find_index_higher_scores(map, num_points=num_points, threshold=threshold) 127 | new_indexes = [] 128 | for ind in indexes: 129 | 130 | scores = map[ind[0], ind[1]] 131 | if order_coord == 'xysr': 132 | tmp = [ind[1], ind[0], scale_value, scores] 133 | elif order_coord == 'yxsr': 134 | tmp = [ind[0], ind[1], scale_value, scores] 135 | 136 | new_indexes.append(tmp) 137 | 138 | indexes = np.asarray(new_indexes) 139 | 140 | return np.asarray(indexes) 141 | 142 | 143 | def get_point_coordinates3D(map, scale_factor=1., up_levels=0, num_points=1000, threshold=-1, order_coord='xysr'): 144 | 145 | indexes = find_index_higher_scores(map, num_points=num_points, threshold=threshold) 146 | new_indexes = [] 147 | for ind in indexes: 148 | scale_value = (scale_factor ** (ind[2] - up_levels)) 149 | scores = map[ind[0], ind[1], ind[2]] 150 | if order_coord == 'xysr': 151 | tmp = [ind[1], ind[0], scale_value, scores] 152 | elif order_coord == 'yxsr': 153 | tmp = [ind[0], ind[1], scale_value, scores] 154 | 155 | new_indexes.append(tmp) 156 | 157 | indexes = np.asarray(new_indexes) 158 | 159 | return np.asarray(indexes) 160 | 161 | 162 | 163 | if __name__ =="__main__": 164 | import numpy as np 165 | import os 166 | path = 'datasets/synth/train_dataset_debug' 167 | keys = os.listdir('datasets/synth/train_dataset_debug/im_src_patch') 168 | 169 | for idx, key in enumerate(sorted(keys)): 170 | src_im = np.load(path+"/im_src_patch/"+key) 171 | trg_im = np.load(path+"/im_dst_patch/"+key) 172 | 173 | h_src_2_dst = np.load(path+'/homography_src_2_dst/'+key) 174 | h_dst_2_src = np.load(path+'/homography_dst_2_src/'+key) 175 | 176 | hom = prepare_homography(torch.from_numpy(h_dst_2_src)[0]) 177 | shape_src = torch.randn(1,192,192).shape 178 | shape_dst = torch.randn(1,192,192).shape 179 | 180 | mask_src, mask_dst = create_common_region_masks(hom, shape_src, shape_dst) 181 | 182 | print(key, mask_src.numpy().shape, mask_dst.numpy().shape, torch.sum(mask_src), torch.sum(mask_dst)) 183 | 184 | dst_pts = [] 185 | a = torch.linspace(0,191, 192)#.astype(np.int) 186 | for i in a: 187 | for j in a: 188 | dst_pts.append([i,j, 1,1]) 189 | dst_pts = torch.tensor(dst_pts) 190 | print(dst_pts.shape) 191 | 192 | dst_to_src_pts = apply_homography_to_points(dst_pts, hom) 193 | 194 | print(dst_to_src_pts) -------------------------------------------------------------------------------- /HSequences_bench/tools/matching_tools.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def create_precision_recall_results(): 5 | return { 6 | 'recall': 0.0, 7 | 'precision': 0.0, 8 | 'correct_matches': [], 9 | 'false_matches': [], 10 | } 11 | 12 | 13 | # retrieve the true correspondences 14 | def compute_matching_based_distance(points_src, points_dst, matches, num_points, pixel_threshold, possible_matches): 15 | dist = np.sqrt((points_src[matches[:, 0], 0] - points_dst[matches[:, 1], 0]) ** 2 + ( 16 | points_src[matches[:, 0], 1] - points_dst[matches[:, 1], 1]) ** 2) 17 | matches_dist = np.sum(np.where(dist < pixel_threshold, 1.0, 0.0)) 18 | match_score = matches_dist / num_points 19 | match_score_corr = matches_dist / (possible_matches+1e-6) 20 | return match_score, match_score_corr, matches_dist 21 | 22 | 23 | def compute_precision_recall(matches, true_matches, num_points, eps=1e-6): 24 | results = create_precision_recall_results() 25 | if len(true_matches) == 0: 26 | return results 27 | 28 | num_correct_matches, num_false_matches = 0.0, 0.0 29 | for match in matches: 30 | found_match = np.where(match[0] == true_matches[:, 0]) == \ 31 | np.where(match[1] == true_matches[:, 1]) 32 | if found_match: 33 | num_correct_matches += 1 34 | results['correct_matches'].append(match) 35 | else: 36 | num_false_matches += 1 37 | results['false_matches'].append(match) 38 | # stack matches 39 | results['correct_matches'] = np.array(results['correct_matches']) 40 | results['false_matches'] = np.array(results['false_matches']) 41 | # compute the actual statistics 42 | num_correspondences = true_matches.shape[0] + eps 43 | sum_matches = num_correct_matches + num_false_matches + eps 44 | # return a dictionary with all the results 45 | results['recall'] = num_correct_matches / num_correspondences 46 | results['recall_total'] = num_correct_matches / num_points 47 | results['precision'] = 1. - num_false_matches / sum_matches 48 | return results 49 | 50 | # find matches 51 | def find_matches(dsc_src, dsc_dst): 52 | 53 | dsc_src = np.reshape(dsc_src, (dsc_src.shape[0], 1, 128)) 54 | dsc_dst = np.reshape(dsc_dst, (1, dsc_dst.shape[0], 128)) 55 | dsc_src = np.repeat(dsc_src, dsc_dst.shape[1], axis=1) 56 | dsc_dst = np.repeat(dsc_dst, dsc_src.shape[0], axis=0) 57 | 58 | l2_matrix = np.sum((dsc_src - dsc_dst)**2, axis=-1) 59 | matches = l2_matrix.argmin(axis=1) 60 | return [np.arange(len(dsc_src)), matches] -------------------------------------------------------------------------------- /HSequences_bench/tools/opencv_matcher.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | 5 | class OpencvBruteForceMatcher(object): 6 | name = 'opencv_brute_force_matcher' 7 | distances = {} 8 | distances['l2'] = cv2.NORM_L2 9 | distances['hamming'] = cv2.NORM_HAMMING 10 | 11 | def __init__(self, distance='l2'): 12 | self._matcher = cv2.BFMatcher(self.distances[distance]) 13 | 14 | def match(self, descs1, descs2): 15 | """Compute brute force matches between two sets of descriptors. 16 | """ 17 | assert isinstance(descs1, np.ndarray), type(descs1) 18 | assert isinstance(descs2, np.ndarray), type(descs2) 19 | assert len(descs1.shape) == 2, descs1.shape 20 | assert len(descs2.shape) == 2, descs2.shape 21 | matches = self._matcher.match(descs1, descs2) 22 | return matches 23 | 24 | def match_putative(self, descs1, descs2, knn=2, threshold_ratio=0.7): 25 | """Compute putatives matches betweem two sets of descriptors. 26 | """ 27 | assert isinstance(descs1, np.ndarray), type(descs1) 28 | assert isinstance(descs2, np.ndarray), type(descs2) 29 | assert len(descs1.shape) == 2, descs1.shape 30 | assert len(descs2.shape) == 2, descs2.shape 31 | matches = self._matcher.knnMatch(descs1, descs2, k=knn) 32 | # apply Lowe's ratio test 33 | good = [] 34 | for m, n in matches: 35 | if m.distance < threshold_ratio * n.distance: 36 | good.append(m) 37 | return good 38 | 39 | def convert_opencv_matches_to_numpy(self, matches): 40 | """Returns a np.ndarray array with points indices correspondences 41 | with the shape of Nx2 which each N feature is a vector containing 42 | the keypoints id [id_ref, id_dst]. 43 | """ 44 | assert isinstance(matches, list), type(matches) 45 | correspondences = [] 46 | for match in matches: 47 | assert isinstance(match, cv2.DMatch), type(match) 48 | correspondences.append([match.queryIdx, match.trainIdx]) 49 | return np.asarray(correspondences) -------------------------------------------------------------------------------- /HSequences_bench/tools/repeatability_tools.py: -------------------------------------------------------------------------------- 1 | ## I keep all this numpy code without translating to PyTorch 2 | import numpy as np 3 | from scipy.ndimage.filters import maximum_filter 4 | import torch 5 | 6 | def check_common_points(kpts, mask): 7 | idx_valid_points = [] 8 | for idx, ktp in enumerate(kpts): 9 | if mask[int(round(ktp[0]))-1, int(round(ktp[1]))-1]: 10 | idx_valid_points.append(idx) 11 | return np.asarray(idx_valid_points) 12 | 13 | 14 | def select_top_k(kpts, k=1000): 15 | scores = -1 * kpts[:, 3] 16 | return np.argsort(scores)[:k] 17 | 18 | 19 | def apply_nms(score_map, size): 20 | 21 | score_map = score_map * (score_map == maximum_filter(score_map, footprint=np.ones((size, size)))) 22 | 23 | return score_map 24 | 25 | 26 | def intersection_area(R, r, d = 0): 27 | """Return the area of intersection of two circles. 28 | 29 | The circles have radii R and r, and their centres are separated by d. 30 | 31 | """ 32 | if d <= abs(R-r): 33 | # One circle is entirely enclosed in the other. 34 | return np.pi * min(R, r)**2 35 | if d >= r + R: 36 | # The circles don't overlap at all. 37 | return 0 38 | 39 | r2, R2, d2 = r**2, R**2, d**2 40 | alpha = np.arccos((d2 + r2 - R2) / (2*d*r)) 41 | beta = np.arccos((d2 + R2 - r2) / (2*d*R)) 42 | 43 | return ( r2 * alpha + R2 * beta - 0.5 * (r2 * np.sin(2*alpha) + R2 * np.sin(2*beta))) 44 | 45 | 46 | def union_area(r, R, intersection): 47 | return (np.pi * (r ** 2)) + (np.pi * (R ** 2)) - intersection 48 | 49 | 50 | def compute_repeatability(src_indexes, dst_indexes, overlap_err=0.4, eps=1e-6, dist_match_thresh=3, radious_size=30.): 51 | 52 | error_overlap_s = 0. 53 | error_overlap_m = 0. 54 | found_points_s = 0 55 | found_points_m = 0 56 | possible_matches = 0 57 | correspondences = [] 58 | correspondences_m = [] 59 | 60 | dst_indexes_num = len(dst_indexes) 61 | src_indexes_num = len(src_indexes) 62 | 63 | matrix_overlaps = np.zeros((len(src_indexes), len(dst_indexes))) 64 | matrix_overlaps_single_scale = np.zeros((len(src_indexes), len(dst_indexes))) 65 | 66 | max_distance = 4 * radious_size 67 | 68 | for idx_ref, point_ref in enumerate(src_indexes): 69 | 70 | radious_ref = point_ref[2] 71 | found_possible_match = False 72 | 73 | for idx_dst, point_dst in enumerate(dst_indexes): 74 | 75 | radious_dst = point_dst[2] 76 | distance = (((point_ref[0] - point_dst[0]) ** 2) + ((point_ref[1] - point_dst[1]) ** 2)) ** 0.5 77 | 78 | if distance <= dist_match_thresh and not found_possible_match: 79 | found_possible_match = True 80 | possible_matches += 1 81 | 82 | if distance > max_distance: 83 | continue 84 | 85 | factor_scale = radious_size / (max(radious_ref, radious_dst) + np.finfo(float).eps) 86 | I = intersection_area(factor_scale*radious_ref, factor_scale*radious_dst, distance) 87 | U = union_area(factor_scale*radious_ref, factor_scale*radious_dst, I) + eps 88 | 89 | matrix_overlaps[idx_ref, idx_dst] = I/U 90 | 91 | I = intersection_area(radious_size, radious_size, distance) 92 | U = union_area(radious_size, radious_size, I) + eps 93 | 94 | matrix_overlaps_single_scale[idx_ref, idx_dst] = I/U 95 | 96 | y_visited = np.zeros(src_indexes.shape[0], dtype=np.uint8) 97 | x_visited = np.zeros(dst_indexes.shape[0], dtype=np.uint8) 98 | 99 | # Multiply matrix to get descendent order 100 | for index in (-1 * matrix_overlaps_single_scale).flatten().argsort(): 101 | y_pos = index // dst_indexes.shape[0] 102 | x_pos = index % dst_indexes.shape[0] 103 | if x_visited[x_pos] or y_visited[y_pos]: 104 | continue 105 | max_overlap = matrix_overlaps_single_scale[y_pos, x_pos] 106 | if max_overlap < (1 - overlap_err): 107 | break 108 | found_points_s += 1 109 | error_overlap_s += (1 - max_overlap) 110 | correspondences.append([x_pos, y_pos]) 111 | # update visited cells 112 | x_visited[x_pos] = 1 113 | y_visited[y_pos] = 1 114 | 115 | matrix_overlaps_single_scale = 0 116 | del matrix_overlaps_single_scale 117 | 118 | y_visited = np.zeros(src_indexes.shape[0], dtype=np.uint8) 119 | x_visited = np.zeros(dst_indexes.shape[0], dtype=np.uint8) 120 | 121 | # Multiply matrix to get descendent order 122 | for index in (-1 * matrix_overlaps).flatten().argsort(): 123 | y_pos = index // dst_indexes.shape[0] 124 | x_pos = index % dst_indexes.shape[0] 125 | if x_visited[x_pos] or y_visited[y_pos]: 126 | continue 127 | max_overlap = matrix_overlaps[y_pos, x_pos] 128 | if max_overlap < (1 - overlap_err): 129 | break 130 | found_points_m += 1 131 | error_overlap_m += (1 - max_overlap) 132 | correspondences_m.append([x_pos, y_pos]) 133 | # update visited cells 134 | x_visited[x_pos] = 1 135 | y_visited[y_pos] = 1 136 | 137 | matrix_overlaps = 0 138 | del matrix_overlaps 139 | 140 | points = dst_indexes_num 141 | if src_indexes_num < points: 142 | points = src_indexes_num 143 | 144 | rep_s = (found_points_s / np.asarray(points, float)) * 100.0 145 | rep_m = (found_points_m / np.asarray(points, float)) * 100.0 146 | 147 | if found_points_m == 0: 148 | error_overlap_m = 0.0 149 | else: 150 | error_overlap_m = error_overlap_m / float(found_points_m+np.finfo(float).eps) 151 | 152 | if found_points_s == 0: 153 | error_overlap_s = 0.0 154 | else: 155 | error_overlap_s = error_overlap_s / float(found_points_s+np.finfo(float).eps) 156 | 157 | return {'rep_single_scale': rep_s, 'rep_multi_scale': rep_m, 'num_points_single_scale': found_points_s, 158 | 'num_points_multi_scale': found_points_m, 'error_overlap_single_scale': error_overlap_s, 159 | 'error_overlap_multi_scale': error_overlap_m, 'total_num_points': points, 160 | 'correspondences': np.asarray(correspondences), 'possible_matches': possible_matches, 161 | 'correspondences_m': np.asarray(correspondences_m)} 162 | 163 | ####### =================================================================================== PyTorch version 164 | 165 | def intersection_area_torch(R, r, d): 166 | """Return the area of intersection of two circles. 167 | 168 | The circles have radii R and r, and their centres are separated by d. 169 | 170 | """ 171 | 172 | intersection = torch.zeros_like(R) 173 | 174 | # One circle is entirely enclosed in the other 175 | min_R_r = np.pi * torch.pow(torch.min(R,r), 2) 176 | intersection += torch.where(d <= torch.abs(R-r), min_R_r, torch.zeros_like(intersection)) 177 | 178 | r2, R2, d2 = torch.pow(r, 2), torch.pow(R, 2), torch.pow(d, 2) 179 | alpha = torch.arccos((d2 + r2 - R2) / (2 * d * r)) 180 | beta = torch.arccos((d2 + R2 - r2) / (2 * d * R)) 181 | overlap = r2 * alpha + R2 * beta - 0.5 * (r2 * torch.sin(2 * alpha) + R2 * torch.sin(2 * beta)) 182 | intersection += torch.where((d > torch.abs(R-r)) & (d < r + R) , overlap, torch.zeros_like(intersection)) 183 | 184 | return intersection 185 | 186 | def union_area_torch(r, R, intersection): 187 | return (np.pi * (r ** 2)) + (np.pi * (R ** 2)) - intersection 188 | 189 | def compute_repeatability_torch(src_indexes, dst_indexes, overlap_err=0.4, eps=1e-6, dist_match_thresh=3, radious_size=30.): 190 | 191 | error_overlap_s = 0. 192 | error_overlap_m = 0. 193 | found_points_s = 0 194 | found_points_m = 0 195 | possible_matches = 0 196 | correspondences = [] 197 | correspondences_m = [] 198 | 199 | if isinstance(src_indexes, np.ndarray): 200 | src_indexes = torch.tensor(src_indexes, dtype=torch.float) 201 | if isinstance(dst_indexes, np.ndarray): 202 | dst_indexes = torch.tensor(dst_indexes, dtype=torch.float) 203 | 204 | dst_indexes_num = len(dst_indexes) # set of (y, x, scale), M 205 | src_indexes_num = len(src_indexes) # set of (y, x, scale), N 206 | 207 | matrix_overlaps = np.zeros((len(src_indexes), len(dst_indexes))) # [N,M] 208 | matrix_overlaps_single_scale = np.zeros((len(src_indexes), len(dst_indexes))) # [N,M] 209 | 210 | max_distance = 4 * radious_size # greater than 30 pixel. 211 | 212 | distance_matrix = torch.cdist(src_indexes[:,0:2], dst_indexes[:,0:2]) 213 | possible_matches = torch.where(((distance_matrix <= dist_match_thresh) & (distance_matrix <= max_distance)), torch.ones_like(distance_matrix), torch.zeros_like(distance_matrix)) 214 | possible_matches = torch.count_nonzero(torch.count_nonzero(possible_matches, dim=1)).sum().item() 215 | 216 | # distance_valid = torch.where((distance_matrix <= max_distance), distance_matrix, torch.zeros_like(distance_matrix)) 217 | 218 | src_radious = src_indexes[:,2] 219 | dst_radious = dst_indexes[:,2] 220 | src_radious_tile = torch.tile(src_radious, (dst_radious.shape[0], 1)) 221 | dst_radious_tile = torch.tile(dst_radious, (src_radious.shape[0], 1)) 222 | factor_scale = radious_size / (torch.max(src_radious_tile, dst_radious_tile.transpose(0,1)) + np.finfo(float).eps) 223 | 224 | I=intersection_area_torch(factor_scale*src_radious_tile, factor_scale*dst_radious_tile.transpose(0,1), distance_matrix) 225 | U = union_area_torch(factor_scale*src_radious_tile, factor_scale*dst_radious_tile.transpose(0,1), I) + eps 226 | matrix_overlaps = I/U 227 | 228 | radious_size_torch = torch.ones_like(src_radious_tile) * radious_size 229 | I = intersection_area_torch(radious_size_torch, radious_size_torch, distance_matrix) 230 | U = union_area_torch(radious_size_torch, radious_size_torch, I) + eps 231 | 232 | matrix_overlaps_single_scale = I/U # overlap between source and destination points for single scale 233 | 234 | y_visited = np.zeros(src_indexes.shape[0], dtype=np.uint8) 235 | x_visited = np.zeros(dst_indexes.shape[0], dtype=np.uint8) 236 | 237 | # Multiply matrix to get descendent order 238 | for index in (-1 * matrix_overlaps_single_scale).flatten().argsort(): # return sorted index array 239 | y_pos = index // dst_indexes.shape[0] # coordinate is flattened like (y * N + x) 240 | x_pos = index % dst_indexes.shape[0] 241 | if x_visited[x_pos] or y_visited[y_pos]: # for one-by-one matching 242 | continue 243 | max_overlap = matrix_overlaps_single_scale[y_pos, x_pos].item() 244 | if max_overlap < (1 - overlap_err): # max overlap should smaller than 0.6(1-0.4) 245 | break 246 | found_points_s += 1 247 | error_overlap_s += (1 - max_overlap) 248 | correspondences.append([x_pos, y_pos]) 249 | # update visited cells 250 | x_visited[x_pos] = 1 251 | y_visited[y_pos] = 1 252 | 253 | matrix_overlaps_single_scale = 0 254 | del matrix_overlaps_single_scale 255 | 256 | y_visited = np.zeros(src_indexes.shape[0], dtype=np.uint8) 257 | x_visited = np.zeros(dst_indexes.shape[0], dtype=np.uint8) 258 | 259 | # Multiply matrix to get descendent order 260 | for index in (-1 * matrix_overlaps).flatten().argsort(): 261 | y_pos = index // dst_indexes.shape[0] 262 | x_pos = index % dst_indexes.shape[0] 263 | if x_visited[x_pos] or y_visited[y_pos]: 264 | continue 265 | max_overlap = matrix_overlaps[y_pos, x_pos].item() 266 | if max_overlap < (1 - overlap_err): 267 | break 268 | found_points_m += 1 269 | error_overlap_m += (1 - max_overlap) 270 | correspondences_m.append([x_pos, y_pos]) 271 | # update visited cells 272 | x_visited[x_pos] = 1 273 | y_visited[y_pos] = 1 274 | 275 | matrix_overlaps = 0 276 | del matrix_overlaps 277 | 278 | points = dst_indexes_num # to calculate repeatability score, use the lower number of keypoints. 279 | if src_indexes_num < points: 280 | points = src_indexes_num 281 | 282 | rep_s = (found_points_s / np.asarray(points, float)) * 100.0 283 | rep_m = (found_points_m / np.asarray(points, float)) * 100.0 284 | 285 | if found_points_m == 0: 286 | error_overlap_m = 0.0 287 | else: 288 | error_overlap_m = error_overlap_m / float(found_points_m+np.finfo(float).eps) 289 | 290 | if found_points_s == 0: 291 | error_overlap_s = 0.0 292 | else: 293 | error_overlap_s = error_overlap_s / float(found_points_s+np.finfo(float).eps) 294 | 295 | return {'rep_single_scale': rep_s, 'rep_multi_scale': rep_m, 'num_points_single_scale': found_points_s, 296 | 'num_points_multi_scale': found_points_m, 'error_overlap_single_scale': error_overlap_s, 297 | 'error_overlap_multi_scale': error_overlap_m, 'total_num_points': points, 298 | 'correspondences': np.asarray(correspondences), 'possible_matches': possible_matches, 299 | 'correspondences_m': np.asarray(correspondences_m)} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters (PyTorch) 2 | 3 | PyTorch reproduce source code for the ICCV19 paper: 4 | 5 | 6 | ```text 7 | "Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters". 8 | Axel Barroso-Laguna, Edgar Riba, Daniel Ponsa, Krystian Mikolajczyk. ICCV 2019. 9 | ``` 10 | [[Paper on arxiv](https://arxiv.org/abs/1904.00889)] 11 | 12 | Most of the source codes are replaced by the original Key.Net [Tensorflow](https://github.com/axelBarroso/Key.Net) source code. 13 | 14 | ## Prerequisite 15 | 16 | ```bash 17 | conda create --name keyNet 18 | conda activate keyNet 19 | conda install pytorch==1.8.0 torchvision==0.9.0 cudatoolkit=11.1 -c pytorch -c conda-forge 20 | pip install opencv-python tqdm scikit-image pandas 21 | pip install torchgeometry 22 | 23 | ``` 24 | 25 | ## Training Key.Net 26 | 27 | Before training Key.Net a synthetic dataset must be generated. In the original paper, we downloaded ImageNet and used it to generate synthetic pairs of images, however, any other dataset would work if it is big enough. Therefore, the first time you run the `train.py` script, `.npy` files for synthetic pairs will be generated at datasets folder, one for training and another for validation. This is only done when the code couldn't find them, thus, the next runs of the script will skip this part. 28 | 29 | ```bash 30 | python train.py --data-dir /path/to/ImageNet 31 | ``` 32 | 33 | Check the arguments to customize your training, some parameters you might want to change are: 34 | 35 | * Dataset parameters: 36 | 37 | * max-angle: The max angle value for generating a synthetic view to train Key.Net. 38 | * max-scale: The max scale value for generating a synthetic view to train Key.Net. 39 | * max-shearing: The max shearing value for generating a synthetic view to train Key.Net. 40 | 41 | * Network Architecture: 42 | 43 | * num-filters: The number of filters in each learnable block. 44 | * num-learnable-blocks: The number of learnable blocks after handcrafted block. 45 | * num-levels-within-net: The number of pyramid levels inside the architecture. 46 | * factor-scaling-pyramid: The scale factor between the multi-scale pyramid levels in the architecture. 47 | * conv-kernel-size: The size of the convolutional filters in each of the learnable blocks. 48 | 49 | 50 | ## Feature Extraction 51 | 52 | `extract_multiscale_features.py` can be used to extract Key.Net features for a given list of images. The list of images must contain the full path to them, if they do not exist, an error will raise. 53 | 54 | The script generates two numpy files, one '.kpt' for keypoints, and a '.dsc' for descriptors. The descriptor used together with Key.Net is [HardNet](https://github.com/DagnyT/hardnet). The output format of the keypoints is as follow: 55 | 56 | - `keypoints` [`N x 4`] array containing the positions of keypoints `x, y`, scales `s` and their scores `sc`. 57 | 58 | 59 | Arguments: 60 | 61 | * list-images: File containing the image paths for extracting features. 62 | * results-dir: The output path to save the extracted features. 63 | * checkpoint-det-dir: The path to the checkpoint file to load the detector weights. Default: Pretrained Key.Net. 64 | * checkpoint-desc-dir: The path to the checkpoint file to load the HardNet descriptor weights. 65 | * num-points: The number of desired features to extract. Default: 1500. 66 | * extract-MS: Set to True if you want to extract multi-scale features. Default: True. 67 | 68 | 69 | Run the following script to generate the keypoint and descriptor numpy files from the image allocated in `test_im` directory. 70 | 71 | ```bash 72 | python extract_multiscale_features.py --list-images test_im/image.txt --results-dir test_im/ 73 | ``` 74 | 75 | ## HSequences Benchmark 76 | 77 | We also provide the benchmark to compute [HSequences](https://github.com/hpatches/hpatches-dataset) repeatability (single- and multi-scale), and MMA metrics. To do so, first download full images (HSequences) from [HPatches repository](http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz). Once downloaded, place it on the root directory of the project. We provide a file `HSequences_bench/HPatches_images.txt` containing the list of images inside HSequences. 78 | 79 | 80 | Run the next script to compute the features from HSequences: 81 | 82 | ```bash 83 | python extract_multiscale_features.py --list-images HSequences_bench/HPatches_images.txt --checkpoint-det-dir keyNet/pretrained_nets/keyNet.pt 84 | ``` 85 | 86 | Once all features have been extracted, to compute repeatability and MMA metrics run: 87 | 88 | ```bash 89 | python hsequences_bench.py --results-dir extracted_features --results-bench-dir HSequences_bench/results --split full 90 | ``` 91 | 92 | Use arguments to set different options: 93 | 94 | * results-bench-dir: The output path to save the results in a pickle file. 95 | * results-dir: The output path to load the extracted features. 96 | * split: The name of the HPatches (HSequences) split. Use full, view or illum. 97 | * top-k-points: The number of top points to use for evaluation. Set to None to use all points. 98 | * pixel-threshold: The distance of pixels for a matching correspondence to be considered correct. 99 | * overlap: The overlap threshold for a correspondence to be considered correct. 100 | * detector-name: Set the name of the detector for which you desire to compute the benchmark (and features have been already extracted). 101 | 102 | ## BibTeX 103 | 104 | If you use this code in your research, please cite the original authors' paper: 105 | 106 | ```bibtex 107 | @InProceedings{Barroso-Laguna2019ICCV, 108 | author = {Barroso-Laguna, Axel and Riba, Edgar and Ponsa, Daniel and Mikolajczyk, Krystian}, 109 | title = {{Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters}}, 110 | booktitle = {Proceedings of the 2019 IEEE/CVF International Conference on Computer Vision}, 111 | year = {2019}, 112 | } 113 | 114 | -------------------------------------------------------------------------------- /extract_multiscale_features.py: -------------------------------------------------------------------------------- 1 | import os, sys, cv2 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from keyNet.config_hpatches import get_config 6 | import keyNet.aux.tools as aux 7 | from keyNet.model.keynet_architecture import keynet 8 | from keyNet.model.hardnet_pytorch import HardNet 9 | ## Network architecture 10 | import keyNet.aux.desc_aux_function as loss_desc 11 | import HSequences_bench.tools.geometry_tools as geo_tools 12 | import HSequences_bench.tools.repeatability_tools as rep_tools 13 | ## image load & pre-processing 14 | from keyNet.datasets.dataset_utils import read_bw_image 15 | from skimage.transform import pyramid_gaussian 16 | from tqdm import tqdm 17 | 18 | def check_directory(dir): 19 | if not os.path.isdir(dir): 20 | os.mkdir(dir) 21 | 22 | def create_result_dir(path): 23 | directories = path.split('/') 24 | tmp = '' 25 | for idx, dir in enumerate(directories): 26 | tmp += (dir + '/') 27 | if idx == len(directories)-1: 28 | continue 29 | check_directory(tmp) 30 | 31 | def extract_features(image, model1, model2, device, levels, point_level, args): 32 | pyramid = pyramid_gaussian(image, max_layer=args.pyramid_levels, downscale=args.scale_factor_levels) 33 | score_maps = {} 34 | for (j, resized) in enumerate(pyramid): 35 | im = resized.reshape(1, resized.shape[0], resized.shape[1], 1) 36 | im = torch.from_numpy(im).to(device).to(torch.float32) 37 | 38 | _, im_scores = model1(im) 39 | im_scores = F.relu(im_scores) 40 | im_scores = geo_tools.remove_borders(im_scores[0,0,:,:].cpu().detach().numpy(), borders=args.border_size) 41 | 42 | score_maps['map_' + str(j + 1 + args.upsampled_levels)] = im_scores[:, :] 43 | 44 | if args.upsampled_levels: 45 | for j in range(args.upsampled_levels): 46 | factor = args.scale_factor_levels ** (args.upsampled_levels - j) 47 | up_image = cv2.resize(image, (0, 0), fx=factor, fy=factor) 48 | 49 | im = np.reshape(up_image, (1, up_image.shape[0], up_image.shape[1], 1)) 50 | im = torch.from_numpy(im).to(device).to(torch.float32) 51 | 52 | _, im_scores = model1(im) 53 | im_scores = F.relu(im_scores) 54 | im_scores = geo_tools.remove_borders(im_scores[0,0,:,:].cpu().detach().numpy(), borders=args.border_size) 55 | 56 | score_maps['map_' + str(j + 1)] = im_scores[:, :] 57 | 58 | ## compute 59 | im_pts = [] 60 | for idx_level in range(levels): 61 | scale_value = (args.scale_factor_levels ** (idx_level - args.upsampled_levels)) 62 | scale_factor = 1. / scale_value 63 | 64 | h_scale = np.asarray([[scale_factor, 0., 0.], [0., scale_factor, 0.], [0., 0., 1.]]) 65 | h_scale_inv = np.linalg.inv(h_scale) 66 | h_scale_inv = h_scale_inv / h_scale_inv[2, 2] 67 | 68 | num_points_level = point_level[idx_level] 69 | if idx_level > 0: 70 | res_points = int(np.asarray([point_level[a] for a in range(0, idx_level + 1)]).sum() - len(im_pts)) 71 | num_points_level = res_points 72 | 73 | im_scores = rep_tools.apply_nms(score_maps['map_' + str(idx_level + 1)], args.nms_size) 74 | 75 | im_pts_tmp = geo_tools.get_point_coordinates(im_scores, num_points=num_points_level, order_coord='xysr') 76 | 77 | im_pts_tmp = geo_tools.apply_homography_to_points(im_pts_tmp, h_scale_inv) 78 | 79 | if not idx_level: 80 | im_pts = im_pts_tmp 81 | else: 82 | im_pts = np.concatenate((im_pts, im_pts_tmp), axis=0) 83 | 84 | 85 | if args.order_coord == 'yxsr': 86 | im_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], im_pts))) 87 | 88 | im_pts = im_pts[(-1 * im_pts[:, 3]).argsort()] 89 | im_pts = im_pts[:args.num_points] 90 | 91 | # Extract descriptor from features 92 | descriptors = [] 93 | im = image.reshape(1, image.shape[0], image.shape[1], 1) 94 | 95 | for idx_desc_batch in range(int(len(im_pts) / 10000 + 1)): 96 | points_batch = im_pts[idx_desc_batch * 10000: (idx_desc_batch + 1) * 10000] 97 | 98 | if not len(points_batch): 99 | break 100 | 101 | kpts_coord = torch.tensor(points_batch[:, :2]).to(torch.float32).cpu() 102 | kpts_batch = torch.zeros(len(points_batch)).to(torch.float32).cpu() 103 | input_network = torch.tensor(im).to(torch.float32).cpu().permute(0,3,1,2) 104 | kpts_scale = torch.tensor(points_batch[:, 2] * args.scale_factor).to(torch.float32).cpu() 105 | 106 | patch_batch = loss_desc.build_patch_extraction(kpts_coord, kpts_batch, input_network, kpts_scale) 107 | patch_batch = np.reshape(patch_batch, (patch_batch.shape[0], 1, 32, 32)) 108 | 109 | data_a = patch_batch.to(device) 110 | with torch.no_grad(): 111 | out_a = model2(data_a) 112 | desc_batch = out_a.data.cpu().numpy().reshape(-1, 128) 113 | if idx_desc_batch == 0: 114 | descriptors = desc_batch 115 | else: 116 | descriptors = np.concatenate([descriptors, desc_batch], axis=0) 117 | 118 | return im_pts, descriptors 119 | 120 | 121 | def extract_multiscale_features(): 122 | args = get_config() 123 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 124 | MSIP_sizes = [8, 16, 24, 32, 40] 125 | MSIP_factor_loss = [256.0, 64.0, 16.0, 4.0, 1.0] 126 | 127 | version_network_name = args.network_version 128 | 129 | if not args.extract_MS: 130 | args.pyramid_levels = 0 131 | args.upsampled_levels = 0 132 | 133 | print('Extract features for : ' + version_network_name) 134 | aux.check_directory(args.results_dir) 135 | aux.check_directory(os.path.join(args.results_dir, version_network_name)) 136 | 137 | ## Define PyTorch Key.Net 138 | model1 = keynet(args, device, MSIP_sizes) 139 | model1.load_state_dict(torch.load(args.checkpoint_det_dir)) 140 | model1.eval() 141 | model1 = model1.to(device) ## use GPU 142 | 143 | kernels = model1.get_kernels(device) ## with GPU 144 | 145 | # Define Pytorch HardNet 146 | model2 = HardNet() 147 | checkpoint = torch.load(args.pytorch_hardnet_dir) 148 | model2.load_state_dict(checkpoint['state_dict']) 149 | model2.eval() 150 | model2.to(device) 151 | 152 | ## points level define 153 | point_level = [] 154 | tmp = 0.0 155 | factor_points = (args.scale_factor_levels ** 2) 156 | levels = args.pyramid_levels + args.upsampled_levels + 1 157 | for idx_level in range(levels): 158 | tmp += factor_points ** (-1 * (idx_level - args.upsampled_levels)) 159 | point_level.append(args.num_points * factor_points ** (-1 * (idx_level - args.upsampled_levels))) 160 | 161 | point_level = np.asarray(list(map(lambda x: int(x/tmp), point_level))) 162 | 163 | ## open images 164 | f = open(args.list_images, "r") 165 | image_list = sorted(f.readlines()) 166 | iterate = tqdm(image_list, total=len(image_list), desc="Key.Net HPatches") 167 | 168 | for path_to_image in iterate: 169 | path = path_to_image.rstrip('\n') 170 | iterate.set_description("Current {}".format('/'.join(path.split('/')[-2:]) )) 171 | 172 | if not os.path.exists(path): 173 | print('[ERROR]: File {0} not found!'.format(path)) 174 | return 175 | 176 | create_result_dir(os.path.join(args.results_dir, version_network_name, path)) 177 | 178 | im = read_bw_image(path) 179 | 180 | im = im.astype(float) / im.max() 181 | with torch.no_grad(): 182 | im_pts, descriptors = extract_features(im, model1, model2, device, levels, point_level, args) 183 | 184 | file_name = os.path.join(args.results_dir, version_network_name, path)+'.kpt' 185 | np.save(file_name, im_pts) 186 | 187 | file_name = os.path.join(args.results_dir, version_network_name, path)+'.dsc' 188 | np.save(file_name, descriptors) 189 | 190 | # Extract Patches from inputs 191 | 192 | 193 | if __name__ == '__main__': 194 | extract_multiscale_features() 195 | -------------------------------------------------------------------------------- /hsequences_bench.py: -------------------------------------------------------------------------------- 1 | import os, pickle 2 | import numpy as np 3 | from tqdm import tqdm 4 | from keyNet.config_hpatches import get_eval_config 5 | import HSequences_bench.tools.aux_tools as aux 6 | import HSequences_bench.tools.geometry_tools as geo_tools 7 | import HSequences_bench.tools.repeatability_tools as rep_tools 8 | import HSequences_bench.tools.matching_tools as match_tools 9 | from HSequences_bench.tools.HSequences_reader import HSequences_dataset 10 | from HSequences_bench.tools.opencv_matcher import OpencvBruteForceMatcher 11 | 12 | def hsequences_metrics(): 13 | args = get_eval_config() 14 | 15 | print(args.detector_name + ': ' + args.split) 16 | aux.check_directory(args.results_bench_dir) 17 | 18 | # create the dataloader 19 | data_loader = HSequences_dataset(args.data_dir, args.split, args.split_path) 20 | results = aux.create_overlapping_results(args.detector_name, args.overlap) 21 | 22 | # matching method 23 | matcher = OpencvBruteForceMatcher('l2') 24 | count_seq = 0 25 | 26 | # load data and compute the keypoints 27 | iterate = tqdm(enumerate(data_loader.extract_hsequences()), total=len(data_loader.sequences), desc="HPatches Eval") 28 | print("HPatches evaluation using Keypoints and Descriptors.") 29 | for sample_id, sample_data in iterate: 30 | sequence = sample_data['sequence_name'] 31 | 32 | count_seq += 1 33 | image_src = sample_data['im_src'] 34 | images_dst = sample_data['images_dst'] 35 | h_src_2_dst = sample_data['h_src_2_dst'] 36 | h_dst_2_src = sample_data['h_dst_2_src'] 37 | 38 | for idx_im in range(len(images_dst)): 39 | 40 | # create the mask to filter out the points outside of the common areas 41 | mask_src, mask_dst = geo_tools.create_common_region_masks(h_dst_2_src[idx_im], image_src.shape, images_dst[idx_im].shape) 42 | 43 | # compute the files paths 44 | src_pts_filename = os.path.join(args.results_dir, args.detector_name, 45 | 'hpatches-sequences-release', '{}/1.ppm.kpt.npy'.format(sample_data['sequence_name'])) 46 | src_dsc_filename = os.path.join(args.results_dir, args.detector_name, 47 | 'hpatches-sequences-release', '{}/1.ppm.dsc.npy'.format(sample_data['sequence_name'])) 48 | dst_pts_filename = os.path.join(args.results_dir, args.detector_name, 49 | 'hpatches-sequences-release', '{}/{}.ppm.kpt.npy'.format(sample_data['sequence_name'], idx_im+2)) 50 | dst_dsc_filename = os.path.join(args.results_dir, args.detector_name, 51 | 'hpatches-sequences-release', '{}/{}.ppm.dsc.npy'.format(sample_data['sequence_name'], idx_im+2)) 52 | 53 | if not os.path.isfile(src_pts_filename): 54 | print("Could not find the file: " + src_pts_filename) 55 | return False 56 | 57 | if not os.path.isfile(src_dsc_filename): 58 | print("Could not find the file: " + src_dsc_filename) 59 | return False 60 | 61 | if not os.path.isfile(dst_pts_filename): 62 | print("Could not find the file: " + dst_pts_filename) 63 | return False 64 | 65 | if not os.path.isfile(dst_dsc_filename): 66 | print("Could not find the file: " + dst_dsc_filename) 67 | return False 68 | 69 | # load the points 70 | src_pts = np.load(src_pts_filename) 71 | src_dsc = np.load(src_dsc_filename) 72 | 73 | dst_pts = np.load(dst_pts_filename) 74 | dst_dsc = np.load(dst_dsc_filename) 75 | 76 | if args.order_coord == 'xysr': 77 | src_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], src_pts))) 78 | dst_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], dst_pts))) 79 | 80 | # Check Common Points 81 | src_idx = rep_tools.check_common_points(src_pts, mask_src) 82 | src_pts = src_pts[src_idx] 83 | src_dsc = src_dsc[src_idx] 84 | 85 | dst_idx = rep_tools.check_common_points(dst_pts, mask_dst) 86 | dst_pts = dst_pts[dst_idx] 87 | dst_dsc = dst_dsc[dst_idx] 88 | 89 | # Select top K points 90 | if args.top_k_points: 91 | src_idx = rep_tools.select_top_k(src_pts, args.top_k_points) 92 | src_pts = src_pts[src_idx] 93 | src_dsc = src_dsc[src_idx] 94 | 95 | dst_idx = rep_tools.select_top_k(dst_pts, args.top_k_points) 96 | dst_pts = dst_pts[dst_idx] 97 | dst_dsc = dst_dsc[dst_idx] 98 | 99 | src_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], src_pts))) 100 | dst_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], dst_pts))) 101 | 102 | src_to_dst_pts = geo_tools.apply_homography_to_points( 103 | src_pts, h_src_2_dst[idx_im]) 104 | 105 | dst_to_src_pts = geo_tools.apply_homography_to_points( 106 | dst_pts, h_dst_2_src[idx_im]) 107 | 108 | if args.dst_to_src_evaluation: 109 | points_src = src_pts 110 | points_dst = dst_to_src_pts 111 | else: 112 | points_src = src_to_dst_pts 113 | points_dst = dst_pts 114 | 115 | # compute repeatability 116 | repeatability_results = rep_tools.compute_repeatability(points_src, points_dst, overlap_err=1-args.overlap, 117 | dist_match_thresh=args.pixel_threshold) 118 | 119 | # match descriptors 120 | matches = matcher.match(src_dsc, dst_dsc) 121 | matches_np = aux.convert_opencv_matches_to_numpy(matches) 122 | 123 | matches_inv = matcher.match(dst_dsc, src_dsc) 124 | matches_inv_np = aux.convert_opencv_matches_to_numpy(matches_inv) 125 | 126 | mask = matches_np[:, 0] == matches_inv_np[matches_np[:, 1], 1] 127 | matches_np = matches_np[mask] 128 | 129 | match_score, match_score_corr, num_matches = {}, {}, {} 130 | 131 | # compute matching based on pixel distance 132 | for th_i in range(1, 11): 133 | match_score_i, match_score_corr_i, num_matches_i = match_tools.compute_matching_based_distance(points_src, points_dst, matches_np, 134 | repeatability_results['total_num_points'], 135 | pixel_threshold=th_i, 136 | possible_matches=repeatability_results['possible_matches']) 137 | match_score[str(th_i)] = match_score_i 138 | match_score_corr[str(th_i)] = match_score_corr_i 139 | num_matches[str(th_i)] = num_matches_i 140 | 141 | mma = np.mean([match_score[str(idx)] for idx in match_score]) 142 | 143 | results['rep_single_scale'].append( 144 | repeatability_results['rep_single_scale']) 145 | results['rep_multi_scale'].append( 146 | repeatability_results['rep_multi_scale']) 147 | results['num_points_single_scale'].append( 148 | repeatability_results['num_points_single_scale']) 149 | results['num_points_multi_scale'].append( 150 | repeatability_results['num_points_multi_scale']) 151 | results['error_overlap_single_scale'].append( 152 | repeatability_results['error_overlap_single_scale']) 153 | results['error_overlap_multi_scale'].append( 154 | repeatability_results['error_overlap_multi_scale']) 155 | 156 | results['mma'].append(match_score[str(args.pixel_threshold)]) 157 | results['mma_corr'].append(match_score_corr[str(args.pixel_threshold)]) 158 | results['num_matches'].append(num_matches[str(args.pixel_threshold)]) 159 | results['num_mutual_corresp'].append(len(matches_np)) 160 | results['avg_mma'].append(mma) 161 | results['num_features'].append(repeatability_results['total_num_points']) 162 | 163 | ## logging 164 | iterate.set_description("{} {} / {} - {} rep_s {:.2f} , rep_m {:.2f}, p_s {:d} , p_m {:d}, eps_s {:.2f}, eps_m {:.2f} " 165 | .format(sequence, count_seq, len(data_loader.sequences), idx_im, 166 | repeatability_results['rep_single_scale'], repeatability_results['rep_multi_scale'], repeatability_results['num_points_single_scale'], 167 | repeatability_results['num_points_multi_scale'], repeatability_results['error_overlap_single_scale'], repeatability_results['error_overlap_multi_scale'] 168 | ) ) 169 | 170 | 171 | # average the results 172 | rep_single = np.array(results['rep_single_scale']).mean() 173 | rep_multi = np.array(results['rep_multi_scale']).mean() 174 | error_overlap_s = np.array(results['error_overlap_single_scale']).mean() 175 | error_overlap_m = np.array(results['error_overlap_multi_scale']).mean() 176 | mma = np.array(results['mma']).mean() 177 | mma_corr = np.array(results['mma_corr']).mean() 178 | num_matches = np.array(results['num_matches']).mean() 179 | num_mutual_corresp = np.array(results['num_mutual_corresp']).mean() 180 | avg_mma = np.array(results['avg_mma']).mean() 181 | num_features = np.array(results['num_features']).mean() 182 | 183 | # Matching Score: Matching Score taking into account all features that have been 184 | # detected in any of the two images. 185 | # Matching Score (possible matches): Matching Score only taking into account those features that have been 186 | # detected in both images. 187 | # MMA Score is computed based on the Matching Score (all detected features) 188 | 189 | print('\n## Overlap @{0}:\n \ 190 | #### Rep. Multi: {1:.4f}\n \ 191 | #### Rep. Single: {2:.4f}\n \ 192 | #### Overlap Multi: {3:.4f}\n \ 193 | #### Overlap Single: {4:.4f}\n \ 194 | #### MMA: {5:.4f}\n \ 195 | #### MMA (possible matches): {6:.4f}\n \ 196 | #### Num matches: {7:.4f}\n \ 197 | #### Num Mutual Correspondences: {8:.4f}\n \ 198 | #### Avg. over Threshold MMA: {9:.4f}\n \ 199 | #### Num Feats: {10:.4f}'.format( 200 | args.overlap, rep_multi, rep_single, error_overlap_s, error_overlap_m, mma, 201 | mma_corr, num_matches, num_mutual_corresp, avg_mma, num_features)) 202 | 203 | # Store data (serialize) 204 | output_file_path = os.path.join(args.results_bench_dir, '{0}_{1}.pickle' 205 | .format(args.detector_name, args.split)) 206 | with open(output_file_path, 'wb') as handle: 207 | pickle.dump(results, handle, protocol=pickle.HIGHEST_PROTOCOL) 208 | 209 | 210 | if __name__ == '__main__': 211 | 212 | hsequences_metrics() -------------------------------------------------------------------------------- /keyNet/aux/desc_aux_function.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def _meshgrid(height, width): 4 | # x_t, y_t = torch.meshgrid(torch.linspace(-1,1,width), torch.linspace(-1,1,height)) 5 | # ones = torch.ones(torch.prod(torch.tensor(x_t.shape))) 6 | # grid = torch.vstack([x_t.flatten(), y_t.flatten(), ones]) 7 | 8 | x_t = torch.matmul(torch.ones([height, 1]), torch.linspace(-1.0,1.0,width).unsqueeze(1).transpose(1,0)) 9 | y_t = torch.matmul(torch.linspace(-1.0,1.0,height).unsqueeze(1), torch.ones([1, width]) ) 10 | 11 | x_t_flat = torch.reshape(x_t, (1, -1)) 12 | y_t_flat = torch.reshape(y_t, (1, -1)) 13 | 14 | ones = torch.ones_like(x_t_flat) 15 | grid = torch.cat([x_t_flat, y_t_flat, ones], dim=0) 16 | return grid 17 | 18 | 19 | def transformer_crop(images, out_size, batch_inds, kpts_xy, kpts_scale=None, kpts_ori=None, thetas=None, 20 | name='SpatialTransformCropper'): 21 | # images : [B,C,H,W] 22 | # out_size : (out_width, out_height) 23 | # batch_inds : [B*K,] torch.int32 [0,B) 24 | # kpts_xy : [B*K,2] torch.float32 or whatever 25 | # kpts_scale : [B*K,] torch.float32 26 | # kpts_ori : [B*K,2] torch.float32 (cos,sin) 27 | if isinstance(out_size, int): 28 | out_width = out_height = out_size 29 | else: 30 | out_width, out_height = out_size 31 | hoW = out_width // 2 32 | hoH = out_height // 2 33 | 34 | num_batch, C, height, width = images.shape 35 | num_kp = kpts_xy.shape[0] 36 | 37 | zero = torch.zeros([], dtype=torch.int32) 38 | max_y = torch.tensor(height - 1).to(torch.int32) 39 | max_x = torch.tensor(width - 1).to(torch.int32) 40 | 41 | grid = _meshgrid(out_height, out_width).to(kpts_xy.device) # normalized -1~1 42 | grid = grid.unsqueeze(0) 43 | grid = torch.reshape(grid, [-1]) 44 | grid = torch.tile(grid, [num_kp]) 45 | grid = torch.reshape(grid, [num_kp, 3, -1]) 46 | 47 | # create 6D affine from scale and orientation 48 | # [s, 0, 0] [cos, -sin, 0] 49 | # [0, s, 0] * [sin, cos, 0] 50 | # [0, 0, 1] [0, 0, 1] 51 | 52 | if thetas is None: 53 | thetas = torch.eye(2, 3, dtype=torch.float32).to(kpts_xy.device) 54 | thetas = torch.tile(thetas, [num_kp, 1, 1]) 55 | if kpts_scale is not None: 56 | thetas = thetas * kpts_scale[:, None, None] 57 | ones = torch.tile(torch.tensor([[[0, 0, 1]]], dtype=torch.float32), [num_kp, 1, 1]).to(kpts_xy.device) 58 | thetas = torch.cat([thetas, ones], dim=1) # [num_kp, 3,3] 59 | 60 | if kpts_ori is not None: 61 | cos = kpts_ori[:, 0] # [num_kp, 1] 62 | sin = kpts_ori[:, 1] 63 | zeros = torch.zeros_like(cos).to(kpts_xy.device) 64 | ones = torch.ones_like(cos).to(kpts_xy.device) 65 | R = torch.cat([cos, -sin, zeros, sin, cos, zeros, zeros, zeros, ones], dim=-1) 66 | R = torch.reshape(R, [-1, 3, 3]) 67 | thetas = torch.matmul(thetas, R) 68 | # Apply transformation to regular grid 69 | T_g = torch.matmul(thetas, grid) # [num_kp,3,3] * [num_kp,3,H*W] 70 | x = T_g[:, 0, :] # [num_kp,1,H*W] 71 | y = T_g[:, 1, :] 72 | 73 | # unnormalization [-1,1] --> [-out_size/2,out_size/2] 74 | x = x * out_width / 2.0 75 | y = y * out_height / 2.0 76 | 77 | kp_x_ofst = kpts_xy[:,0].unsqueeze(1)# [B*K,1,1] 78 | kp_y_ofst = kpts_xy[:,1].unsqueeze(1)# [B*K,1,1] 79 | 80 | # centerize on keypoints 81 | x = x + kp_x_ofst 82 | y = y + kp_y_ofst 83 | x = torch.reshape(x, [-1]) # num_kp*out_height*out_width 84 | y = torch.reshape(y, [-1]) 85 | 86 | # interpolation 87 | x0 = torch.floor(x).to(torch.int32) 88 | x1 = x0 + 1 89 | y0 = torch.floor(y).to(torch.int32) 90 | y1 = y0 + 1 91 | 92 | x0 = torch.clamp(x0, zero, max_x) 93 | x1 = torch.clamp(x1, zero, max_x) 94 | y0 = torch.clamp(y0, zero, max_y) 95 | y1 = torch.clamp(y1, zero, max_y) 96 | 97 | dim2 = width 98 | dim1 = width * height 99 | base = torch.tile(batch_inds[:, None], [1, out_height * out_width]) # [B*K,out_height*out_width] 100 | base = torch.reshape(base, [-1]) * dim1 101 | base_y0 = base + y0 * dim2 102 | base_y1 = base + y1 * dim2 103 | idx_a = base_y0 + x0 104 | idx_b = base_y1 + x0 105 | idx_c = base_y0 + x1 106 | idx_d = base_y1 + x1 107 | 108 | im_flat = torch.reshape(images.permute(0,2,3,1), [-1, C]) # [B*height*width,C] 109 | im_flat = im_flat.to(torch.float32) 110 | 111 | Ia = torch.gather(im_flat, 0, idx_a.to(torch.int64).unsqueeze(1).repeat(1,C)) 112 | Ib = torch.gather(im_flat, 0, idx_b.to(torch.int64).unsqueeze(1).repeat(1,C)) 113 | Ic = torch.gather(im_flat, 0, idx_c.to(torch.int64).unsqueeze(1).repeat(1,C)) 114 | Id = torch.gather(im_flat, 0, idx_d.to(torch.int64).unsqueeze(1).repeat(1,C)) 115 | 116 | x0_f = x0.to(torch.float32) 117 | x1_f = x1.to(torch.float32) 118 | y0_f = y0.to(torch.float32) 119 | y1_f = y1.to(torch.float32) 120 | 121 | wa = ((x1_f - x) * (y1_f - y)).unsqueeze(1) 122 | wb = ((x1_f - x) * (y - y0_f)).unsqueeze(1) 123 | wc = ((x - x0_f) * (y1_f - y)).unsqueeze(1) 124 | wd = ((x - x0_f) * (y - y0_f)).unsqueeze(1) 125 | 126 | output = wa * Ia + wb * Ib + wc * Ic + wd * Id 127 | output = torch.reshape(output, [num_kp, out_height, out_width, C]) 128 | output = output.permute(0,3,1,2) 129 | #output.set_shape([batch_inds.shape[0], out_height, out_width, images.shape[-1]]) 130 | return output 131 | 132 | 133 | def build_patch_extraction(kpts, batch_inds, images, kpts_scale, name='PatchExtract', patch_size=32): 134 | patches = transformer_crop(images, patch_size, batch_inds, kpts, kpts_scale=kpts_scale) 135 | 136 | return patches 137 | -------------------------------------------------------------------------------- /keyNet/aux/logger.py: -------------------------------------------------------------------------------- 1 | r"""Logging""" 2 | import datetime 3 | import logging 4 | import os 5 | 6 | import matplotlib.pyplot as plt 7 | import pandas as pd 8 | import numpy as np 9 | import torch 10 | 11 | class Logger: 12 | r"""Writes results of training/testing""" 13 | @classmethod 14 | def initialize(cls, args): 15 | logtime = datetime.datetime.now().__format__('_%m%d_%H%M%S') 16 | logpath = args.network_version + logtime 17 | cls.logpath = os.path.join(args.weights_dir +'/' + logpath + '.log') 18 | 19 | os.makedirs(cls.logpath) 20 | 21 | logging.basicConfig(filemode='w', 22 | filename=os.path.join(cls.logpath, 'log.txt'), 23 | level=logging.INFO, 24 | format='%(message)s', 25 | datefmt='%m-%d %H:%M:%S') 26 | 27 | # Console log config 28 | console = logging.StreamHandler() 29 | console.setLevel(logging.INFO) 30 | formatter = logging.Formatter('%(message)s') 31 | console.setFormatter(formatter) 32 | logging.getLogger('').addHandler(console) 33 | 34 | # # Tensorboard writer 35 | # cls.tbd_writer = SummaryWriter(os.path.join(cls.logpath, 'tbd/runs')) 36 | 37 | # # Log arguments 38 | logging.info('\n+=========== Key.Net PyTorch Version ============+') 39 | for arg_key in args.__dict__: 40 | logging.info('| %20s: %-24s |' % (arg_key, str(args.__dict__[arg_key]))) 41 | logging.info('+================================================+\n') 42 | 43 | @classmethod 44 | def info(cls, msg): 45 | r"""Writes message to .txt""" 46 | logging.info(msg) 47 | 48 | @classmethod 49 | def save_model(cls, model, epoch, val_rep): 50 | torch.save(model.state_dict(), os.path.join(cls.logpath, 'best_model.pt')) 51 | cls.info('Model saved @%d w/ val. Repeability score: %5.2f.\n' % (epoch, val_rep)) 52 | 53 | -------------------------------------------------------------------------------- /keyNet/aux/tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | def remove_borders(images, borders=3): 4 | ## input [B,C,H,W] 5 | shape = images.shape 6 | 7 | if len(shape) == 4: 8 | for batch_id in range(shape[0]): 9 | images[batch_id, :, 0:borders, :] = 0 10 | images[batch_id, :, :, 0:borders] = 0 11 | images[batch_id, :, shape[2] - borders:shape[2], :] = 0 12 | images[batch_id, :, :, shape[3] - borders:shape[3]] = 0 13 | # elif len(shape) == 3: 14 | # ## C, H, W case 15 | # images[:, 0:borders, :] = 0 16 | # images[:, :, 0:borders] = 0 17 | # images[:, shape[1] - borders:shape[1], :] = 0 18 | # images[:, :, shape[2] - borders:shape[2]] = 0 19 | # else: 20 | # images[0:borders, :] = 0 21 | # images[:, 0:borders] = 0 22 | # images[shape[0] - borders:shape[0], :] = 0 23 | # images[:, shape[1] - borders:shape[1]] = 0 24 | else: 25 | print("Not implemented") 26 | exit() 27 | 28 | return images 29 | 30 | def check_directory(file_path): 31 | if not os.path.exists(file_path): 32 | os.mkdir(file_path) 33 | 34 | 35 | # def check_tensorboard_directory(version_network_name): 36 | # check_directory('keyNet/logs_network') 37 | # check_directory('keyNet/logs_network/' + version_network_name) 38 | 39 | -------------------------------------------------------------------------------- /keyNet/config.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def get_config(): 4 | parser = argparse.ArgumentParser(description='Train Key.Net Architecture') 5 | 6 | ## basic configuration 7 | parser.add_argument('--data-dir', type=str, default='/home/jongmin/datasets/ImageNet2012/ILSVRC2012_img_val', #default='path-to-ImageNet', 8 | help='The root path to the data from which the synthetic dataset will be created.') 9 | parser.add_argument('--synth-dir', type=str, default='datasets/synth/', 10 | help='The path to save the generated sythetic image pairs.') 11 | parser.add_argument('--weights-dir', type=str, default='keyNet/weights', 12 | help='The path to save the Key.Net weights.') 13 | parser.add_argument('--write-summary', type=bool, default=False, 14 | help='Set to True if you desire to save the summary of the training.') 15 | parser.add_argument('--network-version', type=str, default='KeyNet_default', 16 | help='The Key.Net network version name') 17 | parser.add_argument('--random-seed', type=int, default=12345, 18 | help='The random seed value for PyTorch and Numpy.') 19 | parser.add_argument('--num-epochs', type=int, default=30, 20 | help='Number of epochs for training.') 21 | parser.add_argument('--epochs-val', type=int, default=3, 22 | help='Set the number of training epochs between repeteability checks on the validation set.') 23 | ## Dataset generation 24 | parser.add_argument('--patch-size', type=int, default=192, 25 | help='The patch size of the generated dataset.') 26 | parser.add_argument('--max-angle', type=int, default=45, 27 | help='The max angle value for generating a synthetic view to train Key.Net.') 28 | parser.add_argument('--max-scale', type=int, default=2.0, 29 | help='The max scale value for generating a synthetic view to train Key.Net.') 30 | parser.add_argument('--max-shearing', type=int, default=0.8, 31 | help='The max shearing value for generating a synthetic view to train Key.Net.') 32 | parser.add_argument('--is-debugging', type=bool, default=False, 33 | help='Set variable to True if you desire to train network on a smaller dataset.') 34 | parser.add_argument('--load-tfrecord', type=bool, default=False, 35 | help='Load tensorflor tfrecord.') 36 | ## Training 37 | parser.add_argument('--batch-size', type=int, default=32, 38 | help='The batch size for training.') 39 | parser.add_argument('--init-initial-learning-rate', type=float, default=1e-3, 40 | help='The init initial learning rate value.') 41 | parser.add_argument('--num-epochs-before-decay', type=int, default=20, 42 | help='The number of epochs before decay.') 43 | parser.add_argument('--learning-rate-decay-factor', type=float, default=0.5, 44 | help='The learning rate decay factor.') 45 | parser.add_argument('--resume-training', type=str, default='', 46 | help='Set saved model parameters if resume training is desired.') 47 | parser.add_argument('--weight-coordinates', type=bool, default=True, 48 | help='Weighting coordinates by their scores.') 49 | parser.add_argument('--MSIP_sizes', type=str, default="8,16,24,32,40", 50 | help='MSIP sizes.') 51 | parser.add_argument('--MSIP_factor_loss', type=str, default="256.0,64.0,16.0,4.0,1.0", 52 | help='MSIP loss balancing parameters.') 53 | ## network architectures 54 | parser.add_argument('--num-filters', type=int, default=8, 55 | help='The number of filters in each learnable block.') 56 | parser.add_argument('--num-learnable-blocks', type=int, default=3, 57 | help='The number of learnable blocks after handcrafted block.') 58 | parser.add_argument('--num-levels-within-net', type=int, default=3, 59 | help='The number of pyramid levels inside the architecture.') 60 | parser.add_argument('--factor-scaling-pyramid', type=float, default=1.2, 61 | help='The scale factor between the multi-scale pyramid levels in the architecture.') 62 | parser.add_argument('--conv-kernel-size', type=int, default=5, 63 | help='The size of the convolutional filters in each of the learnable blocks.') 64 | parser.add_argument('--nms-size', type=int, default=15, 65 | help='The NMS size for computing the validation repeatability.') 66 | parser.add_argument('--border-size', type=int, default=15, 67 | help='The number of pixels to remove from the borders to compute the repeatability.') 68 | 69 | args = parser.parse_args() 70 | 71 | return args -------------------------------------------------------------------------------- /keyNet/config_hpatches.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | 4 | def get_config(): 5 | parser = argparse.ArgumentParser(description='HSequences Extract Features') 6 | 7 | ## basic configuration 8 | parser.add_argument('--list-images', type=str, 9 | help='File containing the image paths for extracting features.', required=True) 10 | parser.add_argument('--results-dir', type=str, default='extracted_features/', 11 | help='The output path to save the extracted keypoint.') 12 | parser.add_argument('--network-version', type=str, default='KeyNet_default', 13 | help='The Key.Net network version name') 14 | parser.add_argument('--checkpoint-det-dir', type=str, 15 | help='The path to the checkpoint file to load the detector weights.', required=True) 16 | parser.add_argument('--pytorch-hardnet-dir', type=str, default='keyNet/pretrained_nets/HardNet++.pth', 17 | help='The path to the checkpoint file to load the HardNet descriptor weights.') 18 | # Detector Settings 19 | parser.add_argument('--batch-size', type=int, default=1, 20 | help='The batch size for training.') 21 | parser.add_argument('--patch-size', type=int, default=192, 22 | help='The patch size of the generated dataset.') 23 | parser.add_argument('--num-filters', type=int, default=8, 24 | help='The number of filters in each learnable block.') 25 | parser.add_argument('--num-learnable-blocks', type=int, default=3, 26 | help='The number of learnable blocks after handcrafted block.') 27 | parser.add_argument('--num-levels-within-net', type=int, default=3, 28 | help='The number of pyramid levels inside the architecture.') 29 | parser.add_argument('--factor-scaling-pyramid', type=float, default=1.2, 30 | help='The scale factor between the multi-scale pyramid levels in the architecture.') 31 | parser.add_argument('--conv-kernel-size', type=int, default=5, 32 | help='The size of the convolutional filters in each of the learnable blocks.') 33 | # Multi-Scale Extractor Settings 34 | parser.add_argument('--extract-MS', type=bool, default=True, 35 | help='Set to True if you want to extract multi-scale features.') 36 | parser.add_argument('--num-points', type=int, default=1500, 37 | help='The number of desired features to extract.') 38 | parser.add_argument('--nms-size', type=int, default=15, 39 | help='The NMS size for computing the validation repeatability.') 40 | parser.add_argument('--border-size', type=int, default=15, 41 | help='The number of pixels to remove from the borders to compute the repeatability.') 42 | parser.add_argument('--order-coord', type=str, default='xysr', 43 | help='The coordinate order that follows the extracted points. Use yxsr or xysr.') 44 | parser.add_argument('--random-seed', type=int, default=12345, 45 | help='The random seed value for TensorFlow and Numpy.') 46 | parser.add_argument('--pyramid_levels', type=int, default=5, 47 | help='The number of downsample levels in the pyramid.') 48 | parser.add_argument('--upsampled-levels', type=int, default=1, 49 | help='The number of upsample levels in the pyramid.') 50 | parser.add_argument('--scale-factor-levels', type=float, default=np.sqrt(2), 51 | help='The scale factor between the pyramid levels.') 52 | parser.add_argument('--scale-factor', type=float, default=2., 53 | help='The scale factor to extract patches before descriptor.') 54 | args = parser.parse_args() 55 | 56 | return args 57 | 58 | 59 | def get_eval_config(): 60 | 61 | parser = argparse.ArgumentParser(description='HSequences Compute Repeatability') 62 | 63 | parser.add_argument('--data-dir', type=str, default='hpatches-sequences-release/', 64 | help='The root path to HSequences dataset.') 65 | parser.add_argument('--results-bench-dir', type=str, default='HSequences_bench/results/', 66 | help='The output path to save the results.') 67 | parser.add_argument('--detector-name', type=str, default='KeyNet_default', 68 | help='The name of the detector to compute metrics.') 69 | parser.add_argument('--results-dir', type=str, default='extracted_features/', 70 | help='The path to the extracted points.') 71 | 72 | parser.add_argument('--split', type=str, default='full', 73 | help='The name of the HPatches (HSequences) split. Use full, debug_view, debug_illum, view or illum.') 74 | parser.add_argument('--split-path', type=str, default='HSequences_bench/splits.json', 75 | help='The path to the split json file.') 76 | parser.add_argument('--top-k-points', type=int, default=1000, 77 | help='The number of top points to use for evaluation. Set to None to use all points') 78 | parser.add_argument('--overlap', type=float, default=0.6, 79 | help='The overlap threshold for a correspondence to be considered correct.') 80 | parser.add_argument('--pixel-threshold', type=int, default=5, 81 | help='The distance of pixels for a matching correspondence to be considered correct.') 82 | 83 | parser.add_argument('--dst-to-src-evaluation', type=bool, default=True, 84 | help='Order to apply homography to points. Use True for dst to src, False otherwise.') 85 | parser.add_argument('--order-coord', type=str, default='xysr', 86 | help='The coordinate order that follows the extracted points. Use either xysr or yxsr.') 87 | 88 | args = parser.parse_args() 89 | 90 | return args 91 | -------------------------------------------------------------------------------- /keyNet/datasets/dataset_utils.py: -------------------------------------------------------------------------------- 1 | ## Jongmin : I Just used tensorflow version. 2 | import cv2 3 | import numpy as np 4 | from cv2 import warpPerspective as applyH 5 | 6 | perms = ((0, 1, 2), (0, 2, 1), 7 | (1, 0, 2), (1, 2, 0), 8 | (2, 0, 1), (2, 1, 0)) 9 | 10 | def read_bw_image(path): 11 | img = read_color_image(path) 12 | img = to_black_and_white(img) 13 | return img 14 | 15 | def read_color_image(path): 16 | im_c = cv2.imread(path) 17 | return im_c.reshape(im_c.shape[0], im_c.shape[1], 3) 18 | 19 | def apply_h_2_source_image(source_im, h): 20 | shape_source_im = source_im.shape 21 | dst = applyH(source_im, h, (shape_source_im[1], shape_source_im[0])) 22 | return np.reshape(dst, (shape_source_im[0], shape_source_im[1], 1)) 23 | 24 | 25 | def generate_composed_homography(max_angle=45, max_scaling=2.0, max_shearing=0.8): 26 | 27 | # random sample 28 | scale = np.random.uniform(0.5, max_scaling) 29 | angle = np.random.uniform(-max_angle, max_angle) 30 | shear = np.random.uniform(-max_shearing, max_shearing) 31 | 32 | # scale transform 33 | scale_mat = np.eye(3) 34 | scale_mat[0, 0] = scale 35 | scale_mat[1, 1] = scale 36 | # rotation transform 37 | angle = np.deg2rad(angle) 38 | rotation_mat = np.eye(3) 39 | rotation_mat[0, 0] = np.cos(angle) 40 | rotation_mat[0, 1] = -np.sin(angle) 41 | rotation_mat[1, 0] = np.sin(angle) 42 | rotation_mat[1, 1] = np.cos(angle) 43 | # shear transform 44 | shear_mat = np.eye(3) 45 | shear_mat[0, 1] = shear 46 | # compose transforms 47 | h = np.matmul(shear_mat, np.matmul(scale_mat, rotation_mat)) 48 | return h 49 | 50 | 51 | def color_distorsion(im_c): 52 | im_correction = colorDistorsion(im_c) 53 | im = cv2.cvtColor(im_correction, cv2.COLOR_BGR2GRAY) 54 | return im.reshape(im.shape[0], im.shape[1], 1) 55 | 56 | 57 | def to_black_and_white(img): 58 | im = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 59 | return im.reshape(im.shape[0], im.shape[1], 1) 60 | 61 | 62 | def colorDistorsion(image, lower=0.5, upper=1.5, delta=18.0, delta_brigtness=36): 63 | 64 | image = image.astype(float) 65 | 66 | if np.random.randint(2): 67 | delta = np.random.uniform(-delta_brigtness, delta_brigtness) 68 | image += delta 69 | image = check_margins(image) 70 | 71 | contrast = np.random.randint(2) 72 | if contrast: 73 | alpha = np.random.uniform(lower, upper) 74 | image *= alpha 75 | image = check_margins(image) 76 | 77 | image = image.astype(np.uint8) 78 | image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) 79 | image = image.astype(float) 80 | 81 | if np.random.randint(2): 82 | image[:, :, 1] *= np.random.uniform(lower, upper) 83 | image = check_margins(image, axis=1) 84 | if np.random.randint(2): 85 | image[:, :, 0] += np.random.uniform(-delta, delta) 86 | image[:, :, 0][image[:, :, 0] > 360.0] -= 360.0 87 | image[:, :, 0][image[:, :, 0] < 0.0] += 360.0 88 | 89 | image = image.astype(np.uint8) 90 | image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR) 91 | image = image.astype(float) 92 | 93 | if contrast: 94 | alpha = np.random.uniform(lower, upper) 95 | image *= alpha 96 | image = check_margins(image) 97 | 98 | if np.random.randint(2): 99 | swap = perms[np.random.randint(len(perms))] 100 | image = swap_channels(image, swap) # shuffle channels 101 | 102 | return image.astype(np.uint8) 103 | 104 | 105 | def check_margins(img, axis=-1): 106 | if axis == -1: 107 | img[img > 255.0] = 255.0 108 | img[img < 0.0] = 0.0 109 | else: 110 | img[:, :, axis][img[:, :, axis] > 255.0] = 255.0 111 | img[:, :, axis][img[:, :, axis] < 0.0] = 0.0 112 | return img 113 | 114 | 115 | def swap_channels(image, swaps): 116 | image = image[:, :, swaps] 117 | return image 118 | -------------------------------------------------------------------------------- /keyNet/datasets/pytorch_dataset.py: -------------------------------------------------------------------------------- 1 | import os, cv2, logging 2 | import numpy as np 3 | import keyNet.datasets.dataset_utils as tools 4 | from tqdm import tqdm 5 | from torch.utils.data import Dataset 6 | from keyNet.aux.tools import check_directory 7 | 8 | class pytorch_dataset(Dataset): 9 | def __init__(self, data, mode='train'): 10 | self.data =data 11 | 12 | ## Restrict the number of training and validation examples (9000 : 3000 = train : val) 13 | if mode == 'train': 14 | if len(self.data) > 9000: 15 | self.data = self.data[:9000] 16 | elif mode == 'val': 17 | if len(self.data) > 3000: 18 | self.data = self.data[:3000] 19 | 20 | logging.info('mode : {} the number of examples : {}'.format(mode, len(self.data))) 21 | 22 | def __len__(self): 23 | return len(self.data) 24 | 25 | def __getitem__(self,idx): 26 | im_src_patch, im_dst_patch, homography_src_2_dst, homography_dst_2_src = self.data[idx] 27 | # print(im_src_patch.shape, im_dst_patch.shape, homography_src_2_dst, homography_dst_2_src) 28 | 29 | return im_src_patch[0], im_dst_patch[0], homography_src_2_dst[0], homography_dst_2_src[0] 30 | 31 | 32 | class DatasetGeneration(object): 33 | def __init__(self, dataset_root, savepair_root, size_patches, batch_size, max_angle, max_scaling, max_shearing, random_seed, is_debugging=False, load_tfrecord=True): 34 | 35 | self.size_patches = size_patches 36 | self.batch_size = batch_size 37 | self.dataset_root = dataset_root 38 | self.num_examples = 0 39 | self.num_val_examples = 0 40 | self.max_angle = max_angle 41 | self.max_scaling = max_scaling 42 | self.max_shearing = max_shearing 43 | self.is_debugging = is_debugging 44 | 45 | self.savepair_root = savepair_root 46 | 47 | ## Input lists 48 | self.training_data = [] ## input_image_pairs : self.input_image_pairs / self.src2dst_Hs / self.dst2src_Hs 49 | self.validation_data = [] 50 | 51 | if load_tfrecord: 52 | self._load_tfrecord_images('keyNet/tfrecords/train_dataset.npz', is_val=False) 53 | self._load_tfrecord_images('keyNet/tfrecords/val_dataset.npz', is_val=True) 54 | 55 | else: 56 | if is_debugging: 57 | self.save_path = os.path.join(self.savepair_root ,'train_dataset_debug') 58 | self.save_val_path = os.path.join(self.savepair_root , 'val_dataset_debug') 59 | else: 60 | self.save_path = os.path.join(self.savepair_root , 'train_dataset') 61 | self.save_val_path = os.path.join(self.savepair_root , 'val_dataset') 62 | 63 | savepair_exists = self.existence_check(self.savepair_root, is_debugging) 64 | 65 | if not savepair_exists: 66 | check_directory(self.save_path) 67 | check_directory(self.save_val_path) 68 | 69 | self.data_path = self._find_data_path(self.dataset_root) 70 | self.images_info = self._load_data_names(self.data_path) 71 | 72 | print("Total images in directory at \"" , self.dataset_root, "\" is : ", len(self.images_info)) 73 | 74 | self._create_synthetic_pairs(is_val=False) 75 | self._create_synthetic_pairs(is_val=True) 76 | else: 77 | self._load_pair_images(is_val=False) 78 | self._load_pair_images(is_val=True) 79 | 80 | print("# of Training / validation : ", len(self.training_data), len(self.validation_data)) 81 | 82 | def existence_check(self, root, is_debugging): 83 | if is_debugging: 84 | a = os.path.exists(os.path.join(self.savepair_root ,'train_dataset_debug')) 85 | b = os.path.exists(os.path.join(self.savepair_root , 'val_dataset_debug')) 86 | else: 87 | a = os.path.exists(os.path.join(self.savepair_root , 'train_dataset')) 88 | b = os.path.exists(os.path.join(self.savepair_root , 'val_dataset')) 89 | 90 | return a and b 91 | 92 | def get_training_data(self): 93 | return self.training_data 94 | 95 | def get_validation_data(self): 96 | return self.validation_data 97 | 98 | def _find_data_path(self, data_path): 99 | assert os.path.isdir(data_path), \ 100 | "Invalid directory: {}".format(data_path) 101 | return data_path 102 | 103 | def _load_data_names(self, data_path): 104 | count = 0 105 | images_info = [] 106 | 107 | for r, d, f in os.walk(data_path): 108 | for file_name in f: 109 | if file_name.endswith(".JPEG") or file_name.endswith(".jpg") or file_name.endswith(".png"): 110 | images_info.append(os.path.join(data_path, r, file_name)) 111 | count += 1 112 | 113 | src_idx = np.random.permutation(len(np.asarray(images_info))) 114 | images_info = np.asarray(images_info)[src_idx] 115 | return images_info 116 | 117 | 118 | def _create_synthetic_pairs(self, is_val): 119 | 120 | self._create_pair_images(is_val) 121 | 122 | 123 | ## This is main dataset generation and preprocessing function. 124 | def _create_pair_images(self, is_val): 125 | # More stable repeatability when using bigger size patches 126 | if is_val: 127 | size_patches = 2 * self.size_patches 128 | self.counter += 1 129 | else: 130 | size_patches = self.size_patches 131 | self.counter = 0 132 | 133 | counter_patches = 0 134 | 135 | print('Generating Synthetic pairs . . .') 136 | 137 | save_path = self.save_val_path if is_val else self.save_path 138 | 139 | path_im_src_patch = os.path.join(save_path, 'im_src_patch') 140 | path_im_dst_patch = os.path.join(save_path, 'im_dst_patch') 141 | path_homography_src_2_dst = os.path.join(save_path, 'homography_src_2_dst') 142 | path_homography_dst_2_src = os.path.join(save_path, 'homography_dst_2_src') 143 | 144 | check_directory(path_im_src_patch) 145 | check_directory(path_im_dst_patch) 146 | check_directory(path_homography_src_2_dst) 147 | check_directory(path_homography_dst_2_src) 148 | 149 | for path_image_idx in tqdm(range(len(self.images_info))): 150 | name_image_path = self.images_info[(self.counter+path_image_idx) % len(self.images_info)] 151 | 152 | correct_patch = False 153 | counter = -1 154 | while counter < 10: 155 | 156 | counter += 1 157 | incorrect_h = True 158 | 159 | while incorrect_h: 160 | 161 | scr_c = tools.read_color_image(name_image_path) 162 | 163 | source_shape = scr_c.shape 164 | h = tools.generate_composed_homography(self.max_angle, self.max_scaling, self.max_shearing) 165 | 166 | inv_h = np.linalg.inv(h) 167 | inv_h = inv_h / inv_h[2, 2] 168 | 169 | scr = tools.to_black_and_white(scr_c) 170 | dst = tools.color_distorsion(scr_c) 171 | dst = tools.apply_h_2_source_image(dst, inv_h) 172 | 173 | if dst.max() > 0.0: 174 | incorrect_h = False 175 | 176 | scr_sobelx = cv2.Sobel(scr, cv2.CV_64F, 1, 0, ksize=3) 177 | scr_sobelx = abs(scr_sobelx.reshape((scr.shape[0], scr.shape[1], 1))) 178 | scr_sobelx = scr_sobelx.astype(float) / scr_sobelx.max() 179 | dst_sobelx = cv2.Sobel(dst, cv2.CV_64F, 1, 0, ksize=3) 180 | dst_sobelx = abs(dst_sobelx.reshape((dst.shape[0], dst.shape[1], 1))) 181 | dst_sobelx = dst_sobelx.astype(float) / dst_sobelx.max() 182 | 183 | scr = scr.astype(float) / scr.max() 184 | dst = dst.astype(float) / dst.max() 185 | 186 | if size_patches/2 >= scr.shape[0]-size_patches/2 or size_patches/2 >= scr.shape[1]-size_patches/2: 187 | break 188 | 189 | window_point = [scr.shape[0]/2, scr.shape[1]/2] 190 | 191 | # Define points 192 | point_src = [window_point[0], window_point[1], 1.0] 193 | 194 | im_src_patch = scr[int(point_src[0] - size_patches / 2): int(point_src[0] + size_patches / 2), 195 | int(point_src[1] - size_patches / 2): int(point_src[1] + size_patches / 2)] 196 | 197 | point_dst = inv_h.dot([point_src[1], point_src[0], 1.0]) 198 | point_dst = [point_dst[1] / point_dst[2], point_dst[0] / point_dst[2]] 199 | 200 | if (point_dst[0] - size_patches / 2) < 0 or (point_dst[1] - size_patches / 2) < 0: 201 | continue 202 | if (point_dst[0] + size_patches / 2) > source_shape[0] or (point_dst[1] + size_patches / 2) > \ 203 | source_shape[1]: 204 | continue 205 | 206 | h_src_translation = np.asanyarray([[1., 0., -(int(point_src[1]) - size_patches / 2)], 207 | [0., 1., -(int(point_src[0]) - size_patches / 2)], [0., 0., 1.]]) 208 | h_dst_translation = np.asanyarray( 209 | [[1., 0., int(point_dst[1] - size_patches / 2)], [0., 1., int(point_dst[0] - size_patches / 2)], 210 | [0., 0., 1.]]) 211 | 212 | im_dst_patch = dst[int(point_dst[0] - size_patches / 2): int(point_dst[0] + size_patches / 2), 213 | int(point_dst[1] - size_patches / 2): int(point_dst[1] + size_patches / 2)] 214 | label_dst_patch = dst_sobelx[ 215 | int(point_dst[0] - size_patches / 2): int(point_dst[0] + size_patches / 2), 216 | int(point_dst[1] - size_patches / 2): int(point_dst[1] + size_patches / 2)] 217 | label_scr_patch = scr_sobelx[ 218 | int(point_src[0] - size_patches / 2): int(point_src[0] + size_patches / 2), 219 | int(point_src[1] - size_patches / 2): int(point_src[1] + size_patches / 2)] 220 | 221 | if im_src_patch.shape[0] != size_patches or im_src_patch.shape[1] != size_patches: 222 | continue 223 | if label_dst_patch.max() < 0.25: 224 | continue 225 | if label_scr_patch.max() < 0.25: 226 | continue 227 | 228 | correct_patch = True 229 | break 230 | 231 | if correct_patch: 232 | im_src_patch = im_src_patch.reshape((1, im_src_patch.shape[0], im_src_patch.shape[1], 1)) 233 | im_dst_patch = im_dst_patch.reshape((1, im_dst_patch.shape[0], im_dst_patch.shape[1], 1)) 234 | 235 | homography = np.dot(h_src_translation, np.dot(h, h_dst_translation)) 236 | 237 | homography_dst_2_src = homography.astype('float32') 238 | homography_dst_2_src = homography_dst_2_src.flatten() 239 | homography_dst_2_src = homography_dst_2_src / homography_dst_2_src[8] 240 | homography_dst_2_src = homography_dst_2_src[:8] 241 | 242 | homography_src_2_dst = np.linalg.inv(homography) 243 | homography_src_2_dst = homography_src_2_dst.astype('float32') 244 | homography_src_2_dst = homography_src_2_dst.flatten() 245 | homography_src_2_dst = homography_src_2_dst / homography_src_2_dst[8] 246 | homography_src_2_dst = homography_src_2_dst[:8] 247 | 248 | homography_src_2_dst = homography_src_2_dst.reshape((1, homography_src_2_dst.shape[0])) 249 | homography_dst_2_src = homography_dst_2_src.reshape((1, homography_dst_2_src.shape[0])) 250 | 251 | ## Save the patches by np format (For caching) 252 | name_image = name_image_path.split('/')[-1] 253 | np.save(os.path.join(path_im_src_patch, name_image), im_src_patch) 254 | np.save(os.path.join(path_im_dst_patch, name_image), im_dst_patch) 255 | np.save(os.path.join(path_homography_src_2_dst, name_image), homography_src_2_dst) 256 | np.save(os.path.join(path_homography_dst_2_src, name_image), homography_dst_2_src) 257 | 258 | if is_val: 259 | self.validation_data.append([im_src_patch, im_dst_patch, homography_src_2_dst, homography_dst_2_src]) 260 | else: 261 | self.training_data.append([im_src_patch, im_dst_patch, homography_src_2_dst, homography_dst_2_src]) 262 | 263 | if self.is_debugging: 264 | import matplotlib.pyplot as plt 265 | print("Save : ", im_src_patch.shape, im_dst_patch.shape, homography_src_2_dst.shape, homography_dst_2_src.shape) 266 | 267 | """fig = plt.figure() 268 | rows = 1 ; cols = 3 269 | ax1 = fig.add_subplot(rows, cols, 1) 270 | ax1.imshow(scr_c) 271 | ax1.set_title('scr_c (input image)') 272 | ax1.axis("off") 273 | 274 | ax2 = fig.add_subplot(rows, cols, 2) 275 | ax2.imshow(im_src_patch[0,:,:,0], cmap='gray') 276 | ax2.set_title('im_src_patch') 277 | ax2.axis("off") 278 | 279 | ax3 = fig.add_subplot(rows, cols, 3) 280 | ax3.imshow(im_dst_patch[0,:,:,0], cmap='gray') 281 | ax3.set_title('im_dst_patch') 282 | ax3.axis("off") 283 | 284 | plt.show()""" 285 | 286 | counter_patches += 1 287 | 288 | ## Select the number of training patches and validation patches (and debug mode). (original paper : 9000, 3000) 289 | if is_val and counter_patches > 1500: 290 | break 291 | elif counter_patches > 4000: 292 | break 293 | if is_val and self.is_debugging and counter_patches > 100: 294 | break 295 | elif not is_val and self.is_debugging and counter_patches > 400: 296 | break 297 | 298 | self.counter = counter_patches 299 | 300 | def _load_pair_images(self, is_val): 301 | print('Loading Synthetic pairs . . .') 302 | 303 | save_path = self.save_val_path if is_val else self.save_path 304 | 305 | path_im_src_patch = os.path.join(save_path, 'im_src_patch') 306 | path_im_dst_patch = os.path.join(save_path, 'im_dst_patch') 307 | path_homography_src_2_dst = os.path.join(save_path, 'homography_src_2_dst') 308 | path_homography_dst_2_src = os.path.join(save_path, 'homography_dst_2_src') 309 | 310 | save_name_list= os.listdir(path_im_src_patch) 311 | 312 | for name_image in tqdm(save_name_list, total=len(save_name_list)): 313 | if name_image[-8:] != "JPEG.npy": 314 | continue 315 | ## Load the patches by np format (For caching) 316 | im_src_patch = np.load(os.path.join(path_im_src_patch, name_image)) 317 | im_dst_patch = np.load(os.path.join(path_im_dst_patch, name_image)) 318 | homography_src_2_dst = np.load(os.path.join(path_homography_src_2_dst, name_image)) 319 | homography_dst_2_src = np.load(os.path.join(path_homography_dst_2_src, name_image)) 320 | 321 | if is_val: 322 | self.validation_data.append([im_src_patch, im_dst_patch, homography_src_2_dst, homography_dst_2_src]) 323 | else: 324 | self.training_data.append([im_src_patch, im_dst_patch, homography_src_2_dst, homography_dst_2_src]) 325 | 326 | 327 | def _load_tfrecord_images(self, tfrecord_name, is_val=False): 328 | print('Loading Synthetic pairs . . .') 329 | load_dict = np.load(tfrecord_name) 330 | 331 | im_src_patch = load_dict['im_src_patch'] 332 | im_dst_patch = load_dict['im_dst_patch'] 333 | homography_src_2_dst = load_dict['homography_src_2_dst'] 334 | homography_dst_2_src = load_dict['homography_dst_2_src'] 335 | 336 | for a,b,c,d in zip(im_src_patch, im_dst_patch, homography_src_2_dst, homography_dst_2_src): 337 | if is_val: 338 | self.validation_data.append([a[np.newaxis, ...],b[np.newaxis, ...],c[np.newaxis, ...],d[np.newaxis, ...]]) 339 | else: 340 | self.training_data.append([a[np.newaxis, ...],b[np.newaxis, ...],c[np.newaxis, ...],d[np.newaxis, ...]]) 341 | 342 | -------------------------------------------------------------------------------- /keyNet/loss/score_loss_function.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import math 5 | 6 | from collections import OrderedDict 7 | 8 | import keyNet.aux.tools as aux 9 | from torchgeometry.core import warp_perspective 10 | 11 | 12 | 13 | # Index Proposal Layer 14 | def ip_layer(scores, window_size, kernels): 15 | 16 | exponential_value = math.e 17 | 18 | shape_scores = scores.shape 19 | 20 | weights = F.max_pool2d(scores.detach(), kernel_size=[window_size, window_size], stride=[window_size, window_size], padding=0) # padding='VALID' 21 | 22 | max_pool_unpool = F.conv_transpose2d(weights, kernels['upsample_filter_np_'+str(window_size)], stride=[window_size, window_size]) 23 | 24 | exp_map_1 = torch.add(torch.pow(exponential_value, torch.div(scores, max_pool_unpool+1e-6)), -1*(1.-1e-6)) 25 | 26 | sum_exp_map_1 = F.conv2d(exp_map_1, kernels['ones_kernel_'+str(window_size)], stride=[window_size, window_size], padding=0) # padding='VALID' 27 | 28 | indexes_map = F.conv2d(exp_map_1, kernels['indexes_kernel_' + str(window_size)], stride=[window_size, window_size], padding=0) 29 | 30 | indexes_map = torch.div(indexes_map, torch.add(sum_exp_map_1, 1e-6)) 31 | 32 | max_weights = torch.amax(weights, keepdims=True, dim=(1, 2, 3)) 33 | 34 | norm_weights = torch.divide(weights, max_weights + 1e-6) 35 | 36 | return indexes_map, [weights, norm_weights] 37 | 38 | def ip_softscores(scores, window_size, kernels): 39 | 40 | exponential_value = math.e 41 | 42 | shape_scores = scores.shape 43 | 44 | weights = F.max_pool2d(scores, kernel_size=[window_size, window_size], stride=[window_size, window_size], padding=0) 45 | 46 | max_pool_unpool = F.conv_transpose2d(weights, kernels['upsample_filter_np_'+str(window_size)], stride=[window_size, window_size]) 47 | 48 | exp_map_1 = torch.add(torch.pow(exponential_value, torch.div(scores, torch.add(max_pool_unpool, 1e-6))), -1*(1. - 1e-6)) 49 | 50 | sum_exp_map_1 = F.conv2d(exp_map_1, kernels['ones_kernel_'+str(window_size)], stride=[window_size, window_size], padding=0) 51 | 52 | sum_scores_map_1 = F.conv2d(exp_map_1*scores, kernels['ones_kernel_'+str(window_size)], stride=[window_size, window_size], padding=0) 53 | 54 | soft_scores = torch.div(sum_scores_map_1, torch.add(sum_exp_map_1, 1e-6)) 55 | 56 | return soft_scores 57 | 58 | 59 | def grid_indexes_nms_conv(scores, kernels, window_size): 60 | 61 | weights, indexes = F.max_pool2d(scores, kernel_size=(window_size, window_size), padding=0, return_indices=True) ## stride is same as kernel_size as default. 62 | 63 | weights_norm = torch.div(weights, torch.add(weights, torch.finfo(float).eps)) 64 | 65 | score_map = F.max_unpool2d(weights_norm, indexes, kernel_size=[window_size, window_size]) 66 | 67 | indexes_label = F.conv2d(score_map, kernels['indexes_kernel_'+str(window_size)], stride=[window_size, window_size], padding=0) 68 | 69 | ind_rand = torch.randint(low=0, high=window_size, size=indexes_label.shape, dtype=torch.int32).to(torch.float32).to(indexes_label.device) 70 | 71 | indexes_label = torch.where((indexes_label == torch.zeros_like(indexes_label)), ind_rand, indexes_label) 72 | 73 | return indexes_label, weights, score_map 74 | 75 | def loss_ln_indexes_norm(src_indexes, label_indexes, weights_indexes, window_size, n=2): 76 | 77 | norm_sq = torch.sum(((src_indexes-label_indexes)/window_size)**n, dim=1, keepdims=True) 78 | weigthed_norm_sq = 1000*(torch.multiply(weights_indexes, norm_sq)) 79 | loss = torch.mean(weigthed_norm_sq) 80 | 81 | return loss 82 | 83 | def msip_loss_function(src_im, src_score_maps, dst_score_maps, window_size, kernels, h_src_2_dst, h_dst_2_src, 84 | coordinate_weighting, patch_size, mask_borders): 85 | 86 | src_maps = F.relu(src_score_maps) 87 | dst_maps = F.relu(dst_score_maps) 88 | 89 | 90 | # Check if patch size is divisible by the window size 91 | if patch_size % window_size > 0: 92 | batch_shape = src_maps.shape 93 | new_size = patch_size - (patch_size % window_size) 94 | src_maps = src_maps[0:batch_shape[0], 0:batch_shape[1], 0:new_size, 0:new_size] 95 | dst_maps = dst_maps[0:batch_shape[0], 0:batch_shape[1], 0:new_size, 0:new_size] 96 | mask_borders = mask_borders[0:batch_shape[0], 0:batch_shape[1], 0:new_size, 0:new_size] 97 | 98 | # Tensorflow inverts homography 99 | warped_output_shape =src_maps.shape[2:] 100 | 101 | ## use this! https://kornia.readthedocs.io/en/v0.1.2/geometric.html 102 | ## Note that warp_perspective function is not inverse warping -> Original warp parameters! as different with tensorflow.image.transform 103 | src_maps_warped = warp_perspective(src_maps * mask_borders, h_src_2_dst, dsize=warped_output_shape) 104 | src_im_warped = warp_perspective(src_im, h_src_2_dst, dsize=warped_output_shape) 105 | dst_maps_warped = warp_perspective(dst_maps * mask_borders, h_dst_2_src, dsize=warped_output_shape) 106 | visible_src_mask = warp_perspective(mask_borders, h_dst_2_src, dsize=warped_output_shape) 107 | visible_dst_mask = warp_perspective(mask_borders, h_src_2_dst, dsize=warped_output_shape) 108 | 109 | # Remove borders and stop gradients to only backpropagate on the unwarped maps 110 | src_maps_warped = src_maps_warped.detach() ## x.detach() 111 | dst_maps_warped = dst_maps_warped.detach() 112 | visible_src_mask = visible_src_mask * mask_borders 113 | visible_dst_mask = visible_dst_mask * mask_borders 114 | 115 | src_maps *= visible_src_mask 116 | dst_maps *= visible_dst_mask 117 | src_maps_warped *= visible_dst_mask 118 | dst_maps_warped *= visible_src_mask 119 | 120 | # Compute visible coordinates to discard uncommon regions 121 | _, weights_visible_src, map_nms = grid_indexes_nms_conv(visible_src_mask, kernels, window_size) 122 | _, weights_visible_dst, _ = grid_indexes_nms_conv(visible_dst_mask, kernels, window_size) 123 | 124 | # Extract NMS coordinates from warped maps 125 | src_indexes_nms_warped, weights_src_warped, _ = grid_indexes_nms_conv(src_maps_warped, kernels, window_size) 126 | dst_indexes_nms_warped, weights_dst_warped, _ = grid_indexes_nms_conv(dst_maps_warped, kernels, window_size) 127 | 128 | # Use IP Layer to extract soft coordinates 129 | src_indexes, _ = ip_layer(src_maps, window_size, kernels) 130 | dst_indexes, _ = ip_layer(dst_maps, window_size, kernels) 131 | 132 | # Compute soft weights 133 | weights_src = ip_softscores(src_maps, window_size, kernels).detach() 134 | weights_dst = ip_softscores(dst_maps, window_size, kernels).detach() 135 | 136 | if coordinate_weighting: 137 | shape = weights_src.shape 138 | 139 | weights_src = torch.flatten(weights_src) 140 | weights_dst = torch.flatten(weights_dst) 141 | 142 | weights_src = F.softmax(weights_src, dim=0) 143 | weights_dst = F.softmax(weights_dst, dim=0) 144 | 145 | weights_src = 100 * weights_visible_src * torch.reshape(weights_src, shape) 146 | weights_dst = 100 * weights_visible_dst * torch.reshape(weights_dst, shape) 147 | else: 148 | weights_src = weights_visible_src 149 | weights_dst = weights_visible_dst 150 | 151 | loss_src = loss_ln_indexes_norm(src_indexes, dst_indexes_nms_warped, weights_src, window_size, n=2) 152 | loss_dst = loss_ln_indexes_norm(dst_indexes, src_indexes_nms_warped, weights_dst, window_size, n=2) 153 | 154 | loss_indexes = (loss_src + loss_dst) / 2. 155 | 156 | loss_elements = {} 157 | loss_elements['src_im'] = src_im 158 | loss_elements['src_im_warped'] = src_im_warped 159 | loss_elements['map_nms'] = map_nms 160 | loss_elements['src_maps'] = src_maps 161 | loss_elements['dst_maps'] = dst_maps 162 | loss_elements['src_maps_warped'] = src_maps_warped 163 | loss_elements['dst_maps_warped'] = dst_maps_warped 164 | loss_elements['weights_src'] = weights_src 165 | loss_elements['weights_src_warped'] = weights_src_warped 166 | loss_elements['weights_visible_src'] = weights_visible_src 167 | loss_elements['weights_dst'] = weights_dst 168 | loss_elements['weights_visible_dst'] = weights_visible_dst 169 | loss_elements['weights_dst_warped'] = weights_dst_warped 170 | loss_elements['src_indexes'] = src_indexes 171 | loss_elements['dst_indexes'] = dst_indexes 172 | loss_elements['dst_indexes_nms_warped'] = dst_indexes_nms_warped 173 | 174 | return loss_indexes, loss_elements -------------------------------------------------------------------------------- /keyNet/model/hardnet_pytorch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 -utt 2 | # -*- coding: utf-8 -*- 3 | import torch 4 | import torch.nn as nn 5 | from torch.autograd import Variable 6 | 7 | class L2Norm(nn.Module): 8 | def __init__(self): 9 | super(L2Norm, self).__init__() 10 | self.eps = 1e-10 11 | 12 | def forward(self, x): 13 | norm = torch.sqrt(torch.sum(x * x, dim=1) + self.eps) 14 | x = x / norm.unsqueeze(-1).expand_as(x) 15 | return x 16 | 17 | 18 | class L1Norm(nn.Module): 19 | def __init__(self): 20 | super(L1Norm, self).__init__() 21 | self.eps = 1e-10 22 | 23 | def forward(self, x): 24 | norm = torch.sum(torch.abs(x), dim=1) + self.eps 25 | x = x / norm.expand_as(x) 26 | return x 27 | 28 | 29 | class HardNet(nn.Module): 30 | """HardNet model definition 31 | """ 32 | 33 | def __init__(self): 34 | super(HardNet, self).__init__() 35 | 36 | self.features = nn.Sequential( 37 | nn.Conv2d(1, 32, kernel_size=3, padding=1, bias=False), 38 | nn.BatchNorm2d(32, affine=False), 39 | nn.ReLU(), 40 | nn.Conv2d(32, 32, kernel_size=3, padding=1, bias=False), 41 | nn.BatchNorm2d(32, affine=False), 42 | nn.ReLU(), 43 | nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=False), 44 | nn.BatchNorm2d(64, affine=False), 45 | nn.ReLU(), 46 | nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False), 47 | nn.BatchNorm2d(64, affine=False), 48 | nn.ReLU(), 49 | nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, bias=False), 50 | nn.BatchNorm2d(128, affine=False), 51 | nn.ReLU(), 52 | nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False), 53 | nn.BatchNorm2d(128, affine=False), 54 | nn.ReLU(), 55 | nn.Dropout(0.1), 56 | nn.Conv2d(128, 128, kernel_size=8, bias=False), 57 | nn.BatchNorm2d(128, affine=False), 58 | 59 | ) 60 | # self.features.apply(weights_init) 61 | 62 | def input_norm(self, x): 63 | flat = x.view(x.size(0), -1) 64 | mp = torch.mean(flat, dim=1) 65 | sp = torch.std(flat, dim=1) + 1e-7 66 | return (x - mp.detach().unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand_as(x)) / sp.detach().unsqueeze( 67 | -1).unsqueeze(-1).unsqueeze(1).expand_as(x) 68 | 69 | def forward(self, input): 70 | x_features = self.features(self.input_norm(input)) 71 | x = x_features.view(x_features.size(0), -1) 72 | return L2Norm()(x) 73 | 74 | -------------------------------------------------------------------------------- /keyNet/model/keynet_architecture.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | from collections import OrderedDict 8 | 9 | def gaussian_multiple_channels(num_channels, sigma): 10 | 11 | r = 2*sigma 12 | size = 2*r+1 13 | size = int(math.ceil(size)) 14 | x = torch.arange(0, size, 1, dtype=torch.float) 15 | y = x.unsqueeze(1) 16 | x0 = y0 = r 17 | 18 | gaussian = torch.exp(-1 * (((x - x0) ** 2 + (y - y0) ** 2) / (2 * (sigma ** 2)))) / ((2 * math.pi * (sigma ** 2))**0.5) 19 | gaussian = gaussian.to(dtype=torch.float32) 20 | 21 | weights = torch.zeros((num_channels, num_channels, size, size), dtype=torch.float32) 22 | for i in range(num_channels): 23 | weights[i, i, :, :] = gaussian 24 | 25 | return weights 26 | 27 | 28 | def ones_multiple_channels(size, num_channels): 29 | 30 | ones = torch.ones((size, size)) 31 | weights = torch.zeros((num_channels, num_channels, size, size), dtype=torch.float32) 32 | 33 | for i in range(num_channels): 34 | weights[i, i, :, :] = ones 35 | 36 | return weights 37 | 38 | def grid_indexes(size): 39 | 40 | weights = torch.zeros((2, 1, size, size), dtype=torch.float32) 41 | 42 | columns = [] 43 | for idx in range(1, 1+size): 44 | columns.append(torch.ones((size))*idx) 45 | columns = torch.stack(columns) 46 | 47 | rows = [] 48 | for idx in range(1, 1+size): 49 | rows.append(torch.tensor(range(1, 1+size))) 50 | rows = torch.stack(rows) 51 | 52 | weights[0, 0, :, :] = columns 53 | weights[1, 0, :, :] = rows 54 | 55 | return weights 56 | 57 | 58 | 59 | def get_kernel_size(factor): 60 | """ 61 | Find the kernel size given the desired factor of upsampling. 62 | """ 63 | return 2 * factor - factor % 2 64 | 65 | def linear_upsample_weights(half_factor, number_of_classes): 66 | """ 67 | Create weights matrix for transposed convolution with linear filter 68 | initialization. 69 | """ 70 | 71 | filter_size = get_kernel_size(half_factor) 72 | 73 | weights = torch.zeros((number_of_classes, 74 | number_of_classes, 75 | filter_size, 76 | filter_size, 77 | ), dtype=torch.float32) 78 | 79 | upsample_kernel = torch.ones((filter_size, filter_size)) 80 | for i in range(number_of_classes): 81 | weights[i, i, :, :] = upsample_kernel 82 | 83 | return weights 84 | 85 | def create_derivatives_kernel(): 86 | # Sobel derivative 3x3 X 87 | kernel_filter_dx_3 = torch.tensor([[-1, 0, 1], 88 | [-2, 0, 2], 89 | [-1, 0, 1]], dtype=torch.float32) 90 | kernel_filter_dx_3 = kernel_filter_dx_3.unsqueeze(0).unsqueeze(0) 91 | 92 | # Sobel derivative 3x3 Y 93 | kernel_filter_dy_3 = torch.tensor([[-1, -2, -1], 94 | [0, 0, 0], 95 | [1, 2, 1]], dtype=torch.float32) 96 | kernel_filter_dy_3 = kernel_filter_dy_3.unsqueeze(0).unsqueeze(0) 97 | 98 | return kernel_filter_dx_3, kernel_filter_dy_3 99 | 100 | 101 | 102 | class keynet(nn.Module): 103 | def __init__(self, args, device, MSIP_sizes=[]): 104 | super(keynet, self).__init__() 105 | self.pyramid_levels = args.num_levels_within_net 106 | self.factor_scaling = args.factor_scaling_pyramid 107 | self.num_blocks = args.num_learnable_blocks 108 | self.num_filters = args.num_filters 109 | self.conv_kernel_size = args.conv_kernel_size 110 | self.ksize = args.nms_size 111 | 112 | self.batch_size = args.batch_size 113 | self.patch_size = args.patch_size 114 | 115 | channel_of_learner_output = 8 * self.pyramid_levels 116 | 117 | # Smooth Gausian Filter 118 | self.gaussian_avg = gaussian_multiple_channels(1, 1.5) 119 | 120 | # Sobel derivatives 121 | kernel_x, kernel_y = create_derivatives_kernel() 122 | self.kernel_filter_dx = kernel_x 123 | self.kernel_filter_dy = kernel_y 124 | 125 | # create_kernels 126 | self.kernels = {} 127 | 128 | if MSIP_sizes != []: 129 | self.create_kernels(MSIP_sizes) 130 | 131 | if 8 not in MSIP_sizes: 132 | self.create_kernels([8]) 133 | 134 | ## learnable modules initialization 135 | modules = [] 136 | ## first layer using derivative inputs 137 | modules.append(('conv_'+str(0), nn.Conv2d(in_channels=10, out_channels=8, kernel_size=self.conv_kernel_size, stride=1, padding=2, bias=True))) 138 | modules.append(('bn_'+str(0), nn.BatchNorm2d(num_features=8))) 139 | modules.append(('relu_'+str(0), nn.ReLU())) 140 | ## next layers 141 | for idx_layer in range(self.num_blocks - 1): 142 | modules.append(('conv_'+str(idx_layer+1), nn.Conv2d(in_channels=8, out_channels=8, kernel_size=self.conv_kernel_size, stride=1, padding=2, bias=True))) 143 | modules.append(('bn_'+str(idx_layer+1), nn.BatchNorm2d(num_features=8))) 144 | modules.append(('relu_'+str(idx_layer+1), nn.ReLU())) 145 | 146 | self.learner = nn.Sequential( OrderedDict(modules) ) 147 | self.last_layer_learner = nn.Sequential(OrderedDict([ 148 | ('bn_last', nn.BatchNorm2d(num_features=channel_of_learner_output)), 149 | # ('conv_last', nn.Conv2d(in_channels=channel_of_learner_output ,out_channels=1, kernel_size=self.conv_kernel_size, stride=1, padding=2, bias=True), ) ## original paper version 150 | ('conv_last', nn.Conv2d(in_channels=channel_of_learner_output ,out_channels=1, kernel_size=1, bias=True), ) 151 | ])) 152 | 153 | 154 | ## Handcrafted kernels to GPU 155 | self.kernel_filter_dx = self.kernel_filter_dx.to(device) 156 | self.kernel_filter_dy = self.kernel_filter_dy.to(device) 157 | self.gaussian_avg = self.gaussian_avg.to(device) 158 | 159 | 160 | def create_kernels(self, MSIP_sizes): 161 | # Grid Indexes for MSIP 162 | for ksize in MSIP_sizes: 163 | 164 | ones_kernel = ones_multiple_channels(ksize, 1) 165 | indexes_kernel = grid_indexes(ksize) 166 | upsample_filter_np = linear_upsample_weights(int(ksize / 2), 1) 167 | 168 | self.ones_kernel = ones_kernel.requires_grad_(False) 169 | self.kernels['ones_kernel_'+str(ksize)] = self.ones_kernel 170 | 171 | self.upsample_filter_np = upsample_filter_np.requires_grad_(False) 172 | self.kernels['upsample_filter_np_'+str(ksize)] = self.upsample_filter_np 173 | 174 | self.indexes_kernel = indexes_kernel.requires_grad_(False) 175 | self.kernels['indexes_kernel_'+str(ksize)] = self.indexes_kernel 176 | 177 | index_size = int(self.patch_size/ksize) 178 | zeros = torch.zeros((self.batch_size, index_size, index_size, 2), dtype=torch.float32) 179 | zeros = zeros.requires_grad_(False) 180 | self.kernels['zeros_ind_kernel_'+str(ksize)] = zeros 181 | 182 | ones = torch.ones((self.batch_size, index_size, index_size, 2), dtype=torch.float32) 183 | ones = ones.requires_grad_(False) 184 | self.kernels['ones_ind_kernel_'+str(ksize)] = ones 185 | 186 | def get_kernels(self, device): 187 | kernels = {} 188 | for k,v in self.kernels.items(): 189 | kernels[k] = v.to(device) 190 | return kernels 191 | 192 | 193 | def forward(self, input_data, train_score=True, H_vector=[], apply_homography = False): 194 | 195 | features, network = self.compute_features(input_data) 196 | 197 | output = self.last_layer_learner(features) 198 | 199 | if apply_homography: 200 | output = self.transform_map(output, H_vector) 201 | 202 | network['input_data'] = input_data 203 | network['features'] = features 204 | network['output'] = output 205 | 206 | return network, output 207 | 208 | 209 | def compute_features(self, input_data): 210 | input_data = self.tensorflowTensor2pytorchTensor(input_data) 211 | 212 | H,W = input_data.shape[2:] 213 | features = [] 214 | network = {} 215 | 216 | for idx_level in range(self.pyramid_levels): 217 | 218 | if idx_level == 0: 219 | input_data_smooth = input_data 220 | else: 221 | ## (7,7) size gaussian kernel. 222 | input_data_smooth = F.conv2d(input_data, self.gaussian_avg, padding=[3,3]) # padding='SAME' 223 | 224 | target_resize = int(H / (self.factor_scaling ** idx_level)), int(W / (self.factor_scaling ** idx_level)) 225 | 226 | input_data_resized = F.interpolate(input_data_smooth, size=target_resize, align_corners=True, mode='bilinear') 227 | 228 | input_data_resized = self.local_norm_image(input_data_resized) 229 | 230 | features_t, network = self.compute_handcrafted_features(input_data_resized, network, idx_level) 231 | 232 | features_t = self.learner(features_t) 233 | 234 | features_t = F.interpolate(features_t,size=(H,W),align_corners=True, mode='bilinear') 235 | 236 | if not idx_level: 237 | features = features_t 238 | else: 239 | features = torch.cat([features, features_t], axis=1) 240 | 241 | return features, network 242 | 243 | def compute_handcrafted_features(self, image, network, idx): 244 | # Sobel_conv_derivativeX 245 | dx = F.conv2d(image, self.kernel_filter_dx, padding=[1,1]) 246 | dxx = F.conv2d(dx, self.kernel_filter_dx, padding=[1,1]) 247 | dx2 = torch.mul(dx,dx) 248 | 249 | # Sobel_conv_derivativeY 250 | dy = F.conv2d(image, self.kernel_filter_dy, padding=[1,1]) 251 | dyy = F.conv2d(dy, self.kernel_filter_dy, padding=[1,1]) 252 | dy2 = torch.mul(dy,dy) 253 | 254 | dxy = F.conv2d(dx, self.kernel_filter_dy, padding=[1,1]) 255 | 256 | dxdy = torch.mul(dx, dy) 257 | dxxdyy = torch.mul(dxx, dyy) 258 | dxy2 = torch.mul(dxy, dxy) 259 | 260 | # Concatenate Handcrafted Features 261 | features_t = torch.cat([dx, dx2, dxx, dy, dy2, dyy, dxdy, dxxdyy, dxy, dxy2], axis=1) 262 | 263 | network['dx_' + str(idx + 1)] = dx 264 | network['dx2_' + str(idx + 1)] = dx2 265 | network['dy_' + str(idx + 1)] = dy 266 | network['dy2_' + str(idx + 1)] = dy2 267 | network['dxdy_' + str(idx + 1)] = dxdy 268 | network['dxxdyy_' + str(idx + 1)] = dxxdyy 269 | network['dxy_' + str(idx + 1)] = dxy 270 | network['dxy2_' + str(idx + 1)] = dxy2 271 | network['dx2dy2_' + str(idx + 1)] = dx2+dy2 272 | 273 | return features_t,network 274 | 275 | def local_norm_image(self, x, k_size=65, eps=1e-10): 276 | pad = int(k_size / 2) 277 | 278 | x_pad = F.pad(x, (pad,pad,pad,pad), mode='reflect') 279 | x_mean = F.avg_pool2d(x_pad, kernel_size=[k_size, k_size], stride=[1, 1], padding=0) ## padding='valid'==0 280 | x2_mean = F.avg_pool2d(torch.pow(x_pad, 2.0), kernel_size=[k_size, k_size], stride=[1, 1], padding=0) 281 | 282 | x_std = (torch.sqrt(torch.abs(x2_mean - x_mean * x_mean)) + eps) 283 | x_norm = (x - x_mean) / (1.+x_std) 284 | 285 | return x_norm 286 | 287 | def tensorflowTensor2pytorchTensor(self,x): 288 | ## input B, H, W, C 289 | ## output B, C, H, W 290 | return x.permute(0,3,1,2) 291 | 292 | def pytorchTensor2tensorflowTensor(self,x): 293 | ## input B, C, H, W 294 | ## output B, H, W, C 295 | return x.permute(0,2,3,1) 296 | 297 | def state_dict(self): 298 | res = OrderedDict() 299 | res['learner'] = self.learner.state_dict() 300 | res['last_layer_learner'] = self.last_layer_learner.state_dict() 301 | return res 302 | 303 | def load_state_dict(self, state_dict): 304 | self.learner.load_state_dict(state_dict['learner']) 305 | self.last_layer_learner.load_state_dict(state_dict['last_layer_learner']) 306 | 307 | def eval(self): 308 | self.learner.eval() 309 | self.last_layer_learner.eval() 310 | 311 | def train(self): 312 | self.learner.train() 313 | self.last_layer_learner.train() -------------------------------------------------------------------------------- /keyNet/pretrained_nets/HardNet++.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bluedream1121/Key.Net_PyTorch/89697ade1876376d27040784a6144e33dd41c39d/keyNet/pretrained_nets/HardNet++.pth -------------------------------------------------------------------------------- /keyNet/pretrained_nets/keyNet.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bluedream1121/Key.Net_PyTorch/89697ade1876376d27040784a6144e33dd41c39d/keyNet/pretrained_nets/keyNet.pt -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import os, math, cv2, sys, time, torch, logging 2 | import keyNet.config as config 3 | import keyNet.aux.tools as aux 4 | from train_utils import training_epochs, check_val_rep, fix_randseed 5 | 6 | ## Load data 7 | from keyNet.datasets.pytorch_dataset import DatasetGeneration, pytorch_dataset 8 | from torch.utils.data import DataLoader 9 | ## Network architecture & loss / optimizer 10 | from keyNet.model.keynet_architecture import keynet 11 | import torch.optim as optim 12 | ## Training loop 13 | from tqdm import tqdm 14 | import time 15 | from keyNet.aux.logger import Logger 16 | 17 | args = config.get_config() 18 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 19 | 20 | # Check directories 21 | aux.check_directory('keyNet/data') 22 | aux.check_directory(args.weights_dir) 23 | aux.check_directory(args.synth_dir) 24 | 25 | Logger.initialize(args) 26 | 27 | # Set random seeds 28 | fix_randseed(args.random_seed) 29 | 30 | print('Start training Key.Net Architecture') 31 | 32 | # Create Dataset 33 | dataset_generation = DatasetGeneration(args.data_dir, args.synth_dir, args.patch_size, args.batch_size, 34 | args.max_angle, args.max_scale, args.max_shearing, args.random_seed, args.is_debugging, args.load_tfrecord) 35 | 36 | training_data = dataset_generation.get_training_data() 37 | validation_data = dataset_generation.get_validation_data() 38 | 39 | dataset_train = pytorch_dataset(training_data, mode='train') 40 | dataset_val = pytorch_dataset(validation_data, mode='val') 41 | 42 | dataloader_train = DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) 43 | dataloader_val = DataLoader(dataset_val, batch_size=1, shuffle=False) 44 | 45 | 46 | ## network model configuration 47 | # MSIP_sizes = [8, 16, 24, 32, 40] 48 | # MSIP_factor_loss = [256.0, 64.0, 16.0, 4.0, 1.0] 49 | MSIP_sizes = [int(i) for i in args.MSIP_sizes.split(",")] 50 | MSIP_factor_loss =[float(i) for i in args.MSIP_factor_loss.split(",")] 51 | 52 | print("MSIP hyperparameters : ", MSIP_sizes, MSIP_factor_loss) 53 | 54 | model = keynet(args, device, MSIP_sizes) 55 | model = model.to(device) ## use GPU 56 | 57 | kernels = model.get_kernels(device) ## with GPU 58 | 59 | if args.resume_training != '': 60 | model.load_state_dict(torch.load(args.resume_training)) ## Load the PyTorch learnable model parameters. 61 | logging.info("Model paramter : ", args.resume_training , " is loaded.") 62 | 63 | ## training configuration 64 | epochs = args.num_epochs 65 | epochs_val = args.epochs_val 66 | 67 | learning_rate = args.init_initial_learning_rate ## 0.5 after 20 epochs / 30 epochs converged. 68 | decay_rate = args.learning_rate_decay_factor 69 | 70 | ## loss function and optimizer. 71 | optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.1) ## weight decay (l2 regularizer) same as keynet paper. 72 | scheduler = optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=decay_rate) ## lr decay same as keynet paper. 73 | 74 | 75 | ## Count the number of learnable parameters. 76 | logging.info("================ List of Learnable model parameters ================ ") 77 | for n,p in model.named_parameters(): 78 | if p.requires_grad: 79 | logging.info("{} {}".format(n, p.data.shape)) 80 | else: 81 | logging.info("\n\n\n None learnable params {} {}".format( n ,p.data.shape)) 82 | model_parameters = filter(lambda p: p.requires_grad, model.parameters()) 83 | params = sum([torch.prod(torch.tensor(p.size())) for p in model_parameters]) 84 | logging.info("The number of learnable parameters : {} ".format(params.data)) 85 | logging.info("==================================================================== ") 86 | 87 | ## Run training 88 | best_rep_s = 0 89 | best_epoch = 0 90 | times_per_epoch = 0 91 | with torch.no_grad(): 92 | keynet_rep_val,_,_,_,_ = check_val_rep(dataloader_val, model, args.nms_size, device, num_points=25) 93 | best_rep_s = keynet_rep_val 94 | best_epoch = -1 95 | logging.info(('\n Epoch -1 : Repeatability Validation: {:.3f}.'.format(keynet_rep_val))) 96 | 97 | ## training loop 98 | for epoch in range(epochs): 99 | training_epochs(epoch, dataloader_train, model, kernels, optimizer, MSIP_sizes, MSIP_factor_loss, args.weight_coordinates, args.patch_size, device) 100 | with torch.no_grad(): 101 | rep_s, rep_m, error_overlap_s, error_overlap_m, possible_matches = check_val_rep(dataloader_val, model, args.nms_size, device, num_points=25) 102 | logging.info(('Epoch {} (Validation) : Repeatability (rep_s): {:.3f}. '.format(epoch, rep_s))) 103 | logging.info('\trep_m : {:.3f}, error_overlap_s : {:.3f}, error_overlap_m : {:.3f}, possible_matches : {:.3f}. \n'\ 104 | .format( rep_m, error_overlap_s, error_overlap_m, possible_matches)) 105 | 106 | if best_rep_s < rep_s: 107 | best_rep_s = rep_s 108 | best_epoch = epoch 109 | Logger.save_model(model, epoch, rep_s) 110 | 111 | if epochs == args.num_epochs_before_decay: 112 | ## Learning rate decay at epoch 20. 113 | scheduler.step() 114 | 115 | print("Best validation repeatability score : {} at epoch {}. ".format(rep_s, best_epoch)) -------------------------------------------------------------------------------- /train_utils.py: -------------------------------------------------------------------------------- 1 | import torch, random, time, cv2, logging 2 | import torch.nn.functional as F 3 | import numpy as np 4 | import keyNet.aux.tools as aux 5 | from tqdm import tqdm 6 | ## Loss function. 7 | from keyNet.loss.score_loss_function import msip_loss_function 8 | ## validation. 9 | import HSequences_bench.tools.geometry_tools as geo_tools 10 | import HSequences_bench.tools.repeatability_tools as rep_tools 11 | 12 | def training_epochs(epoch, dataloader, model, kernels, optimizer, MSIP_sizes, MSIP_factor_loss, weight_coordinates, patch_size, device): 13 | total_loss_avg = [] 14 | 15 | tic = time.time() 16 | iterate = tqdm(enumerate(dataloader), total=len(dataloader), desc="Key.Net Training") 17 | for idx, batch in iterate: 18 | images_src_batch, images_dst_batch, h_src_2_dst_batch, h_dst_2_src_batch = batch 19 | # print(images_src_batch.shape, images_dst_batch.shape, h_src_2_dst_batch.shape, h_dst_2_src_batch.shape) 20 | 21 | images_src_batch, images_dst_batch, h_src_2_dst_batch, h_dst_2_src_batch = \ 22 | images_src_batch.to(device).type(torch.float32), images_dst_batch.to(device).type(torch.float32), h_src_2_dst_batch.to(device), h_dst_2_src_batch.to(device) 23 | network1, output1 = model(images_src_batch) 24 | network2, output2 = model(images_dst_batch) 25 | 26 | src_score_maps = F.relu(output1) 27 | dst_score_maps = F.relu(output2) 28 | 29 | ## border mask 30 | network_input_size = images_src_batch.permute(0,3,1,2).shape # currently, tensorflow to pytorch 31 | input_border_mask = aux.remove_borders(torch.ones(network_input_size), 16).to(images_src_batch.device) ## static value 32 | 33 | ## resize GT (will be removed after PyTorch style tensor). 34 | ones = torch.ones(images_src_batch.shape[0]).unsqueeze(1).to(images_src_batch.device) 35 | h_src_2_dst_batch = torch.cat([h_src_2_dst_batch, ones], dim=1).reshape(-1, 3, 3).type(torch.float32) 36 | h_dst_2_src_batch = torch.cat([h_dst_2_src_batch, ones], dim=1).reshape(-1, 3, 3).type(torch.float32) 37 | 38 | ## Compute loss 39 | MSIP_elements = {} 40 | loss = 0 41 | for MSIP_idx in range(len(MSIP_sizes)): 42 | MSIP_loss, loss_elements = msip_loss_function(images_src_batch.permute(0,3,1,2), src_score_maps, dst_score_maps, 43 | MSIP_sizes[MSIP_idx], kernels, h_src_2_dst_batch, h_dst_2_src_batch, 44 | weight_coordinates, patch_size, input_border_mask 45 | ) 46 | MSIP_level_name = "MSIP_ws_{}".format(MSIP_sizes[MSIP_idx]) 47 | MSIP_elements[MSIP_level_name] = loss_elements 48 | 49 | loss += MSIP_factor_loss[MSIP_idx] * MSIP_loss 50 | # print("MSIP_level_name {} of MSIP_idx {} : {}, {} ".format(MSIP_level_name, MSIP_idx, MSIP_loss, MSIP_factor_loss[MSIP_idx] * MSIP_loss)) ## logging 51 | 52 | total_loss_avg.append(loss) 53 | iterate.set_description("current loss : {:0.4f}, avg loss : {:0.4f}".format(loss, torch.stack(total_loss_avg).mean() )) 54 | 55 | optimizer.zero_grad() 56 | loss.backward() 57 | optimizer.step() 58 | 59 | if idx % 50 == 0: 60 | # post_fix = 'e'+str(epoch)+'idx'+str(idx) 61 | post_fix = "sample" 62 | 63 | deep_src = aux.remove_borders(output1, 16).cpu().detach().numpy() 64 | deep_dst = aux.remove_borders(output2, 16).cpu().detach().numpy() 65 | 66 | cv2.imwrite('keyNet/data/image_dst_' + post_fix + '.png', 255 * images_dst_batch[0,:,:,0].cpu().detach().numpy()) ## Tensorflow style input (B,H,W,C) 67 | cv2.imwrite('keyNet/data/KeyNet_dst_' + post_fix + '.png', 255 * deep_dst[0,0,:,:] / deep_dst[0,0,:,:].max()) 68 | cv2.imwrite('keyNet/data/image_src_' + post_fix + '.png', 255 * images_src_batch[0,:,:,0].cpu().detach().numpy()) 69 | cv2.imwrite('keyNet/data/KeyNet_src_' + post_fix + '.png', 255 * deep_src[0,0,:,:] / deep_src[0,0,:,:].max()) 70 | 71 | toc = time.time() 72 | total_loss_avg = torch.stack(total_loss_avg) 73 | logging.info("Epoch {} (Training). Loss: {:0.4f}. Time per epoch: {}".format(epoch, torch.mean(total_loss_avg), round(toc-tic,4))) 74 | 75 | 76 | def check_val_rep(dataloader, model, nms_size, device, num_points=25): 77 | rep_s = [] 78 | rep_m = [] 79 | error_overlap_s = [] 80 | error_overlap_m = [] 81 | possible_matches = [] 82 | iterate = tqdm(enumerate(dataloader), total=len(dataloader), desc="Key.Net Validation") 83 | 84 | ba = 0; cb =0; dc= 0; ed= 0; fe=0 85 | for _, batch in iterate: 86 | a = time.time() 87 | images_src_batch, images_dst_batch, h_src_2_dst_batch, h_dst_2_src_batch = batch 88 | images_src_batch, images_dst_batch, h_src_2_dst_batch, h_dst_2_src_batch = \ 89 | images_src_batch.to(device).type(torch.float32), images_dst_batch.to(device).type(torch.float32), h_src_2_dst_batch.to(device), h_dst_2_src_batch.to(device) 90 | network1, output1 = model(images_src_batch) 91 | network2, output2 = model(images_dst_batch) 92 | src_score_maps = F.relu(output1) 93 | dst_score_maps = F.relu(output2) 94 | 95 | b = time.time() 96 | # hom = geo_tools.prepare_homography(h_dst_2_src_batch[0]) 97 | # mask_src, mask_dst = geo_tools.create_common_region_masks(hom, images_src_batch[0].shape, images_dst_batch[0].shape) 98 | hom = geo_tools.prepare_homography(h_dst_2_src_batch[0].cpu().numpy()) 99 | mask_src, mask_dst = geo_tools.create_common_region_masks(hom, images_src_batch[0].cpu().numpy().shape, images_dst_batch[0].cpu().numpy().shape) 100 | 101 | c = time.time() 102 | src_scores = src_score_maps 103 | dst_scores = dst_score_maps 104 | # Apply NMS 105 | src_scores = rep_tools.apply_nms(src_scores[0, 0, :, :].cpu().numpy(), nms_size) 106 | dst_scores = rep_tools.apply_nms(dst_scores[0, 0, :, :].cpu().numpy(), nms_size) 107 | 108 | src_scores = np.multiply(src_scores, mask_src) 109 | dst_scores = np.multiply(dst_scores, mask_dst) 110 | 111 | d = time.time() 112 | 113 | src_pts = geo_tools.get_point_coordinates(src_scores, num_points=num_points, order_coord='xysr') 114 | dst_pts = geo_tools.get_point_coordinates(dst_scores, num_points=num_points, order_coord='xysr') 115 | 116 | dst_to_src_pts = geo_tools.apply_homography_to_points(dst_pts, hom) 117 | 118 | e = time.time() 119 | 120 | repeatability_results = rep_tools.compute_repeatability(src_pts, dst_to_src_pts) 121 | 122 | rep_s.append(repeatability_results['rep_single_scale']) 123 | rep_m.append(repeatability_results['rep_multi_scale']) 124 | error_overlap_s.append(repeatability_results['error_overlap_single_scale']) 125 | error_overlap_m.append(repeatability_results['error_overlap_multi_scale']) 126 | possible_matches.append(repeatability_results['possible_matches']) 127 | 128 | f = time.time() 129 | 130 | ## time count 131 | ba += b-a 132 | cb += c-b 133 | dc += d-c 134 | ed += e-d 135 | fe += f-e 136 | 137 | iterate.set_description("Key.Net Validation time {:0.3f} {:0.3f} {:0.3f} {:0.3f} {:0.3f}".format(ba, cb, dc, ed, fe )) 138 | 139 | return np.asarray(rep_s).mean(), np.asarray(rep_m).mean(), np.asarray(error_overlap_s).mean(),\ 140 | np.asarray(error_overlap_m).mean(), np.asarray(possible_matches).mean() 141 | 142 | 143 | def fix_randseed(randseed): 144 | r"""Fix random seed""" 145 | random.seed(randseed) 146 | np.random.seed(randseed) 147 | torch.manual_seed(randseed) 148 | torch.cuda.manual_seed(randseed) 149 | torch.cuda.manual_seed_all(randseed) 150 | torch.backends.cudnn.benchmark = False 151 | torch.backends.cudnn.deterministic = True --------------------------------------------------------------------------------