├── configs ├── single_frame.ini ├── skip1step2.ini ├── skip3step2.ini ├── skip5step2.ini └── skip7step2.ini ├── data ├── cambridge_init │ ├── shopfacade_coords_mean.txt │ ├── greatcourt_coords_mean.txt │ ├── kingscollege_coords_mean.txt │ ├── oldhospital_coords_mean.txt │ └── stmaryschurch_coords_mean.txt └── 7scenes_init │ ├── 7scenes_fire │ └── coords_mean.txt │ ├── 7scenes_heads │ └── coords_mean.txt │ ├── 7scenes_chess │ └── coords_mean.txt │ ├── 7scenes_office │ └── coords_mean.txt │ ├── 7scenes_pumpkin │ └── coords_mean.txt │ ├── 7scenes_stairs │ └── coords_mean.txt │ └── 7scenes_redkitchen │ └── coords_mean.txt ├── assets ├── teaser.png └── framework.png ├── .gitignore ├── tools └── camera_operator.py ├── LICENSE ├── evaluator └── basic_metric.py ├── optimizer.py ├── dsacstar ├── setup.py ├── thread_rand.cpp ├── dsacstar_types.h ├── stop_watch.h ├── thread_rand.h └── dsacstar_loss.h ├── plot_per_frame ├── results │ ├── MST_shopfacade.txt │ ├── DLTe2e_shopfacade.txt │ ├── DSACSTAR_shopfacade.txt │ ├── DLTe2e_dsacstar_shopfacade.txt │ ├── DLTe2e_ref_shopfacade.txt │ ├── MST_oldhospital.txt │ ├── DLTe2e_oldhospital.txt │ ├── DSACSTAR_oldhospital.txt │ ├── DLTe2e_ref_oldhospital.txt │ ├── DLTe2e_dsacstar_oldhospital.txt │ ├── MST_kingscollege.txt │ ├── DSACSTAR_kingscollege.txt │ ├── DLTe2e_dsacstar_kingscollege.txt │ ├── DLTe2e_kingscollege.txt │ ├── DLTe2e_ref_kingscollege.txt │ └── MST_stmaryschurch.txt └── plotting.py ├── Dense_Nets ├── Super_GNN.py ├── ESAC.py └── ESAC_DROID.py ├── models ├── Dense_networks.py ├── Dense_networks_e2e_GNN_for_unsup.py ├── Dense_networks_e2e.py └── Dense_networks_e2e_GNN.py ├── logger.py ├── banet_track └── ba_module.py ├── README.md ├── README_TRAIN.md ├── datasets ├── sevenscenes.py ├── sevenscenes_for_unsup.py └── cambridge.py └── reloc_pipeline └── utils_func.py /configs/single_frame.ini: -------------------------------------------------------------------------------- 1 | [hyperparameters] 2 | 3 | max_grad_norm = 0 4 | -------------------------------------------------------------------------------- /data/cambridge_init/shopfacade_coords_mean.txt: -------------------------------------------------------------------------------- 1 | 7.8223 5.0106 4.0047 2 | -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_fire/coords_mean.txt: -------------------------------------------------------------------------------- 1 | -0.372936 0.202488 1.716795 2 | -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_heads/coords_mean.txt: -------------------------------------------------------------------------------- 1 | 0.099536 0.143322 0.811527 2 | -------------------------------------------------------------------------------- /data/cambridge_init/greatcourt_coords_mean.txt: -------------------------------------------------------------------------------- 1 | 99.2214 51.9184 6.0653 2 | -------------------------------------------------------------------------------- /data/cambridge_init/kingscollege_coords_mean.txt: -------------------------------------------------------------------------------- 1 | 16.1123 -6.0560 6.9056 2 | -------------------------------------------------------------------------------- /data/cambridge_init/oldhospital_coords_mean.txt: -------------------------------------------------------------------------------- 1 | 9.9827 -6.3675 32.3213 2 | -------------------------------------------------------------------------------- /data/cambridge_init/stmaryschurch_coords_mean.txt: -------------------------------------------------------------------------------- 1 | 10.1952 -5.8895 17.4761 2 | -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_chess/coords_mean.txt: -------------------------------------------------------------------------------- 1 | -0.006378 -0.158068 1.608667 2 | -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_office/coords_mean.txt: -------------------------------------------------------------------------------- 1 | 0.214342 -0.421638 2.137722 2 | -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_pumpkin/coords_mean.txt: -------------------------------------------------------------------------------- 1 | -0.173342 -0.787906 2.342458 2 | -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_stairs/coords_mean.txt: -------------------------------------------------------------------------------- 1 | -0.000601 -0.170128 1.777769 2 | -------------------------------------------------------------------------------- /assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XinWu98/SC-wLS/HEAD/assets/teaser.png -------------------------------------------------------------------------------- /data/7scenes_init/7scenes_redkitchen/coords_mean.txt: -------------------------------------------------------------------------------- 1 | -0.681654 -0.491324 2.455781 2 | -------------------------------------------------------------------------------- /assets/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XinWu98/SC-wLS/HEAD/assets/framework.png -------------------------------------------------------------------------------- /configs/skip1step2.ini: -------------------------------------------------------------------------------- 1 | [hyperparameters] 2 | 3 | max_grad_norm = 0 4 | 5 | skip = 1 6 | variable_skip = no 7 | steps = 2 8 | -------------------------------------------------------------------------------- /configs/skip3step2.ini: -------------------------------------------------------------------------------- 1 | [hyperparameters] 2 | 3 | max_grad_norm = 0 4 | 5 | skip = 3 6 | variable_skip = no 7 | steps = 2 8 | -------------------------------------------------------------------------------- /configs/skip5step2.ini: -------------------------------------------------------------------------------- 1 | [hyperparameters] 2 | 3 | max_grad_norm = 0 4 | 5 | skip = 5 6 | variable_skip = no 7 | steps = 2 8 | -------------------------------------------------------------------------------- /configs/skip7step2.ini: -------------------------------------------------------------------------------- 1 | [hyperparameters] 2 | 3 | max_grad_norm = 0 4 | 5 | skip = 7 6 | variable_skip = no 7 | steps = 2 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /7scenes_e2e_model/* 2 | /7scenes_unsup_model/* 3 | /cambridge_e2e_model/* 4 | /cambridge_unsup_model/* 5 | 6 | __pycache__/ 7 | *.pkl 8 | 9 | removed_file/* -------------------------------------------------------------------------------- /tools/camera_operator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | def camera_pose_inv(R, t): 5 | """ 6 | Compute the inverse pose 7 | """ 8 | Rinv = R.transpose() 9 | Ov = - np.dot(Rinv, t) 10 | Tinv = np.eye(4, dtype=np.float32) 11 | Tinv[:3, :3] = Rinv 12 | Tinv[:3, 3] = Ov 13 | return Tinv[:3, :] 14 | 15 | def torch_camera_pose_inv(R, t): 16 | """ 17 | Compute the inverse pose 18 | """ 19 | Rinv = torch.transpose(R,1,0) 20 | Ov = - torch.mm(Rinv, t) 21 | Tinv = torch.empty([3,4]) 22 | Tinv[:3, :3] = Rinv 23 | Tinv[:3, 3:4] = Ov 24 | return Tinv -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 wuxin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /evaluator/basic_metric.py: -------------------------------------------------------------------------------- 1 | # Note: Evaluation code from DeepTAM https://github.com/lmb-freiburg/deeptam 2 | import numpy as np 3 | import core_math.transfom as trans 4 | import math 5 | import cv2 6 | 7 | def rel_rot_quaternion_deg(q1, q2): 8 | """ 9 | Compute relative error (deg) of two quaternion 10 | :param q1: quaternion 1, (w, x, y, z), dim: (4) 11 | :param q2: quaternion 2, (w, x, y, z), dim: (4) 12 | :return: relative angle in deg 13 | """ 14 | return 2 * 180 * np.arccos(np.clip(np.dot(q1, q2), -1.0, 1.0)) / np.pi 15 | 16 | 17 | def rel_rot_angle(T1, T2): 18 | R1 = T1[:3, :3] 19 | R2 = T2[:3, :3] 20 | q1 = trans.quaternion_from_matrix(R1) 21 | q2 = trans.quaternion_from_matrix(R2) 22 | return rel_rot_quaternion_deg(q1, q2) 23 | 24 | def rel_R(T1, T2): 25 | R1 = T1[:3, :3] 26 | R2 = T2[:3, :3] 27 | r_err = np.matmul(R1, np.transpose(R2)) 28 | r_err = cv2.Rodrigues(r_err)[0] 29 | r_err = np.linalg.norm(r_err) * 180 / math.pi 30 | 31 | return r_err 32 | 33 | 34 | def rel_distance(T1, T2): 35 | R1 = T1[:3, :3] 36 | R2 = T2[:3, :3] 37 | t1 = T1[:3, 3] 38 | t2 = T2[:3, 3] 39 | d = np.dot(R1.T, t1) - np.dot(R2.T, t2) 40 | return np.linalg.norm(d) 41 | 42 | def rel_t(T1, T2): 43 | t1 = T1[:3, 3] 44 | t2 = T2[:3, 3] 45 | return np.linalg.norm(t1 - t2) -------------------------------------------------------------------------------- /optimizer.py: -------------------------------------------------------------------------------- 1 | import torch.optim as optim 2 | 3 | class Optimizer: 4 | """ 5 | Wrapper around torch.optim + learning rate 6 | """ 7 | def __init__(self, params, method, base_lr, weight_decay, **kwargs): 8 | self.method = method 9 | self.base_lr = base_lr 10 | 11 | if self.method == 'sgd': 12 | self.learner = optim.SGD(params, lr=self.base_lr, 13 | weight_decay=weight_decay, **kwargs) 14 | elif self.method == 'adam': 15 | self.learner = optim.Adam(params, lr=self.base_lr, 16 | weight_decay=weight_decay, **kwargs) 17 | elif self.method == 'rmsprop': 18 | self.learner = optim.RMSprop(params, lr=self.base_lr, 19 | weight_decay=weight_decay, **kwargs) 20 | elif self.method == 'Adadelta': 21 | self.learner = optim.Adadelta(params, lr=self.base_lr, weight_decay=weight_decay) 22 | else: 23 | print("No such optimizier!") 24 | 25 | def adjust_lr(self, epoch): # for SGD 26 | if self.method != 'sgd': 27 | return self.base_lr 28 | 29 | decay_factor = 1 30 | for s in self.lr_stepvalues: 31 | if epoch < s: 32 | break 33 | decay_factor *= self.lr_decay 34 | 35 | lr = self.base_lr * decay_factor 36 | 37 | for param_group in self.learner.param_groups: 38 | param_group['lr'] = lr 39 | 40 | return lr 41 | 42 | def mult_lr(self, f): 43 | for param_group in self.learner.param_groups: 44 | param_group['lr'] *= f -------------------------------------------------------------------------------- /dsacstar/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from torch.utils.cpp_extension import CppExtension, BuildExtension 3 | import os 4 | 5 | # opencv_inc_dir = '/usr/local/include' # directory containing OpenCV header files 6 | # opencv_lib_dir = '/usr/local/lib' # directory containing OpenCV library files 7 | 8 | # conda, must have opencv in conda_env/include 9 | #opencv_inc_dir = '' 10 | #opencv_lib_dir = '' 11 | 12 | opencv_inc_dir = '/usr/local/opencv3.4/include' # directory containing OpenCV header files 13 | opencv_lib_dir = '/usr/local/opencv3.4/lib' # directory containing OpenCV library files 14 | 15 | 16 | # if not explicitly provided, we try to locate OpenCV in the current Conda environment 17 | conda_env = os.environ['CONDA_PREFIX'] 18 | 19 | if len(conda_env) > 0 and len(opencv_inc_dir) == 0 and len(opencv_lib_dir) == 0: 20 | print("Detected active conda environment:", conda_env) 21 | 22 | opencv_inc_dir = conda_env + '/include' 23 | opencv_lib_dir = conda_env + '/lib' 24 | 25 | print("Assuming OpenCV dependencies in:") 26 | print(opencv_inc_dir) 27 | print(opencv_lib_dir) 28 | 29 | if len(opencv_inc_dir) == 0: 30 | print("Error: You have to provide an OpenCV include directory. Edit this file.") 31 | exit() 32 | if len(opencv_lib_dir) == 0: 33 | print("Error: You have to provide an OpenCV library directory. Edit this file.") 34 | exit() 35 | 36 | setup( 37 | name='dsacstar', 38 | ext_modules=[CppExtension( 39 | name='dsacstar', 40 | sources=['dsacstar.cpp','thread_rand.cpp'], 41 | include_dirs=[opencv_inc_dir], 42 | library_dirs=[opencv_lib_dir], 43 | libraries=['opencv_core','opencv_calib3d'], 44 | extra_compile_args=['-fopenmp'] 45 | )], 46 | cmdclass={'build_ext': BuildExtension}) 47 | -------------------------------------------------------------------------------- /dsacstar/thread_rand.cpp: -------------------------------------------------------------------------------- 1 | #include "thread_rand.h" 2 | #include 3 | 4 | std::vector ThreadRand::generators; 5 | bool ThreadRand::initialised = false; 6 | 7 | void ThreadRand::forceInit(unsigned seed) 8 | { 9 | initialised = false; 10 | init(seed); 11 | } 12 | 13 | void ThreadRand::init(unsigned seed) 14 | { 15 | #pragma omp critical 16 | { 17 | if(!initialised) 18 | { 19 | unsigned nThreads = omp_get_max_threads(); 20 | 21 | for(unsigned i = 0; i < nThreads; i++) 22 | { 23 | generators.push_back(std::mt19937()); 24 | generators[i].seed(i+seed); 25 | } 26 | 27 | initialised = true; 28 | } 29 | } 30 | } 31 | 32 | int ThreadRand::irand(int min, int max, int tid) 33 | { 34 | std::uniform_int_distribution dist(min, max); 35 | 36 | unsigned threadID = omp_get_thread_num(); 37 | if(tid >= 0) threadID = tid; 38 | 39 | if(!initialised) init(); 40 | 41 | return dist(ThreadRand::generators[threadID]); 42 | } 43 | 44 | double ThreadRand::drand(double min, double max, int tid) 45 | { 46 | std::uniform_real_distribution dist(min, max); 47 | 48 | unsigned threadID = omp_get_thread_num(); 49 | if(tid >= 0) threadID = tid; 50 | 51 | if(!initialised) init(); 52 | 53 | return dist(ThreadRand::generators[threadID]); 54 | } 55 | 56 | double ThreadRand::dgauss(double mean, double stdDev, int tid) 57 | { 58 | std::normal_distribution dist(mean, stdDev); 59 | 60 | unsigned threadID = omp_get_thread_num(); 61 | if(tid >= 0) threadID = tid; 62 | 63 | if(!initialised) init(); 64 | 65 | return dist(ThreadRand::generators[threadID]); 66 | } 67 | 68 | int irand(int incMin, int excMax, int tid) 69 | { 70 | return ThreadRand::irand(incMin, excMax - 1, tid); 71 | } 72 | 73 | double drand(double incMin, double incMax,int tid) 74 | { 75 | return ThreadRand::drand(incMin, incMax, tid); 76 | } 77 | 78 | int igauss(int mean, int stdDev, int tid) 79 | { 80 | return (int) ThreadRand::dgauss(mean, stdDev, tid); 81 | } 82 | 83 | double dgauss(double mean, double stdDev, int tid) 84 | { 85 | return ThreadRand::dgauss(mean, stdDev, tid); 86 | } -------------------------------------------------------------------------------- /plot_per_frame/results/MST_shopfacade.txt: -------------------------------------------------------------------------------- 1 | 0.491155 2.503278 2 | 0.371728 4.651484 3 | 0.455858 3.137205 4 | 0.569942 4.062871 5 | 0.369061 5.288981 6 | 0.122287 4.329111 7 | 0.277527 5.576082 8 | 0.404994 4.258184 9 | 0.826454 1.471901 10 | 0.541960 3.408893 11 | 0.345519 2.952625 12 | 0.260411 6.813945 13 | 0.367614 4.001521 14 | 0.232824 2.861889 15 | 0.872378 2.839924 16 | 1.562843 6.576262 17 | 1.482468 7.018895 18 | 0.848992 4.820431 19 | 1.152538 2.918762 20 | 1.773562 1.952371 21 | 1.571882 nan 22 | 1.137731 2.844331 23 | 1.119359 2.619705 24 | 1.126811 5.454847 25 | 1.537320 4.824652 26 | 1.765052 2.523831 27 | 1.649585 1.624576 28 | 1.619706 2.095464 29 | 1.740567 3.597706 30 | 1.407404 1.545568 31 | 0.799342 nan 32 | 3.012582 11.371234 33 | 3.885919 2.321242 34 | 2.597555 0.993857 35 | 3.963696 2.871174 36 | 5.460175 2.963475 37 | 0.609668 3.227964 38 | 0.409613 1.311026 39 | 0.218103 2.105525 40 | 0.303084 1.324095 41 | 0.341419 nan 42 | 0.710537 3.445893 43 | 0.967712 1.578142 44 | 1.044059 3.844882 45 | 0.951802 8.412525 46 | 0.858883 5.708060 47 | 0.855750 8.550701 48 | 0.913291 5.949465 49 | 1.101232 8.119596 50 | 1.767922 12.636585 51 | 1.668229 27.602278 52 | 4.368564 5.162680 53 | 6.004450 16.955153 54 | 1.449003 4.765383 55 | 1.035764 4.384634 56 | 0.825280 3.710393 57 | 0.947653 10.639953 58 | 0.904122 7.207610 59 | 0.862407 10.527609 60 | 0.835435 5.909451 61 | 0.751759 4.407784 62 | 0.754464 4.502848 63 | 0.549711 4.071726 64 | 0.525929 4.062679 65 | 0.312007 7.793032 66 | 0.580240 6.725448 67 | 0.347348 4.387133 68 | 0.365862 3.822216 69 | 0.168254 3.818527 70 | 0.213027 2.672361 71 | 0.242024 0.679547 72 | 0.114620 5.278460 73 | 0.511468 4.300261 74 | 0.506232 2.693368 75 | 1.170361 4.948659 76 | 0.834950 3.612472 77 | 0.285744 3.155864 78 | 1.018208 1.800972 79 | 0.729048 3.520951 80 | 1.479885 2.858331 81 | 1.004022 nan 82 | 2.449851 3.986233 83 | 1.771624 0.509756 84 | 1.043497 2.236053 85 | 1.081713 0.631798 86 | 0.637362 nan 87 | 1.461871 2.839924 88 | 1.316265 nan 89 | 1.552123 2.769878 90 | 1.196624 3.768173 91 | 1.680136 2.586629 92 | 1.481587 0.783342 93 | 1.537460 2.130654 94 | 1.863843 3.444075 95 | 1.448322 2.156942 96 | 1.332815 3.611171 97 | 0.532130 2.451780 98 | 0.359696 2.956599 99 | 0.095426 2.172851 100 | 0.658105 3.346084 101 | 0.113895 2.887214 102 | 0.372107 1.764531 103 | 0.645020 2.281105 104 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_shopfacade.txt: -------------------------------------------------------------------------------- 1 | 0.065271 0.927157 2 | 0.135466 0.362733 3 | 0.071048 0.861215 4 | 0.083704 0.513196 5 | 0.278896 1.436439 6 | 0.251982 1.371779 7 | 0.109954 3.249622 8 | 0.280456 6.972908 9 | 0.180090 1.261819 10 | 0.445211 0.527560 11 | 0.325861 1.246526 12 | 0.649196 1.109194 13 | 0.449412 1.117024 14 | 0.102206 0.678506 15 | 0.064202 0.869868 16 | 0.447977 1.741958 17 | 0.485445 1.788424 18 | 0.099684 1.065458 19 | 0.091096 0.633224 20 | 0.076748 0.589414 21 | 0.058287 0.833374 22 | 0.161127 0.308270 23 | 0.683441 1.213317 24 | 1.174940 0.815421 25 | 2.034875 0.775329 26 | 2.002770 2.808465 27 | 2.074479 3.961458 28 | 2.341830 6.731510 29 | 2.079826 6.464982 30 | 1.425375 5.945846 31 | 0.839679 1.880174 32 | 2.471097 90.000679 33 | 2.778142 3.322965 34 | 2.545640 1.362665 35 | 5.050936 90.000003 36 | 4.587951 1.743198 37 | 0.078917 0.429199 38 | 0.126489 1.011139 39 | 0.122447 0.768962 40 | 0.086597 0.265004 41 | 0.066967 0.337295 42 | 0.193438 0.292073 43 | 0.097873 0.313626 44 | 0.059298 0.276444 45 | 0.107543 0.425554 46 | 0.038813 0.260321 47 | 0.066320 0.547950 48 | 0.037934 0.185617 49 | 0.050428 0.628940 50 | 0.052412 0.956766 51 | 0.077291 0.425977 52 | 0.028769 0.899844 53 | 0.102989 1.027999 54 | 0.142629 0.794901 55 | 0.123789 0.560573 56 | 0.110857 0.458451 57 | 0.151276 1.105001 58 | 0.101708 0.637416 59 | 0.042726 0.873241 60 | 0.105351 0.338143 61 | 0.220070 0.665925 62 | 0.057207 0.301952 63 | 0.157858 0.756251 64 | 0.069390 0.118679 65 | 0.183630 0.730588 66 | 0.058824 0.354836 67 | 0.074607 0.439640 68 | 0.046578 0.494404 69 | 0.039475 0.373740 70 | 0.059305 0.540266 71 | 0.039291 0.603514 72 | 0.071776 0.680095 73 | 0.069073 0.660959 74 | 0.103242 0.625056 75 | 0.112324 1.406365 76 | 0.056818 0.351090 77 | 0.224722 0.421556 78 | 0.036457 0.055946 79 | 0.101940 0.267734 80 | 0.144851 0.391201 81 | 0.117291 0.295533 82 | 0.136913 0.470979 83 | 0.156891 1.115074 84 | 0.079027 0.559952 85 | 0.027687 0.369363 86 | 0.175464 0.609926 87 | 0.008866 0.634503 88 | 0.076640 1.029681 89 | 0.018637 0.387184 90 | 0.119917 1.337694 91 | 0.223059 1.573809 92 | 0.053897 0.414463 93 | 0.037011 0.680283 94 | 0.054223 0.877237 95 | 0.112034 1.176365 96 | 0.069534 0.355590 97 | 0.093229 0.256993 98 | 0.102901 0.630198 99 | 0.232927 0.856112 100 | 0.200951 0.631390 101 | 0.215456 0.372962 102 | 0.161223 0.747363 103 | 0.215740 0.590463 104 | -------------------------------------------------------------------------------- /plot_per_frame/results/DSACSTAR_shopfacade.txt: -------------------------------------------------------------------------------- 1 | 0.022893 0.170437 2 | 0.057710 0.196276 3 | 0.054460 0.216529 4 | 0.038771 0.092994 5 | 0.052281 0.154133 6 | 0.051234 0.221789 7 | 0.044625 0.133038 8 | 0.064494 0.178150 9 | 0.020219 0.254141 10 | 0.058937 0.231409 11 | 0.049728 0.228271 12 | 0.048076 0.139347 13 | 0.020362 0.273155 14 | 0.045458 0.228678 15 | 0.024371 0.167223 16 | 0.052900 0.307750 17 | 0.034007 0.234358 18 | 0.041289 0.225281 19 | 0.055803 0.258962 20 | 0.049335 0.181814 21 | 0.066365 0.144636 22 | 0.106463 0.457858 23 | 0.089427 0.311804 24 | 0.203089 0.832540 25 | 0.176476 0.705463 26 | 0.526967 2.315679 27 | 0.520673 1.725972 28 | 0.460318 1.695690 29 | 0.346989 1.392222 30 | 0.243634 0.974942 31 | 0.026891 0.174562 32 | 0.149699 0.561710 33 | 0.138994 0.746293 34 | 0.148431 0.867364 35 | 0.280876 1.452823 36 | 0.199166 0.818043 37 | 0.057261 0.339283 38 | 0.074544 0.337480 39 | 0.035908 0.301875 40 | 0.069409 0.162592 41 | 0.085785 0.214685 42 | 0.067317 0.132153 43 | 0.052380 0.054361 44 | 0.064638 0.166098 45 | 0.034833 0.169265 46 | 0.063204 0.120696 47 | 0.070316 0.350614 48 | 0.038397 0.244860 49 | 0.079993 0.363948 50 | 0.109027 0.449885 51 | 0.074357 0.381328 52 | 0.053240 0.212294 53 | 0.116064 0.896979 54 | 0.036853 0.221649 55 | 0.033200 0.256809 56 | 0.044273 0.424310 57 | 0.040017 0.362091 58 | 0.036368 0.329016 59 | 0.030411 0.241083 60 | 0.009683 0.147799 61 | 0.053593 0.108847 62 | 0.008489 0.100205 63 | 0.046857 0.241198 64 | 0.021019 0.096454 65 | 0.172206 0.520962 66 | 0.050888 0.316314 67 | 0.048878 0.439421 68 | 0.019717 0.138418 69 | 0.034465 0.153676 70 | 0.027295 0.132060 71 | 0.030058 0.126923 72 | 0.034631 0.339711 73 | 0.048820 0.230027 74 | 0.080259 0.383659 75 | 0.322896 2.146118 76 | 0.102164 0.830274 77 | 0.055951 0.269233 78 | 0.013580 0.124468 79 | 0.041816 0.325750 80 | 0.024858 0.182582 81 | 0.075811 0.462445 82 | 0.069874 0.349833 83 | 0.129140 0.711600 84 | 0.077252 0.502881 85 | 0.026469 0.183729 86 | 0.049810 0.325278 87 | 0.075480 0.543538 88 | 0.125724 0.942119 89 | 0.054674 0.358143 90 | 0.081144 0.606383 91 | 0.292455 2.339256 92 | 0.060930 0.464070 93 | 0.017764 0.186438 94 | 0.027925 0.137686 95 | 0.037546 0.142669 96 | 0.016234 0.143054 97 | 0.047377 0.218074 98 | 0.025757 0.179786 99 | 0.038848 0.267303 100 | 0.017503 0.131997 101 | 0.027827 0.187775 102 | 0.046996 0.284398 103 | 0.063915 0.205654 104 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_dsacstar_shopfacade.txt: -------------------------------------------------------------------------------- 1 | 0.045647 0.163206 2 | 0.046379 0.229487 3 | 0.032846 0.176864 4 | 0.043971 0.281134 5 | 0.027267 0.242392 6 | 0.063406 0.356345 7 | 0.072161 0.203170 8 | 0.025418 0.121729 9 | 0.014384 0.182819 10 | 0.079252 0.293283 11 | 0.053259 0.187010 12 | 0.025360 0.158487 13 | 0.081629 0.463681 14 | 0.067824 0.237782 15 | 0.078167 0.373515 16 | 0.100322 0.451837 17 | 0.149292 0.710486 18 | 0.096461 0.465763 19 | 0.112840 0.533401 20 | 0.075344 0.257490 21 | 0.085011 0.165294 22 | 0.095611 0.336542 23 | 0.072018 0.287769 24 | 0.162480 0.612855 25 | 0.217747 0.872248 26 | 0.913166 3.701994 27 | 0.379319 1.433308 28 | 0.395091 1.548273 29 | 0.440539 1.791800 30 | 0.501410 2.132267 31 | 0.213848 0.894400 32 | 0.162621 0.748582 33 | 0.128122 0.614370 34 | 0.282756 1.367002 35 | 0.212089 1.003748 36 | 0.105331 0.483895 37 | 0.101923 0.193815 38 | 0.118662 0.089016 39 | 0.113533 0.157214 40 | 0.123298 0.260437 41 | 0.107200 0.274441 42 | 0.059009 0.199514 43 | 0.071368 0.157038 44 | 0.055460 0.208259 45 | 0.038956 0.259036 46 | 0.041974 0.166976 47 | 0.083688 0.439861 48 | 0.051134 0.199160 49 | 0.078711 0.323057 50 | 0.134959 0.439117 51 | 0.104257 0.485301 52 | 0.060097 0.176466 53 | 0.150015 1.162315 54 | 0.065875 0.420572 55 | 0.036708 0.226209 56 | 0.019344 0.157866 57 | 0.077239 0.701662 58 | 0.041176 0.393706 59 | 0.051726 0.303588 60 | 0.057265 0.234788 61 | 0.066691 0.176689 62 | 0.018128 0.220755 63 | 0.046544 0.081308 64 | 0.013973 0.085218 65 | 0.163968 0.503794 66 | 0.047404 0.325274 67 | 0.059539 0.290203 68 | 0.033949 0.151600 69 | 0.050031 0.224228 70 | 0.067731 0.348202 71 | 0.069454 0.352160 72 | 0.045066 0.323282 73 | 0.086223 0.337506 74 | 0.066852 0.315788 75 | 0.067992 0.275617 76 | 0.049666 0.205756 77 | 0.039532 0.217503 78 | 0.026977 0.111215 79 | 0.100715 0.671943 80 | 0.061996 0.371117 81 | 0.052644 0.211600 82 | 0.082103 0.454362 83 | 0.042048 0.157660 84 | 0.043157 0.257774 85 | 0.055765 0.320229 86 | 0.018111 0.163007 87 | 0.018516 0.116015 88 | 0.058962 0.460624 89 | 0.040766 0.293599 90 | 0.062430 0.400826 91 | 0.128126 0.865584 92 | 0.030539 0.176804 93 | 0.015217 0.104094 94 | 0.040701 0.315924 95 | 0.020001 0.257554 96 | 0.066836 0.373771 97 | 0.068213 0.398529 98 | 0.035238 0.133706 99 | 0.041756 0.310748 100 | 0.077698 0.331218 101 | 0.060474 0.165367 102 | 0.029963 0.343626 103 | 0.027001 0.193460 104 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_ref_shopfacade.txt: -------------------------------------------------------------------------------- 1 | 0.043954 0.166665 2 | 0.036801 0.213825 3 | 0.030960 0.175190 4 | 0.034848 0.267038 5 | 0.028494 0.216363 6 | 0.035836 0.357818 7 | 0.051808 0.203716 8 | 0.277266 6.948561 9 | 0.022913 0.217974 10 | 0.028696 0.278461 11 | 0.043869 0.185683 12 | 0.013708 0.158223 13 | 0.040368 0.462742 14 | 0.033415 0.237782 15 | 0.028904 0.394285 16 | 0.087181 0.487545 17 | 0.080160 0.675258 18 | 0.058575 0.443861 19 | 0.075484 0.518465 20 | 0.067518 0.257490 21 | 0.085736 0.171727 22 | 0.074460 0.305967 23 | 0.024788 0.331810 24 | 0.057047 0.598079 25 | 0.043246 0.814757 26 | 0.323776 2.262497 27 | 4.276722 8.177869 28 | 2.015806 13.762961 29 | 1.391974 8.489460 30 | 1.287217 6.510840 31 | 0.042333 0.790527 32 | 2.616088 111.114808 33 | 0.073193 0.614370 34 | 0.037989 1.176173 35 | 4.930067 108.368833 36 | 4.757421 1.761846 37 | 0.083284 0.193741 38 | 0.108357 0.125812 39 | 0.082840 0.142814 40 | 0.055495 0.209637 41 | 0.051488 0.270381 42 | 0.073015 0.207277 43 | 0.050518 0.131680 44 | 0.046361 0.197030 45 | 0.054422 0.264580 46 | 0.041763 0.190044 47 | 0.066636 0.433861 48 | 0.047115 0.199160 49 | 0.077999 0.323452 50 | 0.137995 0.490801 51 | 0.094819 0.479683 52 | 0.047722 0.361463 53 | 0.075531 1.162315 54 | 0.024826 0.418670 55 | 0.041589 0.229460 56 | 0.022370 0.176177 57 | 0.091035 0.710774 58 | 0.047336 0.393969 59 | 0.017629 0.303291 60 | 0.026064 0.234232 61 | 0.077401 0.190395 62 | 0.031243 0.221587 63 | 0.045458 0.075061 64 | 0.017879 0.089631 65 | 0.182508 0.498196 66 | 0.056052 0.324057 67 | 0.046839 0.290528 68 | 0.023854 0.167955 69 | 0.037607 0.247428 70 | 0.036991 0.345139 71 | 0.034120 0.363428 72 | 0.019740 0.326409 73 | 0.044719 0.337857 74 | 0.038115 0.300037 75 | 0.058358 0.275144 76 | 0.023854 0.192095 77 | 0.019537 0.230676 78 | 0.011475 0.110653 79 | 0.067353 0.665514 80 | 0.030498 0.339342 81 | 0.032979 0.208950 82 | 0.024123 0.457725 83 | 0.044027 0.157660 84 | 0.022225 0.268460 85 | 0.005113 0.332455 86 | 0.030684 0.168367 87 | 0.019725 0.112056 88 | 0.037987 0.461894 89 | 0.014362 0.309675 90 | 0.027475 0.390270 91 | 0.035589 0.859803 92 | 0.044104 0.187869 93 | 0.010974 0.081399 94 | 0.025957 0.315786 95 | 0.043694 0.252597 96 | 0.030681 0.362161 97 | 0.038602 0.377436 98 | 0.023447 0.160718 99 | 0.044889 0.314309 100 | 0.042690 0.337873 101 | 0.035548 0.168580 102 | 0.038293 0.338445 103 | 0.016005 0.179034 104 | -------------------------------------------------------------------------------- /dsacstar/dsacstar_types.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based on the DSAC++ and ESAC code. 3 | https://github.com/vislearn/LessMore 4 | https://github.com/vislearn/esac 5 | 6 | Copyright (c) 2016, TU Dresden 7 | Copyright (c) 2020, Heidelberg University 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are met: 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above copyright 15 | notice, this list of conditions and the following disclaimer in the 16 | documentation and/or other materials provided with the distribution. 17 | * Neither the name of the TU Dresden, Heidelberg University nor the 18 | names of its contributors may be used to endorse or promote products 19 | derived from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL TU DRESDEN OR HEIDELBERG UNIVERSITY BE LIABLE FOR ANY 25 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 28 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #pragma once 34 | 35 | #include "opencv2/opencv.hpp" 36 | 37 | /** Several important types used troughout all this code. If types have to be changed, it can be done here, conveniently. */ 38 | 39 | namespace dsacstar 40 | { 41 | // scene pose type (OpenCV convention: axis-angle + translation) 42 | typedef std::pair pose_t; 43 | // camera transformation type (inverted scene pose as 4x4 matrix) 44 | typedef cv::Mat_ trans_t; 45 | // ATen accessor type 46 | typedef at::TensorAccessor coord_t; 47 | typedef at::TensorAccessor ng_t; 48 | 49 | typedef at::TensorAccessor coord_ts; 50 | typedef at::TensorAccessor p_ts; 51 | typedef at::TensorAccessor ng_ts; 52 | 53 | } 54 | -------------------------------------------------------------------------------- /Dense_Nets/Super_GNN.py: -------------------------------------------------------------------------------- 1 | 2 | from copy import deepcopy 3 | import torch 4 | from torch import nn 5 | device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") 6 | 7 | 8 | def conv(in_planes, out_planes, kernel_size, strides): 9 | return nn.Sequential( 10 | nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, padding=(kernel_size - 1) // 2, stride=strides), 11 | nn.ReLU(inplace=True) 12 | ) 13 | 14 | 15 | def MLP(channels: list, do_bn=True): 16 | """ Multi-layer perceptron """ 17 | n = len(channels) 18 | layers = [] 19 | for i in range(1, n): 20 | layers.append( 21 | nn.Conv1d(channels[i - 1], channels[i], kernel_size=1, bias=True)) 22 | if i < (n-1): 23 | if do_bn: 24 | layers.append(nn.BatchNorm1d(channels[i])) 25 | layers.append(nn.ReLU()) 26 | return nn.Sequential(*layers) 27 | 28 | 29 | def attention(query, key, value): 30 | dim = query.shape[1] 31 | scores = torch.einsum('bdhn,bdhm->bhnm', query, key) / dim**.5 32 | prob = torch.nn.functional.softmax(scores, dim=-1) 33 | return torch.einsum('bhnm,bdhm->bdhn', prob, value), prob 34 | 35 | 36 | class MultiHeadedAttention(nn.Module): 37 | """ Multi-head attention to increase model expressivitiy """ 38 | def __init__(self, num_heads: int, d_model: int): 39 | super().__init__() 40 | assert d_model % num_heads == 0 41 | self.dim = d_model // num_heads 42 | self.num_heads = num_heads 43 | self.merge = nn.Conv1d(d_model, d_model, kernel_size=1) 44 | self.proj = nn.ModuleList([deepcopy(self.merge) for _ in range(3)]) 45 | 46 | def forward(self, query, key, value): 47 | batch_dim = query.size(0) 48 | query, key, value = [l(x).view(batch_dim, self.dim, self.num_heads, -1) 49 | for l, x in zip(self.proj, (query, key, value))] 50 | x, _ = attention(query, key, value) 51 | return self.merge(x.contiguous().view(batch_dim, self.dim*self.num_heads, -1)) 52 | 53 | 54 | class AttentionalPropagation(nn.Module): 55 | def __init__(self, feature_dim: int, num_heads: int): 56 | super().__init__() 57 | self.attn = MultiHeadedAttention(num_heads, feature_dim) 58 | self.mlp = MLP([feature_dim*2, feature_dim*2, feature_dim]) 59 | nn.init.constant_(self.mlp[-1].bias, 0.0) 60 | 61 | def forward(self, x, source): 62 | message = self.attn(x, source, source) 63 | return self.mlp(torch.cat([x, message], dim=1)) 64 | 65 | 66 | -------------------------------------------------------------------------------- /models/Dense_networks.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import math 4 | from loss_functions import dsacstar_loss 5 | from Dense_Nets.ESAC import ESAC_SCNet 6 | from Dense_Nets.ESAC_DROID import ESAC_DROID_Net 7 | 8 | # generate grid of target reprojection pixel positions 9 | OUTPUT_SUBSAMPLE = 8 10 | pixel_grid = torch.zeros((2, 11 | math.ceil(5000 / OUTPUT_SUBSAMPLE), # 5000px is max limit of image size, increase if needed 12 | math.ceil(5000 / OUTPUT_SUBSAMPLE))) 13 | 14 | for x in range(0, pixel_grid.size(2)): 15 | for y in range(0, pixel_grid.size(1)): 16 | pixel_grid[0, y, x] = x * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 17 | pixel_grid[1, y, x] = y * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 18 | 19 | device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") 20 | 21 | class DenseSC(nn.Module): 22 | def __init__(self, args, scene_mean,e2e=False): 23 | super(DenseSC, self).__init__() 24 | self.mean = scene_mean.clone() 25 | self.e2e= e2e 26 | 27 | if args.uncertainty: 28 | self.un = True 29 | else: 30 | self.un = False 31 | self.dsacstar = args.dsacstar 32 | 33 | if args.dataset == '7Scenes': 34 | self.network = ESAC_SCNet(self.mean, self.un) 35 | else: 36 | self.network = ESAC_DROID_Net(self.mean, self.un, output_dim=256, norm_fn='none') 37 | 38 | if self.e2e==False: 39 | self.config = { 40 | 'itol': args.inittolerance, 41 | 'mindepth': args.mindepth, 42 | 'maxdepth': args.maxdepth, 43 | 'targetdepth': args.targetdepth, 44 | 'softclamp': args.softclamp, 45 | 'hardclamp': args.hardclamp 46 | } 47 | 48 | def forward(self, X_world, image, Ts, Ks): 49 | """ 50 | :param X_world: corresponding 3D coord (B, H'W',3) 51 | :param used_mask: discard inaccurate depth (B,H'W') 52 | :param Ts: (B, 3, 4) 53 | :param Ks: (B, 3, 3) 54 | :return: pred_world(B,3,H,W) , loss 55 | """ 56 | pred_X, uncertainty = self.network(image) 57 | pred_X_clone = pred_X.clone() 58 | pixel_grid_crop = pixel_grid[:, 0:pred_X.size(2), 0:pred_X.size(3)].clone() 59 | pixel_grid_crop = pixel_grid_crop.view(2, -1).permute(1,0).to(device) 60 | pixel_grid_crop_clone = pixel_grid_crop.clone() 61 | 62 | if self.dsacstar: 63 | loss_1, num_valid_sc = dsacstar_loss(pred_X, X_world, pixel_grid_crop, Ts, Ks, self.config) 64 | loss_1 = torch.unsqueeze(loss_1, 0) 65 | 66 | return loss_1, None, None, pred_X_clone, uncertainty, None, pixel_grid_crop_clone 67 | 68 | 69 | def init_weights(self): 70 | self.network.init_weights() 71 | -------------------------------------------------------------------------------- /dsacstar/stop_watch.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, TU Dresden 3 | Copyright (c) 2017, Heidelberg University 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | * Neither the name of the TU Dresden, Heidelberg University nor the 14 | names of its contributors may be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL TU DRESDEN OR HEIDELBERG UNIVERSITY BE LIABLE FOR ANY 21 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 24 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | */ 28 | 29 | 30 | #pragma once 31 | 32 | #include 33 | 34 | /** 35 | * @brief Class for time measurements. 36 | */ 37 | class StopWatch 38 | { 39 | public: 40 | /** 41 | * @brief Construction. Initializes the stop watch. 42 | */ 43 | StopWatch(){ init(); } 44 | 45 | /** 46 | * @brief Initialization. Starts the time measurement. 47 | * 48 | * @return void 49 | */ 50 | void init() 51 | { 52 | start = std::chrono::high_resolution_clock::now(); 53 | } 54 | 55 | /** 56 | * @brief Stops and restarts the time measurement. 57 | * 58 | * @return float The time in ms since the last init or stop call. 59 | */ 60 | float stop() 61 | { 62 | std::chrono::high_resolution_clock::time_point now; 63 | now = std::chrono::high_resolution_clock::now(); 64 | 65 | std::chrono::high_resolution_clock::duration duration = now - start; 66 | 67 | start = now; 68 | 69 | return static_cast( 70 | 1000.0 * std::chrono::duration_cast>( 71 | duration).count()); 72 | } 73 | 74 | private: 75 | std::chrono::high_resolution_clock::time_point start; // start time of the current measurement. 76 | }; 77 | -------------------------------------------------------------------------------- /logger.py: -------------------------------------------------------------------------------- 1 | from blessings import Terminal 2 | import progressbar 3 | import sys 4 | 5 | 6 | class TermLogger(object): 7 | def __init__(self, n_epochs, train_size, valid_size): 8 | self.n_epochs = n_epochs 9 | self.train_size = train_size 10 | self.valid_size = valid_size 11 | self.t = Terminal() 12 | s = 10 13 | e = 1 # epoch bar position 14 | tr = 3 # train bar position 15 | ts = 6 # valid bar position 16 | h = self.t.height 17 | 18 | for i in range(10): 19 | print('') 20 | 21 | self.epoch_bar = progressbar.ProgressBar(max_value=self.n_epochs, fd=Writer(self.t, (0, h-s+e))) 22 | 23 | self.train_writer = Writer(self.t, (0, h-s+tr)) 24 | self.train_bar_writer = Writer(self.t, (0, h-s+tr+1)) 25 | 26 | self.valid_writer = Writer(self.t, (0, h-s+ts)) 27 | self.valid_bar_writer = Writer(self.t, (0, h-s+ts+1)) 28 | 29 | self.reset_train_bar() 30 | self.reset_valid_bar() 31 | 32 | def reset_train_bar(self): 33 | self.train_bar = progressbar.ProgressBar(max_value=self.train_size, fd=self.train_bar_writer) 34 | 35 | def reset_valid_bar(self): 36 | self.valid_bar = progressbar.ProgressBar(max_value=self.valid_size, fd=self.valid_bar_writer) 37 | class Writer(object): 38 | """Create an object with a write method that writes to a 39 | specific place on the screen, defined at instantiation. 40 | This is the glue between blessings and progressbar. 41 | """ 42 | 43 | def __init__(self, t, location): 44 | """ 45 | Input: location - tuple of ints (x, y), the position 46 | of the bar in the terminal 47 | """ 48 | self.location = location 49 | self.t = t 50 | 51 | def write(self, string): 52 | with self.t.location(*self.location): 53 | sys.stdout.write("\033[K") 54 | print(string) 55 | 56 | def flush(self): 57 | return 58 | class AverageMeter(object): 59 | """Computes and stores the average and current value""" 60 | 61 | def __init__(self, i=1, precision=3): 62 | self.meters = i 63 | self.precision = precision 64 | self.reset(self.meters) 65 | 66 | def reset(self, i): 67 | self.val = [0]*i 68 | self.avg = [0]*i 69 | self.sum = [0]*i 70 | self.count = 0 71 | 72 | def update(self, val, n=1): 73 | if not isinstance(val, list): 74 | val = [val] 75 | assert(len(val) == self.meters) 76 | self.count += n 77 | for i,v in enumerate(val): 78 | self.val[i] = v 79 | self.sum[i] += v * n 80 | self.avg[i] = self.sum[i] / self.count 81 | 82 | def __repr__(self): 83 | val = ' '.join(['{:.{}f}'.format(v, self.precision) for v in self.val]) 84 | avg = ' '.join(['{:.{}f}'.format(a, self.precision) for a in self.avg]) 85 | return '{} ({})'.format(val, avg) 86 | -------------------------------------------------------------------------------- /plot_per_frame/results/MST_oldhospital.txt: -------------------------------------------------------------------------------- 1 | 0.805085 2.819731 2 | 0.926613 1.769846 3 | 1.084620 nan 4 | 0.966667 nan 5 | 0.817430 2.121819 6 | 0.512475 2.946255 7 | 0.478839 2.546371 8 | 0.261060 1.867541 9 | 1.461636 2.024408 10 | 0.668506 4.331642 11 | 2.348264 4.587251 12 | 2.172250 4.237728 13 | 6.885344 2.862436 14 | 0.795524 6.441146 15 | 5.857535 2.793795 16 | 2.354186 3.593352 17 | 5.612744 5.751245 18 | 1.395861 2.969280 19 | 0.901020 5.091057 20 | 1.394436 2.142012 21 | 0.873543 nan 22 | 1.172971 nan 23 | 1.093783 3.132961 24 | 0.780504 nan 25 | 0.522860 3.024393 26 | 0.473477 2.542063 27 | 0.209520 1.698534 28 | 0.196686 0.438794 29 | 0.353895 0.592151 30 | 0.202166 2.629547 31 | 0.423283 nan 32 | 1.118465 1.620717 33 | 0.607457 3.178602 34 | 1.009756 2.209997 35 | 0.650960 1.690219 36 | 1.112729 2.464835 37 | 1.665538 1.506069 38 | 1.332643 2.004202 39 | 1.180105 4.484555 40 | 1.256863 2.070664 41 | 0.644047 nan 42 | 0.759302 nan 43 | 1.104600 1.579629 44 | 0.858431 2.001467 45 | 0.473355 nan 46 | 0.966115 2.327977 47 | 0.906511 nan 48 | 0.747352 2.382809 49 | 0.903688 2.184347 50 | 0.089750 2.743470 51 | 0.884573 4.046267 52 | 1.314343 3.181801 53 | 1.666642 7.782874 54 | 1.401331 6.115229 55 | 1.116611 3.585938 56 | 0.738042 2.918494 57 | 0.892010 1.311026 58 | 1.009577 nan 59 | 0.894976 2.459430 60 | 2.080043 2.047858 61 | 1.122595 2.272166 62 | 0.961169 3.269408 63 | 1.062411 6.566967 64 | 5.018169 16.375444 65 | 3.024093 15.857303 66 | 2.053423 14.272607 67 | 1.403150 6.601815 68 | 1.014683 4.175196 69 | 4.292847 5.309961 70 | 2.692970 4.966347 71 | 2.592367 5.428372 72 | 0.562420 6.048795 73 | 13.266133 13.528734 74 | 16.501450 14.545840 75 | 3.813796 8.819865 76 | 4.866805 7.891415 77 | 3.882900 8.269772 78 | 1.897458 6.192215 79 | 0.943625 8.836461 80 | 1.229858 4.333630 81 | 1.836025 2.574800 82 | 1.791374 3.585719 83 | 1.526469 1.029446 84 | 2.331714 7.389395 85 | 26.902529 4.783912 86 | 26.926178 4.988369 87 | 3.375402 4.036194 88 | 2.533317 3.268929 89 | 3.186031 2.937208 90 | 3.366704 7.421020 91 | 5.519279 11.296530 92 | 3.311846 9.426945 93 | 2.415687 1.941517 94 | 1.895246 4.984758 95 | 6.385947 5.582678 96 | 4.672098 17.883205 97 | 3.511681 11.367441 98 | 4.081940 8.432895 99 | 3.666939 3.841215 100 | 4.087691 1.884230 101 | 3.280676 nan 102 | 3.631056 2.998138 103 | 2.239556 1.522609 104 | 2.066266 1.739513 105 | 3.497108 4.192970 106 | 3.432535 nan 107 | 3.837547 2.106268 108 | 3.608414 2.255917 109 | 7.683396 4.238097 110 | 8.807558 3.839585 111 | 8.752603 2.173211 112 | 7.626711 2.730027 113 | 6.233593 1.123948 114 | 8.740801 3.136208 115 | 7.224060 5.052155 116 | 6.444848 4.013632 117 | 8.952950 4.002499 118 | 9.709784 3.410959 119 | 9.778456 0.381548 120 | 9.962321 3.956862 121 | 8.384307 nan 122 | 5.387632 nan 123 | 5.499712 1.941517 124 | 4.767752 1.087133 125 | 4.043289 nan 126 | 3.752927 2.736328 127 | 2.149341 0.569237 128 | 1.069742 0.872222 129 | 1.399469 1.352171 130 | 2.908992 5.981362 131 | 1.941874 2.737758 132 | 1.393984 3.105356 133 | 1.420271 1.045290 134 | 1.218234 nan 135 | 1.161019 2.918494 136 | 1.210640 2.791272 137 | 0.957944 2.063848 138 | 0.493629 nan 139 | 2.209561 3.561184 140 | 1.988551 3.810523 141 | 1.230418 3.012204 142 | 1.424212 1.848161 143 | 0.581186 1.870891 144 | 0.494418 2.246530 145 | 0.501419 2.416731 146 | 0.999990 3.337417 147 | 2.243617 5.159039 148 | 1.468921 3.128460 149 | 0.629652 nan 150 | 0.796033 1.981421 151 | 2.412049 0.288035 152 | 3.440433 3.055550 153 | 1.881286 2.200413 154 | 1.192883 1.269163 155 | 0.891989 2.798274 156 | 1.166121 2.901547 157 | 1.306197 nan 158 | 1.975342 0.701089 159 | 2.039542 2.722850 160 | 2.605862 3.319779 161 | 2.688395 2.273544 162 | 2.743679 1.691145 163 | 3.075358 2.368972 164 | 2.396047 0.690968 165 | 1.839575 1.487769 166 | 0.213147 3.267492 167 | 1.480071 nan 168 | 0.595062 2.241646 169 | 1.200801 2.572367 170 | 1.069304 nan 171 | 2.481278 2.837994 172 | 2.533131 3.741487 173 | 2.515232 5.056027 174 | 1.958115 3.780204 175 | 1.916299 3.195793 176 | 2.025007 3.267013 177 | 2.029802 3.212162 178 | 2.470360 3.100562 179 | 1.482711 5.122181 180 | 3.145971 6.429465 181 | 2.593937 3.923285 182 | 2.463702 5.609124 183 | -------------------------------------------------------------------------------- /Dense_Nets/ESAC.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | 5 | 6 | class ESAC_SCNet(nn.Module): 7 | ''' 8 | FCN architecture for scene coordiante regression. 9 | The network has two output heads: One predicting a 3d scene coordinate, and a 1d neural guidance weight (if uncertainty is not None). 10 | The network makes dense predictions, but the output is subsampled by a factor of 8 compared to the input. 11 | ''' 12 | 13 | OUTPUT_SUBSAMPLE = 8 14 | 15 | def __init__(self, mean,uncertainty): 16 | ''' 17 | Constructor. 18 | ''' 19 | super(ESAC_SCNet, self).__init__() 20 | print('ESAC_Net') 21 | 22 | self.un = uncertainty 23 | 24 | self.conv1 = nn.Conv2d(3, 32, 3, 1, 1) 25 | self.conv2 = nn.Conv2d(32, 64, 3, 2, 1) 26 | self.conv3 = nn.Conv2d(64, 128, 3, 2, 1) 27 | self.conv4 = nn.Conv2d(128, 256, 3, 2, 1) 28 | 29 | self.res1_conv1 = nn.Conv2d(256, 256, 3, 1, 1) 30 | self.res1_conv2 = nn.Conv2d(256, 256, 1, 1, 0) 31 | self.res1_conv3 = nn.Conv2d(256, 256, 3, 1, 1) 32 | 33 | self.res2_conv1 = nn.Conv2d(256, 512, 3, 1, 1) 34 | self.res2_conv2 = nn.Conv2d(512, 512, 1, 1, 0) 35 | self.res2_conv3 = nn.Conv2d(512, 512, 3, 1, 1) 36 | 37 | self.res2_skip = nn.Conv2d(256, 512, 1, 1, 0) 38 | 39 | self.res3_conv1 = nn.Conv2d(512, 512, 3, 1, 1) 40 | self.res3_conv2 = nn.Conv2d(512, 512, 1, 1, 0) 41 | self.res3_conv3 = nn.Conv2d(512, 512, 3, 1, 1) 42 | 43 | # output head 1, scene coordinates 44 | self.fc1 = nn.Conv2d(512, 512, 1, 1, 0) 45 | self.fc2 = nn.Conv2d(512, 512, 1, 1, 0) 46 | self.fc3 = nn.Conv2d(512, 3, 1, 1, 0) 47 | 48 | # output head 2, neural guidance 49 | if self.un: 50 | self.fc1_1 = nn.Conv2d(512, 512, 1, 1, 0) 51 | self.fc2_1 = nn.Conv2d(512, 512, 1, 1, 0) 52 | self.fc3_1 = nn.Conv2d(512, 1, 1, 1, 0) 53 | 54 | # learned scene coordinates relative to a mean coordinate (e.g. center of the scene) 55 | self.register_buffer('mean', torch.tensor(mean.size()).cuda()) 56 | self.mean = mean.clone() 57 | 58 | def forward(self, inputs): 59 | ''' 60 | Forward pass. 61 | inputs -- 4D data tensor (BxCxHxW) 62 | ''' 63 | batch_size = inputs.size(0) 64 | 65 | x = inputs 66 | x = F.relu(self.conv1(x)) 67 | x = F.relu(self.conv2(x)) 68 | x = F.relu(self.conv3(x)) 69 | res = F.relu(self.conv4(x)) 70 | 71 | x = F.relu(self.res1_conv1(res)) 72 | x = F.relu(self.res1_conv2(x)) 73 | x = F.relu(self.res1_conv3(x)) 74 | 75 | res = res + x 76 | 77 | x = F.relu(self.res2_conv1(res)) 78 | x = F.relu(self.res2_conv2(x)) 79 | x = F.relu(self.res2_conv3(x)) 80 | 81 | res = self.res2_skip(res) + x 82 | 83 | x = F.relu(self.res3_conv1(res)) 84 | x = F.relu(self.res3_conv2(x)) 85 | x = F.relu(self.res3_conv3(x)) 86 | 87 | res = res + x 88 | 89 | # output head 1, scene coordinates 90 | sc = F.relu(self.fc1(res)) 91 | sc = F.relu(self.fc2(sc)) 92 | sc = self.fc3(sc) 93 | 94 | sc[:, 0,:,:] += self.mean[0] 95 | sc[:, 1,:,:] += self.mean[1] 96 | sc[:, 2,:,:] += self.mean[2] 97 | 98 | # output head 2, neural guidance 99 | if self.un: 100 | log_ng = F.relu(self.fc1_1(res)) 101 | log_ng = F.relu(self.fc2_1(log_ng)) 102 | log_ng = self.fc3_1(log_ng) 103 | un = torch.exp(log_ng) 104 | else: 105 | un = None 106 | 107 | return sc,un 108 | 109 | def init_weights(self): 110 | init_modules = self.modules() 111 | for m in init_modules: 112 | if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d) or isinstance(m, nn.Linear): 113 | torch.nn.init.xavier_uniform_(m.weight.data) 114 | if m.bias is not None: 115 | torch.nn.init.constant_(m.bias.data, 0.0) -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_oldhospital.txt: -------------------------------------------------------------------------------- 1 | 0.193393 1.145414 2 | 0.106305 0.475085 3 | 0.107664 0.744667 4 | 0.088216 0.174115 5 | 0.164989 0.581885 6 | 0.130345 1.950807 7 | 0.041283 1.264717 8 | 0.134388 0.839761 9 | 0.097150 1.460071 10 | 0.071005 1.417939 11 | 0.323633 1.181830 12 | 0.489518 2.237292 13 | 0.139427 0.186847 14 | 0.106325 0.691454 15 | 0.064872 1.824043 16 | 0.165340 0.481263 17 | 0.206866 1.871640 18 | 0.283366 0.635046 19 | 0.369714 0.668227 20 | 0.541851 1.043046 21 | 0.232518 0.972342 22 | 0.447620 0.986943 23 | 0.920054 0.583049 24 | 0.419444 0.226823 25 | 0.326941 0.687056 26 | 0.704649 0.150327 27 | 0.412843 1.045608 28 | 0.670235 1.199378 29 | 0.155275 0.691362 30 | 0.071834 0.547297 31 | 0.109266 0.561036 32 | 0.221323 0.431594 33 | 0.111811 0.995838 34 | 0.421212 0.139387 35 | 0.157149 0.110395 36 | 0.978663 0.708990 37 | 0.936006 1.746302 38 | 1.151142 1.057413 39 | 0.761830 1.318895 40 | 1.121849 1.211870 41 | 0.072253 0.584729 42 | 0.519633 1.629684 43 | 1.304828 1.632307 44 | 0.137815 1.045078 45 | 0.353624 0.529624 46 | 0.244801 0.440054 47 | 0.246647 0.518334 48 | 0.738732 0.831554 49 | 0.096585 0.551481 50 | 0.264032 0.852598 51 | 0.412582 0.991493 52 | 0.734329 2.417558 53 | 0.254872 1.991972 54 | 0.335834 1.351215 55 | 0.501611 2.262968 56 | 0.515449 1.527617 57 | 0.615037 1.867186 58 | 0.747939 1.451742 59 | 0.562221 2.318913 60 | 1.137550 4.454172 61 | 0.315981 0.907307 62 | 1.436030 1.106746 63 | 1.643975 0.902542 64 | 0.435561 0.298155 65 | 1.341176 1.787113 66 | 1.420923 0.824033 67 | 1.925817 1.214478 68 | 0.629998 1.560432 69 | 1.143131 2.525301 70 | 0.459218 1.084557 71 | 2.066081 0.663552 72 | 1.188110 0.238691 73 | 1.487498 0.535687 74 | 0.732234 0.151747 75 | 0.184207 1.074741 76 | 0.107156 0.488148 77 | 0.136465 2.406862 78 | 1.362612 1.855687 79 | 0.208911 3.056220 80 | 0.484608 0.547700 81 | 0.217201 3.251067 82 | 0.863600 1.562741 83 | 0.315181 2.573008 84 | 0.270648 2.660323 85 | 5.117162 5.670636 86 | 6.237815 6.599549 87 | 0.511123 2.801069 88 | 0.149390 2.750841 89 | 0.220504 2.868819 90 | 0.954013 3.305140 91 | 1.072369 2.619096 92 | 1.500777 2.780833 93 | 0.413891 3.810060 94 | 1.028116 3.300868 95 | 2.930666 10.999594 96 | 0.806893 4.907998 97 | 1.196857 3.006701 98 | 0.781187 3.889885 99 | 1.398737 4.283160 100 | 1.605515 3.988912 101 | 1.600213 4.708274 102 | 1.384421 5.195045 103 | 0.224565 3.990970 104 | 0.184052 1.422045 105 | 0.752997 3.614505 106 | 0.322170 2.389410 107 | 0.140822 2.468898 108 | 0.105632 1.733997 109 | 0.120519 3.618580 110 | 4.819141 8.531302 111 | 1.055748 4.629969 112 | 0.248085 4.612763 113 | 0.404312 2.184436 114 | 0.119980 3.364966 115 | 1.469618 4.525989 116 | 0.387888 4.437778 117 | 2.169937 6.670256 118 | 1.757021 7.062642 119 | 3.143325 9.331626 120 | 0.173320 4.257365 121 | 0.436447 3.960069 122 | 0.859284 5.383708 123 | 0.442602 4.455968 124 | 0.350102 4.348060 125 | 0.270284 3.402919 126 | 0.763850 4.675877 127 | 0.304659 3.885895 128 | 0.504269 3.234896 129 | 0.243226 3.080950 130 | 0.659827 1.985053 131 | 0.202389 1.865584 132 | 0.037138 1.262059 133 | 0.529582 1.882763 134 | 0.242948 0.857570 135 | 0.485121 2.272674 136 | 0.205043 2.019019 137 | 0.226701 0.674147 138 | 0.059115 1.055103 139 | 0.188830 1.865942 140 | 0.157696 2.042332 141 | 0.148799 1.578037 142 | 0.428026 0.571535 143 | 0.082220 1.266449 144 | 0.309100 0.314352 145 | 0.437025 0.679743 146 | 0.636073 0.267138 147 | 0.289691 0.597613 148 | 0.467205 0.521542 149 | 0.667218 0.897625 150 | 0.752765 1.511964 151 | 0.438340 1.081852 152 | 0.605264 0.891889 153 | 0.487233 0.593949 154 | 0.480691 3.149163 155 | 0.380947 2.293261 156 | 0.164109 0.906267 157 | 0.128837 0.975762 158 | 0.020274 1.233396 159 | 0.257723 1.280484 160 | 0.054487 1.347032 161 | 0.404033 1.385900 162 | 0.374529 1.353624 163 | 0.730418 2.607071 164 | 0.909331 5.054894 165 | 0.956494 5.540342 166 | 1.314448 12.121888 167 | 1.192931 9.004219 168 | 1.517341 9.282542 169 | 1.489411 9.871213 170 | 1.276521 7.285274 171 | 1.976483 10.108467 172 | 2.162889 8.788789 173 | 1.350638 3.450993 174 | 0.750795 2.697020 175 | 0.653208 1.516283 176 | 0.716966 2.619246 177 | 0.603755 2.021020 178 | 0.688043 1.945757 179 | 0.674032 1.101642 180 | 1.126855 3.213069 181 | 1.014035 3.908398 182 | 0.376348 4.251040 183 | -------------------------------------------------------------------------------- /plot_per_frame/results/DSACSTAR_oldhospital.txt: -------------------------------------------------------------------------------- 1 | 0.037754 0.228707 2 | 0.049596 0.284313 3 | 0.049604 0.283300 4 | 0.060827 0.318999 5 | 0.077517 0.397671 6 | 0.045622 0.335604 7 | 0.070260 0.367555 8 | 0.049673 0.120889 9 | 0.119852 0.396911 10 | 0.126002 0.476679 11 | 0.038470 0.073214 12 | 0.219046 0.739787 13 | 0.083246 0.181039 14 | 0.108959 0.308013 15 | 0.027344 0.130864 16 | 0.181544 0.524209 17 | 0.034860 0.163402 18 | 0.139972 0.405578 19 | 0.100077 0.338658 20 | 0.482871 1.284158 21 | 0.217600 0.637732 22 | 0.311421 0.850142 23 | 0.190532 0.521118 24 | 0.180094 0.419381 25 | 0.077636 0.228173 26 | 0.232344 0.532183 27 | 0.069365 0.184289 28 | 0.218505 0.562566 29 | 0.084375 0.219435 30 | 0.198243 0.362594 31 | 0.217923 0.409112 32 | 0.247383 0.567432 33 | 0.294118 0.639142 34 | 0.287781 0.515327 35 | 0.158360 0.280399 36 | 0.308485 0.605879 37 | 0.133548 0.266887 38 | 0.401676 0.715478 39 | 0.158846 0.303383 40 | 0.132223 0.206036 41 | 0.124466 0.174234 42 | 0.108010 0.209460 43 | 0.314976 0.597207 44 | 0.364765 0.678428 45 | 0.165344 0.276677 46 | 0.234157 0.445049 47 | 0.162817 0.264751 48 | 0.124590 0.199416 49 | 0.156925 0.306059 50 | 0.119107 0.243822 51 | 0.207003 0.447292 52 | 0.385344 0.862225 53 | 0.454324 0.798145 54 | 0.538926 1.008233 55 | 0.490907 0.783589 56 | 0.202697 0.348342 57 | 0.224053 0.328954 58 | 0.206677 0.350385 59 | 0.520117 0.801378 60 | 1.441499 2.378522 61 | 0.669179 1.036352 62 | 0.251989 0.390159 63 | 0.370106 0.565946 64 | 0.283747 0.407385 65 | 0.258468 0.532038 66 | 0.511760 0.732857 67 | 0.460764 0.764677 68 | 1.021178 1.424810 69 | 1.684734 2.402488 70 | 0.264510 0.453266 71 | 0.992052 1.461695 72 | 0.619503 0.928854 73 | 0.539099 0.743652 74 | 0.670157 0.950301 75 | 0.848275 1.299420 76 | 0.664679 0.906064 77 | 0.645470 0.907445 78 | 0.248506 0.235576 79 | 0.782301 1.188958 80 | 0.560972 0.769358 81 | 1.493965 2.197359 82 | 0.750025 1.056136 83 | 1.156874 1.640106 84 | 1.133469 1.634380 85 | 0.401920 0.551807 86 | 0.503553 0.743639 87 | 0.702776 0.956167 88 | 1.501416 1.858069 89 | 0.441683 0.525350 90 | 0.729395 0.928459 91 | 0.338998 0.403417 92 | 0.560325 0.706016 93 | 0.882369 1.052705 94 | 0.354282 0.245418 95 | 1.111991 1.234508 96 | 0.589261 0.612629 97 | 0.345438 0.401190 98 | 0.695720 0.826375 99 | 0.420183 0.601390 100 | 0.762623 0.949327 101 | 0.564221 0.795166 102 | 0.432438 0.502668 103 | 0.496521 0.642764 104 | 0.255040 0.375123 105 | 0.142884 0.159393 106 | 0.233519 0.316759 107 | 0.184083 0.327168 108 | 0.276931 0.343704 109 | 0.444570 0.472959 110 | 0.507409 0.754926 111 | 0.679100 0.996026 112 | 0.175049 0.306933 113 | 0.343377 0.425252 114 | 0.298953 0.394197 115 | 0.071411 0.209435 116 | 0.268422 0.430833 117 | 0.166571 0.123503 118 | 0.312290 0.460202 119 | 0.387778 0.408079 120 | 0.069540 0.190665 121 | 0.369028 0.456661 122 | 0.211041 0.413876 123 | 0.265098 0.412258 124 | 0.298160 0.451957 125 | 0.196873 0.351911 126 | 0.168742 0.362138 127 | 0.361127 0.573274 128 | 0.328078 0.427167 129 | 0.111505 0.160128 130 | 0.130128 0.243375 131 | 0.220489 0.400962 132 | 0.171703 0.289439 133 | 0.175624 0.286411 134 | 0.402168 0.657664 135 | 0.137115 0.294401 136 | 0.198134 0.356832 137 | 0.101277 0.266591 138 | 0.116676 0.308482 139 | 0.117426 0.177073 140 | 0.140219 0.377568 141 | 0.104340 0.128560 142 | 0.023538 0.120782 143 | 0.101006 0.230828 144 | 0.182623 0.252737 145 | 0.125144 0.187164 146 | 0.223390 0.487447 147 | 0.116071 0.077783 148 | 0.114218 0.217717 149 | 0.119359 0.250158 150 | 0.130479 0.370518 151 | 0.113258 0.252794 152 | 0.151503 0.458241 153 | 0.200338 0.493874 154 | 0.077343 0.120906 155 | 0.098563 0.241167 156 | 0.140988 0.153759 157 | 0.165451 0.360393 158 | 0.270132 0.244447 159 | 0.166403 0.251364 160 | 0.119110 0.180900 161 | 0.134486 0.362817 162 | 0.172688 0.298366 163 | 0.138258 0.328220 164 | 0.143270 0.245785 165 | 0.055861 0.117900 166 | 0.194969 0.253466 167 | 0.167832 0.344792 168 | 0.156085 0.175338 169 | 0.220517 0.601657 170 | 0.215303 0.405448 171 | 0.204163 0.308073 172 | 0.108761 0.284643 173 | 0.207088 0.355617 174 | 0.125120 0.250086 175 | 0.049282 0.333567 176 | 0.213315 0.365746 177 | 0.201031 0.383018 178 | 0.212040 0.281462 179 | 0.274666 0.402422 180 | 0.262300 0.576296 181 | 0.382517 0.439388 182 | 0.252751 0.271333 183 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_ref_oldhospital.txt: -------------------------------------------------------------------------------- 1 | 0.038932 0.168412 2 | 0.032821 0.167312 3 | 0.038078 0.125021 4 | 0.026073 0.148536 5 | 0.090045 0.333951 6 | 0.095763 0.425909 7 | 0.060418 0.320454 8 | 0.084855 0.265683 9 | 0.013140 0.233957 10 | 0.027381 0.366776 11 | 0.039579 0.506728 12 | 0.057173 0.639975 13 | 0.037501 0.348625 14 | 0.070970 0.236888 15 | 0.098509 0.444961 16 | 0.049670 0.179589 17 | 0.244794 0.435329 18 | 0.048608 0.108146 19 | 0.068701 0.444607 20 | 0.631027 1.779658 21 | 0.050237 0.475767 22 | 0.146049 0.736607 23 | 0.032130 0.544072 24 | 0.182409 0.346487 25 | 0.018229 0.243124 26 | 0.249204 0.580949 27 | 0.044704 0.189471 28 | 0.113248 0.554385 29 | 0.016453 0.183890 30 | 0.025889 0.149939 31 | 0.016756 0.142617 32 | 0.034550 0.512251 33 | 0.179690 0.657002 34 | 0.020654 0.397310 35 | 0.065851 0.576540 36 | 0.351951 0.752699 37 | 0.131024 0.260099 38 | 0.426164 0.641391 39 | 0.147005 0.198604 40 | 0.075158 0.267040 41 | 0.018884 0.087722 42 | 0.113805 0.149578 43 | 0.324830 0.995460 44 | 0.153530 0.825120 45 | 0.203043 0.426266 46 | 0.183327 0.411117 47 | 0.067373 0.359812 48 | 0.022576 0.157522 49 | 0.038815 0.174988 50 | 0.117388 0.316905 51 | 0.089939 0.444412 52 | 0.363046 1.240840 53 | 0.327320 0.821032 54 | 0.186703 0.658739 55 | 0.212487 0.548886 56 | 0.344746 0.402068 57 | 0.385664 0.737894 58 | 0.209916 0.537794 59 | 0.392772 0.759244 60 | 1.295835 2.740918 61 | 0.426660 0.936422 62 | 0.326163 0.406060 63 | 0.416292 0.765007 64 | 0.146914 0.451775 65 | 0.460865 0.916836 66 | 0.675579 0.814714 67 | 0.622064 0.858729 68 | 0.562579 1.098454 69 | 0.710994 2.387387 70 | 0.133799 0.397045 71 | 0.169367 0.344678 72 | 0.158415 0.525084 73 | 0.108219 0.449901 74 | 0.326125 0.761494 75 | 0.373593 1.114031 76 | 0.213251 0.785057 77 | 0.265486 0.701027 78 | 0.027734 0.307631 79 | 0.325402 1.436604 80 | 0.112013 0.713659 81 | 0.348561 2.272415 82 | 0.266779 1.023225 83 | 0.279486 1.460935 84 | 0.514749 1.368709 85 | 0.275119 0.354202 86 | 0.108422 0.529138 87 | 0.391072 1.193572 88 | 0.430056 1.485106 89 | 0.193666 0.200767 90 | 0.026071 0.576866 91 | 0.205632 0.617200 92 | 0.053448 0.322651 93 | 0.350518 1.519046 94 | 0.415778 0.708019 95 | 0.658177 1.733703 96 | 0.325387 1.235135 97 | 0.274278 0.583925 98 | 0.215439 1.354489 99 | 0.081268 1.093693 100 | 0.176345 1.357267 101 | 0.091965 0.702503 102 | 0.280167 1.002540 103 | 0.432137 1.163856 104 | 0.181164 0.424072 105 | 0.045706 0.300099 106 | 0.172407 0.203273 107 | 0.056578 0.588110 108 | 0.444811 0.688176 109 | 0.208064 0.592790 110 | 0.089523 0.633381 111 | 0.135506 1.094111 112 | 0.276606 0.609567 113 | 0.441062 0.531124 114 | 0.281699 0.377882 115 | 0.025962 0.224519 116 | 0.256738 0.438383 117 | 0.090738 0.246868 118 | 0.165407 0.293810 119 | 0.327090 0.239473 120 | 0.116699 0.480618 121 | 0.081845 0.511569 122 | 0.203006 0.883690 123 | 0.056017 0.339751 124 | 0.073753 0.604444 125 | 0.023795 0.503547 126 | 0.056369 0.603119 127 | 0.093981 0.616726 128 | 0.062401 0.111063 129 | 0.147351 0.277930 130 | 0.018734 0.208382 131 | 0.284238 0.806777 132 | 0.208192 0.589294 133 | 0.380289 0.883829 134 | 0.249783 0.485310 135 | 0.135623 0.590619 136 | 0.193043 0.657528 137 | 0.194317 0.569478 138 | 0.065758 0.229583 139 | 0.098982 0.411050 140 | 0.058262 0.546071 141 | 0.081696 0.420550 142 | 0.036140 0.154556 143 | 0.067514 0.340181 144 | 0.140234 0.273075 145 | 0.083315 0.179175 146 | 0.119206 0.516634 147 | 0.056430 0.110821 148 | 0.083272 0.324048 149 | 0.113455 0.321113 150 | 0.086081 0.281593 151 | 0.091802 0.047277 152 | 0.112687 0.284869 153 | 0.091697 0.283899 154 | 0.116378 0.216502 155 | 0.114971 0.140877 156 | 0.046016 0.084116 157 | 0.042425 0.225662 158 | 0.034955 0.057120 159 | 0.039413 0.168530 160 | 0.037030 0.242315 161 | 0.090925 0.378011 162 | 0.032081 0.186258 163 | 0.071325 0.428581 164 | 0.118572 0.173400 165 | 0.087192 0.252914 166 | 1.228540 9.503648 167 | 0.076994 0.158596 168 | 0.089101 0.151282 169 | 0.087069 0.498576 170 | 0.098292 0.241709 171 | 0.077820 0.338290 172 | 0.348545 10.696189 173 | 0.090572 0.146372 174 | 0.065174 0.398746 175 | 0.068923 0.385873 176 | 0.038158 0.250163 177 | 0.053966 0.297829 178 | 0.106080 0.238809 179 | 0.083545 0.224853 180 | 0.150154 0.251752 181 | 0.195415 0.260769 182 | 0.234955 0.531122 183 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_dsacstar_oldhospital.txt: -------------------------------------------------------------------------------- 1 | 0.050781 0.166255 2 | 0.052291 0.167444 3 | 0.033409 0.125021 4 | 0.045875 0.148460 5 | 0.099438 0.333951 6 | 0.110051 0.420085 7 | 0.077396 0.323319 8 | 0.073456 0.265528 9 | 0.053762 0.233956 10 | 0.096517 0.366248 11 | 0.137942 0.506727 12 | 0.193218 0.638275 13 | 0.093771 0.348624 14 | 0.091518 0.237256 15 | 0.120398 0.444960 16 | 0.091383 0.179589 17 | 0.211746 0.474758 18 | 0.050348 0.107947 19 | 0.153616 0.445352 20 | 0.621180 1.780315 21 | 0.172522 0.475768 22 | 0.284736 0.736605 23 | 0.195560 0.535892 24 | 0.150666 0.346487 25 | 0.091718 0.241743 26 | 0.256215 0.580948 27 | 0.075465 0.189471 28 | 0.218774 0.554590 29 | 0.075478 0.183890 30 | 0.064397 0.148041 31 | 0.081614 0.151745 32 | 0.221084 0.512250 33 | 0.329999 0.659366 34 | 0.208366 0.402696 35 | 0.295816 0.573723 36 | 0.462197 0.752699 37 | 0.187714 0.260442 38 | 0.453956 0.641784 39 | 0.161349 0.219761 40 | 0.086588 0.267041 41 | 0.040178 0.087721 42 | 0.143060 0.152216 43 | 0.518545 0.995460 44 | 0.416332 0.825121 45 | 0.272475 0.423975 46 | 0.199007 0.410726 47 | 0.185746 0.356171 48 | 0.101975 0.160696 49 | 0.106069 0.170920 50 | 0.142173 0.316442 51 | 0.196499 0.438125 52 | 0.587084 1.246134 53 | 0.525470 0.821032 54 | 0.360786 0.646402 55 | 0.377937 0.552980 56 | 0.359569 0.398416 57 | 0.493687 0.734513 58 | 0.336535 0.530655 59 | 0.551965 0.792242 60 | 1.699784 2.663861 61 | 0.656417 0.939661 62 | 0.369731 0.406061 63 | 0.550227 0.764425 64 | 0.367232 0.473914 65 | 0.721074 0.914251 66 | 0.740621 0.866860 67 | 0.667617 0.807858 68 | 0.919995 1.160165 69 | 1.627498 2.391793 70 | 0.278177 0.417874 71 | 0.370280 0.461227 72 | 0.378121 0.511720 73 | 0.440222 0.588842 74 | 0.557708 0.780404 75 | 0.740089 1.081839 76 | 0.528975 0.773241 77 | 0.540542 0.788527 78 | 0.169227 0.260235 79 | 0.966109 1.428762 80 | 0.585010 0.797361 81 | 1.569846 2.252266 82 | 0.789348 1.076318 83 | 1.013755 1.455751 84 | 1.022295 1.334557 85 | 0.327625 0.368197 86 | 0.410640 0.532053 87 | 0.900472 1.188265 88 | 1.199888 1.557815 89 | 0.231406 0.193625 90 | 0.426495 0.547912 91 | 0.515375 0.633882 92 | 0.297035 0.300597 93 | 1.208981 1.522988 94 | 0.660629 0.710742 95 | 1.407675 1.681333 96 | 1.031901 1.265021 97 | 0.402406 0.420622 98 | 0.987534 1.134027 99 | 0.782747 0.985516 100 | 1.081439 1.307687 101 | 0.541149 0.684856 102 | 0.920505 1.114327 103 | 0.929612 1.189324 104 | 0.548044 0.700256 105 | 0.249634 0.316771 106 | 0.177167 0.159412 107 | 0.369744 0.545918 108 | 0.645084 0.758023 109 | 0.439035 0.592790 110 | 0.469905 0.631588 111 | 0.758791 1.094112 112 | 0.517600 0.617033 113 | 0.469983 0.428925 114 | 0.316324 0.328964 115 | 0.168345 0.225927 116 | 0.365658 0.411558 117 | 0.171921 0.236029 118 | 0.280466 0.304063 119 | 0.366673 0.259368 120 | 0.333680 0.483960 121 | 0.329047 0.593045 122 | 0.512116 0.883053 123 | 0.201866 0.337230 124 | 0.377889 0.609284 125 | 0.268193 0.486790 126 | 0.316003 0.601019 127 | 0.308814 0.625560 128 | 0.098855 0.201844 129 | 0.191987 0.280365 130 | 0.130644 0.202158 131 | 0.508265 0.808513 132 | 0.356043 0.588464 133 | 0.537640 0.894786 134 | 0.349545 0.541090 135 | 0.308058 0.551449 136 | 0.326889 0.657528 137 | 0.329189 0.571736 138 | 0.105805 0.248021 139 | 0.238570 0.431504 140 | 0.268787 0.536126 141 | 0.209906 0.420551 142 | 0.068902 0.154556 143 | 0.140048 0.340180 144 | 0.158487 0.271476 145 | 0.094125 0.181238 146 | 0.248745 0.538870 147 | 0.056529 0.113337 148 | 0.137477 0.324048 149 | 0.140069 0.327213 150 | 0.114810 0.293536 151 | 0.090740 0.044483 152 | 0.133901 0.287225 153 | 0.099282 0.294660 154 | 0.146394 0.219330 155 | 0.136428 0.164536 156 | 0.072144 0.095966 157 | 0.064923 0.225962 158 | 0.030747 0.062252 159 | 0.045860 0.168262 160 | 0.024190 0.246999 161 | 0.138652 0.407066 162 | 0.065675 0.225496 163 | 0.101433 0.428414 164 | 0.117840 0.190891 165 | 0.089105 0.255628 166 | 0.102685 0.428790 167 | 0.082270 0.164283 168 | 0.080003 0.149937 169 | 0.130556 0.521259 170 | 0.104720 0.242674 171 | 0.087813 0.339068 172 | 0.127531 0.261859 173 | 0.088726 0.166582 174 | 0.108665 0.399863 175 | 0.096798 0.437173 176 | 0.066191 0.252804 177 | 0.084070 0.297828 178 | 0.121647 0.238809 179 | 0.100088 0.258656 180 | 0.147512 0.234431 181 | 0.179987 0.282838 182 | 0.251204 0.533839 183 | -------------------------------------------------------------------------------- /banet_track/ba_module.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | 5 | def x_2d_coords_torch(n, h, w): 6 | # 0-x-W, 1-y-H 7 | x_2d = np.zeros((n, h, w, 2), dtype=np.float32) 8 | for y in range(0, h): 9 | x_2d[:, y, :, 1] = y 10 | for x in range(0, w): 11 | x_2d[:, :, x, 0] = x 12 | return torch.Tensor(x_2d) 13 | 14 | 15 | """ Camera Operations -------------------------------------------------------------------------------------------------- 16 | """ 17 | 18 | 19 | def batched_inv_pose(R, t): 20 | """ 21 | Compute the inverse pose [Verified] 22 | :param R: rotation matrix, dim (N, 3, 3) 23 | :param t: translation vector, dim (N, 3) 24 | :return: inverse pose of [R, t] 25 | """ 26 | N = R.size(0) 27 | Rwc = torch.transpose(R, 1, 2) 28 | tw = -torch.bmm(Rwc, t.view(N, 3, 1)) 29 | return Rwc, tw 30 | 31 | 32 | def batched_pi(K, X): 33 | """ 34 | Projecting the X in camera coordinates to the image plane 35 | :param K: camera intrinsic matrix tensor (N, 3, 3) 36 | :param X: point position in 3D camera coordinates system, is a 3D array with dimension of (N, num_points, 3) 37 | :return: N projected 2D pixel position u (N, num_points, 2) and the depth X (N, num_points, 1) 38 | """ 39 | # cx = W/2 40 | fx, fy, cx, cy = K[:, 0:1, 0:1], K[:, 1:2, 1:2], K[:, 0:1, 2:3], K[:, 1:2, 2:3] 41 | X[:,:,2].clamp_(1e-5) # avoid division by zero 42 | #!!!! /X[:,:,2:3]: Not be zero!!! 43 | u_x = fx * X[:, :, 0:1] / X[:, :, 2:3] + cx 44 | u_y = fy * X[:, :, 1:2] / X[:, :, 2:3] + cy 45 | u = torch.cat([u_x, u_y], dim=-1) 46 | return u, X[:, :, 2:3] 47 | 48 | 49 | def batched_correspond_depth(x_2d, depth): 50 | """ 51 | Geting corresponding depth of points in x_2d 52 | :param x_2d: B*npoints*2 (0-W, 1-H) 53 | :param depth: B*H*W 54 | :return: sparse_d : B*npoints*1 (0 for newly added (-1,-1)points) 55 | """ 56 | B = x_2d.shape[0] 57 | npoints = x_2d.shape[1] 58 | sparse_d = torch.zeros(B, npoints) 59 | x_W = x_2d[:, :, 0].tolist() 60 | y_H = x_2d[:, :, 1].tolist() 61 | for i in range(B): 62 | for j in range(npoints): 63 | h = int(y_H[i][j]) 64 | w = int(x_W[i][j]) 65 | if h >= 0 and w >= 0: 66 | sparse_d[i][j] = depth[i][h][w] 67 | # print(depth[i][h][w]) 68 | # else: 69 | # sparse_d[i][j][0] = 1e-8 70 | 71 | return sparse_d.unsqueeze(2).cuda() 72 | 73 | 74 | def batched_pi_inv(K, x, d): 75 | """ 76 | Projecting the pixel in 2D image plane and the depth to the 3D point in camera coordinate. 77 | :param x: 2d pixel position, a 2D array with dimension of (N, num_points, 2) 78 | :param d: depth at that pixel, a array with dimension of (N, num_points, 1) 79 | :param K: camera intrinsic matrix tensor (N, 3, 3) 80 | :return: 3D point in camera coordinate (N, num_points, 3) 81 | """ 82 | # print(K.size()) 83 | fx, fy, cx, cy = K[:, 0:1, 0:1], K[:, 1:2, 1:2], K[:, 0:1, 2:3], K[:, 1:2, 2:3] 84 | # print(d.device,fx.device, x.device, cx.device) 85 | # print(d.dtype, fx.dtype, cx.dtype,x[0,0,0].dtype) 86 | # mm!! 87 | # d = d * 0.001 88 | # using meters! (attention when load depth!!!) 89 | X_x = d * (x[:, :, 0:1] - cx) / fx 90 | X_y = d * (x[:, :, 1:2] - cy) / fy 91 | X_z = d 92 | X = torch.cat([X_x, X_y, X_z], dim=-1) 93 | return X 94 | 95 | 96 | def batched_transpose(R, t, X): 97 | """ 98 | Pytorch batch version of computing transform of the 3D points 99 | :param R: rotation matrix in dimension of (B, 3, 3) 100 | :param t: translation vector (B, 3) 101 | :param X: points with 3D position, a 2D array with dimension of (B, num_points, 3). (0,0,0)for added data 102 | :return: transformed 3D points 103 | """ 104 | assert R.shape[1] == 3 105 | assert R.shape[2] == 3 106 | assert t.shape[1] == 3 107 | N = R.shape[0] 108 | M = X.shape[1] 109 | # print(R.device, X.device) 110 | # print(R.dtype, X.dtype) 111 | X_after_R = torch.matmul(R, torch.transpose(X, 1, 2)) 112 | X_after_R = torch.transpose(X_after_R, 1, 2) 113 | # print(X_after_R.size(), t.size()) 114 | trans_X = X_after_R + t.view(N, 1, 3).expand(N, M, 3) 115 | return trans_X 116 | -------------------------------------------------------------------------------- /plot_per_frame/plotting.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | TXTPATH = './results' 7 | SAVEPATH_TRANS = './trans/figs_all' 8 | SAVEPATH_ROT = './rot/figs_all' 9 | 10 | # def tmplog(tmparray): 11 | # return np.log(tmparray) 12 | 13 | def get_trans_rot(txtarray): 14 | assert len(txtarray.shape) == 2 and txtarray.shape[1] == 2 15 | seqlen = txtarray.shape[0] 16 | x_seq = np.arange(seqlen) 17 | trans_err = txtarray[:, 0].flatten() 18 | rot_err = txtarray[:, 1].flatten() 19 | return x_seq, trans_err, rot_err 20 | 21 | def scatter_multi(x_seq, y_seqs, labels, colors, savepath, xlabel=None, ylabel=None, title=None, ymax=None): 22 | 23 | plt.subplot(121) 24 | 25 | plt.ylim(-0.025 * ymax, ymax * 1.025) 26 | 27 | for y_seq, one_label, one_color in zip(y_seqs, labels, colors): 28 | plt.scatter(x_seq, y_seq, s=5, c=one_color, alpha=0.7, label=one_label) 29 | 30 | plt.legend() 31 | if xlabel is not None: 32 | plt.xlabel(xlabel) 33 | else: 34 | plt.xlabel('Image number') 35 | if ylabel is not None: 36 | plt.ylabel(ylabel) 37 | if title is not None: 38 | plt.title(title+'_clipped') 39 | 40 | plt.subplot(122) 41 | for y_seq, one_label, one_color in zip(y_seqs, labels, colors): 42 | plt.scatter(x_seq, y_seq, s=5, c=one_color, alpha=0.7, label=one_label) 43 | 44 | plt.legend() 45 | if xlabel is not None: 46 | plt.xlabel(xlabel) 47 | else: 48 | plt.xlabel('Image number') 49 | if ylabel is not None: 50 | plt.ylabel(ylabel) 51 | if title is not None: 52 | plt.title(title+'_unclipped') 53 | 54 | plt.savefig(os.path.join(savepath, '%s.png' % title)) 55 | # plt.show() 56 | plt.clf() 57 | 58 | def main(seqname='pumpkin', ymax_trans=None, ymax_rot=None): 59 | 60 | methods = { 61 | 'MS-Transformer': ('MST_%s.txt', 'skyblue'), 62 | 'DSAC*': ('DSACSTAR_%s.txt', 'm'), 63 | 'Ours(dlt+e2e)': ('DLTe2e_%s.txt', 'r'), 64 | 'Ours(dlt+e2e+ref)': ('DLTe2e_ref_%s.txt', 'y'), 65 | 'Ours(dlt+e2e+DSAC*)': ('DLTe2e_dsacstar_%s.txt', 'g') 66 | } 67 | 68 | x_seq = None 69 | t_errs = [] 70 | r_errs = [] 71 | labels = [] 72 | colors = [] 73 | for k, (v, c) in methods.items(): 74 | try: 75 | txtarray = np.loadtxt(os.path.join(TXTPATH, v % seqname)) 76 | except: 77 | continue 78 | else: 79 | x_seq, t_err, r_err = get_trans_rot(txtarray) 80 | t_errs.append(t_err) 81 | r_errs.append(r_err) 82 | labels.append(k) 83 | colors.append(c) 84 | 85 | os.makedirs(SAVEPATH_TRANS, exist_ok=True) 86 | os.makedirs(SAVEPATH_ROT, exist_ok=True) 87 | 88 | fig = plt.figure(figsize=(24,9)) 89 | scatter_multi(x_seq, t_errs, labels, colors, SAVEPATH_TRANS, ylabel='Translation error(m)', title=seqname, ymax=ymax_trans) 90 | scatter_multi(x_seq, r_errs, labels, colors, SAVEPATH_ROT, ylabel='Rotation error(degree)', title=seqname, ymax=ymax_rot) 91 | 92 | 93 | def plot_final(): 94 | methods = { 95 | 'MS-Transformer': ('MST_%s.txt', 'skyblue'), 96 | 'DSAC*': ('DSACSTAR_%s.txt', 'm'), 97 | 'Ours(dlt+e2e)': ('DLTe2e_%s.txt', 'r'), 98 | 'Ours(dlt+e2e+ref)': ('DLTe2e_ref_%s.txt', 'y'), 99 | 'Ours(dlt+e2e+DSAC*)': ('DLTe2e_dsacstar_%s.txt', 'g') 100 | } 101 | 102 | seqs = [ 103 | ('office', 1.0, 40.0), 104 | ('kingscollege', 5.0, 30.0) 105 | ] 106 | 107 | fig = plt.figure(figsize=(12, 9)) 108 | plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.2, hspace=0.30) 109 | for inds, (seqname, ymax_trans, ymax_rot) in enumerate(seqs): 110 | 111 | x_seq = None 112 | t_errs = [] 113 | r_errs = [] 114 | labels = [] 115 | colors = [] 116 | for k, (v, c) in methods.items(): 117 | txtarray = np.loadtxt(os.path.join(TXTPATH, v % seqname)) 118 | x_seq, t_err, r_err = get_trans_rot(txtarray) 119 | t_errs.append(t_err) 120 | r_errs.append(r_err) 121 | labels.append(k) 122 | colors.append(c) 123 | 124 | plt.subplot(221 + inds*2) 125 | plt.ylim(-0.025 * ymax_trans, ymax_trans * 1.025) 126 | for y_seq, one_label, one_color in zip(t_errs, labels, colors): 127 | plt.scatter(x_seq, y_seq, s=2, c=one_color, alpha=0.7, label=one_label) 128 | plt.legend(loc='upper left') 129 | plt.xlabel('Image number') 130 | plt.ylabel('Translation error(m)') 131 | plt.title(seqname) 132 | 133 | plt.subplot(222 + inds*2) 134 | plt.ylim(-0.025 * ymax_rot, ymax_rot * 1.025) 135 | for y_seq, one_label, one_color in zip(r_errs, labels, colors): 136 | plt.scatter(x_seq, y_seq, s=2, c=one_color, alpha=0.7, label=one_label) 137 | plt.legend(loc='upper left') 138 | plt.xlabel('Image number') 139 | plt.ylabel('Rotation error(degree)') 140 | plt.title(seqname) 141 | plt.savefig('./result_s2.png', bbox_inches='tight') 142 | 143 | 144 | if __name__ == '__main__': 145 | plot_final() 146 | 147 | # seqs = [ 148 | # # ('chess', 0.5, 20.0), 149 | # # ('fire', 1.0, 90.0), 150 | # # ('greatcourt', 125.0, 180.0), 151 | # # ('heads', 0.5, 85.0), 152 | # ('kingscollege', 5.0, 30.0), 153 | # ('office', 1.0, 100.0), 154 | # # ('oldhospital', 10.0, 18.0), 155 | # # ('pumpkin', 2.0, 89.0), 156 | # # ('redkitchen', 1.8, 80.0), 157 | # # ('shopfacade', 2.0, 30.0), 158 | # # ('stairs', 1.0, 30.0), 159 | # # ('stmaryschurch', 10.0, 50.0), 160 | # ] 161 | # for (seq, ymax_trans, ymax_rot) in seqs: 162 | # main(seqname=seq, ymax_trans=ymax_trans, ymax_rot=ymax_rot) 163 | -------------------------------------------------------------------------------- /models/Dense_networks_e2e_GNN_for_unsup.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | import math 5 | from loss_functions import UnsupLoss, DLT_postprocess 6 | from banet_track.ba_module import batched_pi_inv 7 | from Dense_Nets.ESAC import ESAC_SCNet 8 | from Dense_Nets.ESAC_DROID import ESAC_DROID_Net 9 | from Dense_Nets.OAGNN_EigFree import OAGNN 10 | 11 | # generate grid of target reprojection pixel positions 12 | OUTPUT_SUBSAMPLE = 8 13 | pixel_grid = torch.zeros((2, 14 | math.ceil(5000 / OUTPUT_SUBSAMPLE), # 5000px is max limit of image size, increase if needed 15 | math.ceil(5000 / OUTPUT_SUBSAMPLE))) 16 | 17 | for x in range(0, pixel_grid.size(2)): 18 | for y in range(0, pixel_grid.size(1)): 19 | pixel_grid[0, y, x] = x * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 20 | pixel_grid[1, y, x] = y * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 21 | 22 | device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") 23 | 24 | class DenseSC_GNN(nn.Module): 25 | def __init__(self, args, scene_mean): 26 | super(DenseSC_GNN, self).__init__() 27 | self.mean = scene_mean.clone() 28 | 29 | if args.uncertainty: 30 | self.un = True 31 | else: 32 | self.un = False 33 | 34 | if args.dataset == '7Scenes': 35 | self.relu = True 36 | else: 37 | self.relu = False 38 | 39 | if args.dataset == '7Scenes': 40 | self.network = ESAC_SCNet(self.mean, self.un) 41 | else: 42 | self.network = ESAC_DROID_Net(self.mean, self.un, output_dim=256, norm_fn='none') 43 | 44 | self.config_e2e = { 45 | 'net_depth': args.net_depth, 46 | 'clusters': args.clusters, 47 | 'iter_num': args.iter_num, 48 | 'net_channels': args.net_channels 49 | } 50 | self.wDLT_net = OAGNN(self.config_e2e) 51 | 52 | self.config = { 53 | 'padding_mode':'zeros', 54 | 'with_auto_mask':args.auto_mask, 55 | 'with_ssim':args.ssim, 56 | 'photo_weight':args.photo_weight, 57 | 'use_DLT2T':args.use_DLT2T, 58 | 'use_patch':args.use_patch, 59 | 'Hc': 60, 60 | 'Wc': 80 61 | } 62 | 63 | self.config_e2eloss = {} 64 | self.matchloss = UnsupLoss(self.config, self.config_e2eloss, self.relu) 65 | 66 | def forward(self, X_world, image, Ts, Ks, epoch, test=False): 67 | """ 68 | :param X_world: corresponding 3D coord (B,T, H'W',3) 69 | :param image: (B,T, 3,H, W) 70 | :param Ts: (B,T, 3, 4) 71 | :param Ks: (B,T, 3, 3) 72 | :return: pred_world(B,3,H,W) , loss 73 | """ 74 | if test==False: 75 | B, T, _, H, W = image.size() 76 | image = image.contiguous().view(-1,3,H,W) 77 | Ts = Ts.contiguous().view(-1,3,4) 78 | Ks = Ks.contiguous().view(-1,3,3) 79 | else: 80 | B, _, H, W = image.size() 81 | 82 | pred_X, _ = self.network(image) 83 | 84 | BT,_,cH,cW = pred_X.size() 85 | self.matchloss.config_un['Hc'] = cH 86 | self.matchloss.config_un['Wc'] = cW 87 | N = cH * cW 88 | pred_X_clone = pred_X.clone() 89 | pixel_grid_crop = pixel_grid[:, 0:pred_X.size(2), 0:pred_X.size(3)].clone() 90 | pixel_grid_crop = pixel_grid_crop.view(2, -1).permute(1,0).to(device) 91 | pixel_grid_crop_clone = pixel_grid_crop.clone() 92 | 93 | if test==True: 94 | # (B,N,3) 95 | pred_X_seq = pred_X.detach().view(B, 3, -1).permute(0, 2, 1) 96 | pixel_grid_crop_seq = pixel_grid_crop.expand(B,-1,-1) 97 | 98 | depth_norm = torch.ones(B,N,1).to(device) 99 | Pc_norm = batched_pi_inv(Ks, pixel_grid_crop_seq, depth_norm) 100 | sparse_data = torch.cat([Pc_norm[:,:,:2], pred_X_seq], dim=-1) 101 | 102 | res_logits,_ = self.wDLT_net(sparse_data,Ks) 103 | y_hat = res_logits[-1] 104 | e_post = DLT_postprocess(sparse_data, res_logits[-1], Ks, self.relu) 105 | 106 | if self.relu: 107 | y_hat = torch.relu(torch.tanh(y_hat)) 108 | else: 109 | y_hat = torch.exp(F.logsigmoid(y_hat)) 110 | 111 | return pred_X_clone, y_hat, e_post, pixel_grid_crop_clone 112 | else: 113 | image = image.view(B,T,3,H,W) 114 | # (B,N,3) 115 | pred_X_seq = pred_X.detach().view(BT, 3, -1).permute(0, 2, 1) 116 | pixel_grid_crop_seq = pixel_grid_crop.expand(BT,-1,-1) 117 | 118 | depth_norm = torch.ones(BT,N,1).to(device) 119 | Pc_norm = batched_pi_inv(Ks, pixel_grid_crop_seq, depth_norm) 120 | sparse_data = torch.cat([Pc_norm[:,:,:2], pred_X_seq], dim=-1) 121 | 122 | res_logits,_ = self.wDLT_net(sparse_data,Ks) 123 | y_hat = res_logits[-1] 124 | 125 | loss_1 = 0 126 | loss_val = [] 127 | e_hat = [] 128 | num_valid=[] 129 | for i in range(len(res_logits)): 130 | loss_i, geo_loss, cla_loss, e_post, num_valid_sc = self.matchloss.run(epoch, sparse_data, res_logits[i], Ks, image, pixel_grid_crop_seq) 131 | loss_1 += loss_i 132 | loss_val += [geo_loss, cla_loss] 133 | e_hat.append(e_post) 134 | num_valid.append(num_valid_sc) 135 | 136 | loss_1 = torch.unsqueeze(loss_1, 0) 137 | loss_2 = torch.unsqueeze(loss_val[0], 0) 138 | loss_3 = torch.unsqueeze(loss_val[1], 0) 139 | 140 | if self.relu: 141 | y_hat = torch.relu(torch.tanh(y_hat)) 142 | else: 143 | y_hat = torch.exp(F.logsigmoid(y_hat)) 144 | 145 | pred_X_clone = pred_X_clone.view(B,T,3,cH, cW) 146 | y_hat = y_hat.view(B,T,-1) 147 | e_hat[-1] = e_hat[-1].view(B,T,3,4) 148 | return loss_1, loss_2, loss_3, pred_X_clone, y_hat, e_hat[-1], pixel_grid_crop_clone, num_valid[-1] 149 | 150 | def init_weights(self): 151 | self.network.init_weights() 152 | 153 | -------------------------------------------------------------------------------- /models/Dense_networks_e2e.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | import math 5 | from loss_functions import EigFreeLoss, DLT_postprocess 6 | from banet_track.ba_module import batched_pi_inv 7 | from Dense_Nets.ESAC import ESAC_SCNet 8 | from Dense_Nets.ESAC_DROID import ESAC_DROID_Net 9 | from Dense_Nets.OAGNN_EigFree import OANet 10 | 11 | # generate grid of target reprojection pixel positions 12 | OUTPUT_SUBSAMPLE = 8 13 | pixel_grid = torch.zeros((2, 14 | math.ceil(5000 / OUTPUT_SUBSAMPLE), # 5000px is max limit of image size, increase if needed 15 | math.ceil(5000 / OUTPUT_SUBSAMPLE))) 16 | 17 | for x in range(0, pixel_grid.size(2)): 18 | for y in range(0, pixel_grid.size(1)): 19 | pixel_grid[0, y, x] = x * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 20 | pixel_grid[1, y, x] = y * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 21 | 22 | device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") 23 | 24 | class DenseSC(nn.Module): 25 | def __init__(self, args, scene_mean): 26 | super(DenseSC, self).__init__() 27 | self.mean = scene_mean.clone() 28 | self.e2e_training = args.e2e_training 29 | 30 | if args.uncertainty: 31 | self.un = True 32 | else: 33 | self.un = False 34 | if args.dataset == '7Scenes': 35 | self.relu = True 36 | else: 37 | self.relu = False 38 | 39 | if args.dataset == '7Scenes': 40 | self.network = ESAC_SCNet(self.mean, self.un) 41 | else: 42 | self.network = ESAC_DROID_Net(self.mean, self.un, output_dim=256, norm_fn='none') 43 | 44 | 45 | self.config_e2e = { 46 | 'net_depth': args.net_depth, 47 | 'clusters': args.clusters, 48 | 'iter_num': args.iter_num, 49 | 'net_channels': args.net_channels 50 | } 51 | self.wDLT_net = OANet(self.config_e2e) 52 | 53 | 54 | self.config = { 55 | 'itol': args.inittolerance, 56 | 'mindepth': args.mindepth, 57 | 'maxdepth': args.maxdepth, 58 | 'targetdepth': args.targetdepth, 59 | 'softclamp': args.softclamp, 60 | 'hardclamp': args.hardclamp 61 | } 62 | self.config_e2eloss = { 63 | "loss_eigfree" : args.loss_essential, 64 | "loss_classif" : args.loss_classif, 65 | "loss_essential_init_epo": args.loss_essential_init_epo, 66 | 'mindepth': args.mindepth, 67 | 'maxdepth': args.maxdepth, 68 | 'obj_geod_th': args.obj_geod_th, 69 | 'alpha': args.Eig_alpha, 70 | 'beta': args.Eig_beta, 71 | } 72 | 73 | self.matchloss = EigFreeLoss(self.config_e2eloss, self.relu) 74 | 75 | def forward(self, X_world, image, Ts, Ks, epoch, test=False, ds=False): 76 | """ 77 | :param X_world: corresponding 3D coord (B, H'W',3) 78 | :param used_mask: discard inaccurate depth (B,H'W') 79 | :param Ts: (B, 3, 4) 80 | :param Ks: (B, 3, 3) 81 | :return: pred_world(B,3,H,W) , loss 82 | """ 83 | # B*3*H*W 84 | pred_X, _ = self.network(image) 85 | B,_,cH,cW = pred_X.size() 86 | N = cH * cW 87 | 88 | #(2,H,W) 89 | pixel_grid_crop = pixel_grid[:, 0:pred_X.size(2), 0:pred_X.size(3)].clone() 90 | #(N,2) 91 | pixel_grid_crop = pixel_grid_crop.view(2, -1).permute(1,0).to(device) 92 | if ds==True: 93 | return pred_X, None, None, pixel_grid_crop 94 | 95 | if test==True: 96 | pred_X_seq = pred_X.detach().view(B, 3, -1).permute(0, 2, 1) 97 | pixel_grid_crop_seq = pixel_grid_crop.expand(B,-1,-1) 98 | 99 | depth_norm = torch.ones(B,N,1).to(device) 100 | Pc_norm = batched_pi_inv(Ks, pixel_grid_crop_seq, depth_norm) 101 | sparse_data = torch.cat([Pc_norm[:,:,:2], pred_X_seq], dim=-1) 102 | 103 | res_logits,_ = self.wDLT_net(sparse_data,Ks) 104 | y_hat = res_logits[-1] 105 | 106 | e_post = DLT_postprocess(sparse_data, res_logits[-1], Ks, self.relu) 107 | 108 | if self.relu: 109 | y_hat = torch.relu(torch.tanh(y_hat)) 110 | else: 111 | y_hat = torch.exp(F.logsigmoid(y_hat)) 112 | return pred_X, y_hat, e_post, pixel_grid_crop 113 | else: 114 | pred_X_clone = pred_X.clone() 115 | pixel_grid_crop_clone = pixel_grid_crop.clone() 116 | # (B,N,3) 117 | if self.e2e_training: 118 | pred_X_seq = pred_X.view(B, 3, -1).permute(0, 2, 1) 119 | else: 120 | pred_X_seq = pred_X.detach().view(B, 3, -1).permute(0, 2, 1) 121 | 122 | pixel_grid_crop_seq = pixel_grid_crop.expand(B,-1,-1) 123 | 124 | depth_norm = torch.ones(B,N,1).to(device) 125 | Pc_norm = batched_pi_inv(Ks, pixel_grid_crop_seq, depth_norm) 126 | sparse_data = torch.cat([Pc_norm[:,:,:2], pred_X_seq], dim=-1) 127 | 128 | res_logits,_ = self.wDLT_net(sparse_data,Ks) 129 | y_hat = res_logits[-1] 130 | 131 | loss_1 = 0 132 | loss_val = [] 133 | e_hat = [] 134 | dr_term=[] 135 | for i in range(len(res_logits)): 136 | loss_i, geo_loss, cla_loss, e_post, dr = self.matchloss.run(epoch, sparse_data, res_logits[i], Ts, Ks) 137 | loss_1 += loss_i 138 | loss_val += [geo_loss, cla_loss] 139 | e_hat.append(e_post) 140 | dr_term.append(dr) 141 | 142 | loss_1 = torch.unsqueeze(loss_1, 0) 143 | loss_2 = torch.unsqueeze(loss_val[0], 0) 144 | loss_3 = torch.unsqueeze(loss_val[1], 0) 145 | 146 | if self.relu: 147 | y_hat = torch.relu(torch.tanh(y_hat)) 148 | else: 149 | y_hat = torch.exp(F.logsigmoid(y_hat)) 150 | 151 | return loss_1, loss_2, loss_3, pred_X_clone, y_hat, e_hat[-1], pixel_grid_crop_clone, dr_term[-1][0], dr_term[-1][1] 152 | 153 | 154 | def init_weights(self): 155 | self.network.init_weights() 156 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SC-wLS 2 | 3 | This is the official repo for the paper [*SC-wLS: Towards Interpretable Feed-forward Camera Re-localization*]() (ECCV 2022), by Xin Wu\*, Hao Zhao\*, Shunkai Li, Yingdian Cao, and Hongbin Zha. 4 | 5 |
6 | 7 |
8 | 9 | > we propose a feed-forward camera re-localization method termed SC-wLS that exploits scene coordinate estimates for weighted least squares pose regression. This differentiable formulation exploits a weight network imposed on 2D-3D correspondences, and requires pose supervision only. 10 | 11 | For more details, please refer to the [paper](). 12 | 13 | ## Requirements 14 | 15 | The code in this repo has been tested on Ubuntu 20.04, Python 3.8, and PyTorch 1.9.1 with a nvidia 3090. We recommand to create a virtual conda environment to install them. Other dependencies could be easily installed with pip or conda. 16 | 17 | Moreover, the code includes a C++ extension (in the `dsacstar`) of [DSAC\*](https://github.com/vislearn/dsacstar). You could compile and install it by executing: 18 | ```bash 19 | cd dsacstar 20 | python setup.py install 21 | ``` 22 | Note that you need to install opencv (3.4.15) in advance, and change the opencv path in the `setup.py`. 23 | 24 | ## Datasets 25 | 26 | Download the [7Scenes](https://www.microsoft.com/en-us/research/project/rgb-d-dataset-7-scenes/) dataset and the [Cambridge Landmarks](http://mi.eng.cam.ac.uk/projects/relocalisation/#dataset) dataset. 27 | 28 | ## Inference 29 | 30 | Download the [pretrained models](https://drive.google.com/drive/folders/1FWu1r0e-xHgNF2oUWOt70eu7HZZ4H-LG?usp=sharing) into the repo directory. 31 | 32 | To evaluate on a scene, call: 33 | ```bash 34 | python test_e2e_DLT.py --dataset --scene --config_file configs/single_frame.ini --model DenseSC_GNN --pretrained-model <--useLM> 35 | 36 | ``` 37 | the `--useLM` option means whether to use LM-Refine in Section 3.5 of the paper. 38 | 39 | In order to test the end-to-end training model, for example on the chess scene from the 7Scenes dataset: 40 | ```bash 41 | python test_e2e_DLT.py --dataset 7Scenes --scene chess -b1 --config_file configs/single_frame.ini --model DenseSC_GNN --pretrained-model ./7scenes_e2e_model/chess/model.pkl 42 | 43 | ``` 44 | 45 | In order to test the model after test-time adapatation, for example on the chess scene from the 7Scenes dataset: 46 | ```bash 47 | python test_e2e_DLT.py --dataset 7Scenes --scene chess -b1 --config_file configs/single_frame.ini --model DenseSC_GNN --pretrained-model ./7scenes_unsup_model/chess/model.pkl 48 | 49 | ``` 50 | 51 | To plot the per-frame errors of the scenes, please refer to the `plot_per_frame` file. 52 | 53 | ## Training 54 | 55 | The training of SC-wLS includes three stages: scene coordinate initialization, weight initialization and end-to-end optimization. These stages use pose supervision only. Meanwhile, you could also use self-supervised adaptation to enhance the test-time performance. 56 | 57 | #### Scene coordinate initialization 58 | 59 | To train the scene coordinate network on a scene, call: 60 | ```bash 61 | python train_init.py --dataset --scene -b1 --epochs 1500 --config_file configs/single_frame.ini --lr 1e-4 <--cal_training_pose> --model DenseSC 62 | ``` 63 | `--cal_training_pose` decides whether to calculate and record poses when training. 64 | 65 | #### Weight initialization and end-to-end optimization 66 | 67 | Use `--e2e_training` to specify whether to train the weight network only, or do end-to-end training. 68 | Weight initialization: 69 | ```bash 70 | python train_e2e_DLT.py --dataset --scene -b1 --epochs 1500 --config_file configs/single_frame.ini --glr 1e-4 --model DenseSC_GNN --loss_classif 0.5 --loss_essential 10.0 --obj_geod_th 1.0 --loss_essential_init_epo 0 -Ea 5 -Eb 1e-4 <--cal_training_pose> <--useRANSAC> <--log_val_loss> --pretrained-model 71 | ``` 72 | `--useRANSAC` decides whether to output the poses calculated from PnP+RANSAC as well. `--log_val_loss` means whether to record the losses when validation. 73 | 74 | End-to-end training: 75 | ```bash 76 | python train_e2e_DLT.py --dataset --scene -b1 --epochs 1500 --config_file configs/single_frame.ini --lr 1e-5 --glr 1e-5 --model DenseSC_GNN --loss_classif 0.5 --loss_essential 10.0 --obj_geod_th 1.0 --loss_essential_init_epo 0 -Ea 5 -Eb 1e-4 --e2e_training <--cal_training_pose> <--useRANSAC> <--log_val_loss> --pretrained-model --w-model 77 | ``` 78 | 79 | `--loss_classif` and `--loss_essential` are weights used to balance the classification loss and the regression loss. `-Ea` and `-Eb` are the $\alpha$ and $\beta$ in Eq. 11. As illustrated in the supplement, one should select proper hyper-parameters to balance the losses. Please refer to the [detailed settings](README_TRAIN.md). 80 | 81 | #### Test-time adaptation 82 | To validate the self-supervised test-time adaptation on a scene, call: 83 | 84 | ```bash 85 | python train_e2e_DLT_unsup.py --dataset --scene -b1 --epochs 1500 --config_file configs/ --glr 1e-5 --model DenseSC_GNN --pretrained-model --auto_mask --ssim --use_DLT2T --ph 400 <--cal_training_pose> <--useRANSAC> 86 | 87 | ``` 88 | `--config_file` is `skip1step7.ini` for `7Scenes`, and `skip1step2.ini` for `Cambridge`. 89 | 90 | You could also change `--model DenseSC` to train a model based on OANet w/o attention. Ablations could be seen in the supplementary. 91 | 92 | Call the scripts with the `-h` option to see a listing of all supported command line arguments. 93 | 94 | ## Acknowledgements 95 | In this repository, we have used codes from the following repositories. We thank all the authors for sharing great codes. 96 | 97 | - [DSAC\*](https://github.com/vislearn/dsacstar) 98 | - [OANet](https://github.com/zjhthu/OANet) 99 | - [Eig-Free](https://github.com/Dangzheng/Eig-Free-release) 100 | - [SuperGlue](https://github.com/magicleap/SuperGluePretrainedNetwork) 101 | - [SfMLearner](https://github.com/tinghuiz/SfMLearner) 102 | 103 | ## Citation 104 | If you find this repository useful, please cite the following paper and consider giving a star: 105 | ``` 106 | 107 | ``` -------------------------------------------------------------------------------- /README_TRAIN.md: -------------------------------------------------------------------------------- 1 | ## Training 2 | 3 | #### Weight initialization and end-to-end optimization 4 | 5 | ```bash 6 | python train_e2e_DLT.py --dataset --scene -b1 --epochs 1500 --config_file configs/single_frame.ini --glr 1e-4 --model DenseSC_GNN --loss_classif 0.5 --loss_essential 10.0 --obj_geod_th 1.0 --loss_essential_init_epo 0 -Ea 5 -Eb 1e-4 <--cal_training_pose> <--useRANSAC> <--log_val_loss> --pretrained-model 7 | ``` 8 | Use `--e2e_training` to specify whether to train the weight network only, or do end-to-end training. 9 | 10 | `--loss_classif` and `--loss_essential` are weights used to balance the classification loss and the regression loss. `-Ea` and `-Eb` are the $\alpha$ and $\beta$ in Eq. 11. As illustrated in the supplement, one should select proper hyper-parameters to balance the losses. We notice that for scenes with more noisy predicted coordinates (such as greatcourt), a higher weight of classification loss `--loss_classif` is preferred, which effectively prunes outliers. 11 | 12 | Besides, in Eq. 11, `-Ea` governs whether we focus on the first term (reducing pose errors) or the second one (avoiding the non-null space). `-Eb` is set to balance the trace value in the second term, whose magnitude could significantly vary across the training samples. Please refer to Section 4.1 of [1] for detailed theory analysis. One could adjust these hyper-parameters when training to see their influences. 13 | 14 | To reproduce the results in the paper, one could use following settings: 15 | 16 | ##### Weight initialization: 17 | 18 | We list the hyperparameters used in the command line. As mentioned above, the two terms in Eq. 11 play different roles, so at the early stage of weight initialization (index 1) we focus more on the stablity, and later finetune the model (index 2) with smaller leraning rate `--glr` and `--Ea` to enhance the accuracy when the former has converged. 19 | 20 | | Scene | loss_cla1 | loss_ess1 | Ea1 | Eb1 | glr1 | loss_cla2 | loss_ess2 | Ea2 | Eb2 | glr2 | 21 | | :-----------: | :------------------: | :------------------: | :------------: | :------------: | :-------------: | :------------------: | :------------------: | :------------: | :------------: | :-------------: | 22 | | chess | 0.5 | 10 | 5 | 1e-4 | 1e-4 | 0.5 | 10 | 3.5 | 1e-4 | 1e-5 | 23 | | heads | 0.5 | 10 | 5 | 1e-4 | 1e-4 | 2 | 10 | 2 | 1e-4 | 1e-5 | 24 | | fire | 0.5 | 10 | 5 | 1e-4 | 1e-4 | 0.5 | 10 | 1 | 1e-4 | 1e-5 | 25 | | office | 0.5 | 10 | 5 | 1e-4 | 1e-4 | | | | | | 26 | | pumpkin | 0.5 | 10 | 5 | 1e-4 | 1e-4 | 2 | 10 | 2 | 1e-4 | 1e-5 | 27 | | kitchen | 0.5 | 10 | 5 | 1e-4 | 1e-4 | | | | | | 28 | | stairs | 0.5 | 10 | 5 | 1e-4 | 1e-4 | 0.5 | 10 | 3 | 1e-4 | 1e-5 | 29 | | greatcourt | 8 | 10 | 5 | 1e-6 | 1e-4 | 8 | 10 | 2 | 1e-7 | 1e-4 | 30 | | kingscollege | 2 | 10 | 5 | 1e-6 | 1e-4 | | | | | | 31 | | shopfacade | 2 | 10 | 5 | 1e-4 | 1e-4 | 2 | 10 | 2 | 1e-6 | 1e-4 | 32 | | oldhospital | 0.5 | 10 | 5 | 1e-6 | 1e-4 | 2 | 10 | 2 | 1e-5 | 1e-4 | 33 | | stmaryschurch | 2 | 10 | 5 | 1e-6 | 1e-4 | | | | | | 34 | 35 | Moreover, since our weight network takes only 5d pairs as input, it has better capacity of generalization than that relies on RGB input. One could use pretrained model from other scenes in `--w-model` as init to accelerate convergence. 36 | 37 | As for training epoch, since different scene has different size of training set, one could take an early stop according to the validation process, or simply replace `--epoch` with about 1.5M-2M iterations. 38 | 39 | 40 | ##### End-to-end training: 41 | `--lr` and `--glr` are set to 1e-5. 42 | 43 | | Scene | loss_cla | loss_ess | Ea | Eb | 44 | | :-----------: | :------: | :------: | :---: | :---: | 45 | | chess | 0.5 | 10 | 5 | 1e-4 | 46 | | heads | 0.5 | 10 | 1 | 1e-4 | 47 | | fire | 2 | 10 | 2 | 1e-4 | 48 | | office | 0.5 | 10 | 5 | 1e-4 | 49 | | pumpkin | 0.5 | 10 | 5 | 1e-4 | 50 | | kitchen | 2 | 10 | 2 | 1e-4 | 51 | | stairs | 0.5 | 10 | 5 | 1e-4 | 52 | | greatcourt | 8 | 10 | 2 | 1e-7 | 53 | | kingscollege | 2 | 10 | 5 | 1e-6 | 54 | | shopfacade | 2 | 10 | 2 | 1e-4 | 55 | | oldhospital | 2 | 10 | 2 | 1e-5 | 56 | | stmaryschurch | 2 | 10 | 3 | 1e-6 | 57 | 58 | As aforementioned, `--epoch` could be replaced with about 150k iterations. 59 | 60 | 61 | ### References 62 | 1. Dang, Zheng, et al. "Eigendecomposition-free training of deep networks for linear least-square problems." TPAMI (2020). 63 | 64 | -------------------------------------------------------------------------------- /models/Dense_networks_e2e_GNN.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | import numpy as np 5 | import math 6 | from loss_functions import EigFreeLoss, DLT_postprocess 7 | from banet_track.ba_module import batched_pi_inv 8 | from Dense_Nets.ESAC import ESAC_SCNet 9 | from Dense_Nets.ESAC_DROID import ESAC_DROID_Net 10 | from Dense_Nets.OAGNN_EigFree import OAGNN, OANet 11 | 12 | # generate grid of target reprojection pixel positions 13 | OUTPUT_SUBSAMPLE = 8 14 | pixel_grid = torch.zeros((2, 15 | math.ceil(5000 / OUTPUT_SUBSAMPLE), # 5000px is max limit of image size, increase if needed 16 | math.ceil(5000 / OUTPUT_SUBSAMPLE))) 17 | 18 | for x in range(0, pixel_grid.size(2)): 19 | for y in range(0, pixel_grid.size(1)): 20 | pixel_grid[0, y, x] = x * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 21 | pixel_grid[1, y, x] = y * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 22 | 23 | device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") 24 | 25 | class DenseSC_GNN(nn.Module): 26 | def __init__(self, args, scene_mean): 27 | super(DenseSC_GNN, self).__init__() 28 | self.mean = scene_mean.clone() 29 | self.e2e_training = args.e2e_training 30 | 31 | if args.uncertainty: 32 | self.un = True 33 | else: 34 | self.un = False 35 | if args.dataset == '7Scenes': 36 | self.relu = True 37 | else: 38 | self.relu = False 39 | 40 | if args.dataset == '7Scenes': 41 | self.network = ESAC_SCNet(self.mean, self.un) 42 | else: 43 | self.network = ESAC_DROID_Net(self.mean, self.un, output_dim=256, norm_fn='none') 44 | 45 | self.config_e2e = { 46 | 'net_depth': args.net_depth, 47 | 'clusters': args.clusters, 48 | 'iter_num': args.iter_num, 49 | 'net_channels': args.net_channels 50 | } 51 | # GNN, self-attention 52 | self.wDLT_net = OAGNN(self.config_e2e) 53 | 54 | # self.wDLT_net = OANet(self.config_e2e) 55 | 56 | self.config = { 57 | 'itol': args.inittolerance, 58 | 'mindepth': args.mindepth, 59 | 'maxdepth': args.maxdepth, 60 | 'targetdepth': args.targetdepth, 61 | 'softclamp': args.softclamp, 62 | 'hardclamp': args.hardclamp 63 | } 64 | self.config_e2eloss = { 65 | "loss_eigfree" : args.loss_essential, 66 | "loss_classif" : args.loss_classif, 67 | "loss_essential_init_epo": args.loss_essential_init_epo, 68 | 'mindepth': args.mindepth, 69 | 'maxdepth': args.maxdepth, 70 | 'obj_geod_th': args.obj_geod_th, 71 | 'alpha': args.Eig_alpha, 72 | 'beta': args.Eig_beta, 73 | } 74 | self.matchloss = EigFreeLoss(self.config_e2eloss, self.relu) 75 | 76 | 77 | def forward(self, X_world, image, Ts, Ks, epoch, test=False, ds=False): 78 | """ 79 | :param X_world: corresponding 3D coord (B, H'W',3) 80 | :param used_mask: discard inaccurate depth (B,H'W') 81 | :param Ts: (B, 3, 4) 82 | :param Ks: (B, 3, 3) 83 | :return: pred_world(B,3,H,W) , loss 84 | """ 85 | # B*3*H*W 86 | pred_X, _ = self.network(image) 87 | B,_,cH,cW = pred_X.size() 88 | N = cH * cW 89 | 90 | #(2,H,W) 91 | pixel_grid_crop = pixel_grid[:, 0:pred_X.size(2), 0:pred_X.size(3)].clone() 92 | #(N,2) 93 | pixel_grid_crop = pixel_grid_crop.view(2, -1).permute(1,0).to(device) 94 | if ds==True: 95 | return pred_X, None, None, pixel_grid_crop 96 | 97 | if test==True: 98 | pred_X_seq = pred_X.detach().view(B, 3, -1).permute(0, 2, 1) 99 | pixel_grid_crop_seq = pixel_grid_crop.expand(B,-1,-1) 100 | 101 | depth_norm = torch.ones(B,N,1).to(device) 102 | Pc_norm = batched_pi_inv(Ks, pixel_grid_crop_seq, depth_norm) 103 | sparse_data = torch.cat([Pc_norm[:,:,:2], pred_X_seq], dim=-1) 104 | 105 | res_logits,_ = self.wDLT_net(sparse_data,Ks) 106 | y_hat = res_logits[-1] 107 | 108 | e_post = DLT_postprocess(sparse_data, res_logits[-1], Ks, self.relu) 109 | 110 | if self.relu: 111 | y_hat = torch.relu(torch.tanh(y_hat)) 112 | else: 113 | y_hat = torch.exp(F.logsigmoid(y_hat)) 114 | return pred_X, y_hat, e_post, pixel_grid_crop 115 | else: 116 | pred_X_clone = pred_X.clone() 117 | pixel_grid_crop_clone = pixel_grid_crop.clone() 118 | # (B,N,3) 119 | if self.e2e_training: 120 | pred_X_seq = pred_X.view(B, 3, -1).permute(0, 2, 1) 121 | else: 122 | pred_X_seq = pred_X.detach().view(B, 3, -1).permute(0, 2, 1) 123 | 124 | pixel_grid_crop_seq = pixel_grid_crop.expand(B,-1,-1) 125 | 126 | depth_norm = torch.ones(B,N,1).to(device) 127 | Pc_norm = batched_pi_inv(Ks, pixel_grid_crop_seq, depth_norm) 128 | sparse_data = torch.cat([Pc_norm[:,:,:2], pred_X_seq], dim=-1) 129 | 130 | res_logits,_ = self.wDLT_net(sparse_data,Ks) 131 | y_hat = res_logits[-1] 132 | 133 | loss_1 = 0 134 | loss_val = [] 135 | e_hat = [] 136 | dr_term=[] 137 | for i in range(len(res_logits)): 138 | loss_i, geo_loss, cla_loss, e_post, dr = self.matchloss.run(epoch, sparse_data, res_logits[i], Ts, Ks) 139 | loss_1 += loss_i 140 | loss_val += [geo_loss, cla_loss] 141 | e_hat.append(e_post) 142 | dr_term.append(dr) 143 | 144 | loss_1 = torch.unsqueeze(loss_1, 0) 145 | loss_2 = torch.unsqueeze(loss_val[0], 0) 146 | loss_3 = torch.unsqueeze(loss_val[1], 0) 147 | 148 | if self.relu: 149 | y_hat = torch.relu(torch.tanh(y_hat)) 150 | else: 151 | y_hat = torch.exp(F.logsigmoid(y_hat)) 152 | 153 | return loss_1, loss_2, loss_3, pred_X_clone, y_hat, e_hat[-1], pixel_grid_crop_clone, dr_term[-1][0], dr_term[-1][1] 154 | 155 | 156 | 157 | def init_weights(self): 158 | self.network.init_weights() 159 | 160 | -------------------------------------------------------------------------------- /dsacstar/thread_rand.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based on the DSAC++ and ESAC code. 3 | https://github.com/vislearn/LessMore 4 | https://github.com/vislearn/esac 5 | 6 | Copyright (c) 2016, TU Dresden 7 | Copyright (c) 2019, Heidelberg University 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are met: 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above copyright 15 | notice, this list of conditions and the following disclaimer in the 16 | documentation and/or other materials provided with the distribution. 17 | * Neither the name of the TU Dresden, Heidelberg University nor the 18 | names of its contributors may be used to endorse or promote products 19 | derived from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL TU DRESDEN OR HEIDELBERG UNIVERSITY BE LIABLE FOR ANY 25 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 28 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #pragma once 34 | 35 | #include 36 | 37 | /** Classes and methods for generating random numbers in multi-threaded programs. */ 38 | 39 | /** 40 | * @brief Provides random numbers for multiple threads. 41 | * 42 | * Singelton class. Holds a random number generator for each thread and gives random numbers for the current thread. 43 | */ 44 | class ThreadRand 45 | { 46 | public: 47 | /** 48 | * @brief Returns a random integer (uniform distribution). 49 | * 50 | * @param min Minimum value of the random integer (inclusive). 51 | * @param max Maximum value of the random integer (exclusive). 52 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 53 | * @return int Random integer value. 54 | */ 55 | static int irand(int min, int max, int tid = -1); 56 | 57 | /** 58 | * @brief Returns a random double value (uniform distribution). 59 | * 60 | * @param min Minimum value of the random double (inclusive). 61 | * @param max Maximum value of the random double (inclusive). 62 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 63 | * @return double Random double value. 64 | */ 65 | static double drand(double min, double max, int tid = -1); 66 | 67 | /** 68 | * @brief Returns a random double value (Gauss distribution). 69 | * 70 | * @param mean Mean of the Gauss distribution to sample from. 71 | * @param stdDev Standard deviation of the Gauss distribution to sample from. 72 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 73 | * @return double Random double value. 74 | */ 75 | static double dgauss(double mean, double stdDev, int tid = -1); 76 | 77 | /** 78 | * @brief Re-Initialize the object with the given seed. 79 | * 80 | * @param seed Seed to initialize the random number generators (seed is incremented by one for each generator). 81 | * @return void 82 | */ 83 | static void forceInit(unsigned seed); 84 | 85 | /** 86 | * @brief List of random number generators. One for each thread. 87 | * 88 | */ 89 | static std::vector generators; 90 | 91 | /** 92 | * @brief Initialize class with the given seed. 93 | * 94 | * Method will create a random number generator for each thread. The given seed 95 | * will be incremented by one for each generator. This methods is automatically 96 | * called when this calss is used the first time. 97 | * 98 | * @param seed Optional parameter. Seed to be used when initializing the generators. Will be incremented by one for each generator. 99 | * @return void 100 | */ 101 | static void init(unsigned seed = 1305); 102 | 103 | private: 104 | /** 105 | * @brief True if the class has been initialized already 106 | */ 107 | static bool initialised; 108 | }; 109 | 110 | /** 111 | * @brief Returns a random integer (uniform distribution). 112 | * 113 | * This method used the ThreadRand class. 114 | * 115 | * @param min Minimum value of the random integer (inclusive). 116 | * @param max Maximum value of the random integer (exclusive). 117 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 118 | * @return int Random integer value. 119 | */ 120 | int irand(int incMin, int excMax, int tid = -1); 121 | /** 122 | * @brief Returns a random double value (uniform distribution). 123 | * 124 | * This method used the ThreadRand class. 125 | * 126 | * @param min Minimum value of the random double (inclusive). 127 | * @param max Maximum value of the random double (inclusive). 128 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 129 | * @return double Random double value. 130 | */ 131 | double drand(double incMin, double incMax, int tid = -1); 132 | 133 | /** 134 | * @brief Returns a random integer value (Gauss distribution). 135 | * 136 | * This method used the ThreadRand class. 137 | * 138 | * @param mean Mean of the Gauss distribution to sample from. 139 | * @param stdDev Standard deviation of the Gauss distribution to sample from. 140 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 141 | * @return double Random integer value. 142 | */ 143 | int igauss(int mean, int stdDev, int tid = -1); 144 | 145 | /** 146 | * @brief Returns a random double value (Gauss distribution). 147 | * 148 | * This method used the ThreadRand class. 149 | * 150 | * @param mean Mean of the Gauss distribution to sample from. 151 | * @param stdDev Standard deviation of the Gauss distribution to sample from. 152 | * @param tid Optional parameter. ID of the thread to use. If not given, the method will obtain the thread ID itself. 153 | * @return double Random double value. 154 | */ 155 | double dgauss(double mean, double stdDev, int tid = -1); 156 | -------------------------------------------------------------------------------- /plot_per_frame/results/MST_kingscollege.txt: -------------------------------------------------------------------------------- 1 | 0.504584 2.610726 2 | 0.762123 1.957176 3 | 0.817495 nan 4 | 0.582131 1.995592 5 | 0.442598 nan 6 | 0.659373 2.521038 7 | 0.644249 1.105694 8 | 0.718837 0.682993 9 | 1.003599 nan 10 | 0.589760 nan 11 | 1.248796 nan 12 | 0.930666 nan 13 | 1.550344 2.490110 14 | 1.074063 2.090228 15 | 0.444457 nan 16 | 0.737553 1.816981 17 | 0.637226 nan 18 | 0.845474 2.901278 19 | 0.676238 nan 20 | 1.284686 nan 21 | 0.617506 nan 22 | 1.042922 1.506069 23 | 1.565319 nan 24 | 1.367915 0.832742 25 | 0.817830 2.614022 26 | 0.888807 2.024408 27 | 1.583574 2.086105 28 | 0.294888 1.761868 29 | 1.517235 1.908172 30 | 1.139460 2.831643 31 | 1.257345 3.357295 32 | 1.046701 3.082077 33 | 5.143218 12.442644 34 | 35.116119 88.339233 35 | 2.028149 nan 36 | 2.912406 3.751308 37 | 1.148570 2.646460 38 | 0.352730 2.148944 39 | 0.514446 1.177013 40 | 0.265664 3.513606 41 | 0.249502 1.692071 42 | 0.221796 2.195428 43 | 0.964036 3.567553 44 | 0.278333 2.746892 45 | 1.482836 3.184752 46 | 0.783225 3.201667 47 | 0.860706 1.329404 48 | 0.630788 3.027756 49 | 0.744768 2.631927 50 | 0.709635 3.108882 51 | 0.818917 3.361954 52 | 0.947658 3.200688 53 | 1.365277 nan 54 | 2.015765 0.854086 55 | 1.349475 1.767634 56 | 1.238588 nan 57 | 1.225091 nan 58 | 1.200027 1.286924 59 | 1.035183 2.991865 60 | 0.591208 1.642306 61 | 1.108313 1.812669 62 | 1.056888 1.912679 63 | 0.568841 0.794257 64 | 0.769387 nan 65 | 1.598454 nan 66 | 1.148952 nan 67 | 1.695573 nan 68 | 1.252913 1.143280 69 | 0.764634 1.835412 70 | 0.225836 1.926136 71 | 0.591974 1.582600 72 | 0.482928 2.250011 73 | 0.594428 1.787449 74 | 0.389656 2.191502 75 | 0.649460 nan 76 | 0.421602 2.698014 77 | 0.550089 0.951202 78 | 0.378842 2.444106 79 | 1.114159 nan 80 | 0.754248 2.083852 81 | 0.920233 0.503577 82 | 0.784448 nan 83 | 0.992103 1.322320 84 | 0.691139 2.421261 85 | 0.931234 0.807935 86 | 1.139192 nan 87 | 0.930138 1.820424 88 | 1.417112 nan 89 | 1.258089 nan 90 | 0.893197 nan 91 | 0.826154 2.668551 92 | 1.073335 nan 93 | 0.890679 nan 94 | 0.838819 1.860402 95 | 0.477422 2.758267 96 | 0.628335 2.755428 97 | 0.415413 1.801407 98 | 0.441937 3.025428 99 | 0.280864 nan 100 | 0.470506 1.171681 101 | 0.330094 2.058152 102 | 0.389780 2.273888 103 | 0.375969 2.381166 104 | 0.716811 2.298539 105 | 0.419528 nan 106 | 0.316328 2.216716 107 | 0.428463 1.875487 108 | 0.344307 2.151128 109 | 1.752574 3.667741 110 | 1.282313 2.417702 111 | 0.807607 1.692533 112 | 1.286335 nan 113 | 1.886615 1.296618 114 | 1.623520 3.341636 115 | 1.628875 3.044514 116 | 1.535663 0.811801 117 | 1.773781 2.509212 118 | 1.532595 2.691333 119 | 1.231576 nan 120 | 0.867055 1.764975 121 | 0.507838 nan 122 | 0.628216 1.628907 123 | 0.610700 2.393298 124 | 0.305056 nan 125 | 0.685325 2.206807 126 | 0.301322 1.901598 127 | 1.707867 3.965361 128 | 0.828986 2.616416 129 | 0.936685 3.832442 130 | 0.492417 4.364949 131 | 0.559616 nan 132 | 0.616611 1.876738 133 | 0.652655 0.988328 134 | 0.306459 1.971124 135 | 0.609753 nan 136 | 0.226922 2.999443 137 | 0.534199 1.079910 138 | 0.661325 3.164533 139 | 0.261518 nan 140 | 1.036944 1.831569 141 | 1.073904 nan 142 | 0.735601 3.485646 143 | 0.478585 3.147916 144 | 1.281676 nan 145 | 0.648086 2.178247 146 | 1.249397 4.310262 147 | 0.505864 1.087133 148 | 0.595069 3.182293 149 | 0.516352 2.031741 150 | 0.350663 nan 151 | 0.451327 2.450184 152 | 0.278337 2.004593 153 | 0.796618 1.213681 154 | 0.267175 1.468174 155 | 0.571012 1.321136 156 | 0.335234 2.007324 157 | 0.515202 1.271628 158 | 0.560670 0.476422 159 | 0.901346 3.715874 160 | 1.331267 4.555396 161 | 1.695446 3.960620 162 | 1.558428 4.430637 163 | 2.241949 7.376347 164 | 2.729533 7.480414 165 | 1.657455 4.483158 166 | 0.924036 1.446693 167 | 1.220885 1.963963 168 | 1.158902 1.712303 169 | 1.079306 3.256453 170 | 1.058637 3.281596 171 | 1.714716 3.332957 172 | 1.426242 5.007480 173 | 1.616556 4.122551 174 | 0.992200 3.155367 175 | 1.052274 nan 176 | 1.048836 1.870054 177 | 1.070825 1.292386 178 | 0.957158 3.889819 179 | 1.116064 4.244742 180 | 0.409522 3.047855 181 | 0.879881 3.759646 182 | 1.145115 2.067637 183 | 0.577142 nan 184 | 1.289616 2.702941 185 | 1.039571 nan 186 | 0.987768 nan 187 | 1.329136 3.106364 188 | 1.579870 4.412045 189 | 1.222140 3.058622 190 | 1.349489 2.456246 191 | 1.396364 2.896687 192 | 0.795344 0.767189 193 | 0.895262 1.875904 194 | 0.718809 0.816608 195 | 0.471880 2.540832 196 | 1.001552 2.024408 197 | 0.735882 3.355196 198 | 0.486032 2.438656 199 | 0.386727 nan 200 | 0.804747 2.284877 201 | 0.352341 2.810834 202 | 0.944328 1.565694 203 | 0.511453 nan 204 | 1.106357 3.450207 205 | 0.775724 nan 206 | 1.548841 2.301601 207 | 0.873969 1.784819 208 | 0.856244 6.274749 209 | 0.703320 5.781120 210 | 1.520041 nan 211 | 1.342645 2.826386 212 | 4.533825 8.828566 213 | 15.478654 14.407988 214 | 14.521895 11.217488 215 | 10.200562 12.671357 216 | 13.546143 10.721192 217 | 9.204716 8.200234 218 | 6.715014 8.161743 219 | 8.725685 9.434754 220 | 2.331774 5.585763 221 | 1.505117 3.068077 222 | 1.502073 1.806613 223 | 1.748191 nan 224 | 2.192141 1.513845 225 | 3.827774 4.400852 226 | 2.271634 0.903083 227 | 1.610960 1.054979 228 | 1.412907 nan 229 | 1.989365 nan 230 | 0.905420 3.184015 231 | 1.037568 nan 232 | 0.730458 nan 233 | 0.159305 nan 234 | 0.979544 3.520506 235 | 1.397613 3.975613 236 | 0.861857 2.560472 237 | 1.294442 2.675874 238 | 0.399373 3.732480 239 | 1.047206 nan 240 | 12.457541 5.460442 241 | 1.372785 2.850378 242 | 1.396086 1.214326 243 | 1.555586 nan 244 | 0.606750 nan 245 | 0.648620 1.036266 246 | 0.331589 1.230970 247 | 0.100483 2.611625 248 | 0.372186 2.702362 249 | 0.695164 nan 250 | 1.265891 3.105607 251 | 0.748036 nan 252 | 1.297349 2.350063 253 | 0.651668 2.371944 254 | 0.951389 3.230872 255 | 1.749210 2.214597 256 | 0.966911 3.340699 257 | 0.766508 3.348189 258 | 1.022055 2.195428 259 | 0.807321 1.903243 260 | 0.943080 1.673933 261 | 0.706403 1.821714 262 | 0.973561 nan 263 | 0.533149 1.063845 264 | 0.510425 2.719974 265 | 0.583662 nan 266 | 0.119902 nan 267 | 0.036837 2.729167 268 | 0.604266 1.288139 269 | 0.271888 nan 270 | 0.101916 1.177678 271 | 0.569977 nan 272 | 0.521184 2.717670 273 | 0.525891 nan 274 | 0.526270 0.631798 275 | 0.234335 nan 276 | 0.496487 2.785377 277 | 0.875381 1.856611 278 | 0.259766 1.130198 279 | 0.920999 1.639921 280 | 1.055512 0.824240 281 | 1.143703 nan 282 | 1.315241 nan 283 | 0.965735 2.343392 284 | 1.319636 nan 285 | 0.721857 2.301941 286 | 0.778795 2.955275 287 | 1.574401 2.187928 288 | 1.059419 nan 289 | 1.208350 nan 290 | 1.728412 1.687438 291 | 1.171013 2.612524 292 | 0.760134 1.686511 293 | 1.310998 2.210351 294 | 1.028160 1.558177 295 | 1.074661 nan 296 | 0.732076 1.603237 297 | 1.117109 nan 298 | 0.863427 nan 299 | 0.377884 2.030585 300 | 0.636671 0.800148 301 | 0.683854 nan 302 | 0.663100 3.537143 303 | 1.094421 nan 304 | 1.114524 nan 305 | 1.030720 2.123294 306 | 0.238876 2.061192 307 | 0.500054 2.263192 308 | 0.293697 2.282820 309 | 0.595968 nan 310 | 0.105645 nan 311 | 0.354027 2.426751 312 | 0.563336 nan 313 | 0.293059 nan 314 | 0.451178 3.738138 315 | 0.485501 nan 316 | 0.305212 2.861069 317 | 0.449450 2.794075 318 | 0.546548 nan 319 | 1.038964 2.675581 320 | 0.467019 nan 321 | 0.161238 2.649416 322 | 0.581891 nan 323 | 1.078583 1.932627 324 | 0.167096 2.515754 325 | 0.691260 0.923649 326 | 0.461780 3.914295 327 | 1.161633 nan 328 | 0.376335 3.172932 329 | 0.656003 1.834985 330 | 1.679210 1.163637 331 | 0.964214 2.793515 332 | 1.542239 4.624306 333 | 0.728540 0.842089 334 | 0.622529 1.665495 335 | 0.843861 3.688385 336 | 0.549864 1.097879 337 | 0.610587 1.196796 338 | 0.664166 0.487787 339 | 0.780722 nan 340 | 0.494739 nan 341 | 0.772063 2.584207 342 | 0.374001 3.690082 343 | 0.608139 2.404718 344 | -------------------------------------------------------------------------------- /plot_per_frame/results/DSACSTAR_kingscollege.txt: -------------------------------------------------------------------------------- 1 | 0.229275 0.309811 2 | 0.327366 0.343340 3 | 0.546792 0.534442 4 | 0.532685 0.727946 5 | 0.273184 0.483659 6 | 0.410106 0.261850 7 | 0.377200 0.386027 8 | 0.214181 0.456485 9 | 0.385189 0.541385 10 | 0.116924 0.328328 11 | 0.130959 0.334755 12 | 0.124121 0.107082 13 | 0.238439 0.473685 14 | 0.078559 0.135167 15 | 0.245431 0.206579 16 | 0.208913 0.242928 17 | 0.077259 0.073716 18 | 0.219251 0.282209 19 | 0.110823 0.119602 20 | 0.096353 0.126445 21 | 0.058593 0.040388 22 | 0.103700 0.168689 23 | 0.109699 0.124254 24 | 0.187414 0.344932 25 | 0.186186 0.364491 26 | 0.208632 0.346549 27 | 0.201102 0.334841 28 | 0.303198 0.492700 29 | 0.223531 0.397588 30 | 0.321601 0.593975 31 | 0.276728 0.578024 32 | 0.165192 0.243006 33 | 0.170688 0.393126 34 | 20.564455 11.260672 35 | 0.077380 0.127208 36 | 0.218473 0.399729 37 | 0.228830 0.542127 38 | 0.121822 0.290060 39 | 0.109342 0.266025 40 | 0.131310 0.242288 41 | 0.170966 0.329196 42 | 0.079956 0.226859 43 | 0.045902 0.130745 44 | 0.145815 0.293203 45 | 0.141938 0.422975 46 | 0.066748 0.170323 47 | 0.076004 0.144985 48 | 0.062090 0.106553 49 | 0.163161 0.385451 50 | 0.083685 0.208913 51 | 0.031389 0.095888 52 | 0.080796 0.092241 53 | 0.010689 0.141807 54 | 0.053575 0.122137 55 | 0.099670 0.195526 56 | 0.108034 0.199560 57 | 0.097813 0.122653 58 | 0.089366 0.216784 59 | 0.058517 0.184107 60 | 0.130015 0.198515 61 | 0.067183 0.071663 62 | 0.523283 0.345910 63 | 0.947844 0.478571 64 | 0.650286 0.173122 65 | 0.533965 0.402847 66 | 0.299365 0.478061 67 | 0.405529 0.668717 68 | 0.494082 0.119751 69 | 0.646725 0.618865 70 | 0.160734 0.217848 71 | 0.585379 0.317576 72 | 0.388099 0.274795 73 | 0.478106 0.751308 74 | 0.164526 0.127998 75 | 0.498943 0.468246 76 | 0.182116 0.414729 77 | 0.192513 0.217354 78 | 0.118370 0.163314 79 | 0.152117 0.379019 80 | 0.143162 0.301746 81 | 0.400997 0.576874 82 | 0.156705 0.277816 83 | 0.183461 0.169808 84 | 0.179950 0.187403 85 | 0.404993 0.230078 86 | 0.274589 0.250218 87 | 0.194846 0.256369 88 | 0.299014 0.647639 89 | 0.122040 0.211771 90 | 0.235695 0.235012 91 | 0.090254 0.222174 92 | 0.089422 0.230754 93 | 0.077716 0.262446 94 | 0.142801 0.275510 95 | 0.050299 0.226526 96 | 0.215375 0.308533 97 | 0.153180 0.249196 98 | 0.144278 0.371975 99 | 0.086441 0.194751 100 | 0.103291 0.240760 101 | 0.048160 0.099803 102 | 0.017984 0.080278 103 | 0.047649 0.181876 104 | 0.121536 0.247284 105 | 0.071552 0.175997 106 | 0.099958 0.170808 107 | 0.115124 0.280285 108 | 0.072793 0.170631 109 | 0.094551 0.216003 110 | 0.064762 0.111477 111 | 0.063804 0.162233 112 | 0.100370 0.140117 113 | 0.253543 0.574155 114 | 0.195204 0.500750 115 | 0.149433 0.387929 116 | 0.160333 0.302465 117 | 0.098106 0.170444 118 | 0.080552 0.167746 119 | 0.162556 0.408012 120 | 0.165493 0.340889 121 | 0.072038 0.138430 122 | 0.041696 0.170019 123 | 0.092199 0.269970 124 | 0.067807 0.227936 125 | 0.202945 0.374344 126 | 0.107790 0.276531 127 | 0.156456 0.398299 128 | 0.170234 0.442938 129 | 0.116697 0.242354 130 | 0.264105 0.678124 131 | 0.081283 0.272315 132 | 0.203421 0.454644 133 | 0.104358 0.170139 134 | 0.108235 0.223538 135 | 0.287694 0.703845 136 | 0.205271 0.453830 137 | 0.211772 0.505067 138 | 0.245169 0.594072 139 | 0.212939 0.541661 140 | 0.077758 0.206624 141 | 0.256350 0.502669 142 | 0.300547 0.690512 143 | 0.333302 0.707837 144 | 0.199449 0.475282 145 | 0.111982 0.055452 146 | 0.184071 0.414003 147 | 0.279202 0.531984 148 | 0.073024 0.197585 149 | 0.198910 0.306206 150 | 0.169335 0.207808 151 | 0.071806 0.113355 152 | 0.085171 0.151277 153 | 0.103229 0.131005 154 | 0.050571 0.121594 155 | 0.236777 0.507957 156 | 0.181327 0.349393 157 | 0.132401 0.231382 158 | 0.188712 0.245564 159 | 0.253301 0.381290 160 | 0.324688 0.632520 161 | 0.637104 1.095283 162 | 0.338413 0.574252 163 | 0.202891 0.413669 164 | 0.714131 1.247773 165 | 0.469637 0.879207 166 | 0.328550 0.393977 167 | 0.240235 0.390122 168 | 0.320067 0.510864 169 | 0.263444 0.471776 170 | 0.216111 0.365512 171 | 0.172541 0.228212 172 | 0.405769 0.716658 173 | 0.395663 0.602636 174 | 0.124481 0.112630 175 | 0.097179 0.168982 176 | 0.504134 0.511842 177 | 0.126433 0.106880 178 | 0.126611 0.071716 179 | 0.190670 0.092122 180 | 0.040986 0.120013 181 | 0.095860 0.095513 182 | 0.095259 0.182884 183 | 0.502132 0.718567 184 | 0.202290 0.204361 185 | 0.069356 0.093723 186 | 0.288433 0.425254 187 | 0.200607 0.197839 188 | 0.485783 0.714149 189 | 0.368230 0.311957 190 | 0.603229 0.780889 191 | 0.440868 0.510862 192 | 0.229412 0.217076 193 | 0.807477 1.083721 194 | 0.236186 0.266713 195 | 0.310101 0.103476 196 | 0.354133 0.353481 197 | 0.236596 0.311009 198 | 0.267738 0.295344 199 | 0.323441 0.340782 200 | 1.014861 0.585321 201 | 0.377458 0.529766 202 | 0.810040 0.934658 203 | 0.508982 0.451124 204 | 0.363441 0.191702 205 | 0.610689 0.759120 206 | 0.753111 0.616291 207 | 1.020856 0.493757 208 | 0.433829 0.283800 209 | 0.666615 0.636158 210 | 0.802892 0.149454 211 | 0.727087 0.942561 212 | 1.654102 1.409484 213 | 0.788443 0.708289 214 | 1.239048 1.346944 215 | 1.045167 1.029837 216 | 0.946148 0.387202 217 | 1.144798 1.110950 218 | 1.262872 1.543180 219 | 1.091462 1.019772 220 | 0.892283 0.411401 221 | 0.736735 0.538826 222 | 0.320290 0.300476 223 | 0.308437 0.496144 224 | 0.583084 0.898929 225 | 0.623631 0.248969 226 | 0.664804 0.706101 227 | 0.701100 0.636279 228 | 0.409658 0.414874 229 | 0.872724 0.949710 230 | 1.054030 0.433815 231 | 0.899454 1.166415 232 | 0.288634 0.042892 233 | 0.717107 0.307568 234 | 0.966203 0.551058 235 | 0.494832 0.404981 236 | 0.361316 0.172339 237 | 0.403499 0.416420 238 | 0.546564 0.513304 239 | 1.286918 1.450220 240 | 2.626678 3.955925 241 | 0.548227 0.207406 242 | 0.184288 0.263492 243 | 0.260475 0.108836 244 | 0.064846 0.289340 245 | 0.209346 0.187968 246 | 0.040387 0.232194 247 | 0.454712 0.438119 248 | 0.432819 0.792144 249 | 0.259120 0.330907 250 | 0.383001 0.347348 251 | 0.240636 0.923904 252 | 0.558461 0.961538 253 | 0.170364 0.330996 254 | 0.092190 0.187597 255 | 0.140438 0.250244 256 | 0.169533 0.508313 257 | 0.174306 0.368181 258 | 0.124150 0.298183 259 | 0.249031 0.754481 260 | 0.193266 0.539136 261 | 0.113238 0.156244 262 | 0.177897 0.598103 263 | 0.284371 0.765809 264 | 0.133337 0.388526 265 | 0.248911 0.588956 266 | 0.195396 0.378418 267 | 0.137156 0.451351 268 | 0.105988 0.351496 269 | 0.100208 0.086320 270 | 0.070354 0.153594 271 | 0.231622 0.349611 272 | 0.124351 0.481907 273 | 0.057425 0.152949 274 | 0.042985 0.230919 275 | 0.093365 0.147208 276 | 0.050596 0.234957 277 | 0.063117 0.207648 278 | 0.051788 0.213371 279 | 0.101467 0.216808 280 | 0.089410 0.099004 281 | 0.037236 0.117933 282 | 0.036865 0.120195 283 | 0.083397 0.132401 284 | 0.058716 0.098018 285 | 0.032520 0.035356 286 | 0.083411 0.297107 287 | 0.078307 0.178277 288 | 0.041889 0.198901 289 | 0.026021 0.122934 290 | 0.047172 0.200857 291 | 0.042668 0.082772 292 | 0.038589 0.073188 293 | 0.070397 0.182512 294 | 0.062138 0.229476 295 | 0.077307 0.193320 296 | 0.031085 0.159713 297 | 0.052496 0.104711 298 | 0.122072 0.246602 299 | 0.045888 0.101611 300 | 0.102629 0.255780 301 | 0.056012 0.156115 302 | 0.058100 0.103634 303 | 0.120001 0.329407 304 | 0.036500 0.140832 305 | 0.060724 0.233286 306 | 0.074617 0.246592 307 | 0.118831 0.378513 308 | 0.064211 0.021665 309 | 0.207865 0.573646 310 | 0.136424 0.360901 311 | 0.126545 0.365467 312 | 0.194434 0.524238 313 | 0.161342 0.448258 314 | 0.017429 0.091234 315 | 0.162289 0.469370 316 | 0.038226 0.142841 317 | 0.194824 0.536529 318 | 0.180446 0.481801 319 | 0.237983 0.624284 320 | 0.272059 0.668934 321 | 0.208958 0.513149 322 | 0.125431 0.314297 323 | 0.168501 0.369979 324 | 0.146244 0.314692 325 | 0.200437 0.433431 326 | 0.183763 0.459478 327 | 0.322326 0.759944 328 | 0.100198 0.128078 329 | 0.273726 0.653435 330 | 0.206815 0.427404 331 | 0.062323 0.062023 332 | 0.233827 0.571036 333 | 0.153009 0.356836 334 | 0.248746 0.622243 335 | 0.191839 0.399657 336 | 0.063289 0.218720 337 | 0.357310 0.833350 338 | 0.279100 0.678188 339 | 0.179568 0.365472 340 | 0.100667 0.274125 341 | 0.175918 0.379197 342 | 0.127858 0.308770 343 | 0.208861 0.409622 344 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_dsacstar_kingscollege.txt: -------------------------------------------------------------------------------- 1 | 0.203996 0.171502 2 | 0.277312 0.170522 3 | 0.274825 0.120278 4 | 0.462707 0.187170 5 | 0.178884 0.257816 6 | 0.287609 0.465788 7 | 0.702108 1.075711 8 | 0.168087 0.249111 9 | 0.405286 0.425769 10 | 0.103956 0.212926 11 | 0.230085 0.373828 12 | 0.106674 0.103379 13 | 0.122006 0.058315 14 | 0.173763 0.260870 15 | 0.159891 0.170518 16 | 0.249499 0.305263 17 | 0.162689 0.113202 18 | 0.192999 0.224523 19 | 0.118498 0.220591 20 | 0.285458 0.481242 21 | 0.247671 0.331173 22 | 0.225622 0.389940 23 | 0.206240 0.368113 24 | 0.091896 0.105588 25 | 0.198895 0.363442 26 | 0.289714 0.541946 27 | 0.286818 0.564633 28 | 0.158835 0.304201 29 | 0.167933 0.333909 30 | 0.066378 0.128972 31 | 0.048363 0.071305 32 | 0.225746 0.453773 33 | 0.421247 0.686113 34 | 0.661624 0.337946 35 | 0.173319 0.343986 36 | 0.068485 0.119427 37 | 0.166541 0.162893 38 | 0.114784 0.139833 39 | 0.055908 0.110829 40 | 0.114196 0.127614 41 | 0.157675 0.275374 42 | 0.108671 0.237481 43 | 0.200194 0.432370 44 | 0.215968 0.383011 45 | 0.048265 0.136668 46 | 0.102857 0.252808 47 | 0.112557 0.314274 48 | 0.031598 0.089496 49 | 0.125515 0.186233 50 | 0.281080 0.725363 51 | 0.115404 0.367892 52 | 0.040970 0.054376 53 | 0.078018 0.182922 54 | 0.079479 0.081048 55 | 0.059211 0.225094 56 | 0.081798 0.193360 57 | 0.053992 0.084503 58 | 0.119534 0.198885 59 | 0.075863 0.249055 60 | 0.036867 0.078203 61 | 0.066107 0.158468 62 | 0.055842 0.185227 63 | 0.047378 0.117163 64 | 0.053347 0.205293 65 | 0.051609 0.092762 66 | 0.129113 0.268501 67 | 0.134799 0.534130 68 | 0.059168 0.201826 69 | 0.144680 0.454029 70 | 0.107078 0.197821 71 | 0.141123 0.288434 72 | 0.113068 0.232150 73 | 0.142094 0.410485 74 | 0.113325 0.225958 75 | 0.111022 0.261131 76 | 0.106653 0.304820 77 | 0.087824 0.243092 78 | 0.112626 0.304815 79 | 0.126907 0.293965 80 | 0.132535 0.285555 81 | 0.128053 0.226483 82 | 0.085610 0.175342 83 | 0.089897 0.204460 84 | 0.104336 0.148697 85 | 0.097912 0.238180 86 | 0.055408 0.170917 87 | 0.067434 0.157435 88 | 0.105110 0.135006 89 | 0.073468 0.160796 90 | 0.060475 0.140176 91 | 0.107536 0.246319 92 | 0.080138 0.093059 93 | 0.055044 0.122872 94 | 0.070902 0.180878 95 | 0.034856 0.108061 96 | 0.011273 0.091991 97 | 0.046816 0.116724 98 | 0.048954 0.111342 99 | 0.140319 0.228321 100 | 0.137043 0.205015 101 | 0.124094 0.093850 102 | 0.182090 0.209112 103 | 0.159437 0.223295 104 | 0.186026 0.118861 105 | 0.146693 0.102940 106 | 0.151098 0.066700 107 | 0.070374 0.100485 108 | 0.093678 0.073457 109 | 0.065641 0.120669 110 | 0.147672 0.320585 111 | 0.120902 0.239071 112 | 0.217009 0.475211 113 | 0.083858 0.180648 114 | 0.109515 0.261279 115 | 0.082326 0.173261 116 | 0.095046 0.235668 117 | 0.144170 0.324116 118 | 0.086719 0.198989 119 | 0.041380 0.153006 120 | 0.089578 0.254671 121 | 0.034121 0.087089 122 | 0.081170 0.157045 123 | 0.089471 0.187829 124 | 0.095165 0.213975 125 | 0.071493 0.178279 126 | 0.049876 0.061409 127 | 0.148401 0.071538 128 | 0.167909 0.305384 129 | 0.061259 0.124438 130 | 0.218637 0.368470 131 | 0.150230 0.203295 132 | 0.198772 0.359426 133 | 0.149438 0.127680 134 | 0.208848 0.335004 135 | 0.226927 0.430688 136 | 0.332327 0.662461 137 | 0.160853 0.324561 138 | 0.236229 0.479260 139 | 0.179667 0.365118 140 | 0.288144 0.574285 141 | 0.166227 0.222455 142 | 0.153780 0.263450 143 | 0.149252 0.293749 144 | 0.228850 0.421591 145 | 0.149998 0.313885 146 | 0.258804 0.583554 147 | 0.240040 0.393491 148 | 0.258135 0.434829 149 | 0.183634 0.306829 150 | 0.079713 0.080834 151 | 0.094624 0.146990 152 | 0.171866 0.200737 153 | 0.126135 0.143898 154 | 0.138227 0.164089 155 | 0.069678 0.111035 156 | 0.202311 0.301832 157 | 0.232770 0.378991 158 | 0.113774 0.160774 159 | 0.169467 0.268410 160 | 0.308664 0.530401 161 | 0.297788 0.486515 162 | 0.303929 0.423529 163 | 0.104753 0.149708 164 | 0.214216 0.433050 165 | 0.176129 0.194445 166 | 0.437577 0.346301 167 | 0.276474 0.139100 168 | 0.385719 0.220349 169 | 0.305392 0.246283 170 | 0.195538 0.073912 171 | 0.162482 0.126871 172 | 0.174418 0.278775 173 | 0.075089 0.145017 174 | 0.202406 0.294667 175 | 0.311600 0.469734 176 | 0.400388 0.402916 177 | 0.452209 0.651657 178 | 0.190421 0.251196 179 | 0.235433 0.337365 180 | 0.491405 0.739000 181 | 0.288167 0.365895 182 | 0.298152 0.328719 183 | 0.205417 0.259609 184 | 0.220848 0.222158 185 | 0.121895 0.168151 186 | 0.104698 0.157044 187 | 0.244949 0.310142 188 | 0.237311 0.400212 189 | 0.158913 0.150328 190 | 0.548169 0.719975 191 | 0.113596 0.137532 192 | 0.458525 0.608545 193 | 0.201961 0.239960 194 | 0.190853 0.257091 195 | 0.476628 0.633597 196 | 0.306087 0.281833 197 | 0.209472 0.248273 198 | 0.278531 0.228324 199 | 0.060307 0.014244 200 | 0.308349 0.289293 201 | 0.256638 0.170378 202 | 0.380494 0.494916 203 | 0.182732 0.100336 204 | 0.188878 0.190497 205 | 0.307947 0.341142 206 | 0.216483 0.083532 207 | 0.562151 0.648737 208 | 0.467673 0.357030 209 | 0.394955 0.382817 210 | 0.629654 0.516223 211 | 0.324999 0.249399 212 | 1.336098 1.588898 213 | 0.694489 0.891243 214 | 0.793697 0.945288 215 | 0.647754 0.797928 216 | 0.742210 0.800298 217 | 0.714637 0.808121 218 | 0.763787 0.825506 219 | 1.307262 1.465780 220 | 0.872559 1.005846 221 | 0.619329 0.763957 222 | 0.656477 0.804171 223 | 0.410403 0.504748 224 | 0.551236 0.543002 225 | 0.421119 0.505585 226 | 0.462138 0.341898 227 | 0.451423 0.509231 228 | 0.148146 0.164466 229 | 1.160782 1.318543 230 | 0.446985 0.398187 231 | 0.316250 0.230724 232 | 0.733530 0.806623 233 | 0.477249 0.376670 234 | 0.308589 0.358280 235 | 0.405541 0.428541 236 | 0.413233 0.237933 237 | 0.498902 0.355652 238 | 0.389672 0.353442 239 | 0.333953 0.425845 240 | 0.912059 0.351007 241 | 0.071654 0.165814 242 | 0.118422 0.384566 243 | 0.117341 0.189084 244 | 0.077298 0.082546 245 | 0.135829 0.200364 246 | 0.069584 0.077073 247 | 0.097875 0.078484 248 | 0.063570 0.100990 249 | 0.064632 0.087845 250 | 0.096375 0.057468 251 | 0.069166 0.117249 252 | 0.056531 0.068406 253 | 0.101004 0.047999 254 | 0.044339 0.054537 255 | 0.150309 0.125767 256 | 0.033041 0.029073 257 | 0.056954 0.059527 258 | 0.082542 0.289157 259 | 0.114962 0.417487 260 | 0.088332 0.211681 261 | 0.021751 0.060479 262 | 0.089589 0.217962 263 | 0.097697 0.152697 264 | 0.106269 0.291916 265 | 0.145370 0.352823 266 | 0.168687 0.324096 267 | 0.133090 0.390868 268 | 0.133011 0.312213 269 | 0.119899 0.254908 270 | 0.077175 0.181300 271 | 0.089441 0.143108 272 | 0.113406 0.281292 273 | 0.087260 0.183854 274 | 0.107462 0.251450 275 | 0.118544 0.195269 276 | 0.102647 0.260974 277 | 0.132603 0.344423 278 | 0.147940 0.402241 279 | 0.219016 0.506774 280 | 0.134578 0.282669 281 | 0.093801 0.189066 282 | 0.095279 0.056900 283 | 0.054785 0.053107 284 | 0.070246 0.049394 285 | 0.040734 0.112212 286 | 0.017577 0.041236 287 | 0.056625 0.054614 288 | 0.030140 0.084273 289 | 0.047265 0.041520 290 | 0.038959 0.022294 291 | 0.044401 0.084326 292 | 0.040325 0.098729 293 | 0.130275 0.257642 294 | 0.055449 0.180947 295 | 0.076970 0.187471 296 | 0.024698 0.079931 297 | 0.080856 0.091912 298 | 0.060557 0.088068 299 | 0.054131 0.077622 300 | 0.098634 0.106183 301 | 0.088022 0.163319 302 | 0.103434 0.157442 303 | 0.111404 0.204824 304 | 0.061078 0.092622 305 | 0.104964 0.048029 306 | 0.108289 0.174441 307 | 0.131846 0.301895 308 | 0.057910 0.163859 309 | 0.145672 0.374185 310 | 0.092543 0.206117 311 | 0.208605 0.500396 312 | 0.244588 0.598372 313 | 0.111976 0.271443 314 | 0.182059 0.422232 315 | 0.098579 0.283358 316 | 0.191046 0.434073 317 | 0.232899 0.531993 318 | 0.125782 0.312719 319 | 0.088427 0.231851 320 | 0.151122 0.346645 321 | 0.101191 0.251223 322 | 0.167400 0.328273 323 | 0.164498 0.365598 324 | 0.219647 0.491558 325 | 0.163440 0.334959 326 | 0.144661 0.292935 327 | 0.304134 0.635774 328 | 0.189212 0.385223 329 | 0.409298 0.856428 330 | 0.094935 0.168463 331 | 0.157245 0.309199 332 | 0.360126 0.714828 333 | 0.205383 0.414567 334 | 0.313638 0.616625 335 | 0.212510 0.437549 336 | 0.073753 0.125084 337 | 0.276287 0.530596 338 | 0.164574 0.278704 339 | 0.164042 0.263502 340 | 0.081752 0.144417 341 | 0.084321 0.101403 342 | 0.085142 0.145998 343 | 0.113005 0.186260 344 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_kingscollege.txt: -------------------------------------------------------------------------------- 1 | 0.381077 0.709163 2 | 0.117019 0.543022 3 | 0.294285 0.995859 4 | 0.460584 1.360030 5 | 0.244102 1.555615 6 | 0.459149 1.067631 7 | 0.356776 1.073542 8 | 0.072463 0.383405 9 | 0.576789 0.915641 10 | 0.050332 0.602465 11 | 0.450878 0.282741 12 | 0.107947 0.772918 13 | 0.596203 1.030782 14 | 0.402105 0.502579 15 | 0.935917 2.395346 16 | 0.435929 1.029195 17 | 0.648038 1.226984 18 | 0.182412 0.793291 19 | 0.324170 0.709460 20 | 0.493404 1.665820 21 | 0.596192 1.399341 22 | 0.438806 1.074801 23 | 0.082920 1.121889 24 | 0.190558 0.828142 25 | 0.108379 1.255711 26 | 0.143327 0.518868 27 | 0.159064 0.580031 28 | 0.327720 0.780604 29 | 0.072870 1.185555 30 | 0.127409 0.806566 31 | 0.144038 0.772412 32 | 0.126205 1.503320 33 | 2.517473 4.160391 34 | 2.836549 19.525286 35 | 0.338136 0.815338 36 | 0.196796 0.701235 37 | 0.439383 0.849947 38 | 0.099318 0.239022 39 | 0.128423 0.625051 40 | 0.042979 0.479566 41 | 0.046185 0.447715 42 | 0.439243 0.450988 43 | 0.100593 0.443606 44 | 0.137799 0.238999 45 | 0.336916 0.460491 46 | 0.139363 0.423592 47 | 0.167333 0.769467 48 | 0.053150 0.105299 49 | 0.057607 0.651087 50 | 0.348139 0.715413 51 | 0.328000 1.101740 52 | 0.101422 0.402211 53 | 0.059278 0.477094 54 | 0.050270 0.299109 55 | 0.126507 0.115280 56 | 0.053252 0.541628 57 | 0.056575 0.536110 58 | 0.120897 0.737563 59 | 0.025768 0.427354 60 | 0.035072 0.522605 61 | 0.141781 0.508473 62 | 0.225389 0.862232 63 | 0.087029 0.490083 64 | 0.043442 0.604690 65 | 0.022488 0.827551 66 | 0.117858 0.743957 67 | 0.062776 0.753832 68 | 0.049900 0.348562 69 | 0.102975 0.629076 70 | 0.040607 0.123551 71 | 0.191931 0.444495 72 | 0.058101 0.165458 73 | 0.082601 0.450721 74 | 0.078806 0.223016 75 | 0.076323 0.262161 76 | 0.033441 0.609962 77 | 0.042256 0.372774 78 | 0.029544 0.176743 79 | 0.043553 0.267980 80 | 0.122051 0.414572 81 | 0.076034 0.310279 82 | 0.220792 0.680108 83 | 0.128955 0.411051 84 | 0.221977 0.771147 85 | 0.123845 0.371254 86 | 0.044092 0.369286 87 | 0.165496 0.546854 88 | 0.128704 0.753195 89 | 0.047382 0.372599 90 | 0.034793 0.417638 91 | 0.041958 0.331847 92 | 0.056121 0.694498 93 | 0.048576 0.198448 94 | 0.088617 0.639109 95 | 0.043172 0.234355 96 | 0.045050 0.295761 97 | 0.027407 0.122158 98 | 0.033106 0.213439 99 | 0.043742 0.479983 100 | 0.045205 0.619258 101 | 0.044132 0.328741 102 | 0.061558 0.727166 103 | 0.054641 0.479723 104 | 0.063789 0.389550 105 | 0.037981 0.466671 106 | 0.133718 0.109578 107 | 0.079119 0.540071 108 | 0.108780 0.404475 109 | 0.095970 0.689439 110 | 0.147982 0.864103 111 | 0.147148 0.625932 112 | 0.168515 0.579036 113 | 0.111852 0.424572 114 | 0.160287 0.726106 115 | 0.028322 0.046429 116 | 0.153801 0.936620 117 | 0.063197 0.215537 118 | 0.031597 0.160244 119 | 0.033496 0.047448 120 | 0.167639 0.929074 121 | 0.050748 0.199834 122 | 0.103015 0.191015 123 | 0.303923 0.999672 124 | 0.097809 0.621941 125 | 0.245755 0.508158 126 | 0.083851 0.393980 127 | 0.808136 3.167977 128 | 0.352582 0.264154 129 | 0.130955 0.356096 130 | 0.192979 0.214290 131 | 0.157108 0.380024 132 | 0.205457 1.231638 133 | 0.071630 0.275013 134 | 0.171313 0.062892 135 | 0.220266 0.282141 136 | 0.131740 0.597045 137 | 0.283309 0.683120 138 | 0.281041 0.503165 139 | 0.200350 0.246035 140 | 0.246843 0.353838 141 | 0.332963 0.361973 142 | 0.225398 0.579579 143 | 0.372243 0.470190 144 | 0.148719 0.538177 145 | 0.243149 0.744299 146 | 0.405508 0.329717 147 | 0.475262 0.520430 148 | 0.160756 0.346903 149 | 0.205304 0.739994 150 | 0.073535 0.721253 151 | 0.112590 0.319806 152 | 0.208749 0.686471 153 | 0.043950 0.219198 154 | 0.058848 0.175433 155 | 0.176445 0.709934 156 | 0.133817 0.236577 157 | 0.265329 0.916715 158 | 0.066416 0.167149 159 | 0.281549 0.295574 160 | 0.152300 1.185964 161 | 0.232662 1.267314 162 | 0.142588 1.251139 163 | 0.392383 0.480089 164 | 1.034696 1.217700 165 | 0.230155 0.834700 166 | 0.812188 0.842435 167 | 0.312597 0.664156 168 | 0.682021 0.454715 169 | 0.617680 1.127434 170 | 0.765405 0.344349 171 | 0.429653 0.614770 172 | 0.776203 0.756543 173 | 0.403095 0.781877 174 | 0.929043 0.648231 175 | 0.661132 1.027578 176 | 0.055794 0.700633 177 | 0.596667 0.766528 178 | 0.461268 0.648442 179 | 0.052613 0.555100 180 | 0.951813 1.577057 181 | 0.487577 1.051824 182 | 0.223879 0.344717 183 | 1.074674 1.281543 184 | 0.065703 1.396134 185 | 0.910944 2.086276 186 | 0.467965 0.980579 187 | 0.138058 1.057062 188 | 0.490244 1.962024 189 | 0.581601 0.514528 190 | 0.144147 1.487916 191 | 0.752222 0.835962 192 | 0.820121 1.310093 193 | 0.092822 0.994365 194 | 0.414075 0.680856 195 | 0.533263 0.740559 196 | 0.808199 0.787668 197 | 0.144851 1.353093 198 | 0.628030 1.966083 199 | 0.838863 0.378710 200 | 0.734367 1.060255 201 | 0.137874 0.631795 202 | 0.653843 0.670522 203 | 0.080306 0.065862 204 | 0.298972 0.633788 205 | 0.112204 0.232518 206 | 0.696311 0.755988 207 | 0.394668 0.464050 208 | 0.331617 2.960028 209 | 2.340987 3.055359 210 | 0.132966 1.278753 211 | 0.898867 1.980194 212 | 7.604488 13.003809 213 | 6.189873 9.845176 214 | 25.241680 20.998399 215 | 24.461620 18.269269 216 | 25.340256 19.866696 217 | 30.679340 23.529488 218 | 24.466146 6.597058 219 | 37.016819 18.974353 220 | 1.317827 1.674375 221 | 0.577362 0.584093 222 | 0.161006 2.462909 223 | 0.309570 2.312983 224 | 0.413153 2.497425 225 | 4.201724 6.281568 226 | 0.139519 1.226306 227 | 1.459121 0.785538 228 | 0.261579 1.938147 229 | 0.265649 0.693201 230 | 0.172297 1.259906 231 | 0.530209 2.416032 232 | 0.661302 0.267152 233 | 0.186845 2.772324 234 | 0.169072 1.788718 235 | 0.118737 1.487574 236 | 0.366570 1.557799 237 | 0.548581 1.700076 238 | 0.239634 1.199114 239 | 0.240377 2.505308 240 | 2.308797 3.755558 241 | 0.061593 1.521246 242 | 0.060647 1.694861 243 | 0.080165 1.600737 244 | 0.058657 0.931942 245 | 0.218691 1.133914 246 | 0.031511 0.880848 247 | 0.129703 0.919320 248 | 0.056142 1.054302 249 | 0.044142 0.605179 250 | 0.176345 0.561973 251 | 0.076173 0.438877 252 | 0.037776 0.277979 253 | 0.138543 0.414776 254 | 0.085201 0.309445 255 | 0.221681 0.906155 256 | 0.073176 0.749149 257 | 0.127282 0.622700 258 | 0.103050 0.774945 259 | 0.222792 1.143969 260 | 0.097207 0.531548 261 | 0.070302 0.864128 262 | 0.040704 0.983286 263 | 0.008522 1.371892 264 | 0.135419 0.792871 265 | 0.041503 0.722504 266 | 0.069833 0.367146 267 | 0.096701 0.607096 268 | 0.046161 0.498130 269 | 0.013527 0.081472 270 | 0.033335 0.567692 271 | 0.115989 0.436199 272 | 0.138236 0.859939 273 | 0.095046 0.367029 274 | 0.116992 0.334293 275 | 0.149646 0.623369 276 | 0.055131 0.783869 277 | 0.023145 0.233927 278 | 0.062652 0.572020 279 | 0.079418 0.171949 280 | 0.142360 0.294581 281 | 0.113471 0.111090 282 | 0.207980 0.495206 283 | 0.066554 0.225698 284 | 0.132066 0.247244 285 | 0.071456 0.431683 286 | 0.017684 0.389420 287 | 0.042594 0.262219 288 | 0.043413 0.695631 289 | 0.041538 0.610727 290 | 0.073109 0.766929 291 | 0.116402 0.671099 292 | 0.087880 1.222070 293 | 0.117175 0.456431 294 | 0.052176 0.535695 295 | 0.111233 0.928766 296 | 0.125289 1.002227 297 | 0.058322 0.558735 298 | 0.061756 0.471668 299 | 0.059344 0.642308 300 | 0.092031 0.456319 301 | 0.037280 0.293316 302 | 0.018248 0.131573 303 | 0.106201 0.444960 304 | 0.069567 0.387753 305 | 0.260750 1.210843 306 | 0.119845 0.285400 307 | 0.163379 0.298958 308 | 0.192623 1.023892 309 | 0.157644 0.452342 310 | 0.116663 0.648383 311 | 0.127594 1.090221 312 | 0.164171 0.961851 313 | 0.296755 0.712591 314 | 0.158588 0.880318 315 | 0.281387 1.195004 316 | 0.116758 0.902305 317 | 0.368430 1.324791 318 | 0.156272 0.651276 319 | 0.245794 0.754068 320 | 0.129671 0.503435 321 | 0.295677 0.529742 322 | 0.026753 0.288557 323 | 0.445095 0.952483 324 | 0.438191 0.631511 325 | 0.553614 0.337319 326 | 0.176820 0.565176 327 | 0.340955 1.489710 328 | 0.247439 0.556687 329 | 0.229424 0.970328 330 | 0.288836 0.441608 331 | 0.313659 0.210617 332 | 0.636900 0.475160 333 | 0.301602 0.700632 334 | 0.580970 0.507040 335 | 0.792902 0.500336 336 | 0.137749 0.326933 337 | 0.361932 0.792344 338 | 0.149409 0.434250 339 | 0.239501 0.299538 340 | 0.158483 0.135340 341 | 0.086301 0.114261 342 | 0.216026 0.348132 343 | 0.059017 0.622372 344 | -------------------------------------------------------------------------------- /plot_per_frame/results/DLTe2e_ref_kingscollege.txt: -------------------------------------------------------------------------------- 1 | 0.094721 0.175503 2 | 0.185808 0.206182 3 | 0.174320 0.129027 4 | 0.316895 0.184901 5 | 0.107323 0.342842 6 | 0.071458 0.500743 7 | 0.112911 0.778544 8 | 0.092267 0.187845 9 | 0.265667 0.387208 10 | 0.029972 0.225576 11 | 0.018544 0.363581 12 | 0.101380 0.117378 13 | 0.147007 0.067177 14 | 0.057761 0.193880 15 | 0.123328 0.370363 16 | 0.127608 0.315663 17 | 0.153067 0.168011 18 | 0.110624 0.225314 19 | 0.047110 0.270701 20 | 0.031478 0.482440 21 | 0.116079 0.328785 22 | 0.066081 0.415742 23 | 0.047054 0.333663 24 | 0.081890 0.119513 25 | 0.027983 0.406293 26 | 0.027258 0.531235 27 | 0.120246 0.595410 28 | 0.051618 0.266973 29 | 0.047911 0.325368 30 | 0.039525 0.128575 31 | 0.038105 0.066847 32 | 0.089195 0.463818 33 | 0.265621 0.663027 34 | 2.836549 19.525284 35 | 0.056889 0.332572 36 | 0.038834 0.117369 37 | 0.132858 0.162893 38 | 0.099073 0.138098 39 | 0.037003 0.110374 40 | 0.060062 0.146185 41 | 0.045201 0.330567 42 | 0.008518 0.241837 43 | 0.044329 0.392353 44 | 0.041117 0.371043 45 | 0.036552 0.136668 46 | 0.020023 0.253218 47 | 0.048279 0.276041 48 | 0.030035 0.085115 49 | 0.118589 0.198847 50 | 0.194572 0.725631 51 | 0.062915 0.349110 52 | 0.051093 0.054241 53 | 0.047332 0.126926 54 | 0.058821 0.095221 55 | 0.066233 0.221543 56 | 0.036007 0.208624 57 | 0.049979 0.084503 58 | 0.098704 0.163981 59 | 0.047539 0.258306 60 | 0.019816 0.076415 61 | 0.069868 0.170250 62 | 0.049229 0.191546 63 | 0.024804 0.125503 64 | 0.023958 0.205293 65 | 0.022935 0.092010 66 | 0.114950 0.258210 67 | 0.054466 0.449480 68 | 0.046665 0.207369 69 | 0.079419 0.468148 70 | 0.040185 0.197821 71 | 0.085261 0.273868 72 | 0.039143 0.213306 73 | 0.087902 0.414821 74 | 0.078547 0.232262 75 | 0.080294 0.263212 76 | 0.025061 0.301241 77 | 0.039408 0.229139 78 | 0.062960 0.288840 79 | 0.093793 0.293966 80 | 0.103675 0.286449 81 | 0.071099 0.216981 82 | 0.056111 0.174080 83 | 0.036997 0.202787 84 | 0.094892 0.148925 85 | 0.031927 0.233827 86 | 0.024891 0.169362 87 | 0.047094 0.153502 88 | 0.101014 0.136693 89 | 0.048353 0.176502 90 | 0.055197 0.173866 91 | 0.072914 0.243430 92 | 0.069024 0.085284 93 | 0.043686 0.093209 94 | 0.039947 0.181370 95 | 0.030806 0.107733 96 | 0.021048 0.090813 97 | 0.041095 0.117848 98 | 0.036001 0.111463 99 | 0.111955 0.239132 100 | 0.152929 0.242706 101 | 0.123019 0.096336 102 | 0.172776 0.201109 103 | 0.173183 0.220423 104 | 0.182287 0.118087 105 | 0.147079 0.100359 106 | 0.155566 0.066700 107 | 0.053875 0.100485 108 | 0.089174 0.058534 109 | 0.041354 0.110701 110 | 0.052875 0.319859 111 | 0.044657 0.240157 112 | 0.091713 0.443842 113 | 0.018674 0.191752 114 | 0.092138 0.240824 115 | 0.037416 0.186849 116 | 0.104313 0.238387 117 | 0.129854 0.328145 118 | 0.037309 0.198989 119 | 0.050323 0.140173 120 | 0.094527 0.254856 121 | 0.028891 0.087089 122 | 0.061144 0.159833 123 | 0.076306 0.183877 124 | 0.024580 0.197707 125 | 0.076392 0.142151 126 | 0.033168 0.045156 127 | 0.185013 0.148374 128 | 0.161531 0.297331 129 | 0.066729 0.121643 130 | 0.119039 0.368470 131 | 0.126275 0.203295 132 | 0.062553 0.360571 133 | 0.138197 0.113097 134 | 0.207800 0.335005 135 | 0.220279 0.430283 136 | 0.276956 0.674031 137 | 0.201037 0.323971 138 | 0.250075 0.468698 139 | 0.201628 0.359643 140 | 0.162169 0.273268 141 | 0.172830 0.217975 142 | 0.161662 0.263450 143 | 0.178480 0.294022 144 | 0.173964 0.426134 145 | 0.140827 0.282214 146 | 0.185729 0.435124 147 | 0.293639 0.399984 148 | 0.230654 0.433823 149 | 0.146153 0.291915 150 | 0.061436 0.107339 151 | 0.051325 0.136728 152 | 0.089037 0.190065 153 | 0.112626 0.143898 154 | 0.059057 0.170877 155 | 0.045169 0.107505 156 | 0.076723 0.292347 157 | 0.069996 0.356405 158 | 0.068465 0.182304 159 | 0.085252 0.267665 160 | 0.121127 0.525052 161 | 0.144029 0.472106 162 | 0.112653 0.429747 163 | 0.134178 0.149708 164 | 0.226830 0.393172 165 | 0.092484 0.210501 166 | 0.276056 0.337524 167 | 0.248192 0.139100 168 | 0.316024 0.220087 169 | 0.188978 0.246283 170 | 0.202608 0.043307 171 | 0.116599 0.194711 172 | 0.129711 0.273092 173 | 0.089610 0.144948 174 | 0.133726 0.262383 175 | 0.129232 0.482360 176 | 0.285255 0.407424 177 | 0.132570 0.676430 178 | 0.048983 0.252089 179 | 0.052233 0.321439 180 | 0.143914 0.610020 181 | 0.070226 0.389915 182 | 0.078612 0.329448 183 | 0.095798 0.258861 184 | 0.081530 0.219001 185 | 0.049951 0.203873 186 | 0.056502 0.115826 187 | 0.204131 0.375033 188 | 0.238401 0.302253 189 | 0.112199 0.156121 190 | 0.196828 0.714181 191 | 0.099837 0.133711 192 | 0.052706 0.657495 193 | 0.030423 0.300498 194 | 0.083832 0.089650 195 | 0.125557 0.331540 196 | 0.105185 0.253258 197 | 0.057535 0.228416 198 | 0.173136 0.195322 199 | 0.052496 0.014139 200 | 0.216793 0.344245 201 | 0.107452 0.163062 202 | 0.188457 0.280296 203 | 0.116521 0.111338 204 | 0.134986 0.116831 205 | 0.084033 0.188205 206 | 0.189376 0.083990 207 | 0.127408 0.625144 208 | 0.302014 0.368295 209 | 0.230863 0.380319 210 | 0.193235 0.410564 211 | 0.206262 0.221615 212 | 6.407716 14.605951 213 | 7.650635 11.947124 214 | 25.241680 20.998397 215 | 24.461620 18.269269 216 | 25.340256 19.866696 217 | 0.234860 0.724972 218 | 0.177399 0.825974 219 | 37.016819 18.974349 220 | 0.219508 1.006106 221 | 0.239177 0.634842 222 | 0.189392 0.814254 223 | 0.117201 0.635327 224 | 0.178479 0.481405 225 | 0.157350 0.511333 226 | 0.198454 0.344161 227 | 0.042024 0.511751 228 | 0.047410 0.164466 229 | 0.125596 1.279147 230 | 0.123000 0.328852 231 | 0.124551 0.213326 232 | 0.153648 0.808653 233 | 0.273960 0.377799 234 | 0.112183 0.352011 235 | 0.191055 0.399650 236 | 0.107712 0.220265 237 | 0.243921 0.384097 238 | 0.145823 0.353442 239 | 0.183839 0.518062 240 | 0.689134 0.382453 241 | 0.060629 0.182561 242 | 0.049930 0.379504 243 | 0.102160 0.209050 244 | 0.079531 0.078904 245 | 0.112486 0.200364 246 | 0.050667 0.055904 247 | 0.090695 0.086851 248 | 0.065366 0.065257 249 | 0.052250 0.088835 250 | 0.090412 0.050423 251 | 0.065713 0.090765 252 | 0.052443 0.070021 253 | 0.113525 0.050207 254 | 0.039294 0.053815 255 | 0.133313 0.125879 256 | 0.036241 0.035126 257 | 0.057381 0.045856 258 | 0.047346 0.288327 259 | 0.091453 0.418738 260 | 0.036330 0.211681 261 | 0.026668 0.059530 262 | 0.046952 0.222998 263 | 0.072856 0.168559 264 | 0.058029 0.302898 265 | 0.076985 0.368131 266 | 0.106274 0.279954 267 | 0.056317 0.356932 268 | 0.083636 0.311138 269 | 0.027583 0.250148 270 | 0.021654 0.181300 271 | 0.070450 0.140921 272 | 0.041408 0.282223 273 | 0.059264 0.186411 274 | 0.049553 0.254121 275 | 0.083305 0.232297 276 | 0.022669 0.255317 277 | 0.015428 0.348780 278 | 0.027255 0.402116 279 | 0.120577 0.435043 280 | 0.102420 0.278661 281 | 0.076280 0.178146 282 | 0.120598 0.082598 283 | 0.047194 0.039320 284 | 0.064291 0.018893 285 | 0.027439 0.055261 286 | 0.020562 0.041237 287 | 0.051719 0.055018 288 | 0.019816 0.084273 289 | 0.052706 0.034394 290 | 0.031226 0.017376 291 | 0.020769 0.075617 292 | 0.021189 0.085884 293 | 0.090055 0.257642 294 | 0.012158 0.181940 295 | 0.027889 0.185348 296 | 0.008432 0.093282 297 | 0.070866 0.095243 298 | 0.059674 0.088576 299 | 0.049126 0.074658 300 | 0.107836 0.100250 301 | 0.072609 0.166768 302 | 0.105144 0.175926 303 | 0.092505 0.187080 304 | 0.044758 0.089214 305 | 0.105865 0.048030 306 | 0.098481 0.174552 307 | 0.085891 0.300048 308 | 0.014961 0.160665 309 | 0.130222 0.374185 310 | 0.056205 0.206030 311 | 0.114261 0.505684 312 | 0.128263 0.645559 313 | 0.040805 0.274983 314 | 0.121780 0.461423 315 | 0.055512 0.292310 316 | 0.143274 0.480877 317 | 0.190564 0.522772 318 | 0.022927 0.322537 319 | 0.048933 0.231850 320 | 0.038773 0.353584 321 | 0.092563 0.243170 322 | 0.041208 0.333958 323 | 0.019075 0.340047 324 | 0.041972 0.456642 325 | 0.031910 0.334957 326 | 0.089676 0.312529 327 | 0.199781 0.690974 328 | 0.110626 0.385223 329 | 0.234301 0.805735 330 | 0.079131 0.174758 331 | 0.118865 0.285563 332 | 0.403340 0.715609 333 | 0.043549 0.412516 334 | 0.273863 0.614881 335 | 0.198249 0.432727 336 | 0.060435 0.129046 337 | 0.199588 0.503080 338 | 0.084290 0.275154 339 | 0.012647 0.266096 340 | 0.033983 0.144909 341 | 0.032090 0.105269 342 | 0.037485 0.147556 343 | 0.031549 0.230240 344 | -------------------------------------------------------------------------------- /dsacstar/dsacstar_loss.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based on the DSAC++ and ESAC code. 3 | https://github.com/vislearn/LessMore 4 | https://github.com/vislearn/esac 5 | 6 | Copyright (c) 2016, TU Dresden 7 | Copyright (c) 2020, Heidelberg University 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are met: 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above copyright 15 | notice, this list of conditions and the following disclaimer in the 16 | documentation and/or other materials provided with the distribution. 17 | * Neither the name of the TU Dresden, Heidelberg University nor the 18 | names of its contributors may be used to endorse or promote products 19 | derived from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL TU DRESDEN OR HEIDELBERG UNIVERSITY BE LIABLE FOR ANY 25 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 28 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #pragma once 34 | 35 | #define MAXLOSS 10000000.0 // clamp for stability 36 | 37 | namespace dsacstar 38 | { 39 | /** 40 | * @brief Calculates the rotational distance in degree between two transformations. 41 | * Translation will be ignored. 42 | * 43 | * @param trans1 Transformation 1. 44 | * @param trans2 Transformation 2. 45 | * @return Angle in degree. 46 | */ 47 | double calcAngularDistance(const dsacstar::trans_t& trans1, const dsacstar::trans_t& trans2) 48 | { 49 | cv::Mat rot1 = trans1.colRange(0, 3).rowRange(0, 3); 50 | cv::Mat rot2 = trans2.colRange(0, 3).rowRange(0, 3); 51 | 52 | cv::Mat rotDiff= rot2 * rot1.t(); 53 | double trace = cv::trace(rotDiff)[0]; 54 | 55 | trace = std::min(3.0, std::max(-1.0, trace)); 56 | return 180*acos((trace-1.0)/2.0)/PI; 57 | } 58 | 59 | /** 60 | * @brief Weighted average of translational error and rotational error between two pose hypothesis. 61 | * @param h1 Pose 1. 62 | * @param h2 Pose 2. 63 | * @param wRot Weight of rotation error. 64 | * @param wTrans Weight of translation error. 65 | * @param cut Apply soft clamping after this value. 66 | * @return Loss. 67 | */ 68 | double loss( 69 | const dsacstar::trans_t& trans1, 70 | const dsacstar::trans_t& trans2, 71 | double wRot = 1.0, 72 | double wTrans = 1.0, 73 | double cut = 100) 74 | { 75 | double rotErr = dsacstar::calcAngularDistance(trans1, trans2); 76 | double tErr = cv::norm( 77 | trans1.col(3).rowRange(0, 3) - trans2.col(3).rowRange(0, 3)); 78 | 79 | double loss = wRot * rotErr + wTrans * tErr; 80 | 81 | if(loss > cut) 82 | loss = std::sqrt(cut * loss); 83 | 84 | return std::min(loss, MAXLOSS); 85 | } 86 | 87 | /** 88 | * @brief Calculate the derivative of the loss w.r.t. the estimated pose. 89 | * @param est Estimated pose (6 DoF). 90 | * @param gt Ground truth pose (6 DoF). 91 | * @param wRot Weight of rotation error. 92 | * @param wTrans Weight of translation error. 93 | * @param cut Apply soft clamping after this value. 94 | * @return 1x6 Jacobean. 95 | */ 96 | cv::Mat_ dLoss( 97 | const dsacstar::pose_t& est, 98 | const dsacstar::pose_t& gt, 99 | double wRot = 1.0, 100 | double wTrans = 1.0, 101 | double cut = 100) 102 | { 103 | cv::Mat rot1, rot2, dRod; 104 | cv::Rodrigues(est.first, rot1, dRod); 105 | cv::Rodrigues(gt.first, rot2); 106 | 107 | // measure loss of inverted poses (camera pose instead of scene pose) 108 | cv::Mat_ invRot1 = rot1.t(); 109 | cv::Mat_ invRot2 = rot2.t(); 110 | 111 | // get the difference rotation 112 | cv::Mat diffRot = rot1 * invRot2; 113 | 114 | // calculate rotational and translational error 115 | double trace = cv::trace(diffRot)[0]; 116 | trace = std::min(3.0, std::max(-1.0, trace)); 117 | double rotErr = 180*acos((trace-1.0)/2.0)/CV_PI; 118 | 119 | cv::Mat_ invT1 = est.second.clone(); 120 | invT1 = invRot1 * invT1; 121 | 122 | cv::Mat_ invT2 = gt.second.clone(); 123 | invT2 = invRot2 * invT2; 124 | 125 | // zero error, abort 126 | double tErr = cv::norm(invT1 - invT2); 127 | 128 | cv::Mat_ jacobean = cv::Mat_::zeros(1, 6); 129 | 130 | // clamped loss, return zero gradient if loss is bigger than threshold 131 | double loss = wRot * rotErr + wTrans * tErr; 132 | bool cutLoss = false; 133 | 134 | 135 | if(loss > cut) 136 | { 137 | loss = std::sqrt(loss); 138 | cutLoss = true; 139 | } 140 | 141 | if(loss > MAXLOSS) 142 | return jacobean; 143 | 144 | if((tErr + rotErr) < EPS) 145 | return jacobean; 146 | 147 | 148 | // return gradient of translational error 149 | cv::Mat_ dDist_dInvT1(1, 3); 150 | for(unsigned i = 0; i < 3; i++) 151 | dDist_dInvT1(0, i) = (invT1(i, 0) - invT2(i, 0)) / tErr; 152 | 153 | cv::Mat_ dInvT1_dEstT(3, 3); 154 | dInvT1_dEstT = invRot1; 155 | 156 | cv::Mat_ dDist_dEstT = dDist_dInvT1 * dInvT1_dEstT; 157 | jacobean.colRange(3, 6) += dDist_dEstT * wTrans; 158 | 159 | cv::Mat_ dInvT1_dInvRot1 = cv::Mat_::zeros(3, 9); 160 | 161 | dInvT1_dInvRot1(0, 0) = est.second.at(0, 0); 162 | dInvT1_dInvRot1(0, 3) = est.second.at(1, 0); 163 | dInvT1_dInvRot1(0, 6) = est.second.at(2, 0); 164 | 165 | dInvT1_dInvRot1(1, 1) = est.second.at(0, 0); 166 | dInvT1_dInvRot1(1, 4) = est.second.at(1, 0); 167 | dInvT1_dInvRot1(1, 7) = est.second.at(2, 0); 168 | 169 | dInvT1_dInvRot1(2, 2) = est.second.at(0, 0); 170 | dInvT1_dInvRot1(2, 5) = est.second.at(1, 0); 171 | dInvT1_dInvRot1(2, 8) = est.second.at(2, 0); 172 | 173 | dRod = dRod.t(); 174 | 175 | cv::Mat_ dDist_dRod = dDist_dInvT1 * dInvT1_dInvRot1 * dRod; 176 | jacobean.colRange(0, 3) += dDist_dRod * wTrans; 177 | 178 | 179 | // return gradient of rotational error 180 | cv::Mat_ dRotDiff = cv::Mat_::zeros(9, 9); 181 | invRot2.row(0).copyTo(dRotDiff.row(0).colRange(0, 3)); 182 | invRot2.row(1).copyTo(dRotDiff.row(1).colRange(0, 3)); 183 | invRot2.row(2).copyTo(dRotDiff.row(2).colRange(0, 3)); 184 | 185 | invRot2.row(0).copyTo(dRotDiff.row(3).colRange(3, 6)); 186 | invRot2.row(1).copyTo(dRotDiff.row(4).colRange(3, 6)); 187 | invRot2.row(2).copyTo(dRotDiff.row(5).colRange(3, 6)); 188 | 189 | invRot2.row(0).copyTo(dRotDiff.row(6).colRange(6, 9)); 190 | invRot2.row(1).copyTo(dRotDiff.row(7).colRange(6, 9)); 191 | invRot2.row(2).copyTo(dRotDiff.row(8).colRange(6, 9)); 192 | 193 | dRotDiff = dRotDiff.t(); 194 | 195 | cv::Mat_ dTrace = cv::Mat_::zeros(1, 9); 196 | dTrace(0, 0) = 1; 197 | dTrace(0, 4) = 1; 198 | dTrace(0, 8) = 1; 199 | 200 | cv::Mat_ dAngle = (180 / CV_PI * -1 / sqrt(3 - trace * trace + 2 * trace)) * dTrace * dRotDiff * dRod; 201 | 202 | jacobean.colRange(0, 3) += dAngle * wRot; 203 | 204 | if(cutLoss) 205 | jacobean *= 0.5 / loss; 206 | 207 | 208 | if(cv::sum(cv::Mat(jacobean != jacobean))[0] > 0) //check for NaNs 209 | return cv::Mat_::zeros(1, 6); 210 | 211 | return jacobean; 212 | } 213 | 214 | 215 | } 216 | -------------------------------------------------------------------------------- /datasets/sevenscenes.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | import numpy as np 3 | from imageio import imread 4 | import random 5 | import os 6 | import os.path as osp 7 | import torch 8 | import time 9 | from torchvision import transforms 10 | import torch.nn.functional as F 11 | from skimage.transform import rotate, resize 12 | from skimage import io 13 | import math 14 | import tools.camera_operator as cam_opt 15 | 16 | OUTPUT_SUBSAMPLE = 8 17 | # generate grid of target reprojection pixel positions 18 | prediction_grid = torch.zeros((2, 19 | math.ceil(1000 / OUTPUT_SUBSAMPLE), 20 | # 1000px is max limit of image size, increase if needed 21 | math.ceil(1000 / OUTPUT_SUBSAMPLE))) 22 | 23 | # get the centre of 8*8 patch 24 | for x in range(0, prediction_grid.size(2)): 25 | for y in range(0, prediction_grid.size(1)): 26 | prediction_grid[0, y, x] = x * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 27 | prediction_grid[1, y, x] = y * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 28 | 29 | def load_as_float(path): 30 | return imread(path).astype(np.float32) 31 | 32 | 33 | class SevenScenes(data.Dataset): 34 | def __init__(self, data_path, scene='chess', seed=7, train=True, transform=None, target_transform=None, real=False, 35 | skip_images=False, 36 | augment=False, 37 | preload_image=False, 38 | aug_rotation=30, 39 | aug_scale_min=2 / 3, 40 | aug_scale_max=3 / 2, 41 | aug_contrast=0.1, 42 | aug_brightness=0.1, 43 | aug_saturation=0.1, 44 | image_height=480 45 | ): 46 | """ 47 | :param real: If True, load poses from SLAM/integration of VO 48 | :param skip_images: If True, skip loading images and return None instead 49 | """ 50 | np.random.seed(seed) 51 | self.image_height = image_height 52 | self.augment = augment 53 | self.aug_rotation = aug_rotation 54 | self.aug_scale_min = aug_scale_min 55 | self.aug_scale_max = aug_scale_max 56 | self.aug_contrast = aug_contrast 57 | self.aug_brightness = aug_brightness 58 | self.aug_saturation= aug_saturation 59 | self.image_transform = transforms.Compose([ 60 | transforms.ToPILImage(), 61 | transforms.Resize(self.image_height), 62 | transforms.ToTensor(), 63 | transforms.Normalize( 64 | mean=[0.4], # statistics calculated over 7scenes training set, should generalize fairly well 65 | std=[0.25] 66 | ) 67 | ]) 68 | 69 | self.preload_image = preload_image 70 | 71 | self.scene = scene 72 | self.target_transform = target_transform 73 | self.skip_images = skip_images 74 | self.train = train 75 | self.mean = torch.zeros((3)) 76 | 77 | # directories 78 | base_dir = osp.join(osp.expanduser(data_path), scene) 79 | 80 | if train: 81 | split_file = osp.join(base_dir, 'TrainSplit.txt') 82 | else: 83 | split_file = osp.join(base_dir, 'TestSplit.txt') 84 | with open(split_file, 'r') as f: 85 | seqs = [int(l.split('sequence')[-1]) for l in f if not l.startswith('#')] 86 | 87 | self.c_imgs = [] 88 | self.poses = [] 89 | self.gt_idx = np.empty((0,), dtype=np.int) 90 | gt_offset = int(0) 91 | self.names = [] 92 | for seq in seqs: 93 | seq_dir = osp.join(base_dir, 'seq-{:02d}'.format(seq)) 94 | seq_name = 'seq-{:02d}'.format(seq) 95 | p_filenames = [n for n in os.listdir(osp.join(seq_dir, '.')) if 96 | n.find('pose') >= 0] 97 | 98 | frame_idx = np.array(range(len(p_filenames)), dtype=np.int) 99 | pss = [np.loadtxt(osp.join(seq_dir, 'frame-{:06d}.pose.txt'. 100 | format(i))) for i in frame_idx] 101 | self.poses.extend(pss) 102 | self.gt_idx = np.hstack((self.gt_idx, gt_offset + frame_idx)) 103 | gt_offset += len(p_filenames) 104 | c_imgs = [osp.join(seq_dir, 'frame-{:06d}.color.png'.format(i)) 105 | for i in frame_idx] 106 | names = [osp.join(seq_name, 'frame-{:06d}.color.png'.format(i)) 107 | for i in frame_idx] 108 | self.names.extend(names) 109 | self.c_imgs.extend(c_imgs) 110 | 111 | self.img_k = np.asarray([[525, 0, 320], [0, 525, 240], [0, 0, 1]], dtype=np.float32) 112 | 113 | begin = time.time() 114 | 115 | self.loaded_imgs = [] 116 | 117 | self.pose_transform = transforms.Compose([ 118 | transforms.ToTensor() 119 | ]) 120 | 121 | if self.preload_image: 122 | for idx, fn in enumerate(self.c_imgs): 123 | c_img = io.imread(fn) 124 | self.loaded_imgs.append(c_img) 125 | 126 | print("ReadData time:", time.time() - begin) 127 | 128 | def __getitem__(self, index): 129 | ''' 130 | :return frame: a dict in torch form 131 | ''' 132 | if self.skip_images: 133 | frame = None 134 | else: 135 | if self.c_imgs is None: 136 | print("NONE!!!!!!!") 137 | frame = dict() 138 | pose = self.poses[index] 139 | pose = torch.from_numpy(pose).float() 140 | 141 | K = self.img_k.copy() 142 | K_tensor = torch.from_numpy(K) 143 | 144 | if self.preload_image: 145 | image = self.loaded_imgs[index] 146 | else: 147 | image = io.imread(self.c_imgs[index]) 148 | 149 | if self.augment: 150 | scale_factor = random.uniform(self.aug_scale_min, self.aug_scale_max) 151 | angle = random.uniform(-self.aug_rotation, self.aug_rotation) 152 | cur_image_transform = transforms.Compose([ 153 | transforms.ToPILImage(), 154 | transforms.Resize(int(self.image_height * scale_factor)), 155 | transforms.ColorJitter(brightness=self.aug_brightness, contrast=self.aug_contrast), 156 | transforms.ToTensor(), 157 | transforms.Normalize( 158 | mean=[0.4], 159 | std=[0.25] 160 | ) 161 | ]) 162 | 163 | image = cur_image_transform(image) 164 | # scale focal length 165 | K_tensor[0,0] *= scale_factor 166 | K_tensor[1, 1] *= scale_factor 167 | # image center 168 | K_tensor[0, 2] = image.size(2)/2 169 | K_tensor[1, 2] = image.size(1)/2 170 | 171 | # rotate input image, don't change size(resize=False) 172 | def my_rot(t, angle, order, mode='constant'): 173 | t = t.permute(1, 2, 0).numpy() 174 | t = rotate(t, angle, order=order, mode=mode) 175 | t = torch.from_numpy(t).permute(2, 0, 1).float() 176 | return t 177 | #(3,H,W) 178 | image = my_rot(image, angle, 1, 'reflect') 179 | 180 | angle = angle * math.pi / 180 181 | pose_rot = torch.eye(4) 182 | pose_rot[0, 0] = math.cos(angle) 183 | pose_rot[0, 1] = -math.sin(angle) 184 | pose_rot[1, 0] = math.sin(angle) 185 | pose_rot[1, 1] = math.cos(angle) 186 | pose = torch.matmul(pose, pose_rot) 187 | else: 188 | image = self.image_transform(image) 189 | 190 | T_tensor = cam_opt.torch_camera_pose_inv(pose[:3,:3], pose[:3, 3:4]) 191 | frame['T'] = T_tensor 192 | frame['K'] = K_tensor 193 | frame['image'] = image 194 | frame['name'] = self.names[index] 195 | return frame 196 | 197 | def __len__(self): 198 | return len(self.c_imgs) 199 | -------------------------------------------------------------------------------- /reloc_pipeline/utils_func.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import torch 4 | from evaluator.basic_metric import rel_t, rel_R 5 | 6 | 7 | def compute_pnp_for_photometric(query_X_w, query_Ks, pnp_x_2ds, repro_thres=10): 8 | """ 9 | :param query_X_w: BT,N,3 10 | :param query_Ks: (B,3,3) 11 | :param pnp_x_2ds: (BT,N,2) 12 | :param repro_thres: 13 | :return: 14 | """ 15 | lm_pnp_poses = [] 16 | for idx, query_X_3d_w in enumerate(query_X_w): 17 | # (N) 18 | query_K = query_Ks[idx, :, :].squeeze(0) 19 | pnp_x_2d = pnp_x_2ds[idx] 20 | 21 | _, R_vec, t, inliers = cv2.solvePnPRansac(query_X_3d_w.cpu().detach().numpy().reshape(-1, 1, 3), 22 | pnp_x_2d.cpu().detach().numpy().reshape(-1, 1, 2), 23 | query_K.cpu().detach().numpy(), 24 | None, 25 | useExtrinsicGuess=False, 26 | reprojectionError=10.0, 27 | iterationsCount=128, 28 | flags=cv2.SOLVEPNP_P3P 29 | # minInliersCount=200 30 | ) 31 | 32 | R_res, _ = cv2.Rodrigues(R_vec) 33 | lm_pnp_pose = np.eye(4, dtype=np.float32) 34 | lm_pnp_pose[:3, :3] = R_res 35 | lm_pnp_pose[:3, 3] = t.ravel() 36 | 37 | lm_pnp_poses.append(torch.Tensor(lm_pnp_pose[:3,:])) 38 | 39 | return torch.stack(lm_pnp_poses, dim=0) 40 | 41 | 42 | def compute_pose_lm_pnp(gt_Ts, query_X_w, query_Ks, pnp_x_2ds, scene_valid_mask=None, uncertaintys=None, repro_thres=10): 43 | """ 44 | :param gt_Ts: (B,3,4) 45 | :param query_X_w: B,3,H,W / B,3,N 46 | :param query_Ks: (B,3,3) 47 | :param pnp_x_2ds: (B,N,2) 48 | :param scene_valid_mask:(B,N) 49 | :param uncertainty:B,1,H,W 50 | :param repro_thres: 51 | :return: 52 | """ 53 | lm_pnp_poses = [] 54 | R_accs = [] 55 | t_accs = [] 56 | 57 | for idx, query_X_3d_w in enumerate(query_X_w): 58 | # (N) 59 | gt_T = gt_Ts[idx, :, :].squeeze(0) 60 | query_K = query_Ks[idx, :, :].squeeze(0) 61 | pnp_x_2d = pnp_x_2ds[0, :, :] 62 | query_X_3d_w = query_X_3d_w.view(3,-1).permute(1,0) 63 | if uncertaintys is not None: 64 | uncertainty = uncertaintys[idx].view(-1) 65 | if scene_valid_mask is not None: 66 | mask = scene_valid_mask[idx, :] 67 | query_X_3d_w = query_X_3d_w[mask] 68 | pnp_x_2d = pnp_x_2d[mask] 69 | if uncertaintys is not None: 70 | uncertainty = uncertainty[mask] 71 | 72 | if uncertaintys is not None: 73 | # 2400:deciding #of used points 74 | sample_num = min(2400, pnp_x_2d.size(0)) 75 | sample_id = torch.argsort(uncertainty, descending=True)[:sample_num] 76 | 77 | # sample_id=torch.arange(0,uncertainty.numel())[uncertainty.ge(0.9)] 78 | 79 | query_X_3d_w = query_X_3d_w[sample_id] 80 | pnp_x_2d = pnp_x_2d[sample_id] 81 | 82 | # Or use thresh of weight to decide. 83 | # sample_id = np.argwhere(uncertaintys.cpu().numpy() > 0.5) 84 | # query_X_3d_w = query_X_3d_w.cpu()[sample_id] 85 | # pnp_x_2d = pnp_x_2d.cpu()[sample_id] 86 | 87 | # reshape must be consistent with 3D Coords's order(3,N) 88 | _, R_vec, t, inliers = cv2.solvePnPRansac(query_X_3d_w.cpu().detach().numpy().reshape(-1, 1, 3), 89 | pnp_x_2d.cpu().detach().numpy().reshape(-1, 1, 2), 90 | query_K.cpu().detach().numpy(), 91 | None, 92 | useExtrinsicGuess=False, 93 | reprojectionError=10, 94 | iterationsCount=128, 95 | confidence=0.99, 96 | flags=cv2.SOLVEPNP_P3P 97 | # minInliersCount=200 98 | ) 99 | 100 | R_res, _ = cv2.Rodrigues(R_vec) 101 | lm_pnp_pose = np.eye(4, dtype=np.float32) 102 | lm_pnp_pose[:3, :3] = R_res 103 | lm_pnp_pose[:3, 3] = t.ravel() 104 | 105 | # measure accuracy 106 | gt_pose = gt_T.detach().cpu().numpy() 107 | R_acc = rel_R(lm_pnp_pose, gt_pose) 108 | t_acc = rel_t(lm_pnp_pose, gt_pose) 109 | lm_pnp_poses.append(torch.Tensor(lm_pnp_pose[:3,:])) 110 | R_accs.append(R_acc) 111 | t_accs.append(t_acc) 112 | 113 | return R_accs, t_accs, lm_pnp_poses 114 | 115 | def compute_pose_only(gt_Ts, query_X_w, query_Ks, pnp_x_2ds, scene_valid_mask=None, uncertaintys=None, repro_thres=10): 116 | """ 117 | :param gt_Ts: (B,3,4) 118 | :param query_X_w: B,3,H,W / B,3,N 119 | :param query_Ks: (B,3,3) 120 | :param pnp_x_2ds: (B,N,2) 121 | :param scene_valid_mask:(B,N) 122 | :param uncertainty:B,1,H,W 123 | :param repro_thres: 124 | :return: 125 | """ 126 | lm_pnp_poses = [] 127 | 128 | for idx, query_X_3d_w in enumerate(query_X_w): 129 | # (N) 130 | gt_T = gt_Ts[idx, :, :].squeeze(0) 131 | query_K = query_Ks[idx, :, :].squeeze(0) 132 | pnp_x_2d = pnp_x_2ds[0, :, :] 133 | query_X_3d_w = query_X_3d_w.view(3,-1).permute(1,0) 134 | if uncertaintys is not None: 135 | uncertainty = uncertaintys[idx].view(-1) 136 | if scene_valid_mask is not None: 137 | mask = scene_valid_mask[idx, :] 138 | query_X_3d_w = query_X_3d_w[mask] 139 | pnp_x_2d = pnp_x_2d[mask] 140 | if uncertaintys is not None: 141 | uncertainty = uncertainty[mask] 142 | 143 | if uncertaintys is not None: 144 | # 2400:deciding #of used points 145 | sample_num = min(2400, pnp_x_2d.size(0)) 146 | sample_id = torch.argsort(uncertainty, descending=True)[:sample_num] 147 | 148 | # sample_id=torch.arange(0,uncertainty.numel())[uncertainty.ge(0.9)] 149 | 150 | query_X_3d_w = query_X_3d_w[sample_id] 151 | pnp_x_2d = pnp_x_2d[sample_id] 152 | 153 | # reshape must be consistent with 3D Coords's order(3,N) 154 | _, R_vec, t, inliers = cv2.solvePnPRansac(query_X_3d_w.cpu().detach().numpy().reshape(-1, 1, 3), 155 | pnp_x_2d.cpu().detach().numpy().reshape(-1, 1, 2), 156 | query_K.cpu().detach().numpy(), 157 | None, 158 | useExtrinsicGuess=False, 159 | reprojectionError=10, 160 | iterationsCount=128, 161 | confidence=0.99, 162 | flags=cv2.SOLVEPNP_P3P 163 | # minInliersCount=200 164 | ) 165 | 166 | R_res, _ = cv2.Rodrigues(R_vec) 167 | lm_pnp_pose = np.eye(4, dtype=np.float32) 168 | lm_pnp_pose[:3, :3] = R_res 169 | lm_pnp_pose[:3, 3] = t.ravel() 170 | 171 | 172 | lm_pnp_poses.append(torch.Tensor(lm_pnp_pose[:3,:])) 173 | 174 | return torch.stack(lm_pnp_poses, dim=0) 175 | 176 | def compute_err(pred_T, gt_T): 177 | gt_pose = gt_T.detach().cpu().numpy() 178 | pred_pose = pred_T.detach().cpu().numpy() 179 | R_acc = rel_R(pred_pose, gt_pose) 180 | t_acc = rel_t(pred_pose, gt_pose) 181 | 182 | return R_acc, t_acc 183 | 184 | 185 | def compute_err_batched(pred_Ts, gt_Ts): 186 | R_accs = [] 187 | t_accs = [] 188 | 189 | for idx, pred_T in enumerate(pred_Ts): 190 | # (3,4) 191 | gt_T = gt_Ts[idx, :, :].squeeze(0) 192 | # measure accuracy 193 | gt_pose = gt_T.detach().cpu().numpy() 194 | pred_pose = pred_T.detach().cpu().numpy() 195 | R_acc = rel_R(pred_pose, gt_pose) 196 | t_acc = rel_t(pred_pose, gt_pose) 197 | R_accs.append(R_acc) 198 | t_accs.append(t_acc) 199 | return R_accs, t_accs 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /Dense_Nets/ESAC_DROID.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | 5 | 6 | class ResidualBlock(nn.Module): 7 | def __init__(self, in_planes, planes, norm_fn='group', stride=1): 8 | super(ResidualBlock, self).__init__() 9 | 10 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, padding=1, stride=stride) 11 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1) 12 | self.relu = nn.ReLU(inplace=True) 13 | 14 | num_groups = planes // 8 15 | 16 | if norm_fn == 'group': 17 | self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 18 | self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 19 | if not stride == 1: 20 | self.norm3 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 21 | 22 | elif norm_fn == 'batch': 23 | self.norm1 = nn.BatchNorm2d(planes) 24 | self.norm2 = nn.BatchNorm2d(planes) 25 | if not stride == 1: 26 | self.norm3 = nn.BatchNorm2d(planes) 27 | 28 | elif norm_fn == 'instance': 29 | self.norm1 = nn.InstanceNorm2d(planes) 30 | self.norm2 = nn.InstanceNorm2d(planes) 31 | if not stride == 1: 32 | self.norm3 = nn.InstanceNorm2d(planes) 33 | 34 | elif norm_fn == 'none': 35 | self.norm1 = nn.Sequential() 36 | self.norm2 = nn.Sequential() 37 | if not stride == 1: 38 | self.norm3 = nn.Sequential() 39 | 40 | if stride == 1: 41 | self.downsample = None 42 | 43 | else: 44 | self.downsample = nn.Sequential( 45 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride), self.norm3) 46 | 47 | def forward(self, x): 48 | y = x 49 | y = self.relu(self.norm1(self.conv1(y))) 50 | y = self.relu(self.norm2(self.conv2(y))) 51 | 52 | if self.downsample is not None: 53 | x = self.downsample(x) 54 | 55 | return self.relu(x+y) 56 | 57 | 58 | class BottleneckBlock(nn.Module): 59 | def __init__(self, in_planes, planes, norm_fn='group', stride=1): 60 | super(BottleneckBlock, self).__init__() 61 | 62 | self.conv1 = nn.Conv2d(in_planes, planes//4, kernel_size=1, padding=0) 63 | self.conv2 = nn.Conv2d(planes//4, planes//4, kernel_size=3, padding=1, stride=stride) 64 | self.conv3 = nn.Conv2d(planes//4, planes, kernel_size=1, padding=0) 65 | self.relu = nn.ReLU(inplace=True) 66 | 67 | num_groups = planes // 8 68 | 69 | if norm_fn == 'group': 70 | self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=planes//4) 71 | self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=planes//4) 72 | self.norm3 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 73 | if not stride == 1: 74 | self.norm4 = nn.GroupNorm(num_groups=num_groups, num_channels=planes) 75 | 76 | elif norm_fn == 'batch': 77 | self.norm1 = nn.BatchNorm2d(planes//4) 78 | self.norm2 = nn.BatchNorm2d(planes//4) 79 | self.norm3 = nn.BatchNorm2d(planes) 80 | if not stride == 1: 81 | self.norm4 = nn.BatchNorm2d(planes) 82 | 83 | elif norm_fn == 'instance': 84 | self.norm1 = nn.InstanceNorm2d(planes//4) 85 | self.norm2 = nn.InstanceNorm2d(planes//4) 86 | self.norm3 = nn.InstanceNorm2d(planes) 87 | if not stride == 1: 88 | self.norm4 = nn.InstanceNorm2d(planes) 89 | 90 | elif norm_fn == 'none': 91 | self.norm1 = nn.Sequential() 92 | self.norm2 = nn.Sequential() 93 | self.norm3 = nn.Sequential() 94 | if not stride == 1: 95 | self.norm4 = nn.Sequential() 96 | 97 | if stride == 1: 98 | self.downsample = None 99 | 100 | else: 101 | self.downsample = nn.Sequential( 102 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride), self.norm4) 103 | 104 | def forward(self, x): 105 | y = x 106 | y = self.relu(self.norm1(self.conv1(y))) 107 | y = self.relu(self.norm2(self.conv2(y))) 108 | y = self.relu(self.norm3(self.conv3(y))) 109 | 110 | if self.downsample is not None: 111 | x = self.downsample(x) 112 | 113 | return self.relu(x+y) 114 | 115 | DIM=64 116 | 117 | class ESAC_DROID_Net(nn.Module): 118 | ''' 119 | FCN architecture for scene coordiante regression. 120 | The network has two output heads: One predicting a 3d scene coordinate, and a 1d neural guidance weight (if uncertainty is not None). 121 | The network makes dense predictions, but the output is subsampled by a factor of 8 compared to the input. 122 | ''' 123 | 124 | OUTPUT_SUBSAMPLE = 8 125 | 126 | def __init__(self, mean,uncertainty, output_dim=128, norm_fn='batch', dropout=0.0, multidim=False): 127 | ''' 128 | Constructor. 129 | ''' 130 | super(ESAC_DROID_Net, self).__init__() 131 | print('ESAC_DROID_Net') 132 | 133 | self.un = uncertainty 134 | 135 | self.norm_fn = norm_fn 136 | self.multidim = multidim 137 | 138 | if self.norm_fn == 'group': 139 | self.norm1 = nn.GroupNorm(num_groups=8, num_channels=DIM) 140 | 141 | elif self.norm_fn == 'batch': 142 | self.norm1 = nn.BatchNorm2d(DIM) 143 | 144 | elif self.norm_fn == 'instance': 145 | self.norm1 = nn.InstanceNorm2d(DIM) 146 | 147 | elif self.norm_fn == 'none': 148 | self.norm1 = nn.Sequential() 149 | 150 | self.conv1 = nn.Conv2d(3, DIM, kernel_size=7, stride=2, padding=3) 151 | self.relu1 = nn.ReLU(inplace=True) 152 | 153 | self.in_planes = DIM 154 | self.layer1 = self._make_layer(DIM, stride=1) 155 | self.layer2 = self._make_layer(2*DIM, stride=2) 156 | self.layer3 = self._make_layer(4*DIM, stride=2) 157 | 158 | # output convolution 159 | self.conv2 = nn.Conv2d(4*DIM, output_dim, kernel_size=1) 160 | 161 | if self.multidim: 162 | self.layer4 = self._make_layer(256, stride=2) 163 | self.layer5 = self._make_layer(512, stride=2) 164 | 165 | self.in_planes = 256 166 | self.layer6 = self._make_layer(256, stride=1) 167 | 168 | self.in_planes = 128 169 | self.layer7 = self._make_layer(128, stride=1) 170 | 171 | self.up1 = nn.Conv2d(512, 256, 1) 172 | self.up2 = nn.Conv2d(256, 128, 1) 173 | self.conv3 = nn.Conv2d(128, output_dim, kernel_size=1) 174 | 175 | if dropout > 0: 176 | self.dropout = nn.Dropout2d(p=dropout) 177 | else: 178 | self.dropout = None 179 | 180 | self.res1_conv1 = nn.Conv2d(256, 256, 3, 1, 1) 181 | self.res1_conv2 = nn.Conv2d(256, 256, 1, 1, 0) 182 | self.res1_conv3 = nn.Conv2d(256, 256, 3, 1, 1) 183 | 184 | self.res2_conv1 = nn.Conv2d(256, 512, 3, 1, 1) 185 | self.res2_conv2 = nn.Conv2d(512, 512, 1, 1, 0) 186 | self.res2_conv3 = nn.Conv2d(512, 512, 3, 1, 1) 187 | 188 | self.res2_skip = nn.Conv2d(256, 512, 1, 1, 0) 189 | 190 | self.res3_conv1 = nn.Conv2d(512, 512, 3, 1, 1) 191 | self.res3_conv2 = nn.Conv2d(512, 512, 1, 1, 0) 192 | self.res3_conv3 = nn.Conv2d(512, 512, 3, 1, 1) 193 | 194 | # output head 1, scene coordinates 195 | self.fc1 = nn.Conv2d(512, 512, 1, 1, 0) 196 | self.fc2 = nn.Conv2d(512, 512, 1, 1, 0) 197 | self.fc3 = nn.Conv2d(512, 3, 1, 1, 0) 198 | 199 | # output head 2, neural guidance 200 | if self.un: 201 | self.fc1_1 = nn.Conv2d(512, 512, 1, 1, 0) 202 | self.fc2_1 = nn.Conv2d(512, 512, 1, 1, 0) 203 | self.fc3_1 = nn.Conv2d(512, 1, 1, 1, 0) 204 | 205 | # learned scene coordinates relative to a mean coordinate (e.g. center of the scene) 206 | self.register_buffer('mean', torch.tensor(mean.size()).cuda()) 207 | self.mean = mean.clone() 208 | 209 | def _make_layer(self, dim, stride=1): 210 | layer1 = ResidualBlock(self.in_planes, dim, self.norm_fn, stride=stride) 211 | layer2 = ResidualBlock(dim, dim, self.norm_fn, stride=1) 212 | layers = (layer1, layer2) 213 | 214 | self.in_planes = dim 215 | return nn.Sequential(*layers) 216 | 217 | def forward(self, inputs): 218 | ''' 219 | Forward pass. 220 | inputs -- 4D data tensor (BxCxHxW) 221 | ''' 222 | batch_size = inputs.size(0) 223 | 224 | x = inputs 225 | 226 | x = self.conv1(x) 227 | x = self.norm1(x) 228 | x = self.relu1(x) 229 | 230 | x = self.layer1(x) 231 | x = self.layer2(x) 232 | x = self.layer3(x) 233 | 234 | res = self.conv2(x) 235 | 236 | # origin 237 | x = F.relu(self.res1_conv1(res)) 238 | x = F.relu(self.res1_conv2(x)) 239 | x = F.relu(self.res1_conv3(x)) 240 | 241 | res = res + x 242 | 243 | x = F.relu(self.res2_conv1(res)) 244 | x = F.relu(self.res2_conv2(x)) 245 | x = F.relu(self.res2_conv3(x)) 246 | 247 | res = self.res2_skip(res) + x 248 | 249 | x = F.relu(self.res3_conv1(res)) 250 | x = F.relu(self.res3_conv2(x)) 251 | x = F.relu(self.res3_conv3(x)) 252 | 253 | res = res + x 254 | 255 | # output head 1, scene coordinates 256 | sc = F.relu(self.fc1(res)) 257 | sc = F.relu(self.fc2(sc)) 258 | sc = self.fc3(sc) 259 | 260 | sc[:, 0,:,:] += self.mean[0] 261 | sc[:, 1,:,:] += self.mean[1] 262 | sc[:, 2,:,:] += self.mean[2] 263 | 264 | # output head 2, neural guidance 265 | if self.un: 266 | log_ng = F.relu(self.fc1_1(res)) 267 | log_ng = F.relu(self.fc2_1(log_ng)) 268 | log_ng = self.fc3_1(log_ng) 269 | un = torch.exp(log_ng) 270 | 271 | else: 272 | un = None 273 | 274 | return sc,un 275 | 276 | def init_weights(self): 277 | # init_modules = self.modules() 278 | # for m in init_modules: 279 | # if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d) or isinstance(m, nn.Linear): 280 | # torch.nn.init.xavier_uniform_(m.weight.data) 281 | # if m.bias is not None: 282 | # torch.nn.init.constant_(m.bias.data, 0.0) 283 | 284 | for m in self.modules(): 285 | if isinstance(m, nn.Conv2d): 286 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 287 | elif isinstance(m, (nn.BatchNorm2d, nn.InstanceNorm2d, nn.GroupNorm)): 288 | if m.weight is not None: 289 | nn.init.constant_(m.weight, 1) 290 | if m.bias is not None: 291 | nn.init.constant_(m.bias, 0) 292 | -------------------------------------------------------------------------------- /datasets/sevenscenes_for_unsup.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | import numpy as np 3 | from imageio import imread 4 | import random 5 | import os 6 | import os.path as osp 7 | import torch 8 | import time 9 | from torchvision import transforms 10 | import torch.nn.functional as F 11 | from skimage.transform import rotate, resize 12 | from skimage import io 13 | import math 14 | import tools.camera_operator as cam_opt 15 | 16 | 17 | OUTPUT_SUBSAMPLE = 8 18 | # generate grid of target reprojection pixel positions 19 | prediction_grid = torch.zeros((2, 20 | math.ceil(1000 / OUTPUT_SUBSAMPLE), 21 | # 1000px is max limit of image size, increase if needed 22 | math.ceil(1000 / OUTPUT_SUBSAMPLE))) 23 | 24 | # get the centre of 8*8 patch 25 | for x in range(0, prediction_grid.size(2)): 26 | for y in range(0, prediction_grid.size(1)): 27 | prediction_grid[0, y, x] = x * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 28 | prediction_grid[1, y, x] = y * OUTPUT_SUBSAMPLE + OUTPUT_SUBSAMPLE / 2 29 | 30 | 31 | def load_as_float(path): 32 | return imread(path).astype(np.float32) 33 | 34 | 35 | class SevenScenes(data.Dataset): 36 | def __init__(self, data_path, scene='chess', seed=7, train=True, transform=None, target_transform=None, real=False, 37 | skip_images=False, 38 | steps=1, keep_first=True, 39 | skip=1, variable_skip=False, 40 | augment=False, 41 | aug_rotation=30, 42 | aug_scale_min=2 / 3, 43 | aug_scale_max=3 / 2, 44 | aug_contrast=0.1, 45 | aug_brightness=0.1, 46 | image_height=480, 47 | ): 48 | """ 49 | :param real: If True, load poses from SLAM/integration of VO 50 | :param skip_images: If True, skip loading images and return None instead 51 | """ 52 | np.random.seed(seed) 53 | self.image_height = image_height 54 | self.augment = augment 55 | self.aug_rotation = aug_rotation 56 | self.aug_scale_min = aug_scale_min 57 | self.aug_scale_max = aug_scale_max 58 | self.aug_contrast = aug_contrast 59 | self.aug_brightness = aug_brightness 60 | self.image_transform = transforms.Compose([ 61 | transforms.ToPILImage(), 62 | transforms.Resize(self.image_height), 63 | transforms.ToTensor(), 64 | transforms.Normalize( 65 | mean=[0.4], 66 | std=[0.25] 67 | ) 68 | ]) 69 | self.sparse = True 70 | 71 | self.scene = scene 72 | self.target_transform = target_transform 73 | self.skip_images = skip_images 74 | self.train = train 75 | self.mean = torch.zeros((3)) 76 | self.variable_skip = variable_skip 77 | self.steps = steps 78 | self.skip = skip 79 | self.keep_first = keep_first 80 | if self.steps == 1: 81 | self.skip = 0 82 | 83 | # directories 84 | base_dir = osp.join(osp.expanduser(data_path), scene) 85 | 86 | # decide which sequences to use 87 | if train: 88 | split_file = osp.join(base_dir, 'TrainSplit.txt') 89 | else: 90 | split_file = osp.join(base_dir, 'TestSplit.txt') 91 | with open(split_file, 'r') as f: 92 | seqs = [int(l.split('sequence')[-1]) for l in f if not l.startswith('#')] 93 | 94 | self.c_imgs = [] 95 | self.sc_imgs = [] 96 | self.seq_num = [] 97 | self.seq_pose0 = [] 98 | self.samples = [] 99 | 100 | sequence_length = self.skip * (self.steps - 1) 101 | self.poses = [] 102 | self.gt_idx = np.empty((0,), dtype=np.int) 103 | gt_offset = int(0) 104 | def get_indices(index): 105 | if self.variable_skip: 106 | skips = np.random.randint(1, high=self.skip, size=self.steps - 1) 107 | else: 108 | skips = self.skip * np.ones(self.steps - 1) 109 | 110 | offsets = np.insert(skips, 0, 0).cumsum() 111 | # 20,10,0 112 | offsets = offsets.astype(np.int)[::-1] 113 | idx = index - offsets 114 | 115 | return idx 116 | 117 | for seq in seqs: 118 | seq_dir = osp.join(base_dir, 'seq-{:02d}'.format(seq)) 119 | p_filenames = [n for n in os.listdir(osp.join(seq_dir, '.')) if 120 | n.find('pose') >= 0] 121 | 122 | frame_idx = np.array(range(len(p_filenames)), dtype=np.int) 123 | pss = [np.loadtxt(osp.join(seq_dir, 'frame-{:06d}.pose.txt'. 124 | format(i))) for i in frame_idx] 125 | self.poses.extend(pss) 126 | self.seq_num.extend(np.ones_like(frame_idx) * seq) 127 | self.seq_pose0.append(pss[0]) 128 | 129 | self.gt_idx = np.hstack((self.gt_idx, gt_offset + frame_idx)) 130 | c_imgs = [osp.join(seq_dir, 'frame-{:06d}.color.png'.format(i)) 131 | for i in frame_idx] 132 | if self.train: 133 | sc_imgs = [osp.join(data_path, '7scenes_init', '7scenes_{}'.format(scene), 'training', 'init', 134 | 'seq{:02d}_frame-{:06d}.dat'.format(seq, i)) 135 | for i in frame_idx] 136 | 137 | self.c_imgs.extend(c_imgs) 138 | if self.train: 139 | self.sc_imgs.extend(sc_imgs) 140 | 141 | sample_set = [get_indices(i)+gt_offset for i in range(sequence_length, len(p_filenames))] 142 | self.samples.extend(sample_set) 143 | 144 | gt_offset += len(p_filenames) 145 | 146 | self.img_k = np.asarray([[525, 0, 320], [0, 525, 240], [0, 0, 1]], dtype=np.float32) 147 | 148 | begin = time.time() 149 | 150 | self.loaded_imgs = [] 151 | if self.train: 152 | self.loaded_scs = [] 153 | 154 | self.pose_transform = transforms.Compose([ 155 | transforms.ToTensor() 156 | ]) 157 | 158 | for idx, fn in enumerate(self.c_imgs): 159 | c_img = io.imread(fn) 160 | self.loaded_imgs.append(c_img) 161 | 162 | if self.train: 163 | coords = torch.load(self.sc_imgs[idx]) 164 | assert coords is not None 165 | self.loaded_scs.append(coords) 166 | print("ReadData time:", time.time() - begin) 167 | 168 | def __getitem__(self, index): 169 | ''' 170 | :return frame: a dict in torch form 171 | ''' 172 | if self.skip_images: 173 | frame = None 174 | else: 175 | if self.c_imgs is None: 176 | print("NONE!") 177 | 178 | indexs = self.samples[index] 179 | 180 | frame = dict() 181 | poses = torch.stack([torch.from_numpy(self.poses[i]).float() for i in indexs], dim=0) 182 | K_tensor = torch.stack([torch.from_numpy(self.img_k.copy()) for i in indexs], dim=0) 183 | 184 | if self.augment: 185 | scale_factor = random.uniform(self.aug_scale_min, self.aug_scale_max) 186 | angle = random.uniform(-self.aug_rotation, self.aug_rotation) 187 | 188 | # augment input image 189 | cur_image_transform = transforms.Compose([ 190 | transforms.ToPILImage(), 191 | transforms.Resize(int(self.image_height * scale_factor)), 192 | transforms.ColorJitter(brightness=self.aug_brightness, contrast=self.aug_contrast), 193 | transforms.ToTensor(), 194 | transforms.Normalize( 195 | mean=[0.4], 196 | std=[0.25] 197 | ) 198 | ]) 199 | 200 | # rotate input image, don't change size(resize=False) 201 | def my_rot(t, angle, order, mode='constant'): 202 | t = t.permute(1, 2, 0).numpy() 203 | t = rotate(t, angle, order=order, mode=mode) 204 | t = torch.from_numpy(t).permute(2, 0, 1).float() 205 | return t 206 | images = torch.stack([my_rot(cur_image_transform(self.loaded_imgs[i]), angle, 1, 'reflect') for i in indexs],dim=0) 207 | 208 | if self.train: 209 | if self.sparse: 210 | # rotate and scale initalization targets 211 | coords_w = math.ceil(images.size(3) / OUTPUT_SUBSAMPLE) 212 | coords_h = math.ceil(images.size(2) / OUTPUT_SUBSAMPLE) 213 | coords_set = [] 214 | for i in indexs: 215 | # (3,H,W) 216 | coords = self.loaded_scs[i] 217 | coords = F.interpolate(coords.unsqueeze(0), size=(coords_h, coords_w))[0] 218 | coords = self.my_rot(coords, angle, 0).view(3, -1).permute(1, 0) 219 | coords_set.append(coords) 220 | frame['coords'] = torch.stack(coords_set, dim=0) 221 | 222 | angle = angle * math.pi / 180 223 | pose_rot = torch.eye(4) 224 | pose_rot[0, 0] = math.cos(angle) 225 | pose_rot[0, 1] = -math.sin(angle) 226 | pose_rot[1, 0] = math.sin(angle) 227 | pose_rot[1, 1] = math.cos(angle) 228 | # scale focal length 229 | K_tensor[:, 0, 0] *= scale_factor 230 | K_tensor[:, 1, 1] *= scale_factor 231 | # image center 232 | K_tensor[:, 0, 2] = images.size(3) / 2 233 | K_tensor[:, 1, 2] = images.size(2) / 2 234 | #Broadcast 235 | poses = torch.matmul(poses, pose_rot) 236 | 237 | else: 238 | images = torch.stack( 239 | [self.image_transform(self.loaded_imgs[i]) for i in indexs], dim=0) 240 | if self.train: 241 | if self.sparse: 242 | # rotate and scale initalization targets 243 | coords_w = math.ceil(images.size(3) / OUTPUT_SUBSAMPLE) 244 | coords_h = math.ceil(images.size(2) / OUTPUT_SUBSAMPLE) 245 | coords_set = [] 246 | for i in indexs: 247 | # (3,H,W) 248 | coords = self.loaded_scs[i] 249 | coords = F.interpolate(coords.unsqueeze(0), size=(coords_h, coords_w))[0] 250 | coords_set.append(coords) 251 | frame['coords'] = torch.stack(coords_set, dim=0) 252 | 253 | T_tensor = torch.stack([cam_opt.torch_camera_pose_inv(pose[:3, :3], pose[:3, 3:4]) for pose in poses], dim=0) 254 | frame['T'] = T_tensor 255 | frame['K'] = K_tensor 256 | frame['image'] = images 257 | frame['indexs'] = indexs 258 | 259 | return frame 260 | 261 | def __len__(self): 262 | return len(self.samples) -------------------------------------------------------------------------------- /plot_per_frame/results/MST_stmaryschurch.txt: -------------------------------------------------------------------------------- 1 | 1.429751 3.295878 2 | 1.525674 4.311534 3 | 1.762420 5.266284 4 | 1.196382 3.248752 5 | 1.652630 3.896254 6 | 1.870357 1.867122 7 | 1.942303 3.027497 8 | 2.387963 2.953420 9 | 2.176548 1.363125 10 | 2.794662 2.953420 11 | 3.916400 10.434808 12 | 3.079963 7.543078 13 | 3.542306 16.005949 14 | 3.750928 18.192125 15 | 2.522732 4.226443 16 | 2.341926 5.735157 17 | 1.286527 2.137256 18 | 2.046024 4.246586 19 | 2.801468 7.792429 20 | 3.735312 13.381987 21 | 0.860096 nan 22 | 2.875738 2.946787 23 | 1.042162 2.901008 24 | 1.989273 6.047371 25 | 0.781984 7.308504 26 | 0.797751 7.459653 27 | 1.047041 5.008262 28 | 2.550712 3.526504 29 | 2.846679 2.710172 30 | 2.395844 4.148296 31 | 3.212500 6.184116 32 | 2.958321 8.455438 33 | 2.589009 10.698284 34 | 2.589808 9.104766 35 | 1.980357 10.907471 36 | 1.005142 9.315993 37 | 1.347321 13.522238 38 | 1.634836 4.541626 39 | 1.732195 22.146753 40 | 0.955748 5.110091 41 | 0.829453 1.920438 42 | 1.602938 nan 43 | 1.695488 1.833278 44 | 1.815414 2.120343 45 | 1.974804 2.433515 46 | 2.133754 3.963386 47 | 1.776049 2.547908 48 | 2.457586 3.080045 49 | 71.697937 8.827146 50 | 70.990723 14.961363 51 | 70.365562 11.963067 52 | 69.576782 57.295010 53 | 4.622238 7.187153 54 | 5.399360 7.905893 55 | 3.641401 5.410602 56 | 0.658816 nan 57 | 0.405082 3.420127 58 | 0.398971 4.291697 59 | 0.692398 5.682765 60 | 0.740163 4.807095 61 | 0.890126 5.305683 62 | 1.170199 5.002162 63 | 1.287926 2.311443 64 | 1.595962 3.305127 65 | 1.790970 nan 66 | 2.250918 0.684138 67 | 2.563633 3.669874 68 | 1.599015 3.794672 69 | 1.616861 2.077080 70 | 1.549841 2.233951 71 | 1.282686 3.297777 72 | 1.018269 2.109239 73 | 1.677842 nan 74 | 1.814027 4.365487 75 | 1.555694 5.430247 76 | 1.430442 5.126153 77 | 3.036323 6.732431 78 | 2.820103 21.339514 79 | 3.733801 7.667489 80 | 4.068819 7.273054 81 | 2.443661 5.977695 82 | 1.585797 4.685359 83 | 1.370628 3.335775 84 | 1.830008 3.475525 85 | 1.265645 3.746715 86 | 1.284626 3.024393 87 | 1.590973 nan 88 | 1.976925 1.476679 89 | 1.195103 3.156359 90 | 1.587093 2.954215 91 | 1.183807 3.573911 92 | 1.932703 3.950130 93 | 2.988497 4.747937 94 | 2.831316 7.296814 95 | 1.995957 2.403741 96 | 1.584459 6.419225 97 | 1.515420 12.157082 98 | 1.330820 11.144572 99 | 2.724095 12.337451 100 | 2.037601 7.076126 101 | 1.477080 3.973643 102 | 3.151443 4.261308 103 | 2.495626 nan 104 | 1.302514 3.619832 105 | 1.154105 6.156196 106 | 0.960498 4.640531 107 | 1.248708 7.826931 108 | 1.130304 11.381639 109 | 1.847189 11.105817 110 | 2.271558 8.501359 111 | 1.095756 4.190916 112 | 1.537014 5.036944 113 | 0.608726 2.928401 114 | 0.228253 1.556167 115 | 0.723915 4.247508 116 | 1.029273 2.375571 117 | 0.691251 2.738901 118 | 1.391759 2.841577 119 | 0.799191 4.634960 120 | 1.450270 4.647779 121 | 2.197977 6.362493 122 | 2.122320 5.303912 123 | 3.292201 9.188798 124 | 2.898610 7.975734 125 | 3.023744 7.865167 126 | 2.411493 6.437255 127 | 1.900392 4.384456 128 | 2.601560 2.200769 129 | 2.406847 nan 130 | 3.403304 6.711111 131 | 2.229301 9.660452 132 | 1.795454 7.263464 133 | 2.472496 6.724749 134 | 2.698242 5.831958 135 | 2.094746 9.631043 136 | 1.517083 8.708843 137 | 2.194737 10.196473 138 | 2.451198 8.830606 139 | 1.035758 5.669936 140 | 0.517614 3.376591 141 | 2.489521 5.583240 142 | 2.044435 4.115138 143 | 2.176906 6.938092 144 | 2.621421 4.663083 145 | 2.054071 10.543754 146 | 4.123926 11.292852 147 | 3.000303 10.680539 148 | 4.403234 9.272155 149 | 3.413288 5.694465 150 | 3.655350 5.658047 151 | 4.113262 12.076641 152 | 4.074218 11.833576 153 | 3.370411 10.131549 154 | 3.352398 12.807414 155 | 2.104928 6.924533 156 | 2.650557 4.072687 157 | 2.326266 4.859255 158 | 2.413365 5.393208 159 | 2.556503 4.251192 160 | 2.135954 6.118301 161 | 2.919785 5.255568 162 | 2.839961 3.217520 163 | 2.334446 3.243688 164 | 2.234235 nan 165 | 0.895122 1.363125 166 | 0.412585 1.621683 167 | 0.741843 2.834958 168 | 0.839134 6.283977 169 | 0.854322 7.921237 170 | 1.059707 8.620598 171 | 0.523066 3.781653 172 | 0.671775 6.004880 173 | 0.967520 9.117321 174 | 1.220206 13.925948 175 | 1.381730 7.891117 176 | 1.356436 16.840656 177 | 1.441129 21.925501 178 | 1.621541 56.962101 179 | 1.579906 20.827942 180 | 0.386942 6.578524 181 | 0.895928 8.852406 182 | 0.532513 7.590075 183 | 0.089446 3.010124 184 | 0.628903 nan 185 | 0.951151 nan 186 | 0.903675 0.463093 187 | 1.098878 nan 188 | 0.677995 3.500437 189 | 0.914979 3.869642 190 | 0.790094 5.293272 191 | 1.828485 4.920258 192 | 1.664647 4.754363 193 | 1.727189 4.253401 194 | 2.133258 3.859919 195 | 1.067304 4.279274 196 | 2.000914 5.208429 197 | 1.335041 3.959829 198 | 2.363237 7.331829 199 | 3.443693 11.256487 200 | 3.919665 4.666440 201 | 2.870341 7.389607 202 | 3.880992 11.640587 203 | 1.447002 6.717875 204 | 2.279140 1.630828 205 | 1.838215 3.346552 206 | 1.979890 3.237407 207 | 1.844623 1.230334 208 | 1.618832 1.806180 209 | 2.853815 1.838820 210 | 3.510188 4.572379 211 | 3.082852 4.082862 212 | 3.309538 4.085737 213 | 0.907054 3.287078 214 | 0.867785 2.607726 215 | 0.816783 1.514879 216 | 1.426041 5.374448 217 | 1.005324 4.796987 218 | 3.370830 4.237174 219 | 2.068599 nan 220 | 3.225598 3.002573 221 | 1.522094 2.931340 222 | 1.672289 nan 223 | 1.843984 nan 224 | 2.664834 2.564443 225 | 4.116498 1.314007 226 | 4.765567 4.319878 227 | 4.937383 2.002639 228 | 3.944825 2.681718 229 | 1.648638 2.307715 230 | 2.162369 3.609654 231 | 0.551538 2.513264 232 | 0.691349 4.234216 233 | 1.662616 3.029306 234 | 0.696236 2.906668 235 | 0.666983 4.264063 236 | 0.764956 4.347878 237 | 0.761530 3.631922 238 | 1.044641 3.272041 239 | 0.975447 0.966710 240 | 0.644468 2.750025 241 | 0.828878 1.624094 242 | 0.209139 nan 243 | 1.529078 2.084979 244 | 1.152017 0.837428 245 | 1.345451 nan 246 | 1.910341 4.786530 247 | 1.431222 2.919566 248 | 1.399417 nan 249 | 1.333887 5.274751 250 | 2.916887 11.849003 251 | 1.888081 8.286994 252 | 1.444928 6.537086 253 | 1.873634 2.908015 254 | 1.449035 2.657675 255 | 2.383366 2.332344 256 | 0.900525 nan 257 | 2.959596 2.647938 258 | 4.327278 5.998878 259 | 5.011413 7.738060 260 | 4.614757 nan 261 | 0.967196 3.880551 262 | 1.169775 4.128623 263 | 0.656213 2.961360 264 | 0.945210 2.873081 265 | 0.532697 2.182914 266 | 0.214035 nan 267 | 1.457551 6.447223 268 | 0.800785 2.363680 269 | 1.133026 nan 270 | 2.384078 3.346318 271 | 2.021906 4.835999 272 | 1.576764 3.069862 273 | 0.808450 2.457839 274 | 0.853692 1.256769 275 | 1.147222 0.451107 276 | 0.850397 nan 277 | 0.463537 2.499836 278 | 0.856014 1.066050 279 | 0.324793 1.854080 280 | 0.489002 4.045880 281 | 0.563558 3.129211 282 | 1.023231 3.515165 283 | 1.357249 2.835786 284 | 0.957648 0.866821 285 | 1.092947 5.736113 286 | 1.230923 3.139699 287 | 1.036776 6.186522 288 | 0.812057 3.566017 289 | 0.931821 5.020441 290 | 1.113980 5.198799 291 | 1.214320 7.115088 292 | 1.112923 8.850103 293 | 1.311930 10.888914 294 | 0.995916 10.915445 295 | 1.234602 10.957732 296 | 1.590781 10.229774 297 | 1.246732 9.995659 298 | 1.064481 9.745814 299 | 0.519151 6.251745 300 | 0.934098 13.292704 301 | 1.499799 5.603956 302 | 1.426113 10.377417 303 | 1.112210 12.623111 304 | 1.887445 14.844761 305 | 2.788313 13.562268 306 | 2.016481 18.109680 307 | 2.474571 11.525248 308 | 2.678274 7.651946 309 | 1.990512 2.281105 310 | 1.827062 3.291362 311 | 3.062444 8.452287 312 | 4.347993 10.150483 313 | 4.047351 9.506983 314 | 4.099422 14.935373 315 | 1.852125 32.245983 316 | 2.790816 23.866901 317 | 12.089080 32.841843 318 | 7.134390 12.747628 319 | 3.740816 3.775437 320 | 1.277034 5.678767 321 | 1.687390 3.327551 322 | 1.331145 5.631136 323 | 1.557679 2.958187 324 | 1.570260 0.669100 325 | 1.140154 1.372851 326 | 2.040346 2.331672 327 | 1.059951 5.203465 328 | 0.860649 2.428363 329 | 0.652779 2.657970 330 | 0.902730 3.498200 331 | 1.087073 3.896254 332 | 1.440305 3.717770 333 | 0.831192 0.906543 334 | 0.745765 1.152825 335 | 1.498111 3.145678 336 | 1.989197 1.780868 337 | 2.011107 4.735884 338 | 1.713345 nan 339 | 0.971116 1.272858 340 | 0.852413 1.239840 341 | 0.848559 nan 342 | 0.993943 1.396592 343 | 0.720966 2.667377 344 | 0.838386 3.720716 345 | 0.598947 2.540832 346 | 0.859937 nan 347 | 1.204438 nan 348 | 1.306915 3.833872 349 | 1.571455 2.827217 350 | 1.259979 nan 351 | 0.866353 1.871727 352 | 0.762457 1.530812 353 | 0.715726 3.929465 354 | 0.539143 3.127459 355 | 0.999147 3.278971 356 | 1.169765 0.418713 357 | 2.022740 3.915095 358 | 1.787289 3.640534 359 | 1.700474 2.976126 360 | 0.877391 3.560525 361 | 0.977593 1.604701 362 | 2.343807 4.745958 363 | 2.064145 2.458157 364 | 1.566800 4.149994 365 | 3.350121 4.176695 366 | 1.467301 1.919215 367 | 1.406966 1.872981 368 | 0.735969 nan 369 | 0.748139 3.177616 370 | 0.921993 4.130329 371 | 2.460243 7.370503 372 | 1.283235 3.095256 373 | 3.149016 4.462153 374 | 1.643317 5.721762 375 | 1.235957 5.899505 376 | 2.115195 7.940989 377 | 1.686967 7.761610 378 | 2.291221 6.474857 379 | 0.919313 2.929470 380 | 0.872065 nan 381 | 0.880054 2.848731 382 | 1.065994 2.027113 383 | 0.730896 nan 384 | 1.450460 1.026400 385 | 0.972720 nan 386 | 1.030700 2.522901 387 | 0.738421 nan 388 | 0.999128 nan 389 | 1.484171 5.150382 390 | 1.764008 5.873700 391 | 1.574143 7.399776 392 | 1.745281 4.738693 393 | 0.393811 nan 394 | 0.578286 4.374981 395 | 3.262849 24.838800 396 | 9.137860 28.708618 397 | 1.380093 4.762754 398 | 1.939641 2.983219 399 | 1.620992 5.104112 400 | 2.231078 2.604422 401 | 3.387338 2.804422 402 | 1.487148 4.557114 403 | 1.761432 3.294452 404 | 0.944020 3.342573 405 | 0.866986 4.115519 406 | 1.024431 nan 407 | 1.357618 nan 408 | 0.736439 1.817843 409 | 0.650362 3.862757 410 | 0.716862 3.244171 411 | 1.497759 4.589298 412 | 0.857780 3.842846 413 | 1.119646 4.900008 414 | 1.427737 4.772278 415 | 1.575881 5.209782 416 | 1.591722 2.215303 417 | 1.442911 3.685838 418 | 0.621136 3.125957 419 | 0.869296 4.046654 420 | 0.561760 2.359039 421 | 0.595768 4.111522 422 | 1.130355 2.571758 423 | 0.975513 3.840196 424 | 1.235677 2.033667 425 | 2.194328 nan 426 | 2.009800 8.206060 427 | 1.071442 1.665025 428 | 1.030105 3.775645 429 | 1.694898 6.314062 430 | 3.199109 11.915585 431 | 1.785530 12.950468 432 | 3.887781 14.479055 433 | 2.116760 1.153504 434 | 2.553349 3.702789 435 | 5.073319 4.198754 436 | 5.061663 3.759230 437 | 4.614231 nan 438 | 3.083981 6.137725 439 | 2.428013 6.525096 440 | 3.398724 8.407403 441 | 3.187034 7.725194 442 | 2.482896 8.924684 443 | 1.545424 1.601772 444 | 2.407832 3.580913 445 | 2.358695 7.537678 446 | 2.313356 9.967867 447 | 1.590466 14.959527 448 | 3.939276 17.245041 449 | 3.408335 8.822973 450 | 3.585621 12.861679 451 | 4.084225 12.409385 452 | 20.626665 10.992872 453 | 20.156958 13.989469 454 | 4.114372 8.539333 455 | 5.586559 7.915202 456 | 13.555596 10.542713 457 | 5.228999 3.267013 458 | 3.468022 4.734892 459 | 2.484101 1.189580 460 | 2.960812 3.284219 461 | 2.722267 nan 462 | 3.629453 2.861616 463 | 3.937615 nan 464 | 3.893017 4.320421 465 | 3.963320 6.494541 466 | 2.209528 3.447483 467 | 2.510766 4.670464 468 | 2.856390 10.297659 469 | 3.477567 12.095263 470 | 3.629984 15.333240 471 | 3.142948 12.438295 472 | 3.009748 11.498278 473 | 2.739711 10.990805 474 | 2.893128 11.715847 475 | 3.592110 11.104263 476 | 3.407968 9.260486 477 | 2.821233 9.910387 478 | 1.430746 4.889611 479 | 1.724791 4.756503 480 | 2.065938 4.122741 481 | 2.040009 6.170552 482 | 1.728214 5.369785 483 | 1.577791 4.272867 484 | 2.034390 3.246101 485 | 2.875309 3.801267 486 | 3.956327 3.951319 487 | 6.116873 7.493908 488 | 4.980404 9.739622 489 | 4.057969 13.016198 490 | 3.324224 17.356663 491 | 3.603193 21.316963 492 | 2.724294 12.529760 493 | 3.430408 12.508147 494 | 2.952886 15.074415 495 | 2.567289 13.201156 496 | 2.078226 8.353747 497 | 0.657507 1.843496 498 | 0.904966 4.734892 499 | 1.196942 2.626866 500 | 1.297960 3.421957 501 | 0.691820 2.473078 502 | 0.828890 3.557886 503 | 1.020895 nan 504 | 1.530790 nan 505 | 0.923885 3.368235 506 | 1.434075 2.852026 507 | 0.437393 2.559249 508 | 1.196397 3.015580 509 | 1.476819 3.637092 510 | 1.101819 2.809998 511 | 1.299953 3.301336 512 | 1.280612 2.677628 513 | 1.081087 0.347179 514 | 1.148177 nan 515 | 1.001468 1.497731 516 | 0.722806 1.869216 517 | 0.821103 nan 518 | 0.936676 1.235413 519 | 0.982837 4.871645 520 | 2.340729 4.537142 521 | 3.359125 2.542371 522 | 2.834415 2.795756 523 | 2.313590 2.519485 524 | 2.783319 2.193288 525 | 2.297723 2.907746 526 | 2.076203 2.849829 527 | 1.638880 2.502027 528 | 1.433607 8.369392 529 | 1.764773 3.963583 530 | 1.338060 4.889452 531 | -------------------------------------------------------------------------------- /datasets/cambridge.py: -------------------------------------------------------------------------------- 1 | import os.path as osp 2 | from torchvision import transforms 3 | import torch 4 | import numpy as np 5 | from numpy import mat 6 | from torch.utils import data 7 | import math 8 | import cv2 9 | import random 10 | import torch.nn.functional as F 11 | from skimage.transform import rotate, resize 12 | from skimage import io 13 | from utils import qlog 14 | from tools.pose_transformations import quaternion_from_matrix, euler_from_quaternion, euler_from_matrix, \ 15 | quaternion_from_euler, quaternion_matrix 16 | import tools.camera_operator as cam_opt 17 | import time 18 | 19 | OUTPUT_SUBSAMPLE = 8 20 | 21 | def rot2euler(p): 22 | """ 23 | convert 6DoF pose with rotation matrix to 6DoF euler angle 24 | :param p: vector--12 25 | :return: vector--6 26 | """ 27 | R = mat([[p[0], p[1], p[2]], [p[4], p[5], p[6]], [p[8], p[9], p[10]]]) 28 | euler = euler_from_matrix(matrix=R) 29 | return np.array([p[3], p[7], p[11], euler[0], euler[1], euler[2]], np.float) 30 | 31 | 32 | def rot2quat(p): 33 | """" 34 | convert 6DoF pose with rotation matrix to 6DoF quaternion 35 | : param p: vector -- 12 36 | : return vector -- 7 37 | """ 38 | R = mat([[p[0], p[1], p[2]], [p[4], p[5], p[6]], [p[8], p[9], p[10]]]) 39 | quat = quaternion_from_matrix(matrix=R) 40 | return np.array([p[3], p[7], p[11], quat[0], quat[1], quat[2], quat[3]]) 41 | 42 | 43 | def rot2logq(p): 44 | """ 45 | convert 6DoF pose with rotation matrix to 6DoF logq 46 | :param p: vector -- 12 47 | :return: vector -- 6 48 | """ 49 | R = mat([[p[0], p[1], p[2]], [p[4], p[5], p[6]], [p[8], p[9], p[10]]]) 50 | q = np.array(quaternion_from_matrix(matrix=R)) 51 | q *= np.sign(q[0]) 52 | logq = qlog(q) 53 | return np.array([p[3], p[7], p[11], logq[0], logq[1], logq[2]], np.float) 54 | 55 | 56 | def q2logq(p): 57 | q = p[3:] 58 | q *= np.sign(q[0]) 59 | logq = qlog(q) 60 | return np.array([p[0], p[1], p[2], logq[0], logq[1], logq[2]], np.float) 61 | 62 | 63 | def q2T(p): 64 | cam_rot = [float(r) for r in p[3:]] 65 | 66 | # quaternion to axis-angle 67 | angle = 2 * math.acos(cam_rot[0]) 68 | x = cam_rot[1] / math.sqrt(1 - cam_rot[0] ** 2) 69 | y = cam_rot[2] / math.sqrt(1 - cam_rot[0] ** 2) 70 | z = cam_rot[3] / math.sqrt(1 - cam_rot[0] ** 2) 71 | 72 | cam_rot = [x * angle, y * angle, z * angle] 73 | 74 | cam_rot = np.asarray(cam_rot) 75 | cam_rot, _ = cv2.Rodrigues(cam_rot) 76 | cam_trans = [float(r) for r in p[0:3]] 77 | cam_trans = np.asarray([cam_trans]) 78 | cam_trans = np.transpose(cam_trans) 79 | cam_trans = - np.matmul(cam_rot, cam_trans) 80 | 81 | if np.absolute(cam_trans).max() > 10000: 82 | print("Skipping image, Extremely large translation. Outlier?") 83 | print(cam_trans) 84 | return None 85 | cam_pose = np.concatenate((cam_rot, cam_trans), axis=1) 86 | cam_pose = np.concatenate((cam_pose, [[0, 0, 0, 1]]), axis=0) 87 | return cam_pose 88 | 89 | 90 | def q2euler(p): 91 | euler = euler_from_quaternion(p[3:]) 92 | return np.array([p[0], p[1], p[2], euler[0], euler[1], euler[2]], np.float) 93 | 94 | 95 | class Cambridge(data.Dataset): 96 | def __init__(self, scene, data_path, train=True, transform=None, 97 | target_transform=None, seed=7, steps=1, 98 | keep_first=True, 99 | skip=1, variable_skip=False, 100 | preload_image=False, 101 | rot_model='T', 102 | augment=False, 103 | aug_rotation=30, 104 | aug_scale_min=2 / 3, 105 | aug_scale_max=5 / 4, 106 | aug_contrast=0.1, 107 | aug_brightness=0.1, 108 | image_height=480 109 | ): 110 | self.variable_skip = variable_skip 111 | self.transform = transform 112 | self.target_tranform = target_transform 113 | self.steps = steps 114 | self.skip = skip 115 | self.keep_first = keep_first 116 | self.train = train 117 | self.rot_model = rot_model 118 | self.preload_image = preload_image 119 | 120 | np.random.seed(seed) 121 | self.image_height = image_height 122 | self.augment = augment 123 | self.aug_rotation = aug_rotation 124 | self.aug_scale_min = aug_scale_min 125 | self.aug_scale_max = aug_scale_max 126 | self.aug_contrast = aug_contrast 127 | self.aug_brightness = aug_brightness 128 | self.image_transform = transforms.Compose([ 129 | transforms.ToPILImage(), 130 | transforms.Resize(self.image_height), 131 | transforms.ToTensor(), 132 | transforms.Normalize( 133 | mean=[0.4], # statistics calculated over 7scenes training set, should generalize fairly well 134 | std=[0.25] 135 | ) 136 | ]) 137 | 138 | if self.steps == 1: 139 | self.skip = 0 140 | begin = time.time() 141 | 142 | # directories 143 | base_dir = osp.join(osp.expanduser(data_path), scene) 144 | # data_dir = osp.join('data', 'temporal', 'Cambridge', scene) 145 | # if not osp.isdir(data_dir): 146 | # data_dir = osp.join('temporal', 'Cambridge', scene) 147 | 148 | if train: 149 | image_list = osp.join(base_dir, 'dataset_train.txt') 150 | else: 151 | image_list = osp.join(base_dir, 'dataset_test.txt') 152 | 153 | f = open(image_list) 154 | self.camera_list = f.readlines() 155 | f.close() 156 | self.camera_list = self.camera_list[3:] 157 | self.camera_list.sort() 158 | 159 | # read poses and collect image names 160 | self.c_imgs = [] 161 | 162 | self.calibration_files=[] 163 | self.poses = [] 164 | self.gt_idx = np.empty((0,), dtype=np.int) 165 | self.samples = [] 166 | sequence_length = self.skip * (self.steps - 1) 167 | seq_num = [] 168 | seq_name = [] 169 | 170 | def get_indices(index): 171 | if self.variable_skip: 172 | skips = np.random.randint(1, high=self.skip, size=self.steps - 1) 173 | else: 174 | skips = self.skip * np.ones(self.steps - 1) 175 | 176 | offsets = np.insert(skips, 0, 0).cumsum() 177 | offsets = offsets.astype(np.int)[::-1] 178 | idx = index - offsets 179 | 180 | return idx 181 | 182 | last_name = '' 183 | now_num = 0 184 | valid_num = 0 185 | for idx, camera in enumerate(self.camera_list): 186 | elements = camera.split() 187 | image_name = elements[0] 188 | pose_qt = elements[1:] 189 | 190 | pose_inv = q2T(pose_qt) 191 | if pose_inv is None: 192 | print(image_name) 193 | continue 194 | pose = np.linalg.inv(pose_inv) 195 | self.c_imgs.append(osp.join(base_dir, image_name)) 196 | 197 | seq, num = image_name.split('/') 198 | 199 | self.poses.append(pose) 200 | 201 | # Do not use idx,since there is invalid pose and be excluded.Use valid_num. 202 | if valid_num == 0: 203 | last_name = seq 204 | now_num = 0 205 | seq_name.append(seq) 206 | if valid_num == sequence_length: 207 | self.samples.append(get_indices(valid_num)) 208 | else: 209 | if last_name == seq: 210 | now_num += 1 211 | if now_num >= sequence_length: 212 | self.samples.append(get_indices(valid_num)) 213 | else: 214 | last_name = seq 215 | seq_name.append(seq) 216 | seq_num.append(now_num + 1) 217 | now_num = 0 218 | if now_num >= sequence_length: 219 | self.samples.append(get_indices(valid_num)) 220 | valid_num += 1 221 | seq_num.append(now_num + 1) 222 | 223 | if self.preload_image: 224 | self.loaded_cimgs = [io.imread(fn) for fn in self.c_imgs] 225 | 226 | 227 | self.img_k = np.asarray([[744.375, 0, 426], [0, 744.375, 240], [0, 0, 1]], dtype=np.float32) 228 | self.pose_transform = transforms.Compose([ 229 | transforms.ToTensor() 230 | ]) 231 | 232 | print("ReadData time:", time.time() - begin) 233 | 234 | 235 | def download(self): 236 | pass 237 | 238 | def process(self): 239 | pass 240 | 241 | def convert_pose(self, p): 242 | if self.rot_model == 'rot': 243 | poses_rel = p 244 | elif self.rot_model == 'quat': 245 | poses_rel = rot2quat(p) 246 | elif self.rot_model == 'euler': 247 | poses_rel = rot2euler(p) 248 | elif self.rot_model == 'logq': 249 | poses_rel = rot2logq(p) 250 | else: 251 | raise "No such rot mode %s" % self.rot_model 252 | 253 | return poses_rel 254 | 255 | def my_rot(self, t, angle, order, mode='constant'): 256 | t = t.permute(1, 2, 0).numpy() 257 | t = rotate(t, angle, order=order, mode=mode) 258 | t = torch.from_numpy(t).permute(2, 0, 1).float() 259 | return t 260 | 261 | def __getitem__(self, index): 262 | indexs = self.samples[index] 263 | 264 | frame = dict() 265 | poses = torch.stack([torch.from_numpy(self.poses[i]).float() for i in indexs], dim=0) 266 | 267 | img_k = [torch.Tensor(self.img_k.copy()) for i in indexs] 268 | K_tensor = torch.stack(img_k, dim=0) 269 | 270 | if self.preload_image: 271 | images = [self.loaded_cimgs[i] for i in indexs] 272 | else: 273 | images = [io.imread(self.c_imgs[i]) for i in indexs] 274 | if self.augment: 275 | scale_factor = random.uniform(self.aug_scale_min, self.aug_scale_max) 276 | angle = random.uniform(-self.aug_rotation, self.aug_rotation) 277 | 278 | # augment input image 279 | cur_image_transform = transforms.Compose([ 280 | transforms.ToPILImage(), 281 | transforms.Resize(int(self.image_height * scale_factor)), 282 | transforms.ColorJitter(brightness=self.aug_brightness, contrast=self.aug_contrast), 283 | transforms.ToTensor(), 284 | transforms.Normalize( 285 | mean=[0.4], 286 | std=[0.25] 287 | ) 288 | ]) 289 | 290 | images = torch.stack( 291 | [self.my_rot(cur_image_transform(image), angle, 1, 'reflect') for image in images], dim=0) 292 | 293 | angle = angle * math.pi / 180 294 | pose_rot = torch.eye(4) 295 | pose_rot[0, 0] = math.cos(angle) 296 | pose_rot[0, 1] = -math.sin(angle) 297 | pose_rot[1, 0] = math.sin(angle) 298 | pose_rot[1, 1] = math.cos(angle) 299 | # scale focal length 300 | K_tensor[:, 0, 0] *= scale_factor 301 | K_tensor[:, 1, 1] *= scale_factor 302 | # image center 303 | K_tensor[:, 0, 2] = images.size(3) / 2 304 | K_tensor[:, 1, 2] = images.size(2) / 2 305 | # Broadcast 306 | poses = torch.matmul(poses, pose_rot) 307 | else: 308 | images = torch.stack( 309 | [self.image_transform(image) for image in images], dim=0) 310 | 311 | T_tensor = torch.stack([cam_opt.torch_camera_pose_inv(pose[:3, :3], pose[:3, 3:4]) for pose in poses], dim=0) 312 | frame['T'] = T_tensor.squeeze(0) 313 | frame['K'] = K_tensor.squeeze(0) 314 | frame['image'] = images.squeeze(0) 315 | 316 | return frame 317 | 318 | def __len__(self): 319 | return len(self.samples) 320 | 321 | 322 | def test(): 323 | pass 324 | 325 | --------------------------------------------------------------------------------