├── .gitignore ├── README.md ├── dtu_eval.py ├── requirements.txt └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | outputs/ 2 | visualize_outs/ 3 | __pycache__/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DTU Evaluation with Python 2 | When conducting point cloud evaluation on the DTU dataset, it is necessary to intall Matlab and use the official Matlab evaluation code, which is very cumbersome. 3 | Based on https://github.com/jzhangbs/DTUeval-python, Python evaluation code has been implemented 4 | 5 | 6 | ## Usage 7 | 1. Install dependency 8 | ```bash 9 | pip install -r requirements.txt 10 | ``` 11 | 2. Prepare Dataset 12 | - Download the STL [Point clouds](http://roboimagedata2.compute.dtu.dk/data/MVS/Points.zip) and [Sample Set](http://roboimagedata2.compute.dtu.dk/data/MVS/SampleSet.zip) and unzip them, then copy the `Points/Points/stl/stlxxx_total.ply` file in to the `SampleSet/MVS Data/Points/stl` folder. 13 | - Get prediction results of your mvs algorithm, the naming format of predicted ply file is `{method}{scanid}_l3.ply` such as `mvsnet001_l3.ply`,`casmvsnet001_l3.ply`, or you can name it your own way and modify the reding code [here](http://github.com/Gwencong/utils.py/Line134). 14 | After the two steps above, your data directory will like bellow: 15 | ``` 16 | ./SampleSet/MVS Data/ 17 | |--Points 18 | | |--stl 19 | | |--stlxxx_total.ply 20 | |--ObsMask 21 | | |--ObsMaskxxx_10.mat 22 | | |--Planexxx.mat 23 | 24 | ./Predict/ 25 | |--mvsnet 26 | | |--mvsnetxxx_l3.ply 27 | ``` 28 | 29 | 3. evaluation 30 | - Use following command to evaluation: 31 | ```bash 32 | python dtu_eval.py --method mvsnet --pred_dir "./Preidct/mvsnet/" --gt_dir "./SampleSet/MVS Data" 33 | ``` 34 | - Note: 35 | If you encounter a memory shortage issue, it is possible that the version of Python you are installing is 32-bit, which has limitations on memory usage. Therefore, it is recommended to install 64-bit Python 36 | 37 | ## Compare with MATLAB Evalution result 38 | We compare the evaluation results of matlab and python in r-mvsnet and casmvsnet. The results obtained from this implementation are slightly higher than those from MATLAB code but more fast and memory saving, can be used during experiments. The difference in results between the two is mainly due to random shuffling in the code. Results are shown bellow: 39 | 40 | |Method|acc.(mm)|comp.(mm)|overall(mm)| 41 | |------|--------|---------|-----------| 42 | |R-MVSNet(matlab)|0.3835|0.4520|0.4177| 43 | |R-MVSNet(python)|0.3836|0.4581|0.4209| 44 | |CasMVSNet(matlab)|0.3779|0.3636|0.3707| 45 | |CasMVSNet(python)|0.3780|0.3669|0.3739| -------------------------------------------------------------------------------- /dtu_eval.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from utils import compute_scans 3 | 4 | scans = [1,4,9,10,11,12,13,15,23,24,29,32,33,34,48,49,62,75,77,110,114,118] 5 | parser = argparse.ArgumentParser() 6 | parser.add_argument('--scans', type=list, default=scans, help="scans to be evalutation") 7 | parser.add_argument('--method', type=str, default='mvsnet', help="method name, such as mvsnet,casmvsnet") 8 | parser.add_argument('--pred_dir', type=str, default='./Predict/mvsnet', help="predict result ply file path") 9 | parser.add_argument('--gt_dir', type=str, default='./SampleSet/MVS Data',help="groud truth ply file path") 10 | parser.add_argument('--down_dense', type=float, default=0.2, help="downsample density, Min dist between points when reducing") 11 | parser.add_argument('--patch', type=float, default=60, help="patch size") 12 | parser.add_argument('--max_dist', type=float, default=20, help="outlier thresshold of 20 mm") 13 | parser.add_argument('--vis', type=bool, default=False, help="visualization") 14 | parser.add_argument('--vis_thresh', type=float, default=10, help="visualization distance threshold of 10mm") 15 | parser.add_argument('--vis_out_dir', type=str, default="./visualize_outs", help="visualization result save dir") 16 | args = parser.parse_args() 17 | 18 | if __name__ == "__main__": 19 | # args.scans = [1] 20 | # args.pred_dir = "/home/gwc/gwc/data/DTU/eval/Predict/r-mvsnet" 21 | # args.gt_dir = "/home/gwc/gwc/data/DTU/eval/SampleSet/MVS Data" 22 | 23 | scans = args.scans 24 | method = args.method 25 | pred_dir = args.pred_dir 26 | gt_dir = args.gt_dir 27 | vis = args.vis 28 | 29 | exclude = ["scans", "method", "pred_dir", "gt_dir"] 30 | args = vars(args) 31 | args = {key:args[key] for key in args if key not in exclude} 32 | acc, comp, overall = compute_scans(scans, method, pred_dir, gt_dir, **args) 33 | print(f"mean acc:{acc:>12.4f}\nmean comp:{comp:>11.4f}\nmean overall:{overall:>8.4f}") 34 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | plyfile 3 | tqdm 4 | scipy 5 | scikit-learn -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | 5 | from tqdm import tqdm 6 | from pathlib import Path 7 | from scipy.io import loadmat 8 | from sklearn import neighbors as skln 9 | from plyfile import PlyData, PlyElement 10 | 11 | 12 | def read_ply(file): 13 | data = PlyData.read(file) 14 | vertex = data['vertex'] 15 | data_pcd = np.stack([vertex['x'], vertex['y'], vertex['z']], axis=-1) 16 | return data_pcd 17 | 18 | def write_vis_pcd(file, points, colors): 19 | points = np.array([tuple(v) for v in points], dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) 20 | colors = np.array([tuple(v) for v in colors], dtype=[('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]) 21 | 22 | vertex_all = np.empty(len(points), points.dtype.descr + colors.dtype.descr) 23 | for prop in points.dtype.names: 24 | vertex_all[prop] = points[prop] 25 | for prop in colors.dtype.names: 26 | vertex_all[prop] = colors[prop] 27 | 28 | el = PlyElement.describe(vertex_all, 'vertex') 29 | PlyData([el]).write(file) 30 | 31 | 32 | def comput_one_scan(scanid, # the scan id to be computed 33 | pred_ply, # predict points cloud file path, such as "./mvsnet001_l3.ply" 34 | gt_ply, # ground truth points cloud file path, such as "./stl001_total.ply" 35 | mask_file, # obsmask file path, decide which parts of 3D space should be used for evaluation 36 | plane_file, # plane file path, used to destinguise which Stl points are 'used' 37 | down_dense = 0.2, # downsample density, Min dist between points when reducing 38 | patch = 60, # patch size 39 | max_dist = 20, # outlier thresshold of 20 mm 40 | vis = False,# whether save distance visualization result 41 | vis_thresh = 10, # visualization distance threshold of 10mm 42 | vis_out_dir = "./visualize_outs"): 43 | '''Compute accuracy(mm), completeness(mm), overall(mm) for one scan 44 | 45 | scanid: the scan id to be computed 46 | pred_ply: predict points cloud file path, such as "./mvsnet001_l3.ply" 47 | gt_ply: ground truth points cloud file path, such as "./stl001_total.ply" 48 | mask_file: obsmask file path, decide which parts of 3D space should be used for evaluation 49 | plane_file: plane file path, used to destinguise which Stl points are 'used' 50 | down_dense: downsample density, Min dist between points when reducing 51 | patch: patch size 52 | max_dist: outlier thresshold of 20 mm 53 | vis: whether save distance visualization result 54 | vis_thresh: visualization distance threshold of 10mm 55 | vis_out_dir: visualization result save directory 56 | ''' 57 | 58 | thresh = down_dense 59 | pbar = tqdm(total=8) 60 | pbar.set_description(f'[scan{scanid}] read data pcd') 61 | data_pcd = read_ply(pred_ply) 62 | 63 | pbar.update(1) 64 | pbar.set_description(f'[scan{scanid}] random shuffle pcd index') 65 | shuffle_rng = np.random.default_rng() 66 | shuffle_rng.shuffle(data_pcd, axis=0) 67 | 68 | pbar.update(1) 69 | pbar.set_description(f'[scan{scanid}] downsample pcd') 70 | nn_engine = skln.NearestNeighbors(n_neighbors=1, radius=thresh, algorithm='kd_tree', n_jobs=-1) 71 | nn_engine.fit(data_pcd) 72 | rnn_idxs = nn_engine.radius_neighbors(data_pcd, radius=thresh, return_distance=False) 73 | mask = np.ones(data_pcd.shape[0], dtype=np.bool_) 74 | for curr, idxs in enumerate(rnn_idxs): 75 | if mask[curr]: 76 | mask[idxs] = 0 77 | mask[curr] = 1 78 | data_down = data_pcd[mask] 79 | 80 | pbar.update(1) 81 | pbar.set_description(f'[scan{scanid}] masking data pcd') 82 | obs_mask_file = loadmat(mask_file) 83 | ObsMask, BB, Res = [obs_mask_file[attr] for attr in ['ObsMask', 'BB', 'Res']] 84 | BB = BB.astype(np.float32) 85 | 86 | inbound = ((data_down >= BB[:1]-patch) & (data_down < BB[1:]+patch*2)).sum(axis=-1) ==3 87 | data_in = data_down[inbound] 88 | 89 | data_grid = np.around((data_in - BB[:1]) / Res).astype(np.int32) 90 | grid_inbound = ((data_grid >= 0) & (data_grid < np.expand_dims(ObsMask.shape, 0))).sum(axis=-1) ==3 91 | data_grid_in = data_grid[grid_inbound] 92 | in_obs = ObsMask[data_grid_in[:,0], data_grid_in[:,1], data_grid_in[:,2]].astype(np.bool_) 93 | data_in_obs = data_in[grid_inbound][in_obs] 94 | 95 | pbar.update(1) 96 | pbar.set_description(f'[scan{scanid}] read STL pcd') 97 | stl = read_ply(gt_ply) 98 | 99 | pbar.update(1) 100 | pbar.set_description(f'[scan{scanid}] compute data2stl') 101 | nn_engine.fit(stl) 102 | dist_d2s, idx_d2s = nn_engine.kneighbors(data_in_obs, n_neighbors=1, return_distance=True) 103 | max_dist = max_dist 104 | mean_d2s = dist_d2s[dist_d2s < max_dist].mean() 105 | 106 | pbar.update(1) 107 | pbar.set_description(f'[scan{scanid}] compute stl2data') 108 | ground_plane = loadmat(plane_file)['P'] 109 | 110 | stl_hom = np.concatenate([stl, np.ones_like(stl[:,:1])], -1) 111 | above = (ground_plane.reshape((1,4)) * stl_hom).sum(-1) > 0 112 | stl_above = stl[above] 113 | 114 | nn_engine.fit(data_in) 115 | dist_s2d, idx_s2d = nn_engine.kneighbors(stl_above, n_neighbors=1, return_distance=True) 116 | mean_s2d = dist_s2d[dist_s2d < max_dist].mean() 117 | 118 | pbar.update(1) 119 | pbar.set_description(f'[scan{scanid}] visualize error') 120 | if vis: 121 | Path(vis_out_dir).mkdir(parents=True, exist_ok=True) 122 | vis_dist = vis_thresh 123 | R = np.array([[255,0,0]], dtype=np.float64) 124 | G = np.array([[0,255,0]], dtype=np.float64) 125 | B = np.array([[0,0,255]], dtype=np.float64) 126 | W = np.array([[255,255,255]], dtype=np.float64) 127 | data_color = np.tile(B, (data_down.shape[0], 1)) 128 | data_alpha = dist_d2s.clip(max=vis_dist) / vis_dist 129 | data_color[ np.where(inbound)[0][grid_inbound][in_obs] ] = R * data_alpha + W * (1-data_alpha) 130 | data_color[ np.where(inbound)[0][grid_inbound][in_obs][dist_d2s[:,0] >= max_dist] ] = G 131 | write_vis_pcd(f'{vis_out_dir}/vis_{scanid:03}_d2s.ply', data_down, data_color) 132 | stl_color = np.tile(B, (stl.shape[0], 1)) 133 | stl_alpha = dist_s2d.clip(max=vis_dist) / vis_dist 134 | stl_color[ np.where(above)[0] ] = R * stl_alpha + W * (1-stl_alpha) 135 | stl_color[ np.where(above)[0][dist_s2d[:,0] >= max_dist] ] = G 136 | write_vis_pcd(f'{vis_out_dir}/vis_{scanid:03}_s2d.ply', stl, stl_color) 137 | 138 | pbar.update(1) 139 | pbar.set_description(f'[scan{scanid}] done') 140 | pbar.close() 141 | over_all = (mean_d2s + mean_s2d) / 2 142 | print(f"\t\t\tacc.(mm):{mean_d2s:.4f}, comp.(mm):{mean_s2d:.5f}, overall(mm):{over_all:.4f}") 143 | return mean_d2s, mean_s2d, over_all 144 | 145 | def compute_scans(scans, method, pred_dir, gt_dir, **kargs): 146 | t1 = time.time() 147 | acc ,comp ,overall = [], [], [] 148 | for scanid in scans: 149 | pred_ply = os.path.join(pred_dir, f"{method}{scanid:03}_l3.ply") 150 | gt_ply = os.path.join(gt_dir, f"Points/stl/stl{scanid:03}_total.ply") 151 | mask_file = os.path.join(gt_dir, f'ObsMask/ObsMask{scanid}_10.mat') 152 | plane_file = os.path.join(gt_dir, f'ObsMask/Plane{scanid}.mat') 153 | assert os.path.exists(pred_ply), f"File '{pred_ply}' not found" 154 | assert os.path.exists(gt_ply), f"File '{gt_ply}' not found" 155 | assert os.path.exists(mask_file), f"File '{mask_file}' not found" 156 | assert os.path.exists(plane_file), f"File '{plane_file}' not found" 157 | result = comput_one_scan(scanid, pred_ply, gt_ply, mask_file, plane_file, **kargs) 158 | acc.append(result[0]) 159 | comp.append(result[1]) 160 | overall.append(result[2]) 161 | mean_acc = np.mean(acc) 162 | mean_comp = np.mean(comp) 163 | mean_overall = np.mean(overall) 164 | t2 = time.time() 165 | print(f"Finished, total time cost: {t2-t1:.2f}s") 166 | return mean_acc, mean_comp, mean_overall 167 | 168 | --------------------------------------------------------------------------------