├── LICENSE
├── README.md
└── eval.py


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2021 Jingyang Zhang
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # DTUeval-python
 2 | 
 3 | A python implementation of DTU MVS 2014 evaluation. It only takes 1min for each mesh evaluation. And the gap between the two implementations is negligible. 
 4 | 
 5 | ## Setup and Usage
 6 | 
 7 | This script requires the following dependencies.
 8 | 
 9 | ```
10 | numpy open3d scikit-learn tqdm scipy multiprocessing argparse
11 | ```
12 | 
13 | Download the STL point clouds and Sample Set and prepare the ground truth folder as follows.
14 | 
15 | ```
16 | <dataset_dir>
17 | - Points
18 |     - stl
19 |         - stlxxx_total.ply
20 | - ObsMask
21 |     - ObsMaskxxx_10.mat
22 |     - Planexxx.mat
23 | ```
24 | 
25 | Run the evaluation script (e.g. scan24, mesh mode)
26 | ```
27 | python eval.py --data <input> --scan 24 --mode mesh --dataset_dir <dataset_dir> --vis_out_dir <out_dir_for_visualization>
28 | ```
29 | 
30 | ## Discussion on randomness
31 | There is randomness in point cloud downsampling in both versions. It iterates through the points and delete the points with distance < 0.2. So the order of points matters. We randomly shuffle the points before downsampling. 
32 | 
33 | ## Comparison with the official script
34 | We evaluate a set of meshes from Colmap and compare the results. We run our script 10 times and take the average. 
35 | 
36 | |     | diff/official | official | py_avg   | py_std/official |
37 | |-----|---------------|----------|----------|-----------------|
38 | | 24  | 0.0184%       | 0.986317 | 0.986135 | 0.0108%         |
39 | | 37  | 0.0001%       | 2.354124 | 2.354122 | 0.0091%         |
40 | | 40  | 0.0038%       | 0.730464 | 0.730492 | 0.0104%         |
41 | | 55  | 0.0436%       | 0.530899 | 0.531131 | 0.0104%         |
42 | | 63  | 0.0127%       | 1.555828 | 1.556025 | 0.0118%         |
43 | | 65  | 0.0409%       | 1.007686 | 1.008098 | 0.0080%         |
44 | | 69  | 0.0082%       | 0.888434 | 0.888361 | 0.0125%         |
45 | | 83  | 0.0207%       | 1.136882 | 1.137117 | 0.0096%         |
46 | | 97  | 0.0314%       | 0.907528 | 0.907813 | 0.0089%         |
47 | | 105 | 0.0129%       | 1.463337 | 1.463526 | 0.0118%         |
48 | | 106 | 0.1424%       | 0.785527 | 0.786646 | 0.0151%         |
49 | | 110 | 0.0592%       | 1.076125 | 1.075488 | 0.0132%         |
50 | | 114 | 0.0049%       | 0.436169 | 0.436190 | 0.0074%         |
51 | | 118 | 0.1123%       | 0.679574 | 0.680337 | 0.0099%         |
52 | | 122 | 0.0347%       | 0.726771 | 0.726519 | 0.0178%         |
53 | | avg | 0.0153%       | 1.017711 | 1.017867 |                 |
54 | 
55 | 
56 | ## Error visualization
57 | `vis_xxx_d2s.ply` and `vis_xxx_s2d.ply` are error visualizations.
58 | - Blue: Out of bounding box or ObsMask
59 | - Green: Errors larger than threshold (20)
60 | - White to Red: Errors counted in the reported statistics


--------------------------------------------------------------------------------
/eval.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import open3d as o3d
  3 | import sklearn.neighbors as skln
  4 | from tqdm import tqdm
  5 | from scipy.io import loadmat
  6 | import multiprocessing as mp
  7 | import argparse
  8 | 
  9 | def sample_single_tri(input_):
 10 |     n1, n2, v1, v2, tri_vert = input_
 11 |     c = np.mgrid[:n1+1, :n2+1]
 12 |     c += 0.5
 13 |     c[0] /= max(n1, 1e-7)
 14 |     c[1] /= max(n2, 1e-7)
 15 |     c = np.transpose(c, (1,2,0))
 16 |     k = c[c.sum(axis=-1) < 1]  # m2
 17 |     q = v1 * k[:,:1] + v2 * k[:,1:] + tri_vert
 18 |     return q
 19 | 
 20 | def write_vis_pcd(file, points, colors):
 21 |     pcd = o3d.geometry.PointCloud()
 22 |     pcd.points = o3d.utility.Vector3dVector(points)
 23 |     pcd.colors = o3d.utility.Vector3dVector(colors)
 24 |     o3d.io.write_point_cloud(file, pcd)
 25 | 
 26 | if __name__ == '__main__':
 27 |     mp.freeze_support()
 28 | 
 29 |     parser = argparse.ArgumentParser()
 30 |     parser.add_argument('--data', type=str, default='data_in.ply')
 31 |     parser.add_argument('--scan', type=int, default=1)
 32 |     parser.add_argument('--mode', type=str, default='mesh', choices=['mesh', 'pcd'])
 33 |     parser.add_argument('--dataset_dir', type=str, default='.')
 34 |     parser.add_argument('--vis_out_dir', type=str, default='.')
 35 |     parser.add_argument('--downsample_density', type=float, default=0.2)
 36 |     parser.add_argument('--patch_size', type=float, default=60)
 37 |     parser.add_argument('--max_dist', type=float, default=20)
 38 |     parser.add_argument('--visualize_threshold', type=float, default=10)
 39 |     args = parser.parse_args()
 40 | 
 41 |     thresh = args.downsample_density
 42 |     if args.mode == 'mesh':
 43 |         pbar = tqdm(total=9)
 44 |         pbar.set_description('read data mesh')
 45 |         data_mesh = o3d.io.read_triangle_mesh(args.data)
 46 | 
 47 |         vertices = np.asarray(data_mesh.vertices)
 48 |         triangles = np.asarray(data_mesh.triangles)
 49 |         tri_vert = vertices[triangles]
 50 | 
 51 |         pbar.update(1)
 52 |         pbar.set_description('sample pcd from mesh')
 53 |         v1 = tri_vert[:,1] - tri_vert[:,0]
 54 |         v2 = tri_vert[:,2] - tri_vert[:,0]
 55 |         l1 = np.linalg.norm(v1, axis=-1, keepdims=True)
 56 |         l2 = np.linalg.norm(v2, axis=-1, keepdims=True)
 57 |         area2 = np.linalg.norm(np.cross(v1, v2), axis=-1, keepdims=True)
 58 |         non_zero_area = (area2 > 0)[:,0]
 59 |         l1, l2, area2, v1, v2, tri_vert = [
 60 |             arr[non_zero_area] for arr in [l1, l2, area2, v1, v2, tri_vert]
 61 |         ]
 62 |         thr = thresh * np.sqrt(l1 * l2 / area2)
 63 |         n1 = np.floor(l1 / thr)
 64 |         n2 = np.floor(l2 / thr)
 65 | 
 66 |         with mp.Pool() as mp_pool:
 67 |             new_pts = mp_pool.map(sample_single_tri, ((n1[i,0], n2[i,0], v1[i:i+1], v2[i:i+1], tri_vert[i:i+1,0]) for i in range(len(n1))), chunksize=1024)
 68 | 
 69 |         new_pts = np.concatenate(new_pts, axis=0)
 70 |         data_pcd = np.concatenate([vertices, new_pts], axis=0)
 71 |     
 72 |     elif args.mode == 'pcd':
 73 |         pbar = tqdm(total=8)
 74 |         pbar.set_description('read data pcd')
 75 |         data_pcd_o3d = o3d.io.read_point_cloud(args.data)
 76 |         data_pcd = np.asarray(data_pcd_o3d.points)
 77 | 
 78 |     pbar.update(1)
 79 |     pbar.set_description('random shuffle pcd index')
 80 |     shuffle_rng = np.random.default_rng()
 81 |     shuffle_rng.shuffle(data_pcd, axis=0)
 82 | 
 83 |     pbar.update(1)
 84 |     pbar.set_description('downsample pcd')
 85 |     nn_engine = skln.NearestNeighbors(n_neighbors=1, radius=thresh, algorithm='kd_tree', n_jobs=-1)
 86 |     nn_engine.fit(data_pcd)
 87 |     rnn_idxs = nn_engine.radius_neighbors(data_pcd, radius=thresh, return_distance=False)
 88 |     mask = np.ones(data_pcd.shape[0], dtype=np.bool_)
 89 |     for curr, idxs in enumerate(rnn_idxs):
 90 |         if mask[curr]:
 91 |             mask[idxs] = 0
 92 |             mask[curr] = 1
 93 |     data_down = data_pcd[mask]
 94 | 
 95 |     pbar.update(1)
 96 |     pbar.set_description('masking data pcd')
 97 |     obs_mask_file = loadmat(f'{args.dataset_dir}/ObsMask/ObsMask{args.scan}_10.mat')
 98 |     ObsMask, BB, Res = [obs_mask_file[attr] for attr in ['ObsMask', 'BB', 'Res']]
 99 |     BB = BB.astype(np.float32)
100 | 
101 |     patch = args.patch_size
102 |     inbound = ((data_down >= BB[:1]-patch) & (data_down < BB[1:]+patch*2)).sum(axis=-1) ==3
103 |     data_in = data_down[inbound]
104 | 
105 |     data_grid = np.around((data_in - BB[:1]) / Res).astype(np.int32)
106 |     grid_inbound = ((data_grid >= 0) & (data_grid < np.expand_dims(ObsMask.shape, 0))).sum(axis=-1) ==3
107 |     data_grid_in = data_grid[grid_inbound]
108 |     in_obs = ObsMask[data_grid_in[:,0], data_grid_in[:,1], data_grid_in[:,2]].astype(np.bool_)
109 |     data_in_obs = data_in[grid_inbound][in_obs]
110 | 
111 |     pbar.update(1)
112 |     pbar.set_description('read STL pcd')
113 |     stl_pcd = o3d.io.read_point_cloud(f'{args.dataset_dir}/Points/stl/stl{args.scan:03}_total.ply')
114 |     stl = np.asarray(stl_pcd.points)
115 | 
116 |     pbar.update(1)
117 |     pbar.set_description('compute data2stl')
118 |     nn_engine.fit(stl)
119 |     dist_d2s, idx_d2s = nn_engine.kneighbors(data_in_obs, n_neighbors=1, return_distance=True)
120 |     max_dist = args.max_dist
121 |     mean_d2s = dist_d2s[dist_d2s < max_dist].mean()
122 | 
123 |     pbar.update(1)
124 |     pbar.set_description('compute stl2data')
125 |     ground_plane = loadmat(f'{args.dataset_dir}/ObsMask/Plane{args.scan}.mat')['P']
126 | 
127 |     stl_hom = np.concatenate([stl, np.ones_like(stl[:,:1])], -1)
128 |     above = (ground_plane.reshape((1,4)) * stl_hom).sum(-1) > 0
129 |     stl_above = stl[above]
130 | 
131 |     nn_engine.fit(data_in)
132 |     dist_s2d, idx_s2d = nn_engine.kneighbors(stl_above, n_neighbors=1, return_distance=True)
133 |     mean_s2d = dist_s2d[dist_s2d < max_dist].mean()
134 | 
135 |     pbar.update(1)
136 |     pbar.set_description('visualize error')
137 |     vis_dist = args.visualize_threshold
138 |     R = np.array([[1,0,0]], dtype=np.float64)
139 |     G = np.array([[0,1,0]], dtype=np.float64)
140 |     B = np.array([[0,0,1]], dtype=np.float64)
141 |     W = np.array([[1,1,1]], dtype=np.float64)
142 |     data_color = np.tile(B, (data_down.shape[0], 1))
143 |     data_alpha = dist_d2s.clip(max=vis_dist) / vis_dist
144 |     data_color[ np.where(inbound)[0][grid_inbound][in_obs] ] = R * data_alpha + W * (1-data_alpha)
145 |     data_color[ np.where(inbound)[0][grid_inbound][in_obs][dist_d2s[:,0] >= max_dist] ] = G
146 |     write_vis_pcd(f'{args.vis_out_dir}/vis_{args.scan:03}_d2s.ply', data_down, data_color)
147 |     stl_color = np.tile(B, (stl.shape[0], 1))
148 |     stl_alpha = dist_s2d.clip(max=vis_dist) / vis_dist
149 |     stl_color[ np.where(above)[0] ] = R * stl_alpha + W * (1-stl_alpha)
150 |     stl_color[ np.where(above)[0][dist_s2d[:,0] >= max_dist] ] = G
151 |     write_vis_pcd(f'{args.vis_out_dir}/vis_{args.scan:03}_s2d.ply', stl, stl_color)
152 | 
153 |     pbar.update(1)
154 |     pbar.set_description('done')
155 |     pbar.close()
156 |     over_all = (mean_d2s + mean_s2d) / 2
157 |     print(mean_d2s, mean_s2d, over_all)
158 | 


--------------------------------------------------------------------------------