├── 3DMatch ├── __init__.py ├── images │ ├── 3dmatch-fcgf.png │ ├── 3dmatch-test.png │ ├── 3dmatch-d3feat.png │ ├── 7-scenes-stairs.png │ └── 3dmatch-fcgf-color.png ├── test.txt ├── train.txt ├── test_set.py ├── trainset_d3feat.py ├── trainset_fcgf.py ├── README.md └── fuse_fragments_3DMatch.py └── Datasets.md /3DMatch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3DMatch/images/3dmatch-fcgf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhulf0804/3D-PointCloud/HEAD/3DMatch/images/3dmatch-fcgf.png -------------------------------------------------------------------------------- /3DMatch/images/3dmatch-test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhulf0804/3D-PointCloud/HEAD/3DMatch/images/3dmatch-test.png -------------------------------------------------------------------------------- /3DMatch/images/3dmatch-d3feat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhulf0804/3D-PointCloud/HEAD/3DMatch/images/3dmatch-d3feat.png -------------------------------------------------------------------------------- /3DMatch/images/7-scenes-stairs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhulf0804/3D-PointCloud/HEAD/3DMatch/images/7-scenes-stairs.png -------------------------------------------------------------------------------- /3DMatch/images/3dmatch-fcgf-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhulf0804/3D-PointCloud/HEAD/3DMatch/images/3dmatch-fcgf-color.png -------------------------------------------------------------------------------- /3DMatch/test.txt: -------------------------------------------------------------------------------- 1 | 7-scenes-redkitchen 2 | sun3d-home_at-home_at_scan1_2013_jan_1 3 | sun3d-home_md-home_md_scan9_2012_sep_30 4 | sun3d-hotel_uc-scan3 5 | sun3d-hotel_umd-maryland_hotel1 6 | sun3d-hotel_umd-maryland_hotel3 7 | sun3d-mit_76_studyroom-76-1studyroom2 8 | sun3d-mit_lab_hj-lab_hj_tea_nov_2_2012_scan1_erika -------------------------------------------------------------------------------- /3DMatch/train.txt: -------------------------------------------------------------------------------- 1 | sun3d-brown_bm_1-brown_bm_1 2 | sun3d-brown_bm_4-brown_bm_4 3 | sun3d-brown_cogsci_1-brown_cogsci_1 4 | sun3d-brown_cs_2-brown_cs2 5 | sun3d-brown_cs_3-brown_cs3 6 | sun3d-harvard_c3-hv_c3_1 7 | sun3d-harvard_c5-hv_c5_1 8 | sun3d-harvard_c6-hv_c6_1 9 | sun3d-harvard_c8-hv_c8_3 10 | sun3d-harvard_c11-hv_c11_2 11 | sun3d-home_bksh-home_bksh_oct_30_2012_scan2_erika 12 | sun3d-hotel_nips2012-nips_4 13 | sun3d-hotel_sf-scan1 14 | sun3d-mit_32_d507-d507_2 15 | sun3d-mit_46_ted_lab1-ted_lab_2 16 | sun3d-mit_76_417-76-417b 17 | sun3d-mit_dorm_next_sj-dorm_next_sj_oct_30_2012_scan1_erika 18 | sun3d-mit_w20_athena-sc_athena_oct_29_2012_scan1_erika 19 | 7-scenes-chess 20 | 7-scenes-fire 21 | 7-scenes-heads 22 | 7-scenes-office 23 | 7-scenes-pumpkin 24 | 7-scenes-stairs 25 | rgbd-scenes-v2-scene_01 26 | rgbd-scenes-v2-scene_02 27 | rgbd-scenes-v2-scene_03 28 | rgbd-scenes-v2-scene_04 29 | rgbd-scenes-v2-scene_05 30 | rgbd-scenes-v2-scene_06 31 | rgbd-scenes-v2-scene_07 32 | rgbd-scenes-v2-scene_08 33 | rgbd-scenes-v2-scene_09 34 | rgbd-scenes-v2-scene_10 35 | rgbd-scenes-v2-scene_11 36 | rgbd-scenes-v2-scene_12 37 | rgbd-scenes-v2-scene_13 38 | rgbd-scenes-v2-scene_14 39 | bundlefusion-apt0 40 | bundlefusion-apt1 41 | bundlefusion-apt2 42 | bundlefusion-copyroom 43 | bundlefusion-office0 44 | bundlefusion-office1 45 | bundlefusion-office2 46 | bundlefusion-office3 47 | analysis-by-synthesis-apt1-kitchen 48 | analysis-by-synthesis-apt1-living 49 | analysis-by-synthesis-apt2-bed 50 | analysis-by-synthesis-apt2-kitchen 51 | analysis-by-synthesis-apt2-living 52 | analysis-by-synthesis-apt2-luke 53 | analysis-by-synthesis-office2-5a 54 | analysis-by-synthesis-office2-5b -------------------------------------------------------------------------------- /3DMatch/test_set.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This code is for analysing 3DMatch test set 3 | ''' 4 | 5 | import copy 6 | import glob 7 | import numpy as np 8 | import os 9 | import open3d as o3d 10 | import random 11 | 12 | 13 | def basi_info(root): 14 | ''' 15 | 7-scenes-redkitchen: 60 16 | sun3d-home_at-home_at_scan1_2013_jan_1: 60 17 | sun3d-home_md-home_md_scan9_2012_sep_30: 60 18 | sun3d-hotel_uc-scan3: 55 19 | sun3d-hotel_umd-maryland_hotel1: 57 20 | sun3d-hotel_umd-maryland_hotel3: 37 21 | sun3d-mit_76_studyroom-76-1studyroom2: 66 22 | sun3d-mit_lab_hj-lab_hj_tea_nov_2_2012_scan1_erika: 38 23 | ''' 24 | d = {} 25 | clss = [] 26 | for item in sorted(os.listdir(root)): 27 | if os.path.isdir(os.path.join(root, item)): 28 | clss.append(item) 29 | 30 | for cls in clss: 31 | if '-evaluation' in cls: 32 | continue 33 | plys = glob.glob(os.path.join(root, cls, '*.ply')) 34 | d[cls] = len(plys) 35 | return d 36 | 37 | 38 | def read_gt_log(file_path): 39 | ''' 40 | 7-scenes-redkitchen 506 41 | sun3d-home_at-home_at_scan1_2013_jan_1 156 42 | sun3d-home_md-home_md_scan9_2012_sep_30 208 43 | sun3d-hotel_uc-scan3 226 44 | sun3d-hotel_umd-maryland_hotel1 104 45 | sun3d-hotel_umd-maryland_hotel3 54 46 | sun3d-mit_76_studyroom-76-1studyroom2 292 47 | sun3d-mit_lab_hj-lab_hj_tea_nov_2_2012_scan1_erika 77 48 | ''' 49 | Rts = [] 50 | with open(file_path, 'r') as fr: 51 | line = fr.readline() 52 | while line: 53 | i, j, _ = list(map(int, line.split())) 54 | Rt = np.eye(4, dtype=np.float32) 55 | for t in range(4): 56 | cur = fr.readline() 57 | Rt[t, :] = np.fromstring(cur, np.float32, sep='\t') 58 | Rts.append([[i, j], Rt]) 59 | line = fr.readline() 60 | return Rts 61 | 62 | 63 | def vis_gt_log(root, cls, pairs_Rt): 64 | print(cls, pairs_Rt[0]) 65 | i, j = pairs_Rt[0] 66 | Rt = pairs_Rt[1] 67 | ply1_path = os.path.join(root, cls, 'cloud_bin_{}.ply'.format(i)) 68 | ply2_path = os.path.join(root, cls, 'cloud_bin_{}.ply'.format(j)) 69 | ply1 = o3d.io.read_point_cloud(ply1_path, format='ply') 70 | ply2 = o3d.io.read_point_cloud(ply2_path, format='ply') 71 | raw_ply2 = copy.deepcopy(ply2) 72 | ply2.transform(Rt) 73 | ply1.paint_uniform_color([1, 0, 0]) 74 | raw_ply2.paint_uniform_color([0, 1, 0]) 75 | ply2.paint_uniform_color([0, 0, 1]) 76 | o3d.visualization.draw_geometries([ply1, ply2]) 77 | 78 | 79 | if __name__ == '__main__': 80 | root = '/Users/zhulf/data/threedmatch-test' 81 | 82 | name2num = basi_info(root) 83 | print('=' * 20, 'Plys', '=' * 20) 84 | for k, v in name2num.items(): 85 | print(k, v) 86 | print('Total plys numbers: ', sum(name2num.values())) 87 | 88 | print('='*20, 'Correspondences', '='*20) 89 | name2correspondences = {} 90 | ncorrespondences = 0 91 | for k in name2num.keys(): 92 | gt_log_file_path = os.path.join(root, '{}-evaluation'.format(k), 'gt.log') 93 | Rts = read_gt_log(file_path=gt_log_file_path) 94 | name2correspondences[k] = Rts 95 | ncorrespondences += len(Rts) 96 | print(k, len(Rts)) 97 | print('Total correspondences pairs: ', ncorrespondences) 98 | 99 | #cls = random.choice(list(name2correspondences.keys())) 100 | #pairs_Rt = random.choice(name2correspondences[cls]) 101 | cls = '7-scenes-redkitchen' 102 | pairs_Rt = name2correspondences[cls][0] 103 | vis_gt_log(root, cls, pairs_Rt) -------------------------------------------------------------------------------- /3DMatch/trainset_d3feat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import open3d as o3d 4 | import pickle 5 | import random 6 | 7 | 8 | def vis_npys(npys): 9 | pcds = [] 10 | colors = [[1, 0, 0], [0, 1, 0], [0, 0, 1]] 11 | for i, npy in enumerate(npys): 12 | pcd = o3d.geometry.PointCloud() 13 | pcd.points = o3d.utility.Vector3dVector(npy) 14 | if i < 3: 15 | color = colors[i] 16 | else: 17 | color = [random.random() for _ in range(3)] 18 | pcd.paint_uniform_color(color) 19 | pcds.append(pcd) 20 | o3d.visualization.draw_geometries(pcds) 21 | 22 | 23 | def decode_points(pts_filename): 24 | ''' 25 | # 3DMatch_train_0.030_points.pkl: dict 26 | # key: str, sun3d-brown_bm_1-brown_bm_1/seq-01/cloud_bin_0 27 | # value: np.ndarray n x 3, n是变化的 28 | # 3933个点云 29 | # min: (850, 3), max: (197343, 3), mean: 13565 30 | ''' 31 | with open(pts_filename, 'rb') as file: 32 | data = pickle.load(file) 33 | points = [*data.values()] 34 | ids_list = [*data.keys()] 35 | 36 | dims = [] 37 | for i in range(len(points)): 38 | dims.append(points[i].shape[0]) 39 | print('npts min: {}, npts max: {}, npts mean: {}'. 40 | format(min(dims), max(dims), np.mean(dims))) 41 | print('Total number of point cloud: {}'.format(len(dims))) 42 | 43 | return data 44 | 45 | 46 | def decode_overlap(overlap_filename): 47 | ''' 48 | # 3DMatch_train_0.030_overlap.pkl: dict 49 | # 35297 50 | # key: str, '7-scenes-pumpkin/seq-07/cloud_bin_11@7-scenes-pumpkin/seq-08/cloud_bin_2' 51 | # val: float, 0.6015544397826535 52 | # min: 0.30000815461143276, max: 0.9954887218045113, mean: 0.5150335449363996 53 | ''' 54 | with open(overlap_filename, 'rb') as file: 55 | overlap = pickle.load(file) 56 | 57 | scores = [] 58 | for k, v in overlap.items(): 59 | scores.append(v) 60 | print('overlap min: {}, overlap max: {}, overlap mean: {}' 61 | .format(min(scores), max(scores), np.mean(scores))) 62 | print('Total pairs: {}'.format(len(scores))) 63 | 64 | return overlap 65 | 66 | 67 | def decode_keypts(keypts_filename): 68 | ''' 69 | # 3DMatch_train_0.030_keypts.pkl: dict 70 | # 35297 71 | # key: str, analysis-by-synthesis-office2-5b/seq-01/cloud_bin_34@analysis-by-synthesis-office2-5b/seq-01/cloud_bin_35 72 | # val: np.ndarray, m x 2; m是变化的 73 | # min: 445, max: 76307, mean: 8487 74 | 75 | ''' 76 | with open(keypts_filename, 'rb') as file: 77 | correspondences = pickle.load(file) 78 | 79 | pairs = [] 80 | for k, v in correspondences.items(): 81 | pairs.append(v.shape[0]) 82 | print('min: {}, max: {}, mean: {}'.format(min(pairs), max(pairs), np.mean(pairs))) 83 | print('Total pairs: {}'.format(len(pairs))) 84 | 85 | return correspondences 86 | 87 | 88 | if __name__ == '__main__': 89 | root = '/Users/zhulf/Downloads/data/backup' 90 | pts_filename = os.path.join(root, f'3DMatch_train_0.030_points.pkl') 91 | overlap_filename = os.path.join(root, f'3DMatch_train_0.030_overlap.pkl') 92 | keypts_filename = os.path.join(root, f'3DMatch_train_0.030_keypts.pkl') 93 | 94 | assert os.path.exists(pts_filename) 95 | print('='*20, '3DMatch_train_0.030_points.pkl', '='*20) 96 | data = decode_points(pts_filename) 97 | print('=' * 20, '3DMatch_train_0.030_points.pkl completed', '=' * 20, '\n') 98 | 99 | assert os.path.exists(keypts_filename) 100 | print('=' * 20, '3DMatch_train_0.030_overlap.pkl', '=' * 20) 101 | overlap = decode_overlap(overlap_filename) 102 | print('=' * 20, '3DMatch_train_0.030_overlap.pkl completed', '=' * 20, '\n') 103 | 104 | assert os.path.exists(overlap_filename) 105 | print('=' * 20, '3DMatch_train_0.030_keypts.pkl', '=' * 20) 106 | correspondences = decode_keypts(keypts_filename) 107 | print('=' * 20, '3DMatch_train_0.030_keypts.pkl completed', '=' * 20) 108 | 109 | 110 | f1f2 = list(correspondences.keys())[222] 111 | correspondence = correspondences[f1f2] 112 | path1, path2 = f1f2.split('@') 113 | npy1, npy2 = data[path1], data[path2] 114 | npy3, npy4 = npy1[correspondence[:, 0]], npy2[correspondence[:, 1]] 115 | vis_npys([npy1, npy2]) 116 | vis_npys([npy1, npy2, npy3]) 117 | vis_npys([npy1, npy2, npy3]) -------------------------------------------------------------------------------- /3DMatch/trainset_fcgf.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | import os 4 | import open3d as o3d 5 | import random 6 | 7 | 8 | def read_ids(path): 9 | ids = [] 10 | with open(path, 'r') as f: 11 | lines = f.readlines() 12 | for line in lines: 13 | ids.append(line.strip()) 14 | return ids 15 | 16 | 17 | def read_correspondence_pairs(path): 18 | pairs = [] 19 | with open(path, 'r') as f: 20 | lines = f.readlines() 21 | for line in lines: 22 | pairs.append(line.split()) 23 | return pairs 24 | 25 | 26 | def vis_pair(path1, path2): 27 | pc1, pc2 = np.load(path1), np.load(path2) 28 | pcd1 = o3d.geometry.PointCloud() 29 | pcd1.points = o3d.utility.Vector3dVector(pc1['pcd']) 30 | pcd1_uniform_color = copy.deepcopy(pcd1) 31 | pcd1_uniform_color.paint_uniform_color([1, 0, 0]) 32 | pcd1.colors = o3d.utility.Vector3dVector(pc1['color']) 33 | pcd2 = o3d.geometry.PointCloud() 34 | pcd2.points = o3d.utility.Vector3dVector(pc2['pcd']) 35 | pcd2_uniform_color = copy.deepcopy(pcd2) 36 | pcd2_uniform_color.paint_uniform_color([0, 0, 1]) 37 | pcd2.colors = o3d.utility.Vector3dVector(pc2['color']) 38 | o3d.visualization.draw_geometries([pcd1_uniform_color, pcd2_uniform_color]) 39 | o3d.visualization.draw_geometries([pcd1, pcd2]) 40 | 41 | 42 | if __name__ == '__main__': 43 | root = '/Users/zhulf/data/threedmatch' 44 | train_ids = read_ids('./train.txt') 45 | files = sorted(os.listdir(root)) 46 | suffixes, clss = {}, {} 47 | for file in files: 48 | suffix = file.split('.')[-1] 49 | suffixes[suffix] = suffixes.get(suffix, 0) + 1 50 | cls = file.split('@')[0] 51 | if cls not in train_ids: 52 | continue 53 | 54 | if '0.30' in file: 55 | seq_id = file.split('@')[1][:6] 56 | pairs = read_correspondence_pairs(os.path.join(root, file)) 57 | clss[(cls, seq_id)] = clss.get((cls, seq_id), 0) + len(pairs) 58 | print(suffixes) 59 | for cls, num in clss.items(): 60 | print(cls, num) 61 | ids = set([cls[0] for cls, num in clss.items()]) 62 | print('Total class: {}, npairs: {}'.format(len(ids), sum(clss.values()))) 63 | 64 | #id, seq = random.choice(list(clss.keys())) 65 | id, seq = '7-scenes-chess', 'seq-01' 66 | id_seq_path = os.path.join(root, '{}@{}-0.30.txt'.format(id, seq)) 67 | pairs = read_correspondence_pairs(id_seq_path) 68 | #pair = random.choice(pairs) 69 | pair = pairs[0] 70 | path1, path2 = os.path.join(root, pair[0]), os.path.join(root, pair[1]) 71 | print(path1, path2) 72 | vis_pair(path1, path2) 73 | 74 | ''' 75 | /Users/zhulf/anaconda3/bin/python3.7 /Users/zhulf/data/3D-PointCloud/3DMatch/trainset_fcgf.py 76 | {'txt': 401, 'npz': 2189} 77 | ('7-scenes-chess', 'seq-01') 33 78 | ('7-scenes-chess', 'seq-02') 36 79 | ('7-scenes-chess', 'seq-03') 36 80 | ('7-scenes-chess', 'seq-04') 36 81 | ('7-scenes-chess', 'seq-05') 36 82 | ('7-scenes-chess', 'seq-06') 36 83 | ('7-scenes-fire', 'seq-01') 36 84 | ('7-scenes-fire', 'seq-02') 36 85 | ('7-scenes-fire', 'seq-03') 35 86 | ('7-scenes-fire', 'seq-04') 35 87 | ('7-scenes-heads', 'seq-01') 31 88 | ('7-scenes-heads', 'seq-02') 24 89 | ('7-scenes-office', 'seq-01') 26 90 | ('7-scenes-office', 'seq-02') 21 91 | ('7-scenes-office', 'seq-03') 21 92 | ('7-scenes-office', 'seq-04') 17 93 | ('7-scenes-office', 'seq-05') 17 94 | ('7-scenes-office', 'seq-06') 18 95 | ('7-scenes-office', 'seq-07') 26 96 | ('7-scenes-office', 'seq-08') 27 97 | ('7-scenes-office', 'seq-09') 22 98 | ('7-scenes-office', 'seq-10') 31 99 | ('7-scenes-pumpkin', 'seq-01') 28 100 | ('7-scenes-pumpkin', 'seq-02') 35 101 | ('7-scenes-pumpkin', 'seq-03') 33 102 | ('7-scenes-pumpkin', 'seq-06') 29 103 | ('7-scenes-pumpkin', 'seq-07') 36 104 | ('7-scenes-pumpkin', 'seq-08') 36 105 | ('7-scenes-stairs', 'seq-01') 4 106 | ('7-scenes-stairs', 'seq-02') 6 107 | ('7-scenes-stairs', 'seq-03') 6 108 | ('7-scenes-stairs', 'seq-04') 6 109 | ('7-scenes-stairs', 'seq-05') 4 110 | ('7-scenes-stairs', 'seq-06') 6 111 | ('analysis-by-synthesis-apt1-kitchen', 'seq-01') 35 112 | ('analysis-by-synthesis-apt1-living', 'seq-01') 67 113 | ('analysis-by-synthesis-apt2-bed', 'seq-01') 45 114 | ('analysis-by-synthesis-apt2-kitchen', 'seq-01') 32 115 | ('analysis-by-synthesis-apt2-living', 'seq-01') 30 116 | ('analysis-by-synthesis-apt2-luke', 'seq-01') 84 117 | ('analysis-by-synthesis-office2-5a', 'seq-01') 38 118 | ('analysis-by-synthesis-office2-5b', 'seq-01') 74 119 | ('bundlefusion-apt0', 'seq-01') 699 120 | ('bundlefusion-apt1', 'seq-01') 851 121 | ('bundlefusion-apt2', 'seq-01') 219 122 | ('bundlefusion-copyroom', 'seq-01') 301 123 | ('bundlefusion-office0', 'seq-01') 486 124 | ('bundlefusion-office1', 'seq-01') 477 125 | ('bundlefusion-office2', 'seq-01') 139 126 | ('bundlefusion-office3', 'seq-01') 224 127 | ('rgbd-scenes-v2-scene_01', 'seq-01') 15 128 | ('rgbd-scenes-v2-scene_02', 'seq-01') 14 129 | ('rgbd-scenes-v2-scene_03', 'seq-01') 14 130 | ('rgbd-scenes-v2-scene_04', 'seq-01') 15 131 | ('rgbd-scenes-v2-scene_05', 'seq-01') 47 132 | ('rgbd-scenes-v2-scene_06', 'seq-01') 40 133 | ('rgbd-scenes-v2-scene_07', 'seq-01') 31 134 | ('rgbd-scenes-v2-scene_08', 'seq-01') 27 135 | ('rgbd-scenes-v2-scene_09', 'seq-01') 10 136 | ('rgbd-scenes-v2-scene_10', 'seq-01') 12 137 | ('rgbd-scenes-v2-scene_11', 'seq-01') 8 138 | ('rgbd-scenes-v2-scene_12', 'seq-01') 11 139 | ('rgbd-scenes-v2-scene_13', 'seq-01') 6 140 | ('rgbd-scenes-v2-scene_14', 'seq-01') 15 141 | ('sun3d-brown_bm_1-brown_bm_1', 'seq-01') 115 142 | ('sun3d-brown_bm_4-brown_bm_4', 'seq-01') 48 143 | ('sun3d-brown_cogsci_1-brown_cogsci_1', 'seq-01') 64 144 | ('sun3d-brown_cs_2-brown_cs2', 'seq-01') 122 145 | ('sun3d-brown_cs_3-brown_cs3', 'seq-01') 57 146 | ('sun3d-harvard_c11-hv_c11_2', 'seq-01') 10 147 | ('sun3d-harvard_c3-hv_c3_1', 'seq-01') 47 148 | ('sun3d-harvard_c5-hv_c5_1', 'seq-01') 115 149 | ('sun3d-harvard_c6-hv_c6_1', 'seq-01') 38 150 | ('sun3d-harvard_c8-hv_c8_3', 'seq-01') 20 151 | ('sun3d-home_bksh-home_bksh_oct_30_2012_scan2_erika', 'seq-01') 389 152 | ('sun3d-hotel_nips2012-nips_4', 'seq-01') 270 153 | ('sun3d-hotel_sf-scan1', 'seq-01') 519 154 | ('sun3d-mit_32_d507-d507_2', 'seq-01') 301 155 | ('sun3d-mit_46_ted_lab1-ted_lab_2', 'seq-01') 285 156 | ('sun3d-mit_76_417-76-417b', 'seq-01') 185 157 | ('sun3d-mit_dorm_next_sj-dorm_next_sj_oct_30_2012_scan1_erika', 'seq-01') 121 158 | ('sun3d-mit_w20_athena-sc_athena_oct_29_2012_scan1_erika', 'seq-01') 323 159 | Total class: 54, npairs: 7960 160 | ''' -------------------------------------------------------------------------------- /3DMatch/README.md: -------------------------------------------------------------------------------- 1 | ## 关于 2 | 3 | - train.txt: 训练集的场景名(54) 4 | - test.txt: 测试集的场景名(8) 5 | - fuse_fragments_3DMatch.py: 如何由多帧深度图融合得到点云数据(ply) 6 | - trainset_fcgf.py: 3DMatch训练集(FCGF)的统计分析代码 7 | - trainset_d3feat.py: 3DMatch训练集(D3Feat)的统计分析代码 8 | - test_set.py: 测试集的统计分析代码 9 | - images: 可视化截图 10 | 11 | ## 一、3DMatch数据集简介 12 | 13 | 3DMatch数据集收集了来自于62个场景的数据，其中54个场景的数据用于训练，8个场景的数据用于评估，其具体名称查看`train.txt`和`test.txt`。3DMatch数据常用于3D点云的关键点，特征描述子，点云配准等任务。 14 | 15 | [官方主页](https://3dmatch.cs.princeton.edu/) | [3DMatch: Learning Local Geometric Descriptors from RGB-D Reconstructions](https://openaccess.thecvf.com/content_cvpr_2017/papers/Zeng_3DMatch_Learning_Local_CVPR_2017_paper.pdf) [CVPR 2017] 16 | 17 | 3DMatch原始数据集: [下载地址](http://3dmatch.cs.princeton.edu/#rgbd-reconstruction-datasets)，共包括64个.zip文件。 18 | 19 | 以其中一个场景`7-scenes-stairs`为例，介绍其数据格式，如下截图所示，原始的3DMatch数据集包括两个`.txt`文件，多个`seq`文件夹，每个`seq`文件夹下包括多帧的`.color.png`, `.depth.png`, `.pose.txt`，可以看到， 20 | 其本身是不包括点云数据的，但是可以由这些数据生成点云数据(ply)，一般是50帧-100帧生成一个点云数据，生成点云的代码可以参考`fuse_fragments_3DMatch.py`。 21 | 22 | !['7-scenes-stairs'](./images/7-scenes-stairs.png) 23 | 24 | 下面介绍3DMatch的训练集，包括[FCGF](https://github.com/chrischoy/FCGF)和[D3Feat](https://github.com/XuyangBai/D3Feat)处理的训练集。 25 | 26 | ## 二、3DMatch的训练集(FCGF) 27 | 28 | 3DMatch训练集来自54个场景，详细类别名称参见`train.txt`。每个场景均由1个seq或者多个seq的数据组成。这里以[FCGF](https://github.com/chrischoy/FCGF) 29 | 网络使用的数据格式为例介绍3DMatch数据集。 30 | 31 | 首先，从[这里](http://node2.chrischoy.org/data/datasets/registration/threedmatch.tgz)下载训练集，下载解压后可以得到401个`txt`文件和2189个`npz`文件。2189个`npz`对应每个点云数据，包括(x, y, z)及对应的(r, g, b)信息，其中命名规则是`场景@seqid-id`，例如7-scenes-chess@seq-01_000.npz，表示此数据来自7-scenes-chess场景的seq-01，编号为000。401个txt表示这些npz的点云数据是如何关联的，其命名规则为`场景@seqid-overlap`，如7-scenes-chess@seq-01-0.30.txt表示7-scenes-chess场景的seq-01下的overlap大于0.30的数据，打开此文件后，可以看到如下信息: 32 | 33 | ```text 34 | 7-scenes-chess@seq-01_000.npz 7-scenes-chess@seq-01_001.npz 0.886878 35 | 7-scenes-chess@seq-01_000.npz 7-scenes-chess@seq-01_002.npz 0.636459 36 | 7-scenes-chess@seq-01_000.npz 7-scenes-chess@seq-01_003.npz 0.825012 37 | 7-scenes-chess@seq-01_000.npz 7-scenes-chess@seq-01_004.npz 0.783642 38 | ``` 39 | 40 | 每一行表示点云之间的对应关系，如第一行表示点云7-scenes-chess@seq-01_000.npz和7-scenes-chess@seq-01_001.npz具有0.886878的overlap。可视化结果如下，第一个图中的红色的点云是7-scenes-chess@seq-01_000.npz，蓝色的点云是7-scenes-chess@seq-01_001.npz，可以看到两者是对齐的；第二个图是这两个点云的rgb信息的可视化。 41 | 42 | !['3dmatch-fcgf'](./images/3dmatch-fcgf.png) 43 | 44 | !['3dmatch-fcgf-color'](./images/3dmatch-fcgf-color.png) 45 | 46 | 统计了一下，54个场景总共提供了**7960**对点云。 47 | 48 | **上述的统计信息的相关代码在`trainset_fcgf.py`** 49 | 50 | ## 三、3DMatch训练集(D3Feat) 51 | 52 | 下载D3Feat训练使用的[3DMatch数据集](https://drive.google.com/file/d/1Vo-D_NOs4HS9wwAE-55ggDpgaPfzdRRt/view?usp=sharing)，解压后是6个`.pkl`文件，分别是 53 | `3DMatch_train_0.030_keypts.pkl`, `3DMatch_train_0.030_overlap.pkl`, `3DMatch_train_0.030_points.pkl`, `3DMatch_val_0.030_keypts.pkl`, `3DMatch_val_0.030_overlap.pkl`和 54 | `3DMatch_val_0.030_points.pkl`，可以通过[这里](https://github.com/XuyangBai/D3Feat/blob/master/datasets/cal_overlap.py)查看这些`.pkl`文件是如何产生的。 55 | 56 | - `3DMatch_train_0.030_keypts.pkl`存储了什么信息 ? 57 | 58 | 点云名(str, 如`sun3d-brown_bm_1-brown_bm_1/seq-01/cloud_bin_0`) -> 点云(ndarray, n x 3)。 59 | 60 | 总共有`3933`个点云数据，点云点最少的是`850`个，点云点最多的是`197343`，平均点云点数量是`27127`。 61 | - `3DMatch_train_0.030_overlap.pkl`存储了什么信息 ? 62 | 63 | 点云对(str, 如`7-scenes-pumpkin/seq-07/cloud_bin_11@7-scenes-pumpkin/seq-08/cloud_bin_2`) -> overlap值(float) 64 | 65 | 总共有`35297`个点云对，overlap的最小值为`0.30`，最大值`0.995`，平均值为`0.515`。 66 | - `3DMatch_train_0.030_points.pkl`存储了什么信息 ? 67 | 68 | 点云对(str, 如`analysis-by-synthesis-office2-5b/seq-01/cloud_bin_34@analysis-by-synthesis-office2-5b/seq-01/cloud_bin_35`) -> 映射关系(ndarray, m x 2) 69 | - 可视化pairs中的点云和对应关键点 70 | 71 | !['3dmatch-d3feat'](./images/3dmatch-d3feat.png) 72 | 73 | 左图为两个具有overlap的点云的可视化，中间和右边的可视化是分别在红色和绿色点云上添加了对应点(**蓝色区域**)(来自于`3DMatch_train_0.030_points.pkl`)的可视化结果。 74 | 75 | 上述相关代码在`trainset_d3feat.py`。(为什么D3Feat的训练集中点云和点云对数量比FCGF中的点云和点云对数量多这么多 ?还待验证。) 76 | 77 | ## 四、3DMatch的测试集 78 | 79 | 3DMatch的测试集包括以下8个场景，其中每个场景对应两个文件夹。以`7-scenes-redkitchen`为例，它包括7-scenes-redkitchen和7-scenes-redkitchen-evaluation两个文件夹，7-scenes-redkitchen文件夹下存放的是点云数据，命名格式均为`cloud_bin_*.ply`，共包括60个点云数据；7-scenes-redkitchen-evaluation/gt.log存放了correspondences点云对，组织格式为: 80 | 81 | ```text 82 | 0 1 60 83 | 9.96926560e-01 6.68735757e-02 -4.06664421e-02 -1.15576939e-01 84 | -6.61289946e-02 9.97617877e-01 1.94008687e-02 -3.87705398e-02 85 | 4.18675510e-02 -1.66517807e-02 9.98977765e-01 1.14874890e-01 86 | 0.00000000e+00 0.00000000e+00 0.00000000e+00 1.00000000e+00 87 | 0 2 60 88 | 9.54999224e-01 1.08859481e-01 -2.75869135e-01 -3.41060560e-01 89 | -9.89491703e-02 9.93843326e-01 4.96360476e-02 -1.78254668e-01 90 | 2.79581388e-01 -2.01060700e-02 9.59896612e-01 3.54627338e-01 91 | 0.00000000e+00 0.00000000e+00 0.00000000e+00 1.00000000e+00 92 | ``` 93 | 94 | `0 1 60`中的0和1表示cloud_bin_0.ply和cloud_bin_1.ply点云是成对的，60表示总共包括有60个点云数据(前面也提到过)，下面四列表示cloud_bin_1.ply -> cloud_bin_0.ply的变换矩阵。 95 | 96 | **可视化变换后的点云**，红色的表示cloud_bin_0.ply点云，蓝色的表示对cloud_bin_1.ply变换后的点云，可视化结果显示两者基本重叠: 97 | 98 | !['3dmatch-test'](./images/3dmatch-test.png) 99 | 100 | 其它场景的统计数据如下，可视化和统计信息的代码均可通过`test_set.py`实现。 101 | 102 | | 名称 | 点云数量 | Pairs数量 | 103 | | :---: | :---: | :---: | 104 | | 7-scenes-redkitchen | 60 | 506 | 105 | | sun3d-home_at-home_at_scan1_2013_jan_1 | 60 | 156 | 106 | | sun3d-home_md-home_md_scan9_2012_sep_30 | 60 | 208 | 107 | | sun3d-hotel_uc-scan3 | 55 | 226 | 108 | | sun3d-hotel_umd-maryland_hotel1 | 57 | 104 | 109 | | sun3d-hotel_umd-maryland_hotel3 | 37 | 54 | 110 | | sun3d-mit_76_studyroom-76-1studyroom2 | 66 | 292 | 111 | | sun3d-mit_lab_hj-lab_hj_tea_nov_2_2012_scan1_erika | 38 | 77 | 112 | | 总计 | 433 | 1623 | 113 | 114 | **上述的统计信息的相关代码在`test_set.py`** 115 | 116 | ## 五、3DMatch数据集的评估指标 117 | 118 | 评估指标主要基于[FCGF](https://node1.chrischoy.org/data/publications/fcgf/fcgf.pdf) [ICCV 2019]，评估代码请参考[https://github.com/chrischoy/FCGF/blob/master/scripts/benchmark_3dmatch.py](https://github.com/chrischoy/FCGF/blob/master/scripts/benchmark_3dmatch.py)。但Registration Recall的实现我感觉有问题，待日后再做补充吧。 119 | 120 | - Feature-match Recall 121 | 122 | $$R = \frac{1}{M} \Sigma_{s=1}^M 1([\frac{1}{|\Omega_s|}\Sigma_{(i, j) \in \Omega_s}1(||T^* \text{x}_i - \text{y}_j|| < \tau_1) ] > \tau_2)$$ 123 | 124 | $M$表示pairs点云对的数量，$1$表示指示函数，$\Omega_s$是第$s$个pair的correspondences，$T^*$表当前pair点云的G.T.的R,t变换。$\text{y}_j$是$\text{x}_i$在$Y$中选择的**Feature Distance**最小的点，即$\text{y}_j = \arg \min_{\text{y}_j} ||F_{\text{x}_i} - F_{\text{y}_j}||, \text{y}_j \in Y$。$\tau_1$和$\tau_2$是两个超参数，常取值$\tau_1 = 0.1$，$\tau_2 = 0.05$。 125 | 126 | 简单的说，就是有M个点云对，对每一个点云对做一个是特征好/坏的判断: 在真实R,t的情况下，计算$(\text{x}_i, \text{y}_j)$距离小于$\tau_1$的比例r，如果r大于$\tau_2$，则表示这个点云特征好，否则特征是坏的。 127 | 128 | - Registration Recall 129 | 130 | $$E_{\text{RMSE}} = \sqrt {\frac{1}{\Omega_*}\Sigma_{(\text{x}^*, \text{y}^*) \in \Omega_*} ||\hat T_{i, j}\text{x}^* - \text{y}^*||^2}$$ 131 | 132 | $\Omega_*$表示$(i, j)$点云对中correspondences的数量，$(\text{x}^*, \text{y}^*)$表示G.T.的pair，$\hat T_{i, j}$表示基于$(i, j)$点云对预测的R,t变换。 133 | 134 | 对于具有至少30% overlap的点云对，如果$E_\text{RMSE} < 0.2$，则表示这是一个正确的点云对。 135 | -------------------------------------------------------------------------------- /3DMatch/fuse_fragments_3DMatch.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The file is modified by 3 | https://github.com/XuyangBai/PPF-FoldNet/blob/master/script/fuse_fragments_3DMatch.py 4 | 5 | # Fuse rgbd frames into fragments in 3DMatch 6 | # - Use existing camera poses 7 | # - Save colors & normals 8 | ''' 9 | 10 | import argparse 11 | import glob 12 | import math 13 | import numpy as np 14 | import os 15 | import open3d as o3d 16 | 17 | 18 | def list_folders(path): 19 | folders = [] 20 | for cur in os.listdir(path): 21 | if os.path.isdir(os.path.join(path, cur)) and not cur.startswith('.'): 22 | folders.append(cur) 23 | return folders 24 | 25 | 26 | def read_intrinsic(filepath, width, height): 27 | import open3d as o3d 28 | 29 | m = np.loadtxt(filepath, dtype=np.float32) 30 | intrinsic = o3d.camera.PinholeCameraIntrinsic(width, height, m[0, 0], 31 | m[1, 1], m[0, 2], m[1, 2]) 32 | return intrinsic 33 | 34 | 35 | def read_extrinsic(filepath): 36 | m = np.loadtxt(filepath, dtype=np.float32) 37 | if np.isnan(m).any(): 38 | return None 39 | return m # (4, 4) 40 | 41 | 42 | def read_rgbd_image(cfg, color_file, depth_file, convert_rgb_to_intensity): 43 | if color_file is None: 44 | color_file = depth_file # to avoid "Unsupported image format." 45 | # rgbd_image = o3d.RGBDImage() 46 | # rgbd_image.depth = o3d.io.read_image(depth_file) 47 | # return rgbd_image 48 | color = o3d.io.read_image(color_file) 49 | depth = o3d.io.read_image(depth_file) 50 | rgbd_image = o3d.geometry.create_rgbd_image_from_color_and_depth(color, 51 | depth, 52 | cfg.depth_scale, 53 | cfg.depth_trunc, 54 | convert_rgb_to_intensity) 55 | return rgbd_image 56 | 57 | 58 | def process_single_fragment(cfg, color_files, depth_files, frag_id, n_frags, 59 | intrinsic_path, out_folder): 60 | import open3d as o3d 61 | 62 | depth_only_flag = (len(color_files) == 0) 63 | n_frames = len(depth_files) 64 | intrinsic = read_intrinsic(intrinsic_path, cfg.width, cfg.height) 65 | if depth_only_flag: 66 | color_type = o3d.integration.TSDFVolumeColorType.__dict__['None'] 67 | else: 68 | color_type = o3d.integration.TSDFVolumeColorType.__dict__['RGB8'] 69 | 70 | volume = o3d.integration.ScalableTSDFVolume( 71 | voxel_length=cfg.tsdf_cubic_size / 512.0, 72 | sdf_trunc=0.04, 73 | color_type=color_type) 74 | 75 | sid = frag_id * cfg.frames_per_frag 76 | eid = min(sid + cfg.frames_per_frag, n_frames) 77 | pose_base2world = None 78 | pose_base2world_inv = None 79 | for fid in range(sid, eid): 80 | if not depth_only_flag: 81 | color_path = color_files[fid] 82 | else: 83 | color_path = None 84 | depth_path = depth_files[fid] 85 | pose_path = depth_path[:-10] + '.pose.txt' 86 | 87 | pose_cam2world = read_extrinsic(pose_path) 88 | if pose_cam2world is None: 89 | continue 90 | if fid == sid: # Use as base frame 91 | pose_base2world = pose_cam2world 92 | pose_base2world_inv = np.linalg.inv(pose_base2world) 93 | if pose_base2world_inv is None: 94 | break 95 | # Relative camera pose 96 | pose_cam2world = np.matmul(pose_base2world_inv, pose_cam2world) 97 | 98 | rgbd = read_rgbd_image(cfg, color_path, depth_path, False) 99 | volume.integrate(rgbd, intrinsic, np.linalg.inv(pose_cam2world)) 100 | if pose_base2world_inv is None: 101 | return 102 | 103 | pcloud = volume.extract_point_cloud() 104 | o3d.geometry.estimate_normals(pcloud) 105 | o3d.write_point_cloud( 106 | os.path.join(out_folder, 'cloud_bin_{}.ply'.format(frag_id)), pcloud) 107 | 108 | np.save(os.path.join(out_folder, 'cloud_bin_{}.pose.npy'.format(frag_id)), 109 | pose_base2world) 110 | 111 | 112 | # ---------------------------------------------------------------------------- # 113 | # Iterate Folders 114 | # ---------------------------------------------------------------------------- # 115 | def run_seq(cfg, scene, seq): 116 | print(" Start {}".format(seq)) 117 | 118 | seq_folder = os.path.join(cfg.dataset_root, scene, seq) 119 | color_paths = glob.glob(os.path.join(seq_folder, '*.color.png')) 120 | depth_paths = glob.glob(os.path.join(seq_folder, '*.depth.png')) 121 | 122 | 123 | # n_frames = len(color_paths) 124 | n_frames = len(depth_paths) 125 | n_frags = int(math.ceil(float(n_frames) / cfg.frames_per_frag)) 126 | 127 | out_folder = os.path.join(cfg.out_root, scene, seq) 128 | if not os.path.exists(out_folder): 129 | os.makedirs(out_folder) 130 | 131 | intrinsic_path = os.path.join(cfg.dataset_root, scene, 'camera-intrinsics.txt') 132 | 133 | if cfg.threads > 1: 134 | from joblib import Parallel, delayed 135 | import multiprocessing 136 | 137 | Parallel(n_jobs=cfg.threads)( 138 | delayed(process_single_fragment)(cfg, color_paths, depth_paths, 139 | frag_id, n_frags, intrinsic_path, 140 | out_folder) 141 | for frag_id in range(n_frags)) 142 | 143 | else: 144 | for frag_id in range(n_frags): 145 | process_single_fragment(cfg, color_paths, depth_paths, frag_id, 146 | n_frags, intrinsic_path, out_folder) 147 | 148 | print(" Finished {}".format(seq)) 149 | 150 | 151 | def run_scene(cfg, scene): 152 | print(" Start scene {} ".format(scene)) 153 | 154 | scene_folder = os.path.join(cfg.dataset_root, scene) 155 | seqs = list_folders(scene_folder) 156 | print(" {} sequences".format(len(seqs))) 157 | for seq in seqs: 158 | run_seq(cfg, scene, seq) 159 | 160 | print(" Finished scene {} ".format(scene)) 161 | 162 | 163 | def run(cfg): 164 | print("Start making fragments") 165 | 166 | if not os.path.exists(cfg.out_root): 167 | os.makedirs(cfg.out_root) 168 | 169 | scenes = list_folders(cfg.dataset_root) 170 | print("{} scenes".format(len(scenes))) 171 | for scene in scenes: 172 | # if not scene.startswith('analysis'): 173 | # continue 174 | run_scene(cfg, scene) 175 | 176 | print("Finished making fragments") 177 | 178 | 179 | # ---------------------------------------------------------------------------- # 180 | # Arguments 181 | # ---------------------------------------------------------------------------- # 182 | def parse_args(): 183 | parser = argparse.ArgumentParser() 184 | parser.add_argument('--dataset_root', default='/Users/zhulf/data/rgbd') 185 | parser.add_argument('--out_root', default='/Users/zhulf/data/rgbd_fragments/') 186 | parser.add_argument('--depth_scale', type=float, default=1000.0) 187 | parser.add_argument('--depth_trunc', type=float, default=6.0) 188 | parser.add_argument('--frames_per_frag', type=int, default=50) 189 | parser.add_argument('--height', type=int, default=480) 190 | parser.add_argument('--threads', type=int, default=1) 191 | parser.add_argument('--tsdf_cubic_size', type=float, default=3.0) 192 | parser.add_argument('--width', type=int, default=640) 193 | 194 | return parser.parse_args() 195 | 196 | 197 | if __name__ == '__main__': 198 | cfg = parse_args() 199 | run(cfg) -------------------------------------------------------------------------------- /Datasets.md: -------------------------------------------------------------------------------- 1 | ## Datasets 2 | 3 | - [Stanford Bunny](http://graphics.stanford.edu/data/3Dscanrep/): [A Volumetric Method for Building Complex Models from Range Images](https://graphics.stanford.edu/papers/volrange/volrange.pdf) [SIGGRAPH 1996] 4 | - [KITTI](http://www.cvlibs.net/datasets/kitti/): [Are we ready for autonomous driving? the KITTI vision benchmark suite](http://www.cvlibs.net/publications/Geiger2012CVPR.pdf) [CVPR 2012] 5 | - [NYUV2](https://cs.nyu.edu/~silberman/datasets/nyu_depth_v2.html): [Indoor Segmentation and Support Inference 6 | from RGBD Images](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/11/shkf_eccv2012.pdf) [ECCV 2012] 7 | - [FAUST](http://faust.is.tue.mpg.de/): [FAUST: Dataset and evaluation for 3D mesh registration](http://files.is.tue.mpg.de/black/papers/FAUST2014.pdf) [CVPR 2014] 8 | - [ICL-NUIM](https://www.doc.ic.ac.uk/~ahanda/VaFRIC/iclnuim.html): [A Benchmark for RGB-D Visual Odometry, 3D Reconstruction and SLAM](http://mural.maynoothuniversity.ie/8309/1/JM-Benchmark-2014.pdf) [ICRA 2014] 9 | - [Augmented ICL-NUIM](http://redwood-data.org/indoor/dataset.html): [Robust Reconstruction of Indoor Scenes](https://www.researchgate.net/profile/Vladlen_Koltun/publication/279751165_Robust_Reconstruction_of_Indoor_Scenes/links/5599867708ae5d8f393633dc/Robust-Reconstruction-of-Indoor-Scenes.pdf) [CVPR 2015] 10 | - [ModelNet](https://modelnet.cs.princeton.edu): [3d shapenets: A deep representation for volumetric shapes](https://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Wu_3D_ShapeNets_A_2015_CVPR_paper.pdf) [`cls`; CVPR 2015] 11 | - [SUN RGB-D](http://rgbd.cs.princeton.edu/challenge.html): [Sun rgb-d: A rgb-d scene understanding benchmark suite](https://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Song_SUN_RGB-D_A_2015_CVPR_paper.pdf) [`det`; CVPR 2015] 12 | - [SHREC15](http://www.cs.cf.ac.uk/shaperetrieval/shrec15/): [SHREC’15 Track: Non-rigid 3D Shape Retrieval](http://www.cs.cf.ac.uk/shaperetrieval/files/Lian_3DOR_2015.pdf) [Eurographics 2015] 13 | - [ShapeNetCore](https://www.shapenet.org/): [ShapeNet: An Information-Rich 3D Model Repository](https://arxiv.org/pdf/1512.03012.pdf) [`cls`; arXiv 2015] 14 | - [ShapeNet Part](https://cs.stanford.edu/~ericyi/project_page/part_annotation/index.html): [A Scalable Active Framework for Region Annotation in 3D Shape Collections](https://www-cs.stanford.edu/~ericyi/papers/part_annotation_16_small.pdf) [`seg`; SIGGRAPH Asia 2016] 15 | - [SceneNN](http://103.24.77.34/scenenn/home/): [SceneNN: A Scene Meshes Dataset with aNNotations](https://www.researchgate.net/profile/Binh-Son_Hua/publication/311758430_SceneNN_A_Scene_Meshes_Dataset_with_aNNotations/links/5a6078300f7e9bfbc3f753f4/SceneNN-A-Scene-Meshes-Dataset-with-aNNotations.pdf) [3DV 2016] 16 | - [Oxford RobotCar](https://robotcar-dataset.robots.ox.ac.uk/): [1 Year, 1000km: The Oxford RobotCar Dataset](https://robotcar-dataset.robots.ox.ac.uk/images/robotcar_ijrr.pdf) [IJRR 2016] 17 | - [Redwood](http://redwood-data.org/3dscan/): [A large dataset of object scans](https://arxiv.org/pdf/1602.02481.pdf) [arXiv 2016] 18 | - [S3DIS](http://buildingparser.stanford.edu/dataset.html): [3D Semantic Parsing of Large-Scale Indoor Spaces](https://openaccess.thecvf.com/content_cvpr_2016/papers/Armeni_3D_Semantic_Parsing_CVPR_2016_paper.pdf) [CVPR 2016], [Joint 2D-3D-Semantic Data for Indoor Scene Understanding](https://arxiv.org/pdf/1702.01105.pdf) [`seg`; arXiv 2017] 19 | - [3DMatch](http://3dmatch.cs.princeton.edu/): [3DMatch: Learning Local Geometric Descriptors from RGB-D Reconstructions](https://arxiv.org/pdf/1603.08182.pdf) [CVPR 2017] 20 | - [SUNCG](https://sscnet.cs.princeton.edu/): [Semantic Scene Completion from a Single Depth Image](https://openaccess.thecvf.com/content_cvpr_2017/papers/Song_Semantic_Scene_Completion_CVPR_2017_paper.pdf) [CVPR 2017] 21 | - [ScanNet](http://www.scan-net.org/): [Scannet: Richly-annotated 3d reconstructions of indoor scenes](http://openaccess.thecvf.com/content_cvpr_2017/papers/Dai_ScanNet_Richly-Annotated_3D_CVPR_2017_paper.pdf) [`seg`, `det`; CVPR 2017 ] 22 | - [Semantic3D](http://www.semantic3d.net/): [Semantic3D.net: A new Large-scale Point Cloud Classification Benchmark](https://arxiv.org/pdf/1704.03847.pdf) [arXiv 2017] 23 | - [SemanticKITTI](http://semantic-kitti.org/): [SemanticKITTI: A Dataset for Semantic Scene Understanding of LiDAR Sequences](https://arxiv.org/pdf/1904.01416.pdf) [`seg`; ICCV 2019] 24 | - [ScanObjectNN](https://hkust-vgd.github.io/scanobjectnn/): [Revisiting Point Cloud Classification: A New Benchmark Dataset and Classification Model on Real-World Data](https://arxiv.org/pdf/1908.04616.pdf) [ICCV 2019] 25 | - [PartNet](https://cs.stanford.edu/~kaichun/partnet/): [PartNet: A Large-scale Benchmark for Fine-grained and 26 | Hierarchical Part-level 3D Object Understanding](https://arxiv.org/pdf/1812.02713.pdf) [CVPR 2019] 27 | - [Completion3D](https://completion3d.stanford.edu/): [TopNet: Structural Point Cloud Decoder](https://openaccess.thecvf.com/content_CVPR_2019/papers/Tchapmi_TopNet_Structural_Point_Cloud_Decoder_CVPR_2019_paper.pdf) [`completion`; CVPR 2019] 28 | - [Argoverses](https://www.argoverse.org/): [Argoverse: 3D Tracking and Forecasting with Rich Maps](https://openaccess.thecvf.com/content_CVPR_2019/papers/Chang_Argoverse_3D_Tracking_and_Forecasting_With_Rich_Maps_CVPR_2019_paper.pdf) [CVPR 2019] 29 | - [Waymo Open Dataset](https://github.com/waymo-research/waymo-open-dataset): [Scalability in Perception for Autonomous Driving: Waymo Open Dataset](https://openaccess.thecvf.com/content_CVPR_2020/papers/Sun_Scalability_in_Perception_for_Autonomous_Driving_Waymo_Open_Dataset_CVPR_2020_paper.pdf) [CVPR 2020] 30 | - [nuScenes](https://www.nuscenes.org): [nuScenes: A multimodal dataset for autonomous driving](https://openaccess.thecvf.com/content_CVPR_2020/papers/Caesar_nuScenes_A_Multimodal_Dataset_for_Autonomous_Driving_CVPR_2020_paper.pdf) [`det`, `tracking`; CVPR 2020] 31 | - [SensatUrban](https://github.com/QingyongHu/SensatUrban): [Towards Semantic Segmentation of Urban-Scale 3D Point Clouds: A Dataset, Benchmarks and Challenges](https://arxiv.org/pdf/2009.03137.pdf) [CVPR 2021], [SensatUrban: Learning Semantics from Urban-Scale Photogrammetric Point Clouds](https://arxiv.org/pdf/2201.04494.pdf) [IJCV 2022] 32 | - [WAYMO OPEN MOTION DATASET](https://arxiv.org/abs/2104.10133): [Large Scale Interactive Motion Forecasting for Autonomous Driving : The WAYMO OPEN MOTION DATASET](https://arxiv.org/pdf/2104.10133.pdf) [arXiv 2104] 33 | - [Panoptic nuScenes](https://nuscenes.org/panoptic): [Panoptic nuScenes: A Large-Scale Benchmark for LiDAR Panoptic Segmentation and Tracking](https://arxiv.org/pdf/2109.03805.pdf) [arXiv 2109] 34 | - [BuildingNet](https://buildingnet.org/): [BuildingNet: Learning to Label 3D Buildings](https://arxiv.org/pdf/2110.04955.pdf) [ICCV 2021 Oral] 35 | - [ARKitScenes](https://github.com/apple/ARKitScenes): [ARKitScenes - A Diverse Real-World Dataset For 3D Indoor Scene Understanding Using Mobile RGB-D Data](https://arxiv.org/pdf/2111.08897.pdf) [NeurIPS 2021] 36 | - [CODA](https://coda-dataset.github.io): [CODA: A Real-World Road Corner Case Dataset for Object Detection in Autonomous Driving](https://arxiv.org/pdf/2203.07724.pdf) [arXiv 2203] 37 | - [STPLS3D](https://www.stpls3d.com): [STPLS3D: A Large-Scale Synthetic and Real Aerial Photogrammetry 3D Point Cloud Dataset](https://arxiv.org/pdf/2203.09065v1.pdf) [arXiv 2203] 38 | - [TO-Scene: A Large-scale Dataset for Understanding 3D Tabletop Scenes](https://arxiv.org/pdf/2203.09440v1.pdf) [arXiv 2203] 39 | - [Omni3D](https://github.com/facebookresearch/omni3d): [Omni3D: A Large Benchmark and Model for 3D Object Detection in the Wild](https://arxiv.org/pdf/2207.10660.pdf) [arXiv 2207] 40 | - [Rope3D](https://thudair.baai.ac.cn/rope): [Rope3D: TheRoadside Perception Dataset for Autonomous Driving and Monocular 3D Object Detection Task](https://arxiv.org/pdf/2203.13608v1.pdf) [CVPR 2022] 41 | - [DAIR-V2X](https://thudair.baai.ac.cn/index): [DAIR-V2X: A Large-Scale Dataset for Vehicle-Infrastructure Cooperative 3D Object Detection](https://arxiv.org/pdf/2204.05575.pdf) [CVPR 2022] 42 | - [ONCE-3DLanes](https://once-3dlanes.github.io/): [ONCE-3DLanes: Building Monocular 3D Lane Detection](https://arxiv.org/pdf/2205.00301.pdf) [CVPR 2022] 43 | - [Ithaca365](https://ithaca365.mae.cornell.edu): [Ithaca365: Dataset and Driving Perception under Repeated and Challenging Weather Conditions](https://arxiv.org/pdf/2208.01166.pdf) [CVPR 2022] 44 | - [OpenLane](https://github.com/OpenPerceptionX/OpenLane): [PersFormer: 3D Lane Detection via Perspective Transformer and the OpenLane Benchmark](https://arxiv.org/pdf/2203.11089v1.pdf) [ECCV 2022] 45 | - [HM3DSEM](https://arxiv.org/pdf/2210.05633.pdf): [Habitat-Matterport 3D Semantics Dataset](https://arxiv.org/pdf/2210.05633.pdf) [arXiv 2210] 46 | - [Objaverse](https://objaverse.allenai.org): [Objaverse: A Universe of Annotated 3D Objects](https://arxiv.org/pdf/2212.08051.pdf) [arXiv 2212] 47 | - [OmniObject3D](https://omniobject3d.github.io): [OmniObject3D: Large-Vocabulary 3D Object Dataset for Realistic Perception, Reconstruction and Generation](https://arxiv.org/pdf/2301.07525v1.pdf) [arXiv 2301] 48 | - [OpenOccupancy](https://github.com/JeffWang987/OpenOccupancy): [OpenOccupancy: A Large Scale Benchmark for Surrounding Semantic Occupancy Perception](https://arxiv.org/pdf/2303.03991v1.pdf) [arXiv 2303] 49 | - [V2V4Real](https://github.com/ucla-mobility/V2V4Real): [V2V4Real: A Real-world Large-scale Dataset for Vehicle-to-Vehicle Cooperative Perception](https://arxiv.org/pdf/2303.07601v1.pdf) [CVPR 2023] 50 | - [CVPR2023-Occupancy-Prediction-Challenge](https://opendrivelab.com/AD23Challenge.html#Track3): [https://github.com/CVPR2023-3D-Occupancy-Prediction/CVPR2023-3D-Occupancy-Prediction](https://github.com/CVPR2023-3D-Occupancy-Prediction/CVPR2023-3D-Occupancy-Prediction) [CVPR2023 Challenge] 51 | - [WOMD-LiDAR](https://waymo.com/open/data/motion/): [WOMD-LiDAR: Raw Sensor Dataset Benchmark for Motion Forecasting](https://arxiv.org/pdf/2304.03834v1.pdf) [arXiv 2304] 52 | - [Occ3D](https://tsinghua-mars-lab.github.io/Occ3D/): [Occ3D: A Large-Scale 3D Occupancy Prediction Benchmark for Autonomous Driving](https://arxiv.org/pdf/2304.14365v1.pdf) [arXiv 2304] 53 | - [SSCBench](https://github.com/ai4ce/SSCBench): [SSCBench: A Large-Scale 3D Semantic Scene Completion Benchmark for Autonomous Driving](https://arxiv.org/pdf/2306.09001.pdf) [arXiv 2306] 54 | - [UniG3D](https://unig3d.github.io): [UniG3D: A Unified 3D Object Generation Dataset](https://arxiv.org/pdf/2306.10730.pdf) [arXiv 2306] 55 | - [WaterScenes](https://waterscenes.github.io): [WaterScenes: A Multi-Task 4D Radar-Camera Fusion Dataset for Autonomous Driving on Water Surfaces](https://arxiv.org/abs/2307.06505) [arXiv 2307] 56 | 57 |
58 | 59 | ## 数据集概况(更新中..) 60 | 61 | ### 一、ModelNet40(点云分类) 62 | 63 | 普林斯顿ModelNet项目的目标是为计算机视觉、计算机图形学、机器人和认知科学领域的研究者们提供一个全面、干净的三维CAD模型集合, 该数据的主页地址[https://modelnet.cs.princeton.edu](https://modelnet.cs.princeton.edu/), 数据最早发布在论文[3D ShapeNets: A Deep Representation for Volumetric Shapes](https://people.csail.mit.edu/khosla/papers/cvpr2015_wu.pdf) [CVPR 2015]上. 64 | 65 | 相关工作人员从数据中选择了常见的40类和10类构成数组子集, 分别表示为ModelNet40和ModelNet10, 且两个数据集都有orientation aligned的版本。实验中数据用到比较多的是ModelNet40, 有如下三种数据形式: 66 | 67 | | 数据集 | modelnet40_normal_resampled.zip | modelnet40_ply_hdf5_2048.zip | ModelNet40.zip | 68 | | :---: | :---: | :---: | :---: | 69 | | 文件大小 | 1.71G | 435M | 2.04G | 70 | | 内容 | point: x, y, z, normal_x, normal_y, normal_z;
shape: 10k points | point: x, y, z; normal_x, normal_y, normal_z;
shape: 2048 points | off格式, 具体参考[这里](https://segeval.cs.princeton.edu/public/off_format.html) | 71 | | 训练集 / 测试集 | 9843 / 2468 | 9840 / 2468 | 9844 / 2468 | 72 | | 下载地址 | [modelnet40_normal_resampled.zip](https://shapenet.cs.stanford.edu/media/modelnet40_normal_resampled.zip) | [modelnet40_ply_hdf5_2048.zip](https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip) | [ModelNet40.zip](http://modelnet.cs.princeton.edu/ModelNet40.zip) | 73 | 74 | ### 二、ShapeNet Part(点云分割) 75 | 76 | ShapeNet数据集是一个有丰富标注的、大规模的3D图像数据集, 发布于[ShapeNet: An Information-Rich 3D Model Repository](https://arxiv.org/pdf/1512.03012.pdf) [arXiv 2015], 它是普林斯顿大学、斯坦福大学和TTIC研究人员共同努力的结果, 官方主页为[shapenet.org](https://www.shapenet.org/).ShapeNet包括ShapeNetCore和ShapeNetSem子数据集. 77 | 78 | ShapeNet Part是从ShapeNetCore数据集选择了16类并进行语义信息标注的数据集, 用于点云的语义分割任务, 其数据集发表于[A Scalable Active Framework for Region Annotation in 3D Shape Collections](https://www-cs.stanford.edu/~ericyi/papers/part_annotation_16_small.pdf) [SIGGRAPH Asia 2016], 官方主页为 [ShapeNet Part](https://cs.stanford.edu/~ericyi/project_page/part_annotation/index.html). 数据包含几个不同的版本, 其下载链接分别为[shapenetcore_partanno_v0.zip](https://shapenet.cs.stanford.edu/ericyi/shapenetcore_partanno_v0.zip) (1.08G)和[shapenetcore_partanno_segmentation_benchmark_v0.zip](https://shapenet.cs.stanford.edu/ericyi/shapenetcore_partanno_segmentation_benchmark_v0.zip)(635M). 下面就第2个数据集segmentation benchmark进行介绍: 79 | 80 | 从下面表格可以看出, ShapeNet Part总共有16类, 50个parts，总共包括16846个样本。该数据集中样本呈现出不均衡特性，比如Table包括5263个, 而Earphone只有69个。每个样本包含2000多个点, 属于小数据集。该数据集中**训练集**12137个, **验证集**1870个, **测试集**2874个, **总计**16881个。[注意, 这里和下面表格统计的(16846)并不一样, 后来发现是训练集、验证集和测试集有35个重复的样本] 81 | 82 | | 类别 | nparts/shape | nsamples | 平均npoints/shape | 83 | | :---: | :---: | :---: | :---: | 84 | | Airplane | 4 | 2690 | 2577 | 85 | | Bag | 2 | 76 | 2749 | 86 | | Cap | 2 | 55 | 2631 | 87 | | Car | 4 | 898 | 2763 | 88 | | Chair | 4 | 3746 | 2705 | 89 | | Earphone | 3 | 69 | 2496 | 90 | | Guitar | 3 | 787 | 2353 | 91 | | Knife | 2 | 392 | 2156 | 92 | | Lamp | 4 | 1546 | 2198 | 93 | | Laptop | 2 | 445 | 2757 | 94 | | Motorbike | 6 | 202 | 2735 | 95 | | Mug | 2 | 184 | 2816 | 96 | | Pistol | 3 | 275 | 2654 | 97 | | Rocket | 3 | 66 | 2358 | 98 | | Skateboard | 3 | 152 | 2529 | 99 | | Table | 3 | 5263 | 2722 | 100 | | **Total** | **50** | **16846** | **2616** | 101 | 102 | ### 三、S3DIS(语义分割) 103 | 104 | S3DIS是3D室内场景的数据集, 主要用于点云的语义分割任务。主页[http://buildingparser.stanford.edu/dataset.html](http://buildingparser.stanford.edu/dataset.html). (但官方主页我暂时访问不了了, 关于数据集背景的介绍性说明就不写了). 关于S3DIS的论文是[Joint 2D-3D-Semantic Data for Indoor Scene Understanding](https://arxiv.org/pdf/1702.01105.pdf) [arXiv 2017]和[3D Semantic Parsing of Large-Scale Indoor Spaces](http://svl.stanford.edu/assets/papers/3D_Semantic_Parsing.pdf) [CVPR 2016]. S3DIS从3个building的6个Area采集得到, Area1, Area3, Area6属于buidling 1, Area2和Area4属于building 2, Area5属于building 3. 常用的数据下载格式包括如下三种: 105 | 106 | - [Stanford3dDataset_v1.2_Aligned_Version.zip](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1), 比如: [RandLA-Net](https://github.com/QingyongHu/RandLA-Net) 107 | - [Stanford3dDataset_v1.2.zip](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1), 比如: [CloserLook3D](https://github.com/zeliu98/CloserLook3D/tree/master/pytorch) 108 | - [indoor3d_sem_seg_hdf5_data.zip](https://shapenet.cs.stanford.edu/media/indoor3d_sem_seg_hdf5_data.zip), 比如: [PointNet](https://github.com/charlesq34/pointnet) 109 | 110 | 其中[Stanford3dDataset_v1.2_Aligned_Version.zip](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1)和[Stanford3dDataset_v1.2.zip](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1)都是完整场景的数据集, 每个点对应6个维度(x, y, z, r, g, b), 而[indoor3d_sem_seg_hdf5_data.zip](https://shapenet.cs.stanford.edu/media/indoor3d_sem_seg_hdf5_data.zip)是对原始数据场景的切割，把大场景切割成1m x 1m的block: 完整数据集被切割成了23585个block, 每个block是4096个点, 每个点对应9个维度: 除了x, y, z, r, g, b信息外，剩余的3维是相对于所在大场景的位置(归一化坐标). 111 | 112 | 下面是由[Stanford3dDataset_v1.2.zip](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1)数据统计得到的关于S3DIS的信息, 可能和论文中一些结果不太一致。S3DIS数据集由以上6个Area采集得到, 共包含272个场景, 可分为11种不同的场景(括号内为场景数量, 场景大小(点的数量)): office(156, 87w), conference room(11, 142w), hallway(61, 122w), auditorium(2, 817w), open 113 | space(1, 197w), lobby(3, 242w), lounge(3, 146w), pantry(3, 58w), copy room(2, 52w), storage(19, 35w) and WC(11, 70w). 根据语义信息, 上述场景被分成14个类别, 如下表所示. 可以看到不同的类别也是不均衡的, 比如wall有1547个, 但sofa只有55个. 114 | 115 | | **Total** | column | clutter | chair | window | beam | floor | wall | ceiling | door | bookcase | board | table | sofa | stairs | 116 | | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | 117 | | **9833** | 254 | 3882 | 1363 | 168 | 159 | 284 | 1547 | 385 | 543 | 584 | 137 | 455 | 55 | 17 | 118 | 119 | ### 四、3DMatch数据集(关键点、特征描述子、点云配准等) 120 | 121 | 详细信息请查看[3DMatch](https://github.com/zhulf0804/3D-PointCloud/blob/master/3DMatch)文件夹。 122 | --------------------------------------------------------------------------------