├── .gitignore ├── DATASET_GENERATION ├── 1.create_annotated_ply.py ├── 2.create_partial_pcd.py ├── 3.create_partial_jsons.py └── plyfile.py ├── LICENSE ├── README.md ├── data ├── cnet_data.pth ├── cnet_embeddings.pth ├── cnet_labels.pth ├── nyu_labels.pth ├── train.txt └── val.txt └── images ├── arch2.png ├── partial_scenes.png └── teaser.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /DATASET_GENERATION/1.create_annotated_ply.py: -------------------------------------------------------------------------------- 1 | from plyfile import PlyData, PlyElement, PlyProperty 2 | from pathlib import Path 3 | import json 4 | import pandas as pd 5 | import numpy as np 6 | 7 | import open3d as o3d 8 | import ray 9 | import tqdm 10 | import sys 11 | 12 | DEBUG = False 13 | 14 | 15 | @ray.remote 16 | def parse_scene(scene_folder: Path, out_folder: Path): 17 | 18 | original_ply = PlyData.read( 19 | str(scene_folder / f"{scene_folder.stem}_vh_clean_2.labels.ply") 20 | ) 21 | 22 | json_aggregation = json.load( 23 | (scene_folder / f"{scene_folder.stem}.aggregation.json").open("r") 24 | ) 25 | 26 | json_segments = json.load( 27 | (scene_folder / f"{scene_folder.stem}_vh_clean_2.0.010000.segs.json").open("r") 28 | ) 29 | 30 | instance_ids_array = np.ones(original_ply["vertex"]["x"].shape[0], dtype=int) * -1 31 | 32 | for instance_info in json_aggregation["segGroups"]: 33 | instance = instance_info["objectId"] 34 | # print(instance_info["label"]) 35 | seg = instance_info["segments"] 36 | 37 | for s in seg: 38 | ind = np.where(np.array(json_segments["segIndices"]) == s)[0] 39 | instance_ids_array[ind] = instance 40 | 41 | pass 42 | points_indices = np.where(instance_ids_array == instance)[0] 43 | 44 | if DEBUG: 45 | points = np.stack( 46 | [ 47 | original_ply["vertex"]["x"], 48 | original_ply["vertex"]["y"], 49 | original_ply["vertex"]["z"], 50 | ], 51 | 1, 52 | ) 53 | points_instance = points[points_indices] 54 | color = np.array([1.0, 0.0, 0.0], dtype=np.float)[np.newaxis, :].repeat( 55 | points_instance.shape[0], 0 56 | ) 57 | 58 | pcd = o3d.geometry.PointCloud() 59 | pcd.points = o3d.utility.Vector3dVector(points_instance) 60 | pcd.colors = o3d.utility.Vector3dVector(color) 61 | 62 | o3d.visualization.draw_geometries([pcd]) 63 | 64 | v = original_ply["vertex"] 65 | tmp = np.empty(len(v.data), v.data.dtype.descr + [("instance", "i4")]) 66 | for name in v.data.dtype.fields: 67 | tmp[name] = v[name] 68 | tmp["instance"] = instance_ids_array 69 | 70 | v = PlyElement.describe(tmp, "vertex") 71 | p = PlyData([v, original_ply["face"]], text=False) 72 | 73 | p.write(str(out_folder / f"{scene_folder.stem}.ply")) 74 | 75 | pass 76 | 77 | 78 | def main(): 79 | OUT_folder: Path = Path("annotated_ply") 80 | OUT_folder.mkdir(exist_ok=True) 81 | root_folder = Path("scans") 82 | scenes = [x for x in root_folder.glob("*")] 83 | l = [] 84 | for s in tqdm.tqdm(scenes): 85 | l.append(parse_scene.remote(s, OUT_folder)) 86 | 87 | print( 88 | "\n\nCreating the annotated PCD for the full scenes in the folder 'annotated_ply', please wait, this may take a while\n" 89 | ) 90 | with tqdm.tqdm(total=len(scenes)) as pbar: 91 | # for i 92 | # a = ray.get(l) 93 | unfinished = l 94 | num_ret = min(len(unfinished), 3) 95 | while unfinished: 96 | # Returns the first ObjectRef that is ready. 97 | finished, unfinished = ray.wait(unfinished, num_returns=num_ret) 98 | result = ray.get(finished) 99 | pbar.update(num_ret) 100 | sys.stdout.flush() 101 | 102 | 103 | if __name__ == "__main__": 104 | ray.init() 105 | main() 106 | -------------------------------------------------------------------------------- /DATASET_GENERATION/2.create_partial_pcd.py: -------------------------------------------------------------------------------- 1 | # %% 2 | from collections import defaultdict 3 | import open3d as o3d 4 | import open3d.visualization as o3d_vis 5 | import ray 6 | from pathlib import Path 7 | import numpy as np 8 | import copy 9 | from PIL import Image 10 | 11 | import matplotlib.pyplot as plt 12 | from copy import deepcopy 13 | import scipy.stats as stats 14 | import tqdm 15 | import json 16 | import gc 17 | import scipy.spatial.distance as distance 18 | from plyfile import PlyData, PlyElement 19 | import argparse 20 | import sys 21 | import os 22 | 23 | 24 | @ray.remote # (num_cpus=6) 25 | def create_pcd_rgb(scene_path, color, depth, new_width, new_height, intrinsic, extr): 26 | from PIL import Image 27 | import open3d as o3d 28 | import numpy as np 29 | 30 | rgb = color # Image.open(scene_path / "color" / f"{img}.jpg") 31 | depth = depth # Image.open(scene_path / "depth" / f"{img}.png") 32 | 33 | new_rgb = np.array(rgb.resize((new_width, new_height), resample=Image.NEAREST)) 34 | new_depth = np.array(depth.resize((new_width, new_height), resample=Image.NEAREST)) 35 | ### RESOLVE ISSUES WITH BLACK HOLES 36 | 37 | # image_result = inpaint.inpaint_biharmonic(new_depth, mask) 38 | 39 | # fig,ax=plt.subplots(1,3) 40 | 41 | # ax[0].imshow(new_depth) 42 | # ax[1].imshow(mask.astype(np.uint8)) 43 | # ax[2].imshow(image_result) 44 | 45 | # #fig.colorbar() 46 | # plt.show() 47 | 48 | ### extract 49 | 50 | extrinsic = extr # np.loadtxt(scene_path / "pose" / f"{img}.txt") 51 | new_depth[new_depth < 600] = 10000 52 | 53 | # plt.hist(new_depth.reshape(-1)) 54 | # plt.show() 55 | rgb_o3d = o3d.geometry.Image(new_rgb) 56 | depth_o3d = o3d.geometry.Image(new_depth.astype(np.uint16)) 57 | 58 | rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( 59 | rgb_o3d, depth_o3d, convert_rgb_to_intensity=False, 60 | ) 61 | 62 | pcd = o3d.geometry.PointCloud.create_from_rgbd_image( 63 | rgbd_image, 64 | o3d.camera.PinholeCameraIntrinsic( 65 | new_width, 66 | new_height, 67 | intrinsic[0][0], 68 | intrinsic[1][1], 69 | intrinsic[0][2], 70 | intrinsic[1][2], 71 | ), 72 | ) 73 | pcd.transform([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) 74 | pcd.transform(extrinsic) 75 | # pcd_denoised, _ = pcd.remove_statistical_outlier(nb_neighbors=100, std_ratio=1.2) 76 | return ( 77 | (np.asarray(pcd.points), np.asarray(pcd.colors)), 78 | # (np.asarray(pcd_denoised.points), np.asarray(pcd_denoised.colors)), 79 | ) 80 | 81 | 82 | @ray.remote 83 | def create_pcd_sem(scene_path, color, depth, new_width, new_height, intrinsic, extr): 84 | from PIL import Image 85 | import open3d as o3d 86 | import numpy as np 87 | 88 | rgb = color # Image.open(scene_path / "instance" / f"{img}.png") 89 | depth = depth # Image.open(scene_path / "depth" / f"{img}.png") 90 | 91 | new_rgb = np.array(rgb.resize((new_width, new_height), resample=Image.NEAREST)) 92 | new_depth = np.array(depth.resize((new_width, new_height), resample=Image.NEAREST)) 93 | ### RESOLVE ISSUES WITH BLACK HOLES 94 | 95 | extrinsic = extr # np.loadtxt(scene_path / "pose" / f"{img}.txt") 96 | new_depth[new_depth < 600] = 10000 97 | 98 | ##### Semantic 99 | 100 | rgb_o3d = o3d.geometry.Image( 101 | (new_rgb[:, :, np.newaxis] // 1000).repeat(3, 2).astype(np.uint8) 102 | ) 103 | depth_o3d = o3d.geometry.Image(new_depth.astype(np.uint16)) 104 | 105 | rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( 106 | rgb_o3d, depth_o3d, convert_rgb_to_intensity=False, 107 | ) 108 | 109 | pcd_sem = o3d.geometry.PointCloud.create_from_rgbd_image( 110 | rgbd_image, 111 | o3d.camera.PinholeCameraIntrinsic( 112 | new_width, 113 | new_height, 114 | intrinsic[0][0], 115 | intrinsic[1][1], 116 | intrinsic[0][2], 117 | intrinsic[1][2], 118 | ), 119 | ) 120 | 121 | pcd_sem.transform([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) 122 | pcd_sem.transform(extrinsic) 123 | 124 | ### 125 | rgb_o3d = o3d.geometry.Image( 126 | (new_rgb[:, :, np.newaxis] % 1000).repeat(3, 2).astype(np.uint8) 127 | ) 128 | rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( 129 | rgb_o3d, depth_o3d, convert_rgb_to_intensity=False, 130 | ) 131 | 132 | pcd_inst = o3d.geometry.PointCloud.create_from_rgbd_image( 133 | rgbd_image, 134 | o3d.camera.PinholeCameraIntrinsic( 135 | new_width, 136 | new_height, 137 | intrinsic[0][0], 138 | intrinsic[1][1], 139 | intrinsic[0][2], 140 | intrinsic[1][2], 141 | ), 142 | ) 143 | 144 | pcd_inst.transform([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) 145 | pcd_inst.transform(extrinsic) 146 | 147 | assert pcd_sem.points == pcd_inst.points 148 | return ( 149 | ( 150 | np.asarray(pcd_sem.points), 151 | np.asarray(pcd_sem.colors), 152 | np.asarray(pcd_inst.colors), 153 | ), 154 | # (np.asarray(pcd_denoised.points), np.asarray(pcd_denoised.colors)), 155 | ) 156 | 157 | 158 | def pcd_from_points(points, colors): 159 | pcd = o3d.geometry.PointCloud() 160 | pcd.points = o3d.utility.Vector3dVector(deepcopy(points)) 161 | pcd.colors = o3d.utility.Vector3dVector(deepcopy(colors)) 162 | # down=pcd.voxel_down_sample(voxel_size=0.05) 163 | # down,_=down.remove_statistical_outlier(nb_neighbors=30, 164 | # std_ratio=2.0) 165 | return pcd 166 | 167 | 168 | @ray.remote # (num_cpus=6) 169 | def calc_mode(colors, element_list): 170 | return stats.mode(colors[element_list])[0] 171 | 172 | 173 | @ray.remote 174 | def calc_min(xA, xB): 175 | temp = distance.cdist(xA, xB).T.argmin(0) 176 | return temp 177 | 178 | 179 | @ray.remote(num_cpus=os.cpu_count() // 2) 180 | def preprocess_scene(scene_path, new_width, new_height, subseq): 181 | images = (scene_path / "label").glob("*") 182 | file_names = sorted([x.stem for x in images])[:subseq] 183 | 184 | intr_color = np.loadtxt(scene_path / "intrinsics_color.txt") 185 | intr_depth = np.loadtxt(scene_path / "intrinsics_depth.txt") 186 | 187 | new_intr_rgb = rescale_intrinsic(intr_color, new_width, new_height) 188 | new_intr_depth = rescale_intrinsic(intr_depth, new_width, new_height) 189 | ret = [] 190 | for img in file_names: 191 | 192 | ret.append( 193 | create_pcd_rgb.remote( 194 | scene_path, 195 | Image.open(scene_path / "color" / f"{img}.jpg"), 196 | Image.open(scene_path / "depth" / f"{img}.png"), 197 | new_width, 198 | new_height, 199 | new_intr_depth, 200 | np.loadtxt(scene_path / "pose" / f"{img}.txt"), 201 | ) 202 | ) 203 | 204 | # ret_rgb = [ 205 | # create_pcd_rgb.remote( 206 | # scene_path, color, depth, new_width, new_height, new_intr_depth, extr 207 | # ) 208 | # for img in file_names 209 | # ] 210 | res = ray.get(ret) 211 | 212 | pcds_n = [pcd_from_points(x[0][0], x[0][1]) for x in res] 213 | 214 | del ret, res 215 | gc.collect() 216 | # del ret, res 217 | 218 | pcd = o3d.geometry.PointCloud() 219 | while len(pcds_n) > 0: 220 | p = pcds_n.pop(0) 221 | pcd += p 222 | 223 | pcd_down = pcd.voxel_down_sample(voxel_size=0.05) 224 | pcd_downscaled_points = np.array(pcd_down.points) 225 | 226 | pcd_original = PlyData.read(str(Path("annotated_ply") / f"{scene_path.stem}.ply")) 227 | pcd_vertex = pcd_original["vertex"] 228 | original_points = np.stack( 229 | [pcd_vertex["x"], pcd_vertex["y"], pcd_vertex["z"]], 1 230 | ).astype(np.float64) 231 | original_color = ( 232 | np.stack([pcd_vertex["red"], pcd_vertex["green"], pcd_vertex["blue"]], 1) / 255 233 | ) 234 | original_label = pcd_vertex["label"] 235 | original_instance = pcd_vertex["instance"] 236 | 237 | # pcd_original = o3d.io.read_point_cloud( 238 | # str(scene_path / f"{scene_path.stem}_vh_clean_2.labels.ply"), format="ply" 239 | # ) 240 | # pcd_original_points = np.array(pcd_original.points) 241 | # pcd_original_colors = np.array(pcd_original.colors) 242 | 243 | matching = [] 244 | step_size = 100 245 | orig = ray.put(original_points) 246 | rem = [] 247 | for i in range(0, pcd_downscaled_points.shape[0], step_size): 248 | # print(pcd_downscaled_points.dtype, original_points.dtype) 249 | # temp = distance.cdist( 250 | # pcd_downscaled_points[i : i + step_size], original_points 251 | # ).T.argmin(0) 252 | # matching.append(temp) 253 | rem.append(calc_min.remote(pcd_downscaled_points[i : i + step_size], orig)) 254 | matching = ray.get(rem) 255 | new_col = [original_color[x] for x in np.concatenate(matching)] 256 | new_inst = [original_instance[x] for x in np.concatenate(matching)] 257 | new_label = [original_label[x] for x in np.concatenate(matching)] 258 | new_points = [original_points[x] for x in np.concatenate(matching)] 259 | 260 | descr_lab = ["x", "y", "z", "red", "green", "blue", "label", "instance"] 261 | descr_type = ["f", "f", "f", "f", "f", "f", "i4", "i4"] 262 | 263 | types = list(zip(descr_lab, descr_type)) 264 | 265 | tmp = np.empty(len(new_col), types) 266 | 267 | tmp["x"] = np.array(new_points)[:, 0] 268 | tmp["y"] = np.array(new_points)[:, 1] 269 | tmp["z"] = np.array(new_points)[:, 2] 270 | tmp["red"] = np.array(new_col)[:, 0] 271 | tmp["green"] = np.array(new_col)[:, 1] 272 | tmp["blue"] = np.array(new_col)[:, 2] 273 | tmp["label"] = np.array(new_label) 274 | tmp["instance"] = np.array(new_inst) 275 | 276 | new_v = PlyElement.describe(tmp, "vertex") 277 | new_p = PlyData([new_v], text=False) 278 | 279 | out_dir_ply = Path("partial_pcds") / scene_path.stem 280 | out_dir_ply.mkdir(exist_ok=True, parents=True) 281 | new_p.write(str(out_dir_ply / f"{subseq:04d}.ply")) 282 | 283 | # o3d.visualization.draw_geometries([pcd_down]) 284 | 285 | # pcd_down.colors = o3d.utility.Vector3dVector(new_col) 286 | # o3d.visualization.draw_geometries([pcd_down]) 287 | objects = defaultdict(lambda: []) 288 | for obj_idx in set(new_inst): 289 | idx_ = np.where(new_inst == obj_idx)[0] 290 | lab = np.array(new_label)[idx_][0] 291 | pcd_inst = pcd_down.select_by_index(idx_) 292 | bb = pcd_inst.get_axis_aligned_bounding_box() 293 | objects[int(lab)].append( 294 | {"min_bb": bb.min_bound.tolist(), "max_bb": bb.max_bound.tolist()} 295 | ) 296 | 297 | # out_dir_json = Path("out_json_sing") / scene_path.stem 298 | # out_dir_json.mkdir(exist_ok=True, parents=True) 299 | 300 | # json.dump(objects, (out_dir_json / f"{subseq:04d}.json").open("w")) 301 | 302 | pass 303 | 304 | 305 | def rescale_intrinsic(intrinsic, new_w, new_h): 306 | 307 | original_w = 2 * intrinsic[0][2] 308 | original_h = 2 * intrinsic[1][2] 309 | ratio_w = new_w / original_w 310 | ratio_h = new_h / original_h 311 | new_intr = copy.deepcopy(intrinsic) 312 | 313 | new_intr[0] = new_intr[0] * ratio_w 314 | new_intr[1] = new_intr[1] * ratio_h 315 | 316 | return new_intr 317 | 318 | 319 | def main(): 320 | start_index = 0 321 | scenes = sorted(Path("scannet_frames_25k").glob("*")) 322 | jobs = [] 323 | count = 0 324 | with tqdm.tqdm(total=len(scenes)) as pbar: 325 | for id_, s in enumerate(scenes[start_index:1]): 326 | list_subseq = list(s.glob("color/*")) 327 | for subseq in range(1, len(list_subseq) + 1): 328 | 329 | print( 330 | f"{id_+start_index:04d} / {len(scenes)} : {s.stem} - {subseq}/{len(list_subseq)+1}" 331 | ) 332 | 333 | jobs.append(preprocess_scene.remote(s, 1280, 960, subseq)) 334 | 335 | if len(jobs) >= 3: 336 | chunk = jobs[:3] 337 | jobs = jobs[3:] 338 | results = ray.get(chunk) 339 | 340 | pbar.update() 341 | sys.stdout.flush() 342 | 343 | # gc.collect() 344 | # ray.shutdown() 345 | # final = ray.get(jobs) 346 | while len(jobs) > 0: 347 | chunk = jobs[:3] 348 | jobs = jobs[3:] 349 | results = ray.get(chunk) 350 | 351 | 352 | def main2(): 353 | 354 | # start_index = 0 355 | scenes = sorted(Path("scannet_frames_25k").glob("*")) 356 | jobs = [] 357 | count = 0 358 | # list_jobs=[] 359 | args = [] 360 | for id_, s in enumerate(scenes): 361 | list_subseq = list(s.glob("color/*")) 362 | for subseq in range(1, len(list_subseq) + 1): 363 | 364 | # print( 365 | # f"{id_+start_index:04d} / {len(scenes)} : {s.stem} - {subseq}/{len(list_subseq)+1}" 366 | # ) 367 | args.append((s, 1280, 960, subseq)) 368 | 369 | # jobs.append(preprocess_scene.remote(s, 1280, 960, subseq)) 370 | 371 | with tqdm.tqdm(total=len(args)) as pbar: 372 | for arg in args: 373 | 374 | preprocess_scene(*arg) 375 | pbar.update() 376 | sys.stdout.flush() 377 | # for i 378 | # a = ray.get(l) 379 | # unfinished = jobs 380 | # num_ret = min(len(unfinished), 3) 381 | # while unfinished: 382 | ## Returns the first ObjectRef that is ready. 383 | # finished, unfinished = ray.wait(unfinished, num_returns=num_ret) 384 | # result = ray.get(finished) 385 | # pbar.update(num_ret) 386 | # sys.stdout.flush() 387 | 388 | 389 | def main3(): 390 | 391 | scenes = sorted(Path("scannet_frames_25k").glob("*")) 392 | jobs = [] 393 | count = 0 394 | for id_, s in enumerate(scenes): 395 | list_subseq = list(s.glob("color/*")) 396 | for subseq in range(1, len(list_subseq) + 1): 397 | 398 | # print( 399 | # f"{id_+start_index:04d} / {len(scenes)} : {s.stem} - {subseq}/{len(list_subseq)+1}" 400 | # ) 401 | # args.append((s, 1280, 960, subseq)) 402 | 403 | jobs.append(preprocess_scene.remote(s, 1280, 960, subseq)) 404 | 405 | with tqdm.tqdm(total=len(jobs)) as pbar: 406 | # for arg in args: 407 | 408 | # preprocess_scene(*arg) 409 | # pbar.update() 410 | # sys.stdout.flush() 411 | # for i 412 | # a = ray.get(l) 413 | unfinished = jobs 414 | num_ret = min(len(unfinished), 3) 415 | while unfinished: 416 | ## Returns the first ObjectRef that is ready. 417 | finished, unfinished = ray.wait(unfinished, num_returns=num_ret) 418 | # result = ray.get(finished) 419 | pbar.update(num_ret) 420 | sys.stdout.flush() 421 | 422 | 423 | if __name__ == "__main__": 424 | 425 | # parser = argparse.ArgumentParser(description="preprocess scanned scenes") 426 | # parser.add_argument("--redis_pwd", required=True, type=str, help="redis pwd") 427 | 428 | # parser.add_argument("--head_ip", required=True, type=str, help="ray head ip") 429 | # args = parser.parse_args() 430 | # ray.init(address="auto", _redis_password="5241590000000000") 431 | # ray.init(address="auto", _redis_password="5241590000000000") 432 | # ray.init(address='auto', _redis_password='5241590000000000') 433 | # ray.init(address=args.head_ip, _redis_password=args.redis_pwd) 434 | ray.init() 435 | main() 436 | 437 | 438 | # %% 439 | -------------------------------------------------------------------------------- /DATASET_GENERATION/3.create_partial_jsons.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy 3 | from pathlib import Path 4 | from plyfile import PlyData 5 | import open3d as o3d 6 | import matplotlib.pyplot as plt 7 | from sklearn.decomposition import PCA 8 | import copy 9 | 10 | # import mpi4py 11 | from tqdm import tqdm 12 | import json 13 | 14 | import scipy.spatial.distance as dist 15 | import sys 16 | import ray 17 | 18 | 19 | ITEMS_TO_SKIP = [1, 2, 22, 31] # FLOOR, WALL, CEILING, PERSON 20 | 21 | 22 | def get_transf_and_bb(ref_pcd): 23 | 24 | ref_pcd = copy.deepcopy(ref_pcd) 25 | 26 | obj_flat = copy.deepcopy(ref_pcd) 27 | obj_flat[:, 2] = 0 28 | 29 | pca = PCA(3) 30 | pca.fit(obj_flat) 31 | components = pca.components_ 32 | 33 | transf = np.array( 34 | [components[0], [-components[0, 1], components[0, 0], 0], [0, 0, 1]] 35 | ) 36 | mean = pca.mean_ 37 | 38 | rotated_ref_pcd = (ref_pcd) @ transf.T 39 | width = rotated_ref_pcd[:, 0].max() - rotated_ref_pcd[:, 0].min() 40 | length = rotated_ref_pcd[:, 1].max() - rotated_ref_pcd[:, 1].min() 41 | height = rotated_ref_pcd[:, 2].max() - rotated_ref_pcd[:, 2].min() 42 | 43 | return mean, transf, (width, length, height) 44 | 45 | 46 | def transform_pcd(mean, transf, pcd): 47 | pcd = copy.deepcopy(pcd) 48 | 49 | pcd = pcd - mean 50 | 51 | pcd = np.matmul(pcd, transf.T) 52 | 53 | return pcd, transf 54 | 55 | 56 | def transform_pcd_old(ref_pcd, pcd): 57 | ref_pcd = copy.deepcopy(ref_pcd) 58 | pcd = copy.deepcopy(pcd) 59 | 60 | obj_flat = ref_pcd 61 | obj_flat[:, 2] = 0 62 | 63 | pca = PCA(3) 64 | pca.fit(obj_flat) 65 | components = pca.components_ 66 | 67 | transf = np.array( 68 | [components[0], [-components[0, 1], components[0, 0], 0], [0, 0, 1]] 69 | ) 70 | mean = pca.mean_ 71 | pcd = pcd - mean 72 | 73 | pcd = np.matmul(pcd, transf.T) 74 | 75 | rotated_ref_pcd = (ref_pcd) @ transf.T 76 | width = rotated_ref_pcd[:, 0].max() - rotated_ref_pcd[:, 0].min() 77 | length = rotated_ref_pcd[:, 1].max() - rotated_ref_pcd[:, 1].min() 78 | height = rotated_ref_pcd[:, 2].max() - rotated_ref_pcd[:, 2].min() 79 | # ll = np.array([rotated_ref_pcd[:, 0].min(), rotated_ref_pcd[:, 1].min()]) 80 | # new=[ll,ll+[width,0],ll+[width,length],ll+[0,length],ll] 81 | # new=np.array(new) 82 | return pcd, transf, (width, length, height) 83 | 84 | 85 | def test_eval_scene(scene_ply_name): 86 | 87 | ply = PlyData.read(scene_ply_name) 88 | vertex = ply["vertex"] 89 | xyz = np.array([vertex["x"], vertex["y"], vertex["z"]]).T 90 | instance = np.array(vertex["instance"]) 91 | label = np.array(vertex["label"]) 92 | 93 | label_set = set(np.unique(label)) - {0} 94 | 95 | for lab in label_set: 96 | label_ind = label == lab 97 | if lab < 4: 98 | continue 99 | 100 | ind_set = np.unique(instance[label_ind]) 101 | for inst in ind_set: 102 | 103 | instance_ind = instance == inst 104 | 105 | obj = xyz[label_ind & instance_ind] 106 | obj_flat = obj.copy() 107 | obj_flat[:, 2] = 0 108 | pca = PCA(3) 109 | pca.fit(obj_flat) 110 | components = pca.components_ 111 | 112 | axis0 = components[0] # + pca.mean_ 113 | axis0 = np.array([axis0 * -1 + pca.mean_, axis0 * 1 + pca.mean_]) 114 | 115 | axis1 = components[1] # + pca.mean_ 116 | axis1 = np.array([axis1 * -1 + pca.mean_, axis1 * 1 + pca.mean_]) 117 | 118 | axis2 = components[2] # + pca.mean_ 119 | axis2 = np.array([axis2 * -1 + pca.mean_, axis2 * 1 + pca.mean_]) 120 | 121 | print(lab) 122 | fig = plt.figure() 123 | ax = fig.add_subplot(projection="3d") 124 | ax.scatter(obj[:, 0], obj[:, 1], obj[:, 2]) 125 | ax.plot(axis0[:, 0], axis0[:, 1], axis0[:, 2], "r") 126 | ax.plot(axis1[:, 0], axis1[:, 1], axis1[:, 2], "g") 127 | ax.plot(axis2[:, 0], axis2[:, 1], axis2[:, 2], "y") 128 | 129 | transformer_obj = transform_pcd(obj, obj) 130 | ax.scatter( 131 | transformer_obj[:, 0], 132 | transformer_obj[:, 1], 133 | transformer_obj[:, 2], 134 | color="orange", 135 | ) 136 | plt.show() 137 | pass 138 | pass 139 | 140 | 141 | @ray.remote 142 | def calc_scene_distances(list_): # (partial_scene, complete_scene): 143 | # print("start") 144 | partial_scene = list_[0] 145 | complete_scene = list_[1] 146 | out_dir = Path("partial_jsons") 147 | out_dir.mkdir(exist_ok=True) 148 | 149 | # print("ready") 150 | scene_pcd = PlyData.read(str(partial_scene)) 151 | vert = scene_pcd["vertex"] 152 | xyz = np.stack([vert["x"], vert["y"], vert["z"]], 1) 153 | label = np.array(vert["label"]) 154 | instance = np.array(vert["instance"]) 155 | 156 | j = {} 157 | j["center"] = [] 158 | j["labels"] = [] 159 | j["distances"] = [] 160 | j["relative_pos"] = [] 161 | j["masked"] = [] 162 | j["bb_shape"] = [] 163 | 164 | seen_instances = [] 165 | 166 | ############## objects in partial 167 | 168 | for obj_i in range(1, 38): 169 | if obj_i in ITEMS_TO_SKIP: 170 | continue 171 | ind_obj_i = label == obj_i 172 | instances_i = sorted(np.unique(instance[ind_obj_i])) 173 | for inst_i in instances_i: 174 | 175 | pcd_i = xyz[(instance == inst_i) & ind_obj_i] 176 | if pcd_i.shape[0] < 50: 177 | continue 178 | mean, transf, bb = get_transf_and_bb(pcd_i) 179 | 180 | seen_instances.append(inst_i) 181 | kdtree = scipy.spatial.cKDTree(pcd_i) 182 | j["labels"].append(obj_i) 183 | j["center"].append(((pcd_i.max(0) + pcd_i.min(0)) / 2).tolist()) 184 | j["masked"].append(0) 185 | j["bb_shape"].append([bb[0], bb[1], bb[2]]) 186 | tmp_dist = [] 187 | tmp_rel = [] 188 | for obj_j in range(1, 38): 189 | if obj_j in ITEMS_TO_SKIP: 190 | continue 191 | 192 | ind_obj_j = label == obj_j 193 | instances_j = sorted(np.unique(instance[ind_obj_j])) 194 | for inst_j in instances_j: 195 | pcd_j = xyz[(instance == inst_j) & ind_obj_j] 196 | if pcd_j.shape[0] < 50: 197 | continue 198 | 199 | cur_min = np.inf 200 | # for i in range(0, pcd_j.shape[0], 1300): 201 | # cur_min = min( 202 | # dist.cdist(pcd_i, pcd_j[i : i + 300]).min(), cur_min 203 | # ) 204 | cur_min = kdtree.query(pcd_j)[0].min() 205 | # pcd_j_new, _ = transform_pcd_old(pcd_i, pcd_j) 206 | pcd_j_new, _ = transform_pcd(mean, transf, pcd_j) 207 | 208 | rel = ((pcd_j_new.max(0) + pcd_j_new.min(0)) / 2).tolist() 209 | tmp_dist.append(cur_min) 210 | tmp_rel.append(rel) 211 | j["distances"].append(tmp_dist) 212 | 213 | j["relative_pos"].append(tmp_rel) 214 | 215 | ########### complete scene for missing items 216 | 217 | complete_pcd = PlyData.read(str(complete_scene)) 218 | complete_vert = complete_pcd["vertex"] 219 | complete_xyz = np.stack( 220 | [complete_vert["x"], complete_vert["y"], complete_vert["z"]], 1 221 | ) 222 | complete_label = np.array(complete_vert["label"]) 223 | complete_instance = np.array(complete_vert["instance"]) 224 | 225 | instances_j_seen = [] 226 | j["masked_center"] = [] 227 | j["masked_label"] = [] 228 | j["distance_to_masked"] = [] 229 | j["relative_pos_to_masked"] = [] 230 | 231 | for obj_i in range(1, 38): 232 | if obj_i in ITEMS_TO_SKIP: 233 | continue 234 | 235 | ind_obj_i = label == obj_i 236 | instances_i = sorted(np.unique(instance[ind_obj_i])) 237 | for inst_i in instances_i: 238 | pcd_i = xyz[(instance == inst_i) & ind_obj_i] 239 | if pcd_i.shape[0] < 50: 240 | continue 241 | 242 | mean, transf, bb = get_transf_and_bb(pcd_i) 243 | kdtree = scipy.spatial.cKDTree(pcd_i) 244 | tmp_dist = [] 245 | tmp_rel = [] 246 | for obj_j in range(1, 38): 247 | if obj_j in ITEMS_TO_SKIP: 248 | continue 249 | ind_obj_j = complete_label == obj_j 250 | instances_j = sorted(np.unique(complete_instance[ind_obj_j])) 251 | for inst_j in instances_j: 252 | if inst_j in seen_instances: 253 | continue 254 | 255 | pcd_j = complete_xyz[(complete_instance == inst_j) & ind_obj_j] 256 | if inst_j not in instances_j_seen: 257 | j["masked_center"].append( 258 | ((pcd_j.max(0) + pcd_j.min(0)) / 2).tolist() 259 | ) 260 | j["masked_label"].append(obj_j) 261 | instances_j_seen.append(inst_j) 262 | 263 | cur_min = np.inf 264 | # for i in range(0, pcd_j.shape[0], 1300): 265 | # cur_min = min( 266 | # dist.cdist(pcd_i, pcd_j[i : i + 300]).min(), cur_min 267 | # ) 268 | cur_min = kdtree.query(pcd_j)[0].min() 269 | # pcd_j_new, _ = transform_pcd_old(pcd_i, pcd_j) 270 | pcd_j_new, _ = transform_pcd(mean, transf, pcd_j) 271 | 272 | rel = ((pcd_j_new.max(0) + pcd_j_new.min(0)) / 2).tolist() 273 | tmp_dist.append(cur_min) 274 | tmp_rel.append(rel) 275 | j["distance_to_masked"].append(tmp_dist) 276 | 277 | j["relative_pos_to_masked"].append(tmp_rel) 278 | 279 | if len(j['masked'])==0 or len(j['labels'])<3: 280 | return 281 | 282 | with (out_dir / f"{complete_scene.stem}_{partial_scene.stem}.json").open("w") as f: 283 | json.dump(j, f) 284 | 285 | return j 286 | 287 | 288 | def main(): 289 | scene_ply_name = sorted(list(Path("annotated_ply").glob("*.ply"))) 290 | 291 | ROOT_PARTIAL_DIR = Path("partial_pcds") 292 | SCENE_DIR = sorted([x for x in ROOT_PARTIAL_DIR.iterdir() if x.is_dir()]) 293 | ROOT_COMPLETE_DIR = Path("annotated_ply") 294 | jobs = [] 295 | arg_list = [] 296 | for scene in SCENE_DIR: 297 | partial_scenes = sorted(list(scene.glob("*.ply"))) 298 | scene_name = scene.stem 299 | 300 | complete = ROOT_COMPLETE_DIR / f"{scene_name}.ply" 301 | 302 | for partial in partial_scenes: 303 | jobs.append(calc_scene_distances.remote((partial, complete))) 304 | 305 | pass 306 | 307 | pass 308 | 309 | with tqdm(total=len(jobs)) as pbar: 310 | 311 | unfinished = jobs 312 | num_ret = min(len(unfinished), 3) 313 | while unfinished: 314 | 315 | num_ret = min(len(unfinished), 3) 316 | ## Returns the first ObjectRef that is ready. 317 | # print(len(unfinished)) 318 | # print(num_ret) 319 | finished, unfinished = ray.wait(unfinished, num_returns=num_ret) 320 | result = ray.get(finished) 321 | pbar.update(num_ret) 322 | sys.stdout.flush() 323 | 324 | 325 | if __name__ == "__main__": 326 | ray.init() 327 | main() 328 | -------------------------------------------------------------------------------- /DATASET_GENERATION/plyfile.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2020 Darsh Ranjan 2 | # 3 | # This file is part of python-plyfile. 4 | # 5 | # python-plyfile is free software: you can redistribute it and/or 6 | # modify it under the terms of the GNU General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # python-plyfile is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with python-plyfile. If not, see 17 | # . 18 | 19 | from itertools import islice as _islice 20 | from sys import byteorder as _byteorder 21 | 22 | import numpy as _np 23 | 24 | # try: 25 | # _range = xrange 26 | # except NameError: 27 | _range = range 28 | 29 | 30 | # Many-many relation 31 | _data_type_relation = [ 32 | ("int8", "i1"), 33 | ("char", "i1"), 34 | ("uint8", "u1"), 35 | ("uchar", "b1"), 36 | ("uchar", "u1"), 37 | ("int16", "i2"), 38 | ("short", "i2"), 39 | ("uint16", "u2"), 40 | ("ushort", "u2"), 41 | ("int32", "i4"), 42 | ("int", "i4"), 43 | ("uint32", "u4"), 44 | ("uint", "u4"), 45 | ("float32", "f4"), 46 | ("float", "f4"), 47 | ("float64", "f8"), 48 | ("double", "f8"), 49 | ] 50 | 51 | _data_types = dict(_data_type_relation) 52 | _data_type_reverse = dict((b, a) for (a, b) in _data_type_relation) 53 | 54 | _types_list = [] 55 | _types_set = set() 56 | for (_a, _b) in _data_type_relation: 57 | if _a not in _types_set: 58 | _types_list.append(_a) 59 | _types_set.add(_a) 60 | if _b not in _types_set: 61 | _types_list.append(_b) 62 | _types_set.add(_b) 63 | 64 | 65 | _byte_order_map = {"ascii": "=", "binary_little_endian": "<", "binary_big_endian": ">"} 66 | 67 | _byte_order_reverse = {"<": "binary_little_endian", ">": "binary_big_endian"} 68 | 69 | _native_byte_order = {"little": "<", "big": ">"}[_byteorder] 70 | 71 | 72 | def _lookup_type(type_str): 73 | if type_str not in _data_type_reverse: 74 | try: 75 | type_str = _data_types[type_str] 76 | except KeyError: 77 | raise ValueError("field type %r not in %r" % (type_str, _types_list)) 78 | 79 | return _data_type_reverse[type_str] 80 | 81 | 82 | def make2d(array, cols=None, dtype=None): 83 | """ 84 | Make a 2D array from an array of arrays. The `cols' and `dtype' 85 | arguments can be omitted if the array is not empty. 86 | 87 | """ 88 | if not len(array): 89 | if cols is None or dtype is None: 90 | raise RuntimeError("cols and dtype must be specified for empty array") 91 | return _np.empty((0, cols), dtype=dtype) 92 | return _np.vstack(array) 93 | 94 | 95 | class _PlyHeaderParser(object): 96 | def __init__(self): 97 | self.format = None 98 | self.elements = [] 99 | self.comments = [] 100 | self.obj_info = [] 101 | self.lines = 0 102 | self._allowed = ["ply"] 103 | 104 | def consume(self, raw_line): 105 | self.lines += 1 106 | if not raw_line: 107 | self._error("early end-of-file") 108 | 109 | line = raw_line.decode("ascii").strip() 110 | try: 111 | keyword = line.split(None, 1)[0] 112 | except IndexError: 113 | self._error() 114 | 115 | if keyword not in self._allowed: 116 | self._error("expected one of {%s}" % ", ".join(self._allowed)) 117 | 118 | getattr(self, "parse_" + keyword)(line[(len(keyword) + 1) :]) 119 | return self._allowed 120 | 121 | def _error(self, message="parse error"): 122 | raise PlyHeaderParseError(message, self.lines) 123 | 124 | def parse_ply(self, data): 125 | if data: 126 | self._error("unexpected characters after 'ply'") 127 | self._allowed = ["format", "comment", "obj_info"] 128 | 129 | def parse_format(self, data): 130 | fields = data.strip().split() 131 | if len(fields) != 2: 132 | self._error('expected "format {format} 1.0"') 133 | 134 | self.format = fields[0] 135 | if self.format not in _byte_order_map: 136 | self._error("don't understand format %r" % format) 137 | 138 | if fields[1] != "1.0": 139 | self._error("expected version '1.0'") 140 | 141 | self._allowed = ["element", "comment", "obj_info", "end_header"] 142 | 143 | def parse_comment(self, data): 144 | if not self.elements: 145 | self.comments.append(data) 146 | else: 147 | self.elements[-1][3].append(data) 148 | 149 | def parse_obj_info(self, data): 150 | self.obj_info.append(data) 151 | 152 | def parse_element(self, data): 153 | fields = data.strip().split() 154 | if len(fields) != 2: 155 | self._error('expected "element {name} {count}"') 156 | 157 | name = fields[0] 158 | try: 159 | count = int(fields[1]) 160 | except ValueError: 161 | self._error("expected integer count") 162 | 163 | self.elements.append((name, [], count, [])) 164 | self._allowed = ["element", "comment", "property", "end_header"] 165 | 166 | def parse_property(self, data): 167 | properties = self.elements[-1][1] 168 | fields = data.strip().split() 169 | if len(fields) < 2: 170 | self._error("bad 'property' line") 171 | 172 | if fields[0] == "list": 173 | if len(fields) != 4: 174 | self._error('expected "property list ' '{len_type} {val_type} {name}"') 175 | 176 | try: 177 | properties.append(PlyListProperty(fields[3], fields[1], fields[2])) 178 | except ValueError as e: 179 | self._error(str(e)) 180 | 181 | else: 182 | if len(fields) != 2: 183 | self._error('expected "property {type} {name}"') 184 | 185 | try: 186 | properties.append(PlyProperty(fields[1], fields[0])) 187 | except ValueError as e: 188 | self._error(str(e)) 189 | 190 | def parse_end_header(self, data): 191 | if data: 192 | self._error("unexpected data after 'end_header'") 193 | self._allowed = [] 194 | 195 | 196 | class PlyParseError(Exception): 197 | 198 | """ 199 | Base class for PLY parsing errors. 200 | 201 | """ 202 | 203 | pass 204 | 205 | 206 | class PlyElementParseError(PlyParseError): 207 | 208 | """ 209 | Raised when a PLY element cannot be parsed. 210 | 211 | The attributes `element', `row', `property', and `message' give 212 | additional information. 213 | 214 | """ 215 | 216 | def __init__(self, message, element=None, row=None, prop=None): 217 | self.message = message 218 | self.element = element 219 | self.row = row 220 | self.prop = prop 221 | 222 | s = "" 223 | if self.element: 224 | s += "element %r: " % self.element.name 225 | if self.row is not None: 226 | s += "row %d: " % self.row 227 | if self.prop: 228 | s += "property %r: " % self.prop.name 229 | s += self.message 230 | 231 | Exception.__init__(self, s) 232 | 233 | def __repr__(self): 234 | return "%s(%r, element=%r, row=%r, prop=%r)" % ( 235 | self.__class__.__name__, 236 | self.message, 237 | self.element, 238 | self.row, 239 | self.prop, 240 | ) 241 | 242 | 243 | class PlyHeaderParseError(PlyParseError): 244 | 245 | """ 246 | Raised when a PLY header cannot be parsed. 247 | 248 | The attribute `line' provides additional information. 249 | 250 | """ 251 | 252 | def __init__(self, message, line=None): 253 | self.message = message 254 | self.line = line 255 | 256 | s = "" 257 | if self.line: 258 | s += "line %r: " % self.line 259 | s += self.message 260 | 261 | Exception.__init__(self, s) 262 | 263 | def __repr__(self): 264 | return "%s(%r, line=%r)" % (self.__class__.__name__, self.message, self.line) 265 | 266 | 267 | class PlyData(object): 268 | 269 | """ 270 | PLY file header and data. 271 | 272 | A PlyData instance is created in one of two ways: by the static 273 | method PlyData.read (to read a PLY file), or directly from __init__ 274 | given a sequence of elements (which can then be written to a PLY 275 | file). 276 | 277 | """ 278 | 279 | def __init__( 280 | self, elements=[], text=False, byte_order="=", comments=[], obj_info=[] 281 | ): 282 | """ 283 | elements: sequence of PlyElement instances. 284 | 285 | text: whether the resulting PLY file will be text (True) or 286 | binary (False). 287 | 288 | byte_order: '<' for little-endian, '>' for big-endian, or '=' 289 | for native. This is only relevant if `text' is False. 290 | 291 | comments: sequence of strings that will be placed in the header 292 | between the 'ply' and 'format ...' lines. 293 | 294 | obj_info: like comments, but will be placed in the header with 295 | "obj_info ..." instead of "comment ...". 296 | 297 | """ 298 | if byte_order == "=" and not text: 299 | byte_order = _native_byte_order 300 | 301 | self.byte_order = byte_order 302 | self.text = text 303 | 304 | self.comments = comments 305 | self.obj_info = obj_info 306 | self.elements = elements 307 | 308 | def _get_elements(self): 309 | return self._elements 310 | 311 | def _set_elements(self, elements): 312 | self._elements = tuple(elements) 313 | self._index() 314 | 315 | elements = property(_get_elements, _set_elements) 316 | 317 | def _get_byte_order(self): 318 | return self._byte_order 319 | 320 | def _set_byte_order(self, byte_order): 321 | if byte_order not in ["<", ">", "="]: 322 | raise ValueError("byte order must be '<', '>', or '='") 323 | 324 | self._byte_order = byte_order 325 | 326 | byte_order = property(_get_byte_order, _set_byte_order) 327 | 328 | def _index(self): 329 | self._element_lookup = dict((elt.name, elt) for elt in self._elements) 330 | if len(self._element_lookup) != len(self._elements): 331 | raise ValueError("two elements with same name") 332 | 333 | def _get_comments(self): 334 | return list(self._comments) 335 | 336 | def _set_comments(self, comments): 337 | _check_comments(comments) 338 | self._comments = list(comments) 339 | 340 | comments = property(_get_comments, _set_comments) 341 | 342 | def _get_obj_info(self): 343 | return list(self._obj_info) 344 | 345 | def _set_obj_info(self, obj_info): 346 | _check_comments(obj_info) 347 | self._obj_info = list(obj_info) 348 | 349 | obj_info = property(_get_obj_info, _set_obj_info) 350 | 351 | @staticmethod 352 | def _parse_header(stream): 353 | """ 354 | Parse a PLY header from a readable file-like stream. 355 | 356 | """ 357 | parser = _PlyHeaderParser() 358 | while parser.consume(stream.readline()): 359 | pass 360 | 361 | return PlyData( 362 | [PlyElement(*e) for e in parser.elements], 363 | parser.format == "ascii", 364 | _byte_order_map[parser.format], 365 | parser.comments, 366 | parser.obj_info, 367 | ) 368 | 369 | @staticmethod 370 | def read(stream, mmap=True): 371 | """ 372 | Read PLY data from a readable file-like object or filename. 373 | 374 | mmap: whether to allow element data to be memory-mapped when 375 | possible. The default is True, which allows memory mapping. 376 | Using False will prevent memory mapping. 377 | """ 378 | (must_close, stream) = _open_stream(stream, "read") 379 | try: 380 | data = PlyData._parse_header(stream) 381 | for elt in data: 382 | elt._read(stream, data.text, data.byte_order, mmap) 383 | finally: 384 | if must_close: 385 | stream.close() 386 | 387 | return data 388 | 389 | def write(self, stream): 390 | """ 391 | Write PLY data to a writeable file-like object or filename. 392 | 393 | """ 394 | (must_close, stream) = _open_stream(stream, "write") 395 | try: 396 | stream.write(self.header.encode("ascii")) 397 | stream.write(b"\n") 398 | for elt in self: 399 | elt._write(stream, self.text, self.byte_order) 400 | finally: 401 | if must_close: 402 | stream.close() 403 | 404 | @property 405 | def header(self): 406 | """ 407 | Provide PLY-formatted metadata for the instance. 408 | 409 | """ 410 | lines = ["ply"] 411 | 412 | if self.text: 413 | lines.append("format ascii 1.0") 414 | else: 415 | lines.append("format " + _byte_order_reverse[self.byte_order] + " 1.0") 416 | 417 | # Some information is lost here, since all comments are placed 418 | # between the 'format' line and the first element. 419 | for c in self.comments: 420 | lines.append("comment " + c) 421 | 422 | for c in self.obj_info: 423 | lines.append("obj_info " + c) 424 | 425 | lines.extend(elt.header for elt in self.elements) 426 | lines.append("end_header") 427 | return "\n".join(lines) 428 | 429 | def __iter__(self): 430 | return iter(self.elements) 431 | 432 | def __len__(self): 433 | return len(self.elements) 434 | 435 | def __contains__(self, name): 436 | return name in self._element_lookup 437 | 438 | def __getitem__(self, name): 439 | return self._element_lookup[name] 440 | 441 | def __str__(self): 442 | return self.header 443 | 444 | def __repr__(self): 445 | return "PlyData(%r, text=%r, byte_order=%r, " "comments=%r, obj_info=%r)" % ( 446 | self.elements, 447 | self.text, 448 | self.byte_order, 449 | self.comments, 450 | self.obj_info, 451 | ) 452 | 453 | 454 | def _open_stream(stream, read_or_write): 455 | if hasattr(stream, read_or_write): 456 | return (False, stream) 457 | try: 458 | return (True, open(stream, read_or_write[0] + "b")) 459 | except TypeError: 460 | raise RuntimeError("expected open file or filename") 461 | 462 | 463 | class PlyElement(object): 464 | 465 | """ 466 | PLY file element. 467 | 468 | A client of this library doesn't normally need to instantiate this 469 | directly, so the following is only for the sake of documenting the 470 | internals. 471 | 472 | Creating a PlyElement instance is generally done in one of two ways: 473 | as a byproduct of PlyData.read (when reading a PLY file) and by 474 | PlyElement.describe (before writing a PLY file). 475 | 476 | """ 477 | 478 | def __init__(self, name, properties, count, comments=[]): 479 | """ 480 | This is not part of the public interface. The preferred methods 481 | of obtaining PlyElement instances are PlyData.read (to read from 482 | a file) and PlyElement.describe (to construct from a numpy 483 | array). 484 | 485 | """ 486 | _check_name(name) 487 | self._name = str(name) 488 | self._count = count 489 | 490 | self._properties = tuple(properties) 491 | self._index() 492 | 493 | self.comments = comments 494 | 495 | self._have_list = any(isinstance(p, PlyListProperty) for p in self.properties) 496 | 497 | @property 498 | def count(self): 499 | return self._count 500 | 501 | def _get_data(self): 502 | return self._data 503 | 504 | def _set_data(self, data): 505 | self._data = data 506 | self._count = len(data) 507 | self._check_sanity() 508 | 509 | data = property(_get_data, _set_data) 510 | 511 | def _check_sanity(self): 512 | for prop in self.properties: 513 | if prop.name not in self._data.dtype.fields: 514 | raise ValueError("dangling property %r" % prop.name) 515 | 516 | def _get_properties(self): 517 | return self._properties 518 | 519 | def _set_properties(self, properties): 520 | self._properties = tuple(properties) 521 | self._check_sanity() 522 | self._index() 523 | 524 | properties = property(_get_properties, _set_properties) 525 | 526 | def _get_comments(self): 527 | return list(self._comments) 528 | 529 | def _set_comments(self, comments): 530 | _check_comments(comments) 531 | self._comments = list(comments) 532 | 533 | comments = property(_get_comments, _set_comments) 534 | 535 | def _index(self): 536 | self._property_lookup = dict((prop.name, prop) for prop in self._properties) 537 | if len(self._property_lookup) != len(self._properties): 538 | raise ValueError("two properties with same name") 539 | 540 | def ply_property(self, name): 541 | return self._property_lookup[name] 542 | 543 | @property 544 | def name(self): 545 | return self._name 546 | 547 | def dtype(self, byte_order="="): 548 | """ 549 | Return the numpy dtype of the in-memory representation of the 550 | data. (If there are no list properties, and the PLY format is 551 | binary, then this also accurately describes the on-disk 552 | representation of the element.) 553 | 554 | """ 555 | return _np.dtype( 556 | [(prop.name, prop.dtype(byte_order)) for prop in self.properties] 557 | ) 558 | 559 | @staticmethod 560 | def describe(data, name, len_types={}, val_types={}, comments=[]): 561 | """ 562 | Construct a PlyElement from an array's metadata. 563 | 564 | len_types and val_types can be given as mappings from list 565 | property names to type strings (like 'u1', 'f4', etc., or 566 | 'int8', 'float32', etc.). These can be used to define the length 567 | and value types of list properties. List property lengths 568 | always default to type 'u1' (8-bit unsigned integer), and value 569 | types default to 'i4' (32-bit integer). 570 | 571 | """ 572 | if not isinstance(data, _np.ndarray): 573 | raise TypeError("only numpy arrays are supported") 574 | 575 | if len(data.shape) != 1: 576 | raise ValueError("only one-dimensional arrays are " "supported") 577 | 578 | count = len(data) 579 | 580 | properties = [] 581 | descr = data.dtype.descr 582 | 583 | for t in descr: 584 | if not isinstance(t[1], str): 585 | raise ValueError("nested records not supported") 586 | 587 | if not t[0]: 588 | raise ValueError("field with empty name") 589 | 590 | if len(t) != 2 or t[1][1] == "O": 591 | # non-scalar field, which corresponds to a list 592 | # property in PLY. 593 | 594 | if t[1][1] == "O": 595 | if len(t) != 2: 596 | raise ValueError("non-scalar object fields not " "supported") 597 | 598 | len_str = _data_type_reverse[len_types.get(t[0], "u1")] 599 | if t[1][1] == "O": 600 | val_type = val_types.get(t[0], "i4") 601 | val_str = _lookup_type(val_type) 602 | else: 603 | val_str = _lookup_type(t[1][1:]) 604 | 605 | prop = PlyListProperty(t[0], len_str, val_str) 606 | else: 607 | val_str = _lookup_type(t[1][1:]) 608 | prop = PlyProperty(t[0], val_str) 609 | 610 | properties.append(prop) 611 | 612 | elt = PlyElement(name, properties, count, comments) 613 | elt.data = data 614 | 615 | return elt 616 | 617 | def _read(self, stream, text, byte_order, mmap): 618 | """ 619 | Read the actual data from a PLY file. 620 | 621 | """ 622 | dtype = self.dtype(byte_order) 623 | if text: 624 | self._read_txt(stream) 625 | elif mmap and _can_mmap(stream) and not self._have_list: 626 | # Loading the data is straightforward. We will memory map 627 | # the file in copy-on-write mode. 628 | num_bytes = self.count * dtype.itemsize 629 | offset = stream.tell() 630 | stream.seek(0, 2) 631 | max_bytes = stream.tell() - offset 632 | if max_bytes < num_bytes: 633 | raise PlyElementParseError( 634 | "early end-of-file", self, max_bytes // dtype.itemsize 635 | ) 636 | self._data = _np.memmap(stream, dtype, "c", offset, self.count) 637 | # Fix stream position 638 | stream.seek(offset + self.count * dtype.itemsize) 639 | else: 640 | # A simple load is impossible. 641 | self._read_bin(stream, byte_order) 642 | 643 | self._check_sanity() 644 | 645 | def _write(self, stream, text, byte_order): 646 | """ 647 | Write the data to a PLY file. 648 | 649 | """ 650 | if text: 651 | self._write_txt(stream) 652 | else: 653 | if self._have_list: 654 | # There are list properties, so serialization is 655 | # slightly complicated. 656 | self._write_bin(stream, byte_order) 657 | else: 658 | # no list properties, so serialization is 659 | # straightforward. 660 | stream.write(self.data.astype(self.dtype(byte_order), copy=False).data) 661 | 662 | def _read_txt(self, stream): 663 | """ 664 | Load a PLY element from an ASCII-format PLY file. The element 665 | may contain list properties. 666 | 667 | """ 668 | self._data = _np.empty(self.count, dtype=self.dtype()) 669 | 670 | k = 0 671 | for line in _islice(iter(stream.readline, b""), self.count): 672 | fields = iter(line.strip().split()) 673 | for prop in self.properties: 674 | try: 675 | self._data[prop.name][k] = prop._from_fields(fields) 676 | except StopIteration: 677 | raise PlyElementParseError("early end-of-line", self, k, prop) 678 | except ValueError: 679 | raise PlyElementParseError("malformed input", self, k, prop) 680 | try: 681 | next(fields) 682 | except StopIteration: 683 | pass 684 | else: 685 | raise PlyElementParseError("expected end-of-line", self, k) 686 | k += 1 687 | 688 | if k < self.count: 689 | del self._data 690 | raise PlyElementParseError("early end-of-file", self, k) 691 | 692 | def _write_txt(self, stream): 693 | """ 694 | Save a PLY element to an ASCII-format PLY file. The element may 695 | contain list properties. 696 | 697 | """ 698 | for rec in self.data: 699 | fields = [] 700 | for prop in self.properties: 701 | fields.extend(prop._to_fields(rec[prop.name])) 702 | 703 | _np.savetxt(stream, [fields], "%.18g", newline="\n") 704 | 705 | def _read_bin(self, stream, byte_order): 706 | """ 707 | Load a PLY element from a binary PLY file. The element may 708 | contain list properties. 709 | 710 | """ 711 | self._data = _np.empty(self.count, dtype=self.dtype(byte_order)) 712 | 713 | for k in _range(self.count): 714 | for prop in self.properties: 715 | try: 716 | self._data[prop.name][k] = prop._read_bin(stream, byte_order) 717 | except StopIteration: 718 | raise PlyElementParseError("early end-of-file", self, k, prop) 719 | 720 | def _write_bin(self, stream, byte_order): 721 | """ 722 | Save a PLY element to a binary PLY file. The element may 723 | contain list properties. 724 | 725 | """ 726 | for rec in self.data: 727 | for prop in self.properties: 728 | prop._write_bin(rec[prop.name], stream, byte_order) 729 | 730 | @property 731 | def header(self): 732 | """ 733 | Format this element's metadata as it would appear in a PLY 734 | header. 735 | 736 | """ 737 | lines = ["element %s %d" % (self.name, self.count)] 738 | 739 | # Some information is lost here, since all comments are placed 740 | # between the 'element' line and the first property definition. 741 | for c in self.comments: 742 | lines.append("comment " + c) 743 | 744 | lines.extend(list(map(str, self.properties))) 745 | 746 | return "\n".join(lines) 747 | 748 | def __len__(self): 749 | return self.count 750 | 751 | def __contains__(self, name): 752 | return name in self._property_lookup 753 | 754 | def __getitem__(self, key): 755 | return self.data[key] 756 | 757 | def __setitem__(self, key, value): 758 | self.data[key] = value 759 | 760 | def __str__(self): 761 | return self.header 762 | 763 | def __repr__(self): 764 | return "PlyElement(%r, %r, count=%d, comments=%r)" % ( 765 | self.name, 766 | self.properties, 767 | self.count, 768 | self.comments, 769 | ) 770 | 771 | 772 | def _check_comments(comments): 773 | for comment in comments: 774 | for char in comment: 775 | if not 0 <= ord(char) < 128: 776 | raise ValueError("non-ASCII character in comment") 777 | if char == "\n": 778 | raise ValueError("embedded newline in comment") 779 | 780 | 781 | class PlyProperty(object): 782 | 783 | """ 784 | PLY property description. This class is pure metadata; the data 785 | itself is contained in PlyElement instances. 786 | 787 | """ 788 | 789 | def __init__(self, name, val_dtype): 790 | _check_name(name) 791 | self._name = str(name) 792 | self.val_dtype = val_dtype 793 | 794 | def _get_val_dtype(self): 795 | return self._val_dtype 796 | 797 | def _set_val_dtype(self, val_dtype): 798 | self._val_dtype = _data_types[_lookup_type(val_dtype)] 799 | 800 | val_dtype = property(_get_val_dtype, _set_val_dtype) 801 | 802 | @property 803 | def name(self): 804 | return self._name 805 | 806 | def dtype(self, byte_order="="): 807 | """ 808 | Return the numpy dtype description for this property (as a tuple 809 | of strings). 810 | 811 | """ 812 | return byte_order + self.val_dtype 813 | 814 | def _from_fields(self, fields): 815 | """ 816 | Parse from generator. Raise StopIteration if the property could 817 | not be read. 818 | 819 | """ 820 | return _np.dtype(self.dtype()).type(next(fields)) 821 | 822 | def _to_fields(self, data): 823 | """ 824 | Return generator over one item. 825 | 826 | """ 827 | yield _np.dtype(self.dtype()).type(data) 828 | 829 | def _read_bin(self, stream, byte_order): 830 | """ 831 | Read data from a binary stream. Raise StopIteration if the 832 | property could not be read. 833 | 834 | """ 835 | try: 836 | return _read_array(stream, self.dtype(byte_order), 1)[0] 837 | except IndexError: 838 | raise StopIteration 839 | 840 | def _write_bin(self, data, stream, byte_order): 841 | """ 842 | Write data to a binary stream. 843 | 844 | """ 845 | _write_array(stream, _np.dtype(self.dtype(byte_order)).type(data)) 846 | 847 | def __str__(self): 848 | val_str = _data_type_reverse[self.val_dtype] 849 | return "property %s %s" % (val_str, self.name) 850 | 851 | def __repr__(self): 852 | return "PlyProperty(%r, %r)" % (self.name, _lookup_type(self.val_dtype)) 853 | 854 | 855 | class PlyListProperty(PlyProperty): 856 | 857 | """ 858 | PLY list property description. 859 | 860 | """ 861 | 862 | def __init__(self, name, len_dtype, val_dtype): 863 | PlyProperty.__init__(self, name, val_dtype) 864 | 865 | self.len_dtype = len_dtype 866 | 867 | def _get_len_dtype(self): 868 | return self._len_dtype 869 | 870 | def _set_len_dtype(self, len_dtype): 871 | self._len_dtype = _data_types[_lookup_type(len_dtype)] 872 | 873 | len_dtype = property(_get_len_dtype, _set_len_dtype) 874 | 875 | def dtype(self, byte_order="="): 876 | """ 877 | List properties always have a numpy dtype of "object". 878 | 879 | """ 880 | return "|O" 881 | 882 | def list_dtype(self, byte_order="="): 883 | """ 884 | Return the pair (len_dtype, val_dtype) (both numpy-friendly 885 | strings). 886 | 887 | """ 888 | return (byte_order + self.len_dtype, byte_order + self.val_dtype) 889 | 890 | def _from_fields(self, fields): 891 | (len_t, val_t) = self.list_dtype() 892 | 893 | n = int(_np.dtype(len_t).type(next(fields))) 894 | 895 | data = _np.loadtxt(list(_islice(fields, n)), val_t, ndmin=1) 896 | if len(data) < n: 897 | raise StopIteration 898 | 899 | return data 900 | 901 | def _to_fields(self, data): 902 | """ 903 | Return generator over the (numerical) PLY representation of the 904 | list data (length followed by actual data). 905 | 906 | """ 907 | (len_t, val_t) = self.list_dtype() 908 | 909 | data = _np.asarray(data, dtype=val_t).ravel() 910 | 911 | yield _np.dtype(len_t).type(data.size) 912 | for x in data: 913 | yield x 914 | 915 | def _read_bin(self, stream, byte_order): 916 | (len_t, val_t) = self.list_dtype(byte_order) 917 | 918 | try: 919 | n = _read_array(stream, _np.dtype(len_t), 1)[0] 920 | except IndexError: 921 | raise StopIteration 922 | 923 | data = _read_array(stream, _np.dtype(val_t), n) 924 | if len(data) < n: 925 | raise StopIteration 926 | 927 | return data 928 | 929 | def _write_bin(self, data, stream, byte_order): 930 | """ 931 | Write data to a binary stream. 932 | 933 | """ 934 | (len_t, val_t) = self.list_dtype(byte_order) 935 | 936 | data = _np.asarray(data, dtype=val_t).ravel() 937 | 938 | _write_array(stream, _np.array(data.size, dtype=len_t)) 939 | _write_array(stream, data) 940 | 941 | def __str__(self): 942 | len_str = _data_type_reverse[self.len_dtype] 943 | val_str = _data_type_reverse[self.val_dtype] 944 | return "property list %s %s %s" % (len_str, val_str, self.name) 945 | 946 | def __repr__(self): 947 | return "PlyListProperty(%r, %r, %r)" % ( 948 | self.name, 949 | _lookup_type(self.len_dtype), 950 | _lookup_type(self.val_dtype), 951 | ) 952 | 953 | 954 | def _check_name(name): 955 | for char in name: 956 | if not 0 <= ord(char) < 128: 957 | raise ValueError("non-ASCII character in name %r" % name) 958 | if char.isspace(): 959 | raise ValueError("space character(s) in name %r" % name) 960 | 961 | 962 | def _read_array(stream, dtype, n): 963 | try: 964 | size = int(_np.dtype(dtype).itemsize * n) 965 | return _np.frombuffer(stream.read(size), dtype) 966 | except Exception: 967 | raise StopIteration 968 | 969 | 970 | def _write_array(stream, array): 971 | stream.write(array.tostring()) 972 | 973 | 974 | def _can_mmap(stream): 975 | try: 976 | pos = stream.tell() 977 | try: 978 | _np.memmap(stream, "u1", "c") 979 | stream.seek(pos) 980 | return True 981 | except Exception: 982 | stream.seek(pos) 983 | return False 984 | except Exception: 985 | return False 986 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 IIT-PAVIS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Spatial Commonsense Graph for Object Localisation in Partial Scenes](http://fgiuliari.github.io/projects/SpatialCommonsenseGraph/) 2 | ### [Project Website](http://fgiuliari.github.io/projects/SpatialCommonsenseGraph/) 3 | 4 | 5 | 6 | ### This repo contains the code and the dataset for our paper **[Spatial Commonsense graph for Object Localisation in Partial Scenes](https://arxiv.org/abs/2203.05380), Accepted at CVPR 2022** 7 | 8 | 9 | ## Abstract 10 | ![teaser](images/teaser.png) 11 | We solve object localisation in partial scenes, a new problem of estimating the unknown position of an object (e.g. where is the bag?) given a partial 3D scan of a scene. The proposed solution is based on a novel scene graph model, the Spatial Commonsense Graph (SCG), where objects are the nodes and edges define pairwise distances between them, enriched by concept nodes and relationships from a commonsense knowledge base. This allows SCG to better generalise its spatial inference over unknown 3D scenes. The SCG is used to estimate the unknown position of the target object in two steps: first, we feed the SCG into a novel Proximity Prediction Network, a graph neural network that uses attention to perform distance prediction between the node representing the target object and the nodes representing the observed objects in the SCG; second, we propose a Localisation Module based on circular intersection to estimate the object position using all the predicted pairwise distances in order to be independent of any reference system. We create a new dataset of partially reconstructed scenes to benchmark our method and baselines for object localisation in partial scenes, where our proposed method achieves the best localisation performance. 12 | # Table of Contents 13 | 14 | - [Dataset Generation](#dataset-preparation) 15 | - [SCG- Object Localiser](#scg---object-localiser) 16 | 17 | 18 | *** 19 | ## Dataset generation 20 | 21 | The following section describe how to generate our dataset of partial pcd 22 | ![partial pcd](images/partial_scenes.png) 23 | ### Retrieve the necessary data from ScanNet 24 | 25 | First you need to sign the necessary agreement and obtain the **download-scannet.py** script to download the ScanNet data. 26 | To do so follow the instruction in the original [ScanNet repository](https://github.com/ScanNet/ScanNet). 27 | 28 | 29 | The necessary data to download is the **scannet_frames_25k.zip** and **Scannet v2** for both the training and validation set 30 | next you should organize your folder structure so that in main folder you have: 31 | ``` 32 | This Repo 33 | | 34 | |--scannet_25_k\ 35 | | | 36 | | |--scene0000_00\ 37 | | | | 38 | | | |--color\ 39 | | | | |-- 0000000.jpg 40 | | | | |-- 0000100.jpg 41 | | | | |-- ... 42 | | | |-- depth\ 43 | | | | |-- 0000000.jpg 44 | | | | |-- 0000100.jpg 45 | | | | |-- ... 46 | | | |-- instance\ 47 | | | | |-- 0000000.jpg 48 | | | | |-- 0000100.jpg 49 | | | | |-- ... 50 | | | |-- label\ 51 | | | | |-- 0000000.jpg 52 | | | | |-- 0000100.jpg 53 | | | | |-- ... 54 | | | |-- etc 55 | | | |-- etc 56 | | |--scene0000_01\ 57 | | | |-- color\ 58 | | | |-- depth\ 59 | | | |-- ...\ 60 | | ... 61 | | 62 | |--scans\ 63 | | |--scene0000_00\ 64 | | | |-- scene0000_00.txt 65 | | | |-- scene0000_00.aggregation.json 66 | | | |-- scene0000_00_vh_clean_2.0.010000.segs.json 67 | | | |-- scene0000_00_vh_clean_2.labels.ply 68 | | | |-- scene0000_00_vh_clean_2.ply 69 | | | 70 | | |--scene0000_01\ 71 | | | |-- scene0000_01.txt 72 | | | |-- scene0000_01.aggregation.json 73 | | | |-- scene0000_01_vh_clean_2.0.010000.segs.json 74 | | | |-- scene0000_01_vh_clean_2.labels.ply 75 | | | |-- scene0000_01_vh_clean_2.ply 76 | 77 | ``` 78 | 79 | ### Processing the data 80 | 81 | To process the data we will use [ray](https://docs.ray.io/en/latest/index.html) to handle the multiprocessing on multiple clusters. Ray can also be used on single machine. If you have troubles running it, it should be easy to convert the code to use the python standard multiprocessing library. 82 | 83 | This processing steps will take some hours to run, please be patient. 84 | 85 | #### 1) Preparing the annotated ply files 86 | 87 | In this step we will create the scene pcd with semantic and instance information (the one provided by scannet do not directly provide the information regarding the instances) 88 | 89 | ```bash 90 | python DATASET_GENERATION/1.created_annotated_ply.py 91 | ``` 92 | 93 | 94 | #### 2) Creating the point cloud for the partial scenes 95 | 96 | ```bash 97 | python DATASET_GENERATION/2.create_partial_pcd.py 98 | ``` 99 | 100 | 101 | #### 3) Create the jsons 102 | 103 | ```bash 104 | python DATASET_GENERATION/3.create_partial_jsons 105 | ``` 106 | 107 | ### Conceptnet data 108 | 109 | 110 | The data folder contains the files with the information to create the SCGs 111 | 112 | . **cnet_data.pth** - contains a dict with the related concepts for each object class 113 | 114 | . **cnet_embeddings.pth** - contains a dict with the pretrained embedding for each related concept 115 | 116 | . **nyu_labels.pth** - contains the mapping from the object class name to a unique integer 117 | 118 | . **cnet_labels.pth** - contains the mapping from the concept name to a unique integer 119 | 120 | *** 121 | ## SCG - Object Localiser 122 | ![scgOL](images/arch2.png) 123 | 124 | Code for our **SCG-OL** will be released soon! 125 | -------------------------------------------------------------------------------- /data/cnet_data.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/data/cnet_data.pth -------------------------------------------------------------------------------- /data/cnet_embeddings.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/data/cnet_embeddings.pth -------------------------------------------------------------------------------- /data/cnet_labels.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/data/cnet_labels.pth -------------------------------------------------------------------------------- /data/nyu_labels.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/data/nyu_labels.pth -------------------------------------------------------------------------------- /data/train.txt: -------------------------------------------------------------------------------- 1 | scene0000_00 2 | scene0000_01 3 | scene0000_02 4 | scene0001_00 5 | scene0001_01 6 | scene0002_00 7 | scene0002_01 8 | scene0003_00 9 | scene0003_01 10 | scene0003_02 11 | scene0004_00 12 | scene0005_00 13 | scene0005_01 14 | scene0006_00 15 | scene0006_01 16 | scene0006_02 17 | scene0007_00 18 | scene0008_00 19 | scene0009_00 20 | scene0009_01 21 | scene0009_02 22 | scene0010_00 23 | scene0010_01 24 | scene0012_00 25 | scene0012_01 26 | scene0012_02 27 | scene0013_00 28 | scene0013_01 29 | scene0013_02 30 | scene0014_00 31 | scene0016_00 32 | scene0016_01 33 | scene0016_02 34 | scene0017_00 35 | scene0017_01 36 | scene0017_02 37 | scene0018_00 38 | scene0020_00 39 | scene0020_01 40 | scene0021_00 41 | scene0022_00 42 | scene0022_01 43 | scene0023_00 44 | scene0024_00 45 | scene0024_01 46 | scene0024_02 47 | scene0026_00 48 | scene0027_00 49 | scene0027_01 50 | scene0027_02 51 | scene0028_00 52 | scene0029_00 53 | scene0029_01 54 | scene0029_02 55 | scene0031_00 56 | scene0031_01 57 | scene0031_02 58 | scene0032_00 59 | scene0032_01 60 | scene0033_00 61 | scene0034_00 62 | scene0034_01 63 | scene0034_02 64 | scene0035_00 65 | scene0035_01 66 | scene0036_00 67 | scene0036_01 68 | scene0037_00 69 | scene0038_00 70 | scene0038_01 71 | scene0038_02 72 | scene0039_00 73 | scene0039_01 74 | scene0040_00 75 | scene0040_01 76 | scene0041_00 77 | scene0041_01 78 | scene0042_00 79 | scene0042_01 80 | scene0042_02 81 | scene0043_00 82 | scene0043_01 83 | scene0044_00 84 | scene0044_01 85 | scene0044_02 86 | scene0045_00 87 | scene0045_01 88 | scene0047_00 89 | scene0048_00 90 | scene0048_01 91 | scene0049_00 92 | scene0051_00 93 | scene0051_01 94 | scene0051_02 95 | scene0051_03 96 | scene0052_00 97 | scene0052_01 98 | scene0052_02 99 | scene0053_00 100 | scene0054_00 101 | scene0055_00 102 | scene0055_01 103 | scene0055_02 104 | scene0056_00 105 | scene0056_01 106 | scene0057_00 107 | scene0057_01 108 | scene0058_00 109 | scene0058_01 110 | scene0059_00 111 | scene0059_01 112 | scene0059_02 113 | scene0060_00 114 | scene0060_01 115 | scene0061_00 116 | scene0061_01 117 | scene0062_00 118 | scene0062_01 119 | scene0062_02 120 | scene0065_00 121 | scene0065_01 122 | scene0065_02 123 | scene0066_00 124 | scene0067_00 125 | scene0067_01 126 | scene0067_02 127 | scene0068_00 128 | scene0068_01 129 | scene0069_00 130 | scene0070_00 131 | scene0071_00 132 | scene0072_00 133 | scene0072_01 134 | scene0072_02 135 | scene0073_00 136 | scene0073_01 137 | scene0073_02 138 | scene0073_03 139 | scene0074_00 140 | scene0074_01 141 | scene0074_02 142 | scene0075_00 143 | scene0076_00 144 | scene0078_00 145 | scene0078_01 146 | scene0078_02 147 | scene0079_00 148 | scene0079_01 149 | scene0080_00 150 | scene0080_01 151 | scene0080_02 152 | scene0082_00 153 | scene0083_00 154 | scene0083_01 155 | scene0085_00 156 | scene0085_01 157 | scene0087_00 158 | scene0087_01 159 | scene0087_02 160 | scene0089_00 161 | scene0089_01 162 | scene0089_02 163 | scene0090_00 164 | scene0091_00 165 | scene0092_00 166 | scene0092_01 167 | scene0092_02 168 | scene0092_03 169 | scene0092_04 170 | scene0093_00 171 | scene0093_01 172 | scene0093_02 173 | scene0094_00 174 | scene0096_00 175 | scene0096_01 176 | scene0096_02 177 | scene0097_00 178 | scene0098_00 179 | scene0098_01 180 | scene0099_00 181 | scene0099_01 182 | scene0101_00 183 | scene0101_01 184 | scene0101_02 185 | scene0101_03 186 | scene0101_04 187 | scene0101_05 188 | scene0102_00 189 | scene0102_01 190 | scene0103_00 191 | scene0103_01 192 | scene0104_00 193 | scene0105_00 194 | scene0105_01 195 | scene0105_02 196 | scene0106_00 197 | scene0106_01 198 | scene0106_02 199 | scene0107_00 200 | scene0108_00 201 | scene0109_00 202 | scene0109_01 203 | scene0110_00 204 | scene0110_01 205 | scene0110_02 206 | scene0111_00 207 | scene0111_01 208 | scene0111_02 209 | scene0112_00 210 | scene0112_01 211 | scene0112_02 212 | scene0113_00 213 | scene0113_01 214 | scene0114_00 215 | scene0114_01 216 | scene0114_02 217 | scene0115_00 218 | scene0115_01 219 | scene0115_02 220 | scene0116_00 221 | scene0116_01 222 | scene0116_02 223 | scene0117_00 224 | scene0118_00 225 | scene0118_01 226 | scene0118_02 227 | scene0119_00 228 | scene0120_00 229 | scene0120_01 230 | scene0121_00 231 | scene0121_01 232 | scene0121_02 233 | scene0122_00 234 | scene0122_01 235 | scene0123_00 236 | scene0123_01 237 | scene0123_02 238 | scene0124_00 239 | scene0124_01 240 | scene0125_00 241 | scene0126_00 242 | scene0126_01 243 | scene0126_02 244 | scene0127_00 245 | scene0127_01 246 | scene0128_00 247 | scene0129_00 248 | scene0130_00 249 | scene0132_00 250 | scene0132_01 251 | scene0132_02 252 | scene0133_00 253 | scene0134_00 254 | scene0134_01 255 | scene0134_02 256 | scene0135_00 257 | scene0136_00 258 | scene0136_01 259 | scene0136_02 260 | scene0137_00 261 | scene0137_01 262 | scene0137_02 263 | scene0138_00 264 | scene0140_00 265 | scene0140_01 266 | scene0141_00 267 | scene0141_01 268 | scene0141_02 269 | scene0142_00 270 | scene0142_01 271 | scene0143_00 272 | scene0143_01 273 | scene0143_02 274 | scene0145_00 275 | scene0147_00 276 | scene0147_01 277 | scene0148_00 278 | scene0150_00 279 | scene0150_01 280 | scene0150_02 281 | scene0151_00 282 | scene0151_01 283 | scene0152_00 284 | scene0152_01 285 | scene0152_02 286 | scene0154_00 287 | scene0155_00 288 | scene0155_01 289 | scene0155_02 290 | scene0156_00 291 | scene0157_00 292 | scene0157_01 293 | scene0158_00 294 | scene0158_01 295 | scene0158_02 296 | scene0159_00 297 | scene0160_00 298 | scene0160_01 299 | scene0160_02 300 | scene0160_03 301 | scene0160_04 302 | scene0161_00 303 | scene0161_01 304 | scene0161_02 305 | scene0162_00 306 | scene0163_00 307 | scene0163_01 308 | scene0165_00 309 | scene0165_01 310 | scene0165_02 311 | scene0166_00 312 | scene0166_01 313 | scene0166_02 314 | scene0167_00 315 | scene0168_00 316 | scene0168_01 317 | scene0168_02 318 | scene0170_00 319 | scene0170_01 320 | scene0170_02 321 | scene0171_00 322 | scene0171_01 323 | scene0172_00 324 | scene0172_01 325 | scene0173_00 326 | scene0173_01 327 | scene0173_02 328 | scene0174_00 329 | scene0174_01 330 | scene0175_00 331 | scene0176_00 332 | scene0177_00 333 | scene0177_01 334 | scene0177_02 335 | scene0178_00 336 | scene0179_00 337 | scene0180_00 338 | scene0181_00 339 | scene0181_01 340 | scene0181_02 341 | scene0181_03 342 | scene0182_00 343 | scene0182_01 344 | scene0182_02 345 | scene0183_00 346 | scene0184_00 347 | scene0185_00 348 | scene0186_00 349 | scene0186_01 350 | scene0188_00 351 | scene0189_00 352 | scene0190_00 353 | scene0191_00 354 | scene0191_01 355 | scene0191_02 356 | scene0192_00 357 | scene0192_01 358 | scene0192_02 359 | scene0194_00 360 | scene0195_00 361 | scene0195_01 362 | scene0195_02 363 | scene0197_00 364 | scene0197_01 365 | scene0197_02 366 | scene0198_00 367 | scene0199_00 368 | scene0200_00 369 | scene0200_01 370 | scene0200_02 371 | scene0201_00 372 | scene0201_01 373 | scene0201_02 374 | scene0202_00 375 | scene0204_00 376 | scene0204_01 377 | scene0204_02 378 | scene0205_00 379 | scene0205_01 380 | scene0205_02 381 | scene0206_00 382 | scene0206_01 383 | scene0206_02 384 | scene0209_00 385 | scene0209_01 386 | scene0209_02 387 | scene0210_00 388 | scene0210_01 389 | scene0211_00 390 | scene0211_01 391 | scene0211_02 392 | scene0211_03 393 | scene0212_00 394 | scene0212_01 395 | scene0212_02 396 | scene0213_00 397 | scene0214_00 398 | scene0214_01 399 | scene0214_02 400 | scene0215_00 401 | scene0215_01 402 | scene0216_00 403 | scene0218_00 404 | scene0218_01 405 | scene0219_00 406 | scene0220_00 407 | scene0220_01 408 | scene0220_02 409 | scene0223_00 410 | scene0223_01 411 | scene0223_02 412 | scene0224_00 413 | scene0225_00 414 | scene0226_00 415 | scene0226_01 416 | scene0227_00 417 | scene0228_00 418 | scene0229_00 419 | scene0229_01 420 | scene0229_02 421 | scene0230_00 422 | scene0232_00 423 | scene0232_01 424 | scene0232_02 425 | scene0233_00 426 | scene0233_01 427 | scene0234_00 428 | scene0235_00 429 | scene0236_00 430 | scene0236_01 431 | scene0237_00 432 | scene0237_01 433 | scene0238_00 434 | scene0238_01 435 | scene0239_00 436 | scene0239_01 437 | scene0239_02 438 | scene0240_00 439 | scene0241_00 440 | scene0241_01 441 | scene0241_02 442 | scene0242_00 443 | scene0242_01 444 | scene0242_02 445 | scene0243_00 446 | scene0244_00 447 | scene0244_01 448 | scene0245_00 449 | scene0247_00 450 | scene0247_01 451 | scene0248_00 452 | scene0248_01 453 | scene0248_02 454 | scene0250_00 455 | scene0250_01 456 | scene0250_02 457 | scene0252_00 458 | scene0253_00 459 | scene0254_00 460 | scene0254_01 461 | scene0255_00 462 | scene0255_01 463 | scene0255_02 464 | scene0258_00 465 | scene0259_00 466 | scene0259_01 467 | scene0260_00 468 | scene0260_01 469 | scene0260_02 470 | scene0261_00 471 | scene0261_01 472 | scene0261_02 473 | scene0261_03 474 | scene0262_00 475 | scene0262_01 476 | scene0263_00 477 | scene0263_01 478 | scene0264_00 479 | scene0264_01 480 | scene0264_02 481 | scene0265_00 482 | scene0265_01 483 | scene0265_02 484 | scene0266_00 485 | scene0266_01 486 | scene0267_00 487 | scene0268_00 488 | scene0268_01 489 | scene0268_02 490 | scene0269_00 491 | scene0269_01 492 | scene0269_02 493 | scene0270_00 494 | scene0270_01 495 | scene0270_02 496 | scene0271_00 497 | scene0271_01 498 | scene0272_00 499 | scene0272_01 500 | scene0273_00 501 | scene0273_01 502 | scene0274_00 503 | scene0274_01 504 | scene0274_02 505 | scene0275_00 506 | scene0276_00 507 | scene0276_01 508 | scene0279_00 509 | scene0279_01 510 | scene0279_02 511 | scene0280_00 512 | scene0280_01 513 | scene0280_02 514 | scene0281_00 515 | scene0282_00 516 | scene0282_01 517 | scene0282_02 518 | scene0283_00 519 | scene0284_00 520 | scene0285_00 521 | scene0286_00 522 | scene0286_01 523 | scene0286_02 524 | scene0286_03 525 | scene0287_00 526 | scene0288_00 527 | scene0288_01 528 | scene0288_02 529 | scene0289_00 530 | scene0289_01 531 | scene0290_00 532 | scene0291_00 533 | scene0291_01 534 | scene0291_02 535 | scene0292_00 536 | scene0292_01 537 | scene0293_00 538 | scene0293_01 539 | scene0294_00 540 | scene0294_01 541 | scene0294_02 542 | scene0295_00 543 | scene0295_01 544 | scene0296_00 545 | scene0296_01 546 | scene0297_00 547 | scene0297_01 548 | scene0297_02 549 | scene0298_00 550 | scene0299_00 551 | scene0299_01 552 | scene0301_00 553 | scene0301_01 554 | scene0301_02 555 | scene0302_00 556 | scene0302_01 557 | scene0303_00 558 | scene0303_01 559 | scene0303_02 560 | scene0305_00 561 | scene0305_01 562 | scene0306_00 563 | scene0306_01 564 | scene0308_00 565 | scene0309_00 566 | scene0309_01 567 | scene0310_00 568 | scene0310_01 569 | scene0310_02 570 | scene0311_00 571 | scene0312_00 572 | scene0312_01 573 | scene0312_02 574 | scene0313_00 575 | scene0313_01 576 | scene0313_02 577 | scene0315_00 578 | scene0317_00 579 | scene0317_01 580 | scene0318_00 581 | scene0319_00 582 | scene0320_00 583 | scene0320_01 584 | scene0320_02 585 | scene0320_03 586 | scene0321_00 587 | scene0322_00 588 | scene0323_00 589 | scene0323_01 590 | scene0324_00 591 | scene0324_01 592 | scene0325_00 593 | scene0325_01 594 | scene0326_00 595 | scene0327_00 596 | scene0330_00 597 | scene0331_00 598 | scene0331_01 599 | scene0332_00 600 | scene0332_01 601 | scene0332_02 602 | scene0333_00 603 | scene0335_00 604 | scene0335_01 605 | scene0335_02 606 | scene0336_00 607 | scene0336_01 608 | scene0337_00 609 | scene0337_01 610 | scene0337_02 611 | scene0339_00 612 | scene0340_00 613 | scene0340_01 614 | scene0340_02 615 | scene0341_00 616 | scene0341_01 617 | scene0344_00 618 | scene0344_01 619 | scene0345_00 620 | scene0345_01 621 | scene0346_00 622 | scene0346_01 623 | scene0347_00 624 | scene0347_01 625 | scene0347_02 626 | scene0348_00 627 | scene0348_01 628 | scene0348_02 629 | scene0349_00 630 | scene0349_01 631 | scene0350_00 632 | scene0350_01 633 | scene0350_02 634 | scene0352_00 635 | scene0352_01 636 | scene0352_02 637 | scene0358_00 638 | scene0358_01 639 | scene0358_02 640 | scene0359_00 641 | scene0359_01 642 | scene0360_00 643 | scene0361_00 644 | scene0361_01 645 | scene0361_02 646 | scene0362_00 647 | scene0362_01 648 | scene0362_02 649 | scene0362_03 650 | scene0363_00 651 | scene0364_00 652 | scene0364_01 653 | scene0365_00 654 | scene0365_01 655 | scene0365_02 656 | scene0366_00 657 | scene0367_00 658 | scene0367_01 659 | scene0368_00 660 | scene0368_01 661 | scene0369_00 662 | scene0369_01 663 | scene0369_02 664 | scene0370_00 665 | scene0370_01 666 | scene0370_02 667 | scene0371_00 668 | scene0371_01 669 | scene0372_00 670 | scene0373_00 671 | scene0373_01 672 | scene0374_00 673 | scene0375_00 674 | scene0375_01 675 | scene0375_02 676 | scene0376_00 677 | scene0376_01 678 | scene0376_02 679 | scene0379_00 680 | scene0380_00 681 | scene0380_01 682 | scene0380_02 683 | scene0381_00 684 | scene0381_01 685 | scene0381_02 686 | scene0383_00 687 | scene0383_01 688 | scene0383_02 689 | scene0384_00 690 | scene0385_00 691 | scene0385_01 692 | scene0385_02 693 | scene0386_00 694 | scene0387_00 695 | scene0387_01 696 | scene0387_02 697 | scene0388_00 698 | scene0388_01 699 | scene0390_00 700 | scene0391_00 701 | scene0392_00 702 | scene0392_01 703 | scene0392_02 704 | scene0393_00 705 | scene0393_01 706 | scene0393_02 707 | scene0394_00 708 | scene0394_01 709 | scene0395_00 710 | scene0395_01 711 | scene0395_02 712 | scene0396_00 713 | scene0396_01 714 | scene0396_02 715 | scene0397_00 716 | scene0397_01 717 | scene0398_00 718 | scene0398_01 719 | scene0399_00 720 | scene0399_01 721 | scene0400_00 722 | scene0400_01 723 | scene0401_00 724 | scene0402_00 725 | scene0403_00 726 | scene0403_01 727 | scene0404_00 728 | scene0404_01 729 | scene0404_02 730 | scene0405_00 731 | scene0407_00 732 | scene0407_01 733 | scene0408_00 734 | scene0408_01 735 | scene0409_00 736 | scene0409_01 737 | scene0410_00 738 | scene0410_01 739 | scene0411_00 740 | scene0411_01 741 | scene0411_02 742 | scene0413_00 743 | scene0415_00 744 | scene0415_01 745 | scene0415_02 746 | scene0416_00 747 | scene0416_01 748 | scene0416_02 749 | scene0416_03 750 | scene0416_04 751 | scene0417_00 752 | scene0418_00 753 | scene0418_01 754 | scene0418_02 755 | scene0419_00 756 | scene0419_01 757 | scene0419_02 758 | scene0420_00 759 | scene0420_01 760 | scene0420_02 761 | scene0421_00 762 | scene0421_01 763 | scene0421_02 764 | scene0422_00 765 | scene0424_00 766 | scene0424_01 767 | scene0424_02 768 | scene0425_00 769 | scene0425_01 770 | scene0428_00 771 | scene0428_01 772 | scene0429_00 773 | scene0431_00 774 | scene0433_00 775 | scene0434_00 776 | scene0434_01 777 | scene0434_02 778 | scene0436_00 779 | scene0437_00 780 | scene0437_01 781 | scene0438_00 782 | scene0439_00 783 | scene0439_01 784 | scene0440_00 785 | scene0440_01 786 | scene0440_02 787 | scene0442_00 788 | scene0443_00 789 | scene0444_00 790 | scene0444_01 791 | scene0445_00 792 | scene0445_01 793 | scene0446_00 794 | scene0446_01 795 | scene0447_00 796 | scene0447_01 797 | scene0447_02 798 | scene0448_00 799 | scene0448_01 800 | scene0448_02 801 | scene0449_00 802 | scene0449_01 803 | scene0449_02 804 | scene0450_00 805 | scene0451_00 806 | scene0451_01 807 | scene0451_02 808 | scene0451_03 809 | scene0451_04 810 | scene0451_05 811 | scene0452_00 812 | scene0452_01 813 | scene0452_02 814 | scene0453_00 815 | scene0453_01 816 | scene0454_00 817 | scene0455_00 818 | scene0456_00 819 | scene0456_01 820 | scene0457_00 821 | scene0457_01 822 | scene0457_02 823 | scene0459_00 824 | scene0459_01 825 | scene0460_00 826 | scene0463_00 827 | scene0463_01 828 | scene0464_00 829 | scene0465_00 830 | scene0465_01 831 | scene0466_00 832 | scene0466_01 833 | scene0467_00 834 | scene0468_00 835 | scene0468_01 836 | scene0468_02 837 | scene0469_00 838 | scene0469_01 839 | scene0469_02 840 | scene0470_00 841 | scene0470_01 842 | scene0471_00 843 | scene0471_01 844 | scene0471_02 845 | scene0472_00 846 | scene0472_01 847 | scene0472_02 848 | scene0473_00 849 | scene0473_01 850 | scene0475_00 851 | scene0475_01 852 | scene0475_02 853 | scene0476_00 854 | scene0476_01 855 | scene0476_02 856 | scene0477_00 857 | scene0477_01 858 | scene0478_00 859 | scene0478_01 860 | scene0479_00 861 | scene0479_01 862 | scene0479_02 863 | scene0480_00 864 | scene0480_01 865 | scene0481_00 866 | scene0481_01 867 | scene0482_00 868 | scene0482_01 869 | scene0483_00 870 | scene0484_00 871 | scene0484_01 872 | scene0485_00 873 | scene0486_00 874 | scene0487_00 875 | scene0487_01 876 | scene0489_00 877 | scene0489_01 878 | scene0489_02 879 | scene0491_00 880 | scene0492_00 881 | scene0492_01 882 | scene0493_00 883 | scene0493_01 884 | scene0495_00 885 | scene0497_00 886 | scene0498_00 887 | scene0498_01 888 | scene0498_02 889 | scene0499_00 890 | scene0501_00 891 | scene0501_01 892 | scene0501_02 893 | scene0502_00 894 | scene0502_01 895 | scene0502_02 896 | scene0503_00 897 | scene0504_00 898 | scene0505_00 899 | scene0505_01 900 | scene0505_02 901 | scene0505_03 902 | scene0505_04 903 | scene0506_00 904 | scene0507_00 905 | scene0508_00 906 | scene0508_01 907 | scene0508_02 908 | scene0509_00 909 | scene0509_01 910 | scene0509_02 911 | scene0510_00 912 | scene0510_01 913 | scene0510_02 914 | scene0511_00 915 | scene0511_01 916 | scene0512_00 917 | scene0513_00 918 | scene0514_00 919 | scene0514_01 920 | scene0515_00 921 | scene0515_01 922 | scene0515_02 923 | scene0516_00 924 | scene0516_01 925 | scene0517_00 926 | scene0517_01 927 | scene0517_02 928 | scene0519_00 929 | scene0520_00 930 | scene0520_01 931 | scene0521_00 932 | scene0522_00 933 | scene0523_00 934 | scene0523_01 935 | scene0523_02 936 | scene0524_00 937 | scene0524_01 938 | scene0525_00 939 | scene0525_01 940 | scene0525_02 941 | scene0526_00 942 | scene0526_01 943 | scene0528_00 944 | scene0528_01 945 | scene0529_00 946 | scene0529_01 947 | scene0529_02 948 | scene0530_00 949 | scene0531_00 950 | scene0532_00 951 | scene0532_01 952 | scene0533_00 953 | scene0533_01 954 | scene0534_00 955 | scene0534_01 956 | scene0536_00 957 | scene0536_01 958 | scene0536_02 959 | scene0537_00 960 | scene0538_00 961 | scene0539_00 962 | scene0539_01 963 | scene0539_02 964 | scene0540_00 965 | scene0540_01 966 | scene0540_02 967 | scene0541_00 968 | scene0541_01 969 | scene0541_02 970 | scene0542_00 971 | scene0543_00 972 | scene0543_01 973 | scene0543_02 974 | scene0544_00 975 | scene0545_00 976 | scene0545_01 977 | scene0545_02 978 | scene0546_00 979 | scene0547_00 980 | scene0547_01 981 | scene0547_02 982 | scene0548_00 983 | scene0548_01 984 | scene0548_02 985 | scene0551_00 986 | scene0554_00 987 | scene0554_01 988 | scene0555_00 989 | scene0556_00 990 | scene0556_01 991 | scene0557_00 992 | scene0557_01 993 | scene0557_02 994 | scene0560_00 995 | scene0561_00 996 | scene0561_01 997 | scene0562_00 998 | scene0563_00 999 | scene0564_00 1000 | scene0566_00 1001 | scene0567_00 1002 | scene0567_01 1003 | scene0569_00 1004 | scene0569_01 1005 | scene0570_00 1006 | scene0570_01 1007 | scene0570_02 1008 | scene0571_00 1009 | scene0571_01 1010 | scene0572_00 1011 | scene0572_01 1012 | scene0572_02 1013 | scene0573_00 1014 | scene0573_01 1015 | scene0576_00 1016 | scene0576_01 1017 | scene0576_02 1018 | scene0577_00 1019 | scene0579_00 1020 | scene0579_01 1021 | scene0579_02 1022 | scene0581_00 1023 | scene0581_01 1024 | scene0581_02 1025 | scene0582_00 1026 | scene0582_01 1027 | scene0582_02 1028 | scene0584_00 1029 | scene0584_01 1030 | scene0584_02 1031 | scene0585_00 1032 | scene0585_01 1033 | scene0586_00 1034 | scene0586_01 1035 | scene0586_02 1036 | scene0587_00 1037 | scene0587_01 1038 | scene0587_02 1039 | scene0587_03 1040 | scene0588_00 1041 | scene0588_01 1042 | scene0588_02 1043 | scene0588_03 1044 | scene0589_00 1045 | scene0589_01 1046 | scene0589_02 1047 | scene0590_00 1048 | scene0590_01 1049 | scene0592_00 1050 | scene0592_01 1051 | scene0594_00 1052 | scene0596_00 1053 | scene0596_01 1054 | scene0596_02 1055 | scene0597_00 1056 | scene0597_01 1057 | scene0597_02 1058 | scene0600_00 1059 | scene0600_01 1060 | scene0600_02 1061 | scene0601_00 1062 | scene0601_01 1063 | scene0602_00 1064 | scene0603_00 1065 | scene0603_01 1066 | scene0604_00 1067 | scene0604_01 1068 | scene0604_02 1069 | scene0605_00 1070 | scene0605_01 1071 | scene0610_00 1072 | scene0610_01 1073 | scene0610_02 1074 | scene0611_00 1075 | scene0611_01 1076 | scene0612_00 1077 | scene0612_01 1078 | scene0613_00 1079 | scene0613_01 1080 | scene0613_02 1081 | scene0614_00 1082 | scene0614_01 1083 | scene0614_02 1084 | scene0615_00 1085 | scene0615_01 1086 | scene0617_00 1087 | scene0619_00 1088 | scene0620_00 1089 | scene0620_01 1090 | scene0622_00 1091 | scene0622_01 1092 | scene0623_00 1093 | scene0623_01 1094 | scene0624_00 1095 | scene0625_00 1096 | scene0625_01 1097 | scene0626_00 1098 | scene0626_01 1099 | scene0626_02 1100 | scene0627_00 1101 | scene0627_01 1102 | scene0628_00 1103 | scene0628_01 1104 | scene0628_02 1105 | scene0630_00 1106 | scene0630_01 1107 | scene0630_02 1108 | scene0630_03 1109 | scene0630_04 1110 | scene0630_05 1111 | scene0630_06 1112 | scene0631_00 1113 | scene0631_01 1114 | scene0631_02 1115 | scene0632_00 1116 | scene0634_00 1117 | scene0635_00 1118 | scene0635_01 1119 | scene0636_00 1120 | scene0637_00 1121 | scene0638_00 1122 | scene0639_00 1123 | scene0640_00 1124 | scene0640_01 1125 | scene0640_02 1126 | scene0641_00 1127 | scene0642_00 1128 | scene0642_01 1129 | scene0642_02 1130 | scene0642_03 1131 | scene0646_00 1132 | scene0646_01 1133 | scene0646_02 1134 | scene0649_00 1135 | scene0649_01 1136 | scene0650_00 1137 | scene0654_00 1138 | scene0654_01 1139 | scene0656_00 1140 | scene0656_01 1141 | scene0656_02 1142 | scene0656_03 1143 | scene0657_00 1144 | scene0659_00 1145 | scene0659_01 1146 | scene0661_00 1147 | scene0662_00 1148 | scene0662_01 1149 | scene0662_02 1150 | scene0666_00 1151 | scene0666_01 1152 | scene0666_02 1153 | scene0667_00 1154 | scene0667_01 1155 | scene0667_02 1156 | scene0668_00 1157 | scene0669_00 1158 | scene0669_01 1159 | scene0672_00 1160 | scene0672_01 1161 | scene0673_00 1162 | scene0673_01 1163 | scene0673_02 1164 | scene0673_03 1165 | scene0673_04 1166 | scene0673_05 1167 | scene0674_00 1168 | scene0674_01 1169 | scene0675_00 1170 | scene0675_01 1171 | scene0676_00 1172 | scene0676_01 1173 | scene0677_00 1174 | scene0677_01 1175 | scene0677_02 1176 | scene0679_00 1177 | scene0679_01 1178 | scene0680_00 1179 | scene0680_01 1180 | scene0681_00 1181 | scene0682_00 1182 | scene0683_00 1183 | scene0687_00 1184 | scene0688_00 1185 | scene0691_00 1186 | scene0691_01 1187 | scene0692_00 1188 | scene0692_01 1189 | scene0692_02 1190 | scene0692_03 1191 | scene0692_04 1192 | scene0694_00 1193 | scene0694_01 1194 | scene0698_00 1195 | scene0698_01 1196 | scene0703_00 1197 | scene0703_01 1198 | scene0705_00 1199 | scene0705_01 1200 | scene0705_02 1201 | scene0706_00 1202 | -------------------------------------------------------------------------------- /data/val.txt: -------------------------------------------------------------------------------- 1 | scene0011_00 2 | scene0011_01 3 | scene0015_00 4 | scene0019_00 5 | scene0019_01 6 | scene0025_00 7 | scene0025_01 8 | scene0025_02 9 | scene0030_00 10 | scene0030_01 11 | scene0030_02 12 | scene0046_00 13 | scene0046_01 14 | scene0046_02 15 | scene0050_00 16 | scene0050_01 17 | scene0050_02 18 | scene0063_00 19 | scene0064_00 20 | scene0064_01 21 | scene0077_00 22 | scene0077_01 23 | scene0081_00 24 | scene0081_01 25 | scene0081_02 26 | scene0084_00 27 | scene0084_01 28 | scene0084_02 29 | scene0086_00 30 | scene0086_01 31 | scene0086_02 32 | scene0088_00 33 | scene0088_01 34 | scene0088_02 35 | scene0088_03 36 | scene0095_00 37 | scene0095_01 38 | scene0100_00 39 | scene0100_01 40 | scene0100_02 41 | scene0131_00 42 | scene0131_01 43 | scene0131_02 44 | scene0139_00 45 | scene0144_00 46 | scene0144_01 47 | scene0146_00 48 | scene0146_01 49 | scene0146_02 50 | scene0149_00 51 | scene0153_00 52 | scene0153_01 53 | scene0164_00 54 | scene0164_01 55 | scene0164_02 56 | scene0164_03 57 | scene0169_00 58 | scene0169_01 59 | scene0187_00 60 | scene0187_01 61 | scene0193_00 62 | scene0193_01 63 | scene0196_00 64 | scene0203_00 65 | scene0203_01 66 | scene0203_02 67 | scene0207_00 68 | scene0207_01 69 | scene0207_02 70 | scene0208_00 71 | scene0217_00 72 | scene0221_00 73 | scene0221_01 74 | scene0222_00 75 | scene0222_01 76 | scene0231_00 77 | scene0231_01 78 | scene0231_02 79 | scene0246_00 80 | scene0249_00 81 | scene0251_00 82 | scene0256_00 83 | scene0256_01 84 | scene0256_02 85 | scene0257_00 86 | scene0277_00 87 | scene0277_01 88 | scene0277_02 89 | scene0278_00 90 | scene0278_01 91 | scene0300_00 92 | scene0300_01 93 | scene0304_00 94 | scene0307_00 95 | scene0307_01 96 | scene0307_02 97 | scene0314_00 98 | scene0316_00 99 | scene0328_00 100 | scene0329_00 101 | scene0329_01 102 | scene0329_02 103 | scene0334_00 104 | scene0334_01 105 | scene0334_02 106 | scene0338_00 107 | scene0338_01 108 | scene0338_02 109 | scene0342_00 110 | scene0343_00 111 | scene0351_00 112 | scene0351_01 113 | scene0353_00 114 | scene0353_01 115 | scene0353_02 116 | scene0354_00 117 | scene0355_00 118 | scene0355_01 119 | scene0356_00 120 | scene0356_01 121 | scene0356_02 122 | scene0357_00 123 | scene0357_01 124 | scene0377_00 125 | scene0377_01 126 | scene0377_02 127 | scene0378_00 128 | scene0378_01 129 | scene0378_02 130 | scene0382_00 131 | scene0382_01 132 | scene0389_00 133 | scene0406_00 134 | scene0406_01 135 | scene0406_02 136 | scene0412_00 137 | scene0412_01 138 | scene0414_00 139 | scene0423_00 140 | scene0423_01 141 | scene0423_02 142 | scene0426_00 143 | scene0426_01 144 | scene0426_02 145 | scene0426_03 146 | scene0427_00 147 | scene0430_00 148 | scene0430_01 149 | scene0432_00 150 | scene0432_01 151 | scene0435_00 152 | scene0435_01 153 | scene0435_02 154 | scene0435_03 155 | scene0441_00 156 | scene0458_00 157 | scene0458_01 158 | scene0461_00 159 | scene0462_00 160 | scene0474_00 161 | scene0474_01 162 | scene0474_02 163 | scene0474_03 164 | scene0474_04 165 | scene0474_05 166 | scene0488_00 167 | scene0488_01 168 | scene0490_00 169 | scene0494_00 170 | scene0496_00 171 | scene0500_00 172 | scene0500_01 173 | scene0518_00 174 | scene0527_00 175 | scene0535_00 176 | scene0549_00 177 | scene0549_01 178 | scene0550_00 179 | scene0552_00 180 | scene0552_01 181 | scene0553_00 182 | scene0553_01 183 | scene0553_02 184 | scene0558_00 185 | scene0558_01 186 | scene0558_02 187 | scene0559_00 188 | scene0559_01 189 | scene0559_02 190 | scene0565_00 191 | scene0568_00 192 | scene0568_01 193 | scene0568_02 194 | scene0574_00 195 | scene0574_01 196 | scene0574_02 197 | scene0575_00 198 | scene0575_01 199 | scene0575_02 200 | scene0578_00 201 | scene0578_01 202 | scene0578_02 203 | scene0580_00 204 | scene0580_01 205 | scene0583_00 206 | scene0583_01 207 | scene0583_02 208 | scene0591_00 209 | scene0591_01 210 | scene0591_02 211 | scene0593_00 212 | scene0593_01 213 | scene0595_00 214 | scene0598_00 215 | scene0598_01 216 | scene0598_02 217 | scene0599_00 218 | scene0599_01 219 | scene0599_02 220 | scene0606_00 221 | scene0606_01 222 | scene0606_02 223 | scene0607_00 224 | scene0607_01 225 | scene0608_00 226 | scene0608_01 227 | scene0608_02 228 | scene0609_00 229 | scene0609_01 230 | scene0609_02 231 | scene0609_03 232 | scene0616_00 233 | scene0616_01 234 | scene0618_00 235 | scene0621_00 236 | scene0629_00 237 | scene0629_01 238 | scene0629_02 239 | scene0633_00 240 | scene0633_01 241 | scene0643_00 242 | scene0644_00 243 | scene0645_00 244 | scene0645_01 245 | scene0645_02 246 | scene0647_00 247 | scene0647_01 248 | scene0648_00 249 | scene0648_01 250 | scene0651_00 251 | scene0651_01 252 | scene0651_02 253 | scene0652_00 254 | scene0653_00 255 | scene0653_01 256 | scene0655_00 257 | scene0655_01 258 | scene0655_02 259 | scene0658_00 260 | scene0660_00 261 | scene0663_00 262 | scene0663_01 263 | scene0663_02 264 | scene0664_00 265 | scene0664_01 266 | scene0664_02 267 | scene0665_00 268 | scene0665_01 269 | scene0670_00 270 | scene0670_01 271 | scene0671_00 272 | scene0671_01 273 | scene0678_00 274 | scene0678_01 275 | scene0678_02 276 | scene0684_00 277 | scene0684_01 278 | scene0685_00 279 | scene0685_01 280 | scene0685_02 281 | scene0686_00 282 | scene0686_01 283 | scene0686_02 284 | scene0689_00 285 | scene0690_00 286 | scene0690_01 287 | scene0693_00 288 | scene0693_01 289 | scene0693_02 290 | scene0695_00 291 | scene0695_01 292 | scene0695_02 293 | scene0695_03 294 | scene0696_00 295 | scene0696_01 296 | scene0696_02 297 | scene0697_00 298 | scene0697_01 299 | scene0697_02 300 | scene0697_03 301 | scene0699_00 302 | scene0700_00 303 | scene0700_01 304 | scene0700_02 305 | scene0701_00 306 | scene0701_01 307 | scene0701_02 308 | scene0702_00 309 | scene0702_01 310 | scene0702_02 311 | scene0704_00 312 | scene0704_01 313 | -------------------------------------------------------------------------------- /images/arch2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/images/arch2.png -------------------------------------------------------------------------------- /images/partial_scenes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/images/partial_scenes.png -------------------------------------------------------------------------------- /images/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IIT-PAVIS/SpatialCommonsenseGraph/2022d7d5ebed74872aa530ddd807a070b4445e99/images/teaser.png --------------------------------------------------------------------------------