├── openfungraph ├── __init__.py ├── dataset │ ├── __init__.py │ └── dataconfigs │ │ ├── fungraph3d │ │ └── fungraph3d.yaml │ │ └── scenefun3d │ │ └── scenefun3d.yaml ├── llava │ ├── __init__.py │ └── llava_model_16.py ├── scripts │ ├── __init__.py │ ├── ana_rigid_objs.py │ ├── pyviz3d_interactable_results.py │ ├── generate_part_gsa_results.py │ └── generate_gsa_results.py ├── slam │ ├── __init__.py │ ├── mapping.py │ ├── slam_classes.py │ └── cfslam_pipeline_batch.py ├── utils │ ├── __init__.py │ ├── general_utils.py │ ├── model_utils.py │ ├── vis.py │ ├── ious.py │ └── colmap.py ├── scenegraph │ ├── detection_fungraph3d.sh │ ├── detection_scenefun3d.sh │ └── GPTPrompt.py ├── configs │ └── slam_pipeline │ │ └── base.yaml └── eval │ ├── eval_node.py │ └── eval_triplet.py ├── assets ├── teaser.png └── teaser_top.jpg ├── setup.py ├── env_vars.bash.template ├── .gitignore └── README.md /openfungraph/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openfungraph/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openfungraph/llava/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openfungraph/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openfungraph/slam/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openfungraph/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangCYG/OpenFunGraph/HEAD/assets/teaser.png -------------------------------------------------------------------------------- /assets/teaser_top.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangCYG/OpenFunGraph/HEAD/assets/teaser_top.jpg -------------------------------------------------------------------------------- /openfungraph/dataset/dataconfigs/fungraph3d/fungraph3d.yaml: -------------------------------------------------------------------------------- 1 | dataset_name: 'fungraph3d' 2 | camera_params: 3 | image_height: 1440 4 | image_width: 1920 5 | fx: 1580 6 | fy: 1580 7 | cx: 950 8 | cy: 722 9 | png_depth_scale: 1000 #for depth image in png format 10 | crop_edge: 0 -------------------------------------------------------------------------------- /openfungraph/dataset/dataconfigs/scenefun3d/scenefun3d.yaml: -------------------------------------------------------------------------------- 1 | dataset_name: 'scenefun3d' 2 | camera_params: 3 | image_height: 1440 4 | image_width: 1920 5 | fx: 1592 6 | fy: 1592 7 | cx: 952 8 | cy: 742 9 | png_depth_scale: 1000 #for depth image in png format 10 | crop_edge: 0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='openfungraph', 5 | version='1.0.0', 6 | description='Open-Vocabulary Functional 3D Scene Graphs for Real-World Indoor Spaces', 7 | author='See https://openfungraph.github.io/', 8 | packages=find_packages(), 9 | ) -------------------------------------------------------------------------------- /env_vars.bash.template: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Example script setting up the rnv variables needed for running OpenFunGraph 3 | # Please adapt it to your own paths! 4 | 5 | cd openfungraph 6 | 7 | conda activate openfungraph 8 | 9 | export FG_FOLDER=../ 10 | 11 | export GSA_PATH=../Grounded-Segment-Anything 12 | 13 | export FUNGRAPH3D_ROOT= 14 | export FUNGRAPH3D_CONFIG_PATH=${FG_FOLDER}/openfungraph/dataset/dataconfigs/fungraph3d/fungraph3d.yaml 15 | export SCENEFUN3D_ROOT= # for SceneFun3D, it should be with dev / test 16 | export SCENEFUN3D_CONFIG_PATH=${FG_FOLDER}/openfungraph/dataset/dataconfigs/scenefun3d/scenefun3d.yaml 17 | 18 | export SCENE_NAME= 19 | 20 | export THRESHOLD=1.2 21 | 22 | export CLASS_SET=ram 23 | 24 | export OPENAI_API_KEY= 25 | 26 | -------------------------------------------------------------------------------- /openfungraph/scenegraph/detection_fungraph3d.sh: -------------------------------------------------------------------------------- 1 | # generate object 2D detection 2 | CUDA_VISIBLE_DEVICES=0 python scripts/generate_gsa_results.py --dataset_root $FUNGRAPH3D_ROOT --dataset_config $FUNGRAPH3D_CONFIG_PATH --scene_id $SCENE_NAME --class_set $CLASS_SET --box_threshold 0.25 --text_threshold 0.25 --stride 1 --add_bg_classes --accumu_classes --exp_suffix withbg_allclasses 3 | 4 | # fuse general objects 5 | python slam/cfslam_pipeline_batch.py dataset_root=$FUNGRAPH3D_ROOT dataset_config=$FUNGRAPH3D_CONFIG_PATH stride=1 scene_id=$SCENE_NAME spatial_sim_type=overlap mask_conf_threshold=0.3 match_method=sim_sum sim_threshold=${THRESHOLD} dbscan_eps=0.1 gsa_variant=ram_withbg_allclasses skip_bg=False max_bbox_area_ratio=0.9 merge_overlap_thresh=0.9 save_suffix=overlap_maskconf0.3_bbox0.9_simsum${THRESHOLD}_dbscan.1 merge_visual_sim_thresh=0.75 merge_text_sim_thresh=0.7 6 | 7 | # detect 2D parts 8 | CUDA_VISIBLE_DEVICES=0 python scripts/generate_part_gsa_results.py --dataset_root $FUNGRAPH3D_ROOT --dataset_config $FUNGRAPH3D_CONFIG_PATH --scene_id $SCENE_NAME --class_set $CLASS_SET --box_threshold 0.15 --text_threshold 0.15 --stride 1 --add_bg_classes --accumu_classes --exp_suffix withbg_allclasses 9 | 10 | # fuse parts 11 | python slam/cfslam_pipeline_batch.py dataset_root=$FUNGRAPH3D_ROOT dataset_config=$FUNGRAPH3D_CONFIG_PATH stride=1 scene_id=$SCENE_NAME spatial_sim_type=overlap mask_conf_threshold=0.15 match_method=sim_sum sim_threshold=${THRESHOLD} dbscan_eps=0.1 gsa_variant=ram_withbg_allclasses skip_bg=False max_bbox_area_ratio=0.1 save_suffix=overlap_maskconf0.15_bbox0.1_simsum${THRESHOLD}_dbscan.1_parts part_reg=True 12 | 13 | python scripts/ana_rigid_objs.py --result_path $FUNGRAPH3D_ROOT'/'$SCENE_NAME'/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.3_bbox0.9_simsum1.2_dbscan.1_post.pkl.gz' --part_result_path $FUNGRAPH3D_ROOT'/'$SCENE_NAME'/part/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.15_bbox0.1_simsum1.2_dbscan.1_parts_post.pkl.gz' -------------------------------------------------------------------------------- /openfungraph/scenegraph/detection_scenefun3d.sh: -------------------------------------------------------------------------------- 1 | # generate object 2D detection 2 | CUDA_VISIBLE_DEVICES=0 python scripts/generate_gsa_results.py --dataset_root $SCENEFUN3D_ROOT --dataset_config $SCENEFUN3D_CONFIG_PATH --scene_id $SCENE_NAME --class_set $CLASS_SET --box_threshold 0.25 --text_threshold 0.25 --stride 1 --add_bg_classes --accumu_classes --exp_suffix withbg_allclasses 3 | 4 | # fuse general objects 5 | python slam/cfslam_pipeline_batch.py dataset_root=$SCENEFUN3D_ROOT dataset_config=$SCENEFUN3D_CONFIG_PATH stride=1 scene_id=$SCENE_NAME spatial_sim_type=overlap mask_conf_threshold=0.3 match_method=sim_sum sim_threshold=${THRESHOLD} dbscan_eps=0.1 gsa_variant=ram_withbg_allclasses skip_bg=False max_bbox_area_ratio=0.9 merge_overlap_thresh=0.9 save_suffix=overlap_maskconf0.3_bbox0.9_simsum${THRESHOLD}_dbscan.1 merge_visual_sim_thresh=0.75 merge_text_sim_thresh=0.7 6 | 7 | # detect 2D parts 8 | CUDA_VISIBLE_DEVICES=0 python scripts/generate_part_gsa_results.py --dataset_root $SCENEFUN3D_ROOT --dataset_config $SCENEFUN3D_CONFIG_PATH --scene_id $SCENE_NAME --class_set $CLASS_SET --box_threshold 0.15 --text_threshold 0.15 --stride 1 --add_bg_classes --accumu_classes --exp_suffix withbg_allclasses 9 | 10 | # fuse parts 11 | python slam/cfslam_pipeline_batch.py dataset_root=$SCENEFUN3D_ROOT dataset_config=$SCENEFUN3D_CONFIG_PATH stride=1 scene_id=$SCENE_NAME spatial_sim_type=overlap mask_conf_threshold=0.15 match_method=sim_sum sim_threshold=${THRESHOLD} dbscan_eps=0.1 gsa_variant=ram_withbg_allclasses skip_bg=False max_bbox_area_ratio=0.1 save_suffix=overlap_maskconf0.15_bbox0.1_simsum${THRESHOLD}_dbscan.1_parts part_reg=True 12 | 13 | python scripts/ana_rigid_objs.py --result_path $SCENEFUN3D_ROOT'/'$SCENE_NAME'/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.3_bbox0.9_simsum1.2_dbscan.1_post.pkl.gz' --part_result_path $SCENEFUN3D_ROOT'/'$SCENE_NAME'/part/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.15_bbox0.1_simsum1.2_dbscan.1_parts_post.pkl.gz' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | datasets 2 | openfungraph/outputs 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | .pdm.toml 89 | 90 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 91 | __pypackages__/ 92 | 93 | # Celery stuff 94 | celerybeat-schedule 95 | celerybeat.pid 96 | 97 | # SageMath parsed files 98 | *.sage.py 99 | 100 | # Environments 101 | .env 102 | .venv 103 | env/ 104 | venv/ 105 | ENV/ 106 | env.bak/ 107 | venv.bak/ 108 | 109 | # Spyder project settings 110 | .spyderproject 111 | .spyproject 112 | 113 | # Rope project settings 114 | .ropeproject 115 | 116 | # mkdocs documentation 117 | /site 118 | 119 | # mypy 120 | .mypy_cache/ 121 | .dmypy.json 122 | dmypy.json 123 | 124 | # Pyre type checker 125 | .pyre/ 126 | 127 | # pytype static type analyzer 128 | .pytype/ 129 | 130 | # Cython debug symbols 131 | cython_debug/ 132 | 133 | -------------------------------------------------------------------------------- /openfungraph/configs/slam_pipeline/base.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | dataset_root: /home/kuwajerw/NAS3/MyStuff/Projects/gradslam-vlm/data/ai2thor/ 3 | dataset_config: /home/kuwajerw/repos/CFSLAM/cfslam/dataset/dataconfigs/ai2thor/ai2thor.yaml 4 | scene_id: train_3_interact 5 | start: 0 6 | end: -1 7 | stride: 1 8 | image_height: null # if null, it will be determined by dataconfig 9 | image_width: null # if null, it will be determined by dataconfig 10 | 11 | # Input detections 12 | gsa_variant: ram 13 | detection_folder_name: gsa_detections_${gsa_variant} 14 | det_vis_folder_name: gsa_vis_${gsa_variant} 15 | color_file_name: gsa_classes_${gsa_variant} 16 | 17 | device: cuda 18 | 19 | use_iou: !!bool True 20 | spatial_sim_type: iou # "iou", "giou", "overlap" 21 | phys_bias: 0.0 22 | match_method: "sep_thresh" # "sep_thresh", "sim_sum" 23 | # Only when match_method=="sep_thresh" 24 | semantic_threshold: 0.5 25 | physical_threshold: 0.5 26 | # Only when match_method=="sim_sum" 27 | sim_threshold: 0 28 | 29 | # For contain_number 30 | use_contain_number: !!bool False 31 | contain_area_thresh: 0.95 32 | contain_mismatch_penalty: 0.5 33 | 34 | # Selection criteria on the 2D masks 35 | mask_area_threshold: 25 # mask with pixel area less than this will be skipped 36 | mask_conf_threshold: 0.2 # mask with lower confidence score will be skipped 37 | max_bbox_area_ratio: 1.0 # boxes with larger areas than this will be skipped 38 | skip_bg: !!bool True 39 | min_points_threshold: 16 # projected and sampled pcd with less points will be skipped 40 | 41 | # point cloud processing 42 | downsample_voxel_size: 0.01 43 | dbscan_remove_noise: !!bool True 44 | dbscan_eps: 0.05 45 | dbscan_min_points: 10 46 | 47 | # Selection criteria of the fused object point cloud 48 | obj_min_points: 0 49 | obj_min_detections: 9 50 | 51 | # For merge_overlap_objects() function 52 | merge_overlap_thresh: 0.7 # -1 means do not perform the merge_overlap_objects() 53 | merge_visual_sim_thresh: 0.7 # Merge only if the visual similarity is larger 54 | merge_text_sim_thresh: 0.7 # Merge only if the text cosine sim is larger 55 | 56 | # Periodically perform post-process operations every k frame 57 | # -1 means not perform them during the run. They are performed at the end anyway. 58 | denoise_interval: 20 # Run DBSCAN every k frame. This operation is heavy 59 | filter_interval: -1 # Filter objects that have too few associations or are too small 60 | merge_interval: -1 # Merge objects based on geometric and semantic similarity 61 | 62 | # Output point cloud 63 | save_pcd: !!bool True 64 | save_suffix: exp 65 | 66 | # Visualization 67 | debug_render: !!bool False # If True, the vis.run() will be called and used for debugging 68 | class_agnostic: !!bool False # If set, the color will be set by instance, rather than most common class 69 | 70 | render_camera_path: "replica_room0.json" 71 | 72 | # part recognition 73 | part_reg: !!bool False -------------------------------------------------------------------------------- /openfungraph/slam/mapping.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | from openfungraph.slam.slam_classes import MapObjectList, DetectionList 5 | from openfungraph.utils.general_utils import Timer 6 | from openfungraph.utils.ious import ( 7 | compute_iou_batch, 8 | compute_giou_batch, 9 | compute_3d_iou_accuracte_batch, 10 | compute_3d_giou_accurate_batch, 11 | ) 12 | from openfungraph.slam.utils import ( 13 | merge_obj2_into_obj1, 14 | compute_overlap_matrix_2set 15 | ) 16 | 17 | def compute_spatial_similarities(cfg, detection_list: DetectionList, objects: MapObjectList) -> torch.Tensor: 18 | ''' 19 | Compute the spatial similarities between the detections and the objects 20 | 21 | Args: 22 | detection_list: a list of M detections 23 | objects: a list of N objects in the map 24 | Returns: 25 | A MxN tensor of spatial similarities 26 | ''' 27 | det_bboxes = detection_list.get_stacked_values_torch('bbox') 28 | obj_bboxes = objects.get_stacked_values_torch('bbox') 29 | 30 | if cfg.spatial_sim_type == "iou": 31 | spatial_sim = compute_iou_batch(det_bboxes, obj_bboxes) 32 | elif cfg.spatial_sim_type == "giou": 33 | spatial_sim = compute_giou_batch(det_bboxes, obj_bboxes) 34 | elif cfg.spatial_sim_type == "iou_accurate": 35 | spatial_sim = compute_3d_iou_accuracte_batch(det_bboxes, obj_bboxes) 36 | elif cfg.spatial_sim_type == "giou_accurate": 37 | spatial_sim = compute_3d_giou_accurate_batch(det_bboxes, obj_bboxes) 38 | elif cfg.spatial_sim_type == "overlap": 39 | spatial_sim = compute_overlap_matrix_2set(cfg, objects, detection_list) 40 | spatial_sim = torch.from_numpy(spatial_sim).T 41 | else: 42 | raise ValueError(f"Invalid spatial similarity type: {cfg.spatial_sim_type}") 43 | 44 | return spatial_sim 45 | 46 | def compute_visual_similarities(cfg, detection_list: DetectionList, objects: MapObjectList) -> torch.Tensor: 47 | ''' 48 | Compute the visual similarities between the detections and the objects 49 | 50 | Args: 51 | detection_list: a list of M detections 52 | objects: a list of N objects in the map 53 | Returns: 54 | A MxN tensor of visual similarities 55 | ''' 56 | det_fts = detection_list.get_stacked_values_torch('clip_ft') # (M, D) 57 | obj_fts = objects.get_stacked_values_torch('clip_ft') # (N, D) 58 | 59 | det_fts = det_fts.unsqueeze(-1) # (M, D, 1) 60 | obj_fts = obj_fts.T.unsqueeze(0) # (1, D, N) 61 | 62 | visual_sim = F.cosine_similarity(det_fts, obj_fts, dim=1) # (M, N) 63 | 64 | return visual_sim 65 | 66 | def aggregate_similarities(cfg, spatial_sim: torch.Tensor, visual_sim: torch.Tensor) -> torch.Tensor: 67 | ''' 68 | Aggregate spatial and visual similarities into a single similarity score 69 | 70 | Args: 71 | spatial_sim: a MxN tensor of spatial similarities 72 | visual_sim: a MxN tensor of visual similarities 73 | Returns: 74 | A MxN tensor of aggregated similarities 75 | ''' 76 | if cfg.match_method == "sim_sum": 77 | sims = (1 + cfg.phys_bias) * spatial_sim + (1 - cfg.phys_bias) * visual_sim # (M, N) 78 | else: 79 | raise ValueError(f"Unknown matching method: {cfg.match_method}") 80 | 81 | return sims 82 | 83 | def merge_detections_to_objects( 84 | cfg, 85 | detection_list: DetectionList, 86 | objects: MapObjectList, 87 | agg_sim: torch.Tensor 88 | ) -> MapObjectList: 89 | # Iterate through all detections and merge them into objects 90 | for i in range(agg_sim.shape[0]): 91 | # If not matched to any object, add it as a new object 92 | if agg_sim[i].max() == float('-inf'): 93 | objects.append(detection_list[i]) 94 | # Merge with most similar existing object 95 | else: 96 | j = agg_sim[i].argmax() 97 | matched_det = detection_list[i] 98 | matched_obj = objects[j] 99 | merged_obj = merge_obj2_into_obj1(cfg, matched_obj, matched_det, run_dbscan=False) 100 | objects[j] = merged_obj 101 | 102 | return objects -------------------------------------------------------------------------------- /openfungraph/utils/general_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import torch 3 | import numpy as np 4 | import time 5 | 6 | class Timer: 7 | def __init__(self, heading = "", verbose = True): 8 | self.verbose = verbose 9 | if not self.verbose: 10 | return 11 | self.heading = heading 12 | 13 | def __enter__(self): 14 | if not self.verbose: 15 | return self 16 | self.start = time.time() 17 | return self 18 | 19 | def __exit__(self, *args): 20 | if not self.verbose: 21 | return 22 | self.end = time.time() 23 | self.interval = self.end - self.start 24 | print(self.heading, self.interval) 25 | 26 | def to_numpy(tensor): 27 | if isinstance(tensor, np.ndarray): 28 | return tensor 29 | return tensor.detach().cpu().numpy() 30 | 31 | def to_tensor(numpy_array, device=None): 32 | if isinstance(numpy_array, torch.Tensor): 33 | return numpy_array 34 | if device is None: 35 | return torch.from_numpy(numpy_array) 36 | else: 37 | return torch.from_numpy(numpy_array).to(device) 38 | 39 | def to_scalar(d: np.ndarray | torch.Tensor | float) -> int | float: 40 | ''' 41 | Convert the d to a scalar 42 | ''' 43 | if isinstance(d, float): 44 | return d 45 | 46 | elif "numpy" in str(type(d)): 47 | assert d.size == 1 48 | return d.item() 49 | 50 | elif isinstance(d, torch.Tensor): 51 | assert d.numel() == 1 52 | return d.item() 53 | 54 | else: 55 | raise TypeError(f"Invalid type for conversion: {type(d)}") 56 | 57 | def prjson(input_json, indent=0): 58 | """ Pretty print a json object """ 59 | if not isinstance(input_json, list): 60 | input_json = [input_json] 61 | 62 | print("[") 63 | for i, entry in enumerate(input_json): 64 | print(" {") 65 | for j, (key, value) in enumerate(entry.items()): 66 | terminator = "," if j < len(entry) - 1 else "" 67 | if isinstance(value, str): 68 | formatted_value = value.replace("\\n", "\n").replace("\\t", "\t") 69 | print(' "{}": "{}"{}'.format(key, formatted_value, terminator)) 70 | else: 71 | print(f' "{key}": {value}{terminator}') 72 | print(" }" + ("," if i < len(input_json) - 1 else "")) 73 | print("]") 74 | 75 | def cfg_to_dict(input_cfg): 76 | """ Convert a json object to a dictionary representation """ 77 | # Ensure input is a list for uniform processing 78 | if not isinstance(input_cfg, list): 79 | input_cfg = [input_cfg] 80 | 81 | result = [] # Initialize the result list to hold our dictionaries 82 | 83 | for entry in input_cfg: 84 | entry_dict = {} # Dictionary to store current entry's data 85 | for key, value in entry.items(): 86 | # Replace escaped newline and tab characters in strings 87 | if isinstance(value, str): 88 | formatted_value = value.replace("\\n", "\n").replace("\\t", "\t") 89 | else: 90 | formatted_value = value 91 | # Add the key-value pair to the current entry dictionary 92 | entry_dict[key] = formatted_value 93 | # Append the current entry dictionary to the result list 94 | result.append(entry_dict) 95 | 96 | # Return the result in dictionary format if it's a single entry or list of dictionaries otherwise 97 | return result[0] if len(result) == 1 else result 98 | 99 | def measure_time(func): 100 | def wrapper(*args, **kwargs): 101 | start_time = time.time() 102 | # print(f"Starting {func.__name__}...") 103 | result = func(*args, **kwargs) # Call the function with any arguments it was called with 104 | end_time = time.time() 105 | elapsed_time = end_time - start_time 106 | print(f"Done! Execution time of {func.__name__} function: {elapsed_time:.2f} seconds") 107 | return result # Return the result of the function call 108 | return wrapper 109 | 110 | def save_hydra_config(hydra_cfg, exp_out_path): 111 | with open(exp_out_path / "config_params.json", "w") as f: 112 | json.dump(cfg_to_dict(hydra_cfg), f, indent=2) -------------------------------------------------------------------------------- /openfungraph/llava/llava_model_16.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import torch 3 | from PIL import Image 4 | 5 | from llava.constants import ( 6 | IMAGE_TOKEN_INDEX, 7 | DEFAULT_IMAGE_TOKEN, 8 | DEFAULT_IM_START_TOKEN, 9 | DEFAULT_IM_END_TOKEN, 10 | IMAGE_PLACEHOLDER, 11 | ) 12 | from llava.conversation import conv_templates, SeparatorStyle 13 | from llava.model.builder import load_pretrained_model 14 | from llava.utils import disable_torch_init 15 | from llava.mm_utils import ( 16 | process_images, 17 | tokenizer_image_token, 18 | get_model_name_from_path, 19 | ) 20 | 21 | from PIL import Image 22 | 23 | import requests 24 | from PIL import Image 25 | from io import BytesIO 26 | import re 27 | 28 | class LlavaModel16(): 29 | def __init__( 30 | self, 31 | model_path, 32 | model_base, 33 | conv_mode_input, 34 | ) -> None: 35 | disable_torch_init() 36 | 37 | model_name = get_model_name_from_path(model_path) 38 | tokenizer, model, image_processor, context_len = load_pretrained_model( 39 | model_path, model_base, model_name 40 | ) 41 | 42 | if "llama-2" in model_name.lower(): 43 | conv_mode = "llava_llama_2" 44 | elif "mistral" in model_name.lower(): 45 | conv_mode = "mistral_instruct" 46 | elif "v1.6-34b" in model_name.lower(): 47 | conv_mode = "chatml_direct" 48 | elif "v1" in model_name.lower(): 49 | conv_mode = "llava_v1" 50 | elif "mpt" in model_name.lower(): 51 | conv_mode = "mpt" 52 | else: 53 | conv_mode = "llava_v0" 54 | 55 | if conv_mode_input is not None and conv_mode != conv_mode_input: 56 | print( 57 | "[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}".format( 58 | conv_mode, conv_mode_input, conv_mode_input 59 | ) 60 | ) 61 | conv_mode = conv_mode_input 62 | 63 | self.model = model 64 | self.tokenizer = tokenizer 65 | self.image_processor = image_processor 66 | self.context_len = context_len 67 | self.conv_mode = conv_mode 68 | 69 | def infer( 70 | self, 71 | query: str, 72 | images: list[Image.Image], 73 | top_p = None, 74 | num_beams: int = 1, 75 | max_new_tokens: int = 512, 76 | temperature: float = 0.0, 77 | ): 78 | qs = query 79 | image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN 80 | if IMAGE_PLACEHOLDER in qs: 81 | if self.model.config.mm_use_im_start_end: 82 | qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs) 83 | else: 84 | qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs) 85 | else: 86 | if self.model.config.mm_use_im_start_end: 87 | qs = image_token_se + "\n" + qs 88 | else: 89 | qs = DEFAULT_IMAGE_TOKEN + "\n" + qs 90 | 91 | conv = conv_templates[self.conv_mode].copy() 92 | conv.append_message(conv.roles[0], qs) 93 | conv.append_message(conv.roles[1], None) 94 | prompt = conv.get_prompt() 95 | 96 | image_sizes = [x.size for x in images] 97 | images_tensor = process_images( 98 | images, 99 | self.image_processor, 100 | self.model.config 101 | ).to(self.model.device, dtype=torch.float16) 102 | 103 | input_ids = ( 104 | tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") 105 | .unsqueeze(0) 106 | .cuda() 107 | ) 108 | 109 | with torch.inference_mode(): 110 | output_ids = self.model.generate( 111 | input_ids, 112 | images=images_tensor, 113 | image_sizes=image_sizes, 114 | do_sample=True if temperature > 0 else False, 115 | temperature=temperature, 116 | top_p=top_p, 117 | num_beams=num_beams, 118 | max_new_tokens=max_new_tokens, 119 | use_cache=True, 120 | ) 121 | 122 | outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() 123 | return outputs -------------------------------------------------------------------------------- /openfungraph/scripts/ana_rigid_objs.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import pickle 3 | import gzip 4 | import argparse 5 | import numpy as np 6 | import open3d as o3d 7 | 8 | from openfungraph.slam.slam_classes import MapObjectList 9 | 10 | 11 | def get_parser(): 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("--result_path", type=str, required=True) 14 | parser.add_argument("--part_result_path", type=str, required=True) 15 | 16 | return parser 17 | 18 | 19 | def get_classes_colors(classes): 20 | class_colors = {} 21 | 22 | # Generate a random color for each class 23 | for class_idx, class_name in enumerate(classes): 24 | # Generate random RGB values between 0 and 255 25 | r = np.random.randint(0, 256)/255.0 26 | g = np.random.randint(0, 256)/255.0 27 | b = np.random.randint(0, 256)/255.0 28 | 29 | # Assign the RGB values as a tuple to the class in the dictionary 30 | class_colors[class_name] = (r, g, b) 31 | 32 | class_colors[-1] = (0, 0, 0) 33 | 34 | return class_colors 35 | 36 | 37 | def compute_overlap_ratio(source, target, distance_threshold=0.02): 38 | # source: part 39 | # target: object 40 | 41 | # source_tree = o3d.geometry.KDTreeFlann(source) 42 | target_tree = o3d.geometry.KDTreeFlann(target) 43 | 44 | overlap_count = 0 45 | for point in source.points: 46 | [_, idx, _] = target_tree.search_radius_vector_3d(point, distance_threshold) 47 | if len(idx) > 0: 48 | overlap_count += 1 49 | 50 | overlap_ratio = overlap_count / len(source.points) 51 | return overlap_ratio 52 | 53 | 54 | if __name__ == "__main__": 55 | parser = get_parser() 56 | args = parser.parse_args() 57 | 58 | result_path = args.result_path 59 | part_result_path = args.part_result_path 60 | 61 | with gzip.open(result_path, "rb") as f: 62 | results = pickle.load(f) 63 | 64 | with gzip.open(part_result_path, "rb") as fp: 65 | part_results = pickle.load(fp) 66 | 67 | objects = MapObjectList() 68 | objects.load_serializable(results['objects']) 69 | 70 | parts = MapObjectList() 71 | parts.load_serializable(part_results['objects']) 72 | 73 | # Run the post-processing filtering and merging in instructed to do so 74 | cfg = copy.deepcopy(results['cfg']) 75 | 76 | parts_interest = ["knob", "button", "handle"] 77 | 78 | rigid_inter_id_candidate = [] 79 | part_inter_id_candidate = [] 80 | 81 | for inter_idx, obj_inter in enumerate(objects): 82 | obj_inter['connected_parts'] = [] 83 | 84 | for inter_idx, obj_inter in enumerate(objects): 85 | obj_classes_inter = np.asarray(obj_inter['class_name']) 86 | values_inter, counts_inter = np.unique(obj_classes_inter, return_counts=True) 87 | obj_class_inter = values_inter[np.argmax(counts_inter)] 88 | tag = False 89 | for obj_idx, obj in enumerate(parts): 90 | obj_classes = np.asarray(obj['class_name']) 91 | values, counts = np.unique(obj_classes, return_counts=True) 92 | obj_class = values[np.argmax(counts)] 93 | if obj_class in parts_interest: 94 | # an interactable part 95 | # detect nearby objects of interest 96 | points_part = obj['pcd'] 97 | points_obj_inter = obj_inter['pcd'] 98 | iou = compute_overlap_ratio(points_part, points_obj_inter, 0.02) 99 | # fusion based on inter objects: 1 object many parts and object must be big enough 100 | obj_box_extent = obj_inter['bbox'].extent 101 | part_box_extent = obj['bbox'].extent 102 | if iou > 0.7 and obj_box_extent.mean() > 3 * part_box_extent.mean(): 103 | print(obj_class_inter, ' ', obj_class, ' ', iou) 104 | if 'connected_parts' not in obj_inter: 105 | # obj_inter['ori_id'] = inter_idx 106 | obj_inter['connected_parts'] = [] 107 | obj_inter['connected_parts'].append(obj_idx) 108 | part_inter_id_candidate.append(obj_idx) 109 | tag = True 110 | else: 111 | obj_inter['connected_parts'].append(obj_idx) 112 | part_inter_id_candidate.append(obj_idx) 113 | tag = True 114 | if tag: 115 | rigid_inter_id_candidate.append(inter_idx) 116 | 117 | updated_results = { 118 | 'objects': objects.to_serializable(), 119 | 'cfg': results['cfg'], 120 | 'class_names': results['class_names'], 121 | 'class_colors': results['class_colors'], 122 | 'inter_id_candidate': rigid_inter_id_candidate 123 | } 124 | 125 | save_path = result_path 126 | 127 | with gzip.open(save_path, "wb") as f: 128 | pickle.dump(updated_results, f) 129 | print(f"Saved full point cloud to {save_path}") 130 | 131 | updated_results = { 132 | 'objects': parts.to_serializable(), 133 | 'cfg': part_results['cfg'], 134 | 'class_names': part_results['class_names'], 135 | 'class_colors': part_results['class_colors'], 136 | 'part_inter_id_candidate': part_inter_id_candidate 137 | } 138 | 139 | save_path = part_result_path 140 | 141 | with gzip.open(save_path, "wb") as f: 142 | pickle.dump(updated_results, f) 143 | print(f"Saved full point cloud to {save_path}") -------------------------------------------------------------------------------- /openfungraph/slam/slam_classes.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable 2 | import copy 3 | import matplotlib 4 | import torch 5 | import torch.nn.functional as F 6 | import numpy as np 7 | import open3d as o3d 8 | 9 | def to_numpy(tensor): 10 | if isinstance(tensor, np.ndarray): 11 | return tensor 12 | return tensor.detach().cpu().numpy() 13 | 14 | def to_tensor(numpy_array, device=None): 15 | if isinstance(numpy_array, torch.Tensor): 16 | return numpy_array 17 | if device is None: 18 | return torch.from_numpy(numpy_array) 19 | else: 20 | return torch.from_numpy(numpy_array).to(device) 21 | 22 | class DetectionList(list): 23 | def get_values(self, key, idx:int=None): 24 | if idx is None: 25 | return [detection[key] for detection in self] 26 | else: 27 | return [detection[key][idx] for detection in self] 28 | 29 | def get_stacked_values_torch(self, key, idx:int=None): 30 | values = [] 31 | for detection in self: 32 | v = detection[key] 33 | if idx is not None: 34 | v = v[idx] 35 | if isinstance(v, o3d.geometry.OrientedBoundingBox) or \ 36 | isinstance(v, o3d.geometry.AxisAlignedBoundingBox): 37 | v = np.asarray(v.get_box_points()) 38 | if isinstance(v, np.ndarray): 39 | v = torch.from_numpy(v) 40 | values.append(v) 41 | return torch.stack(values, dim=0) 42 | 43 | def get_stacked_values_numpy(self, key, idx:int=None): 44 | values = self.get_stacked_values_torch(key, idx) 45 | return to_numpy(values) 46 | 47 | def __add__(self, other): 48 | new_list = copy.deepcopy(self) 49 | new_list.extend(other) 50 | return new_list 51 | 52 | def __iadd__(self, other): 53 | self.extend(other) 54 | return self 55 | 56 | def slice_by_indices(self, index: Iterable[int]): 57 | ''' 58 | Return a sublist of the current list by indexing 59 | ''' 60 | new_self = type(self)() 61 | for i in index: 62 | new_self.append(self[i]) 63 | return new_self 64 | 65 | def slice_by_mask(self, mask: Iterable[bool]): 66 | ''' 67 | Return a sublist of the current list by masking 68 | ''' 69 | new_self = type(self)() 70 | for i, m in enumerate(mask): 71 | if m: 72 | new_self.append(self[i]) 73 | return new_self 74 | 75 | def get_most_common_class(self) -> list[int]: 76 | classes = [] 77 | for d in self: 78 | values, counts = np.unique(np.asarray(d['class_name']), return_counts=True) 79 | most_common_class = values[np.argmax(counts)] 80 | classes.append(most_common_class) 81 | return classes 82 | 83 | def color_by_most_common_classes(self, colors_dict: dict[str, list[float]], color_bbox: bool=True): 84 | ''' 85 | Color the point cloud of each detection by the most common class 86 | ''' 87 | classes = self.get_most_common_class() 88 | for d, c in zip(self, classes): 89 | color = colors_dict[str(c)] 90 | d['pcd'].paint_uniform_color(color) 91 | if color_bbox: 92 | d['bbox'].color = color 93 | 94 | def color_by_instance(self): 95 | if len(self) == 0: 96 | # Do nothing 97 | return 98 | 99 | if "inst_color" in self[0]: 100 | for d in self: 101 | d['pcd'].paint_uniform_color(d['inst_color']) 102 | d['bbox'].color = d['inst_color'] 103 | else: 104 | cmap = matplotlib.colormaps.get_cmap("turbo") 105 | instance_colors = cmap(np.linspace(0, 1, len(self))) 106 | instance_colors = instance_colors[:, :3] 107 | for i in range(len(self)): 108 | self[i]['pcd'].paint_uniform_color(instance_colors[i]) 109 | self[i]['bbox'].color = instance_colors[i] 110 | 111 | 112 | class MapObjectList(DetectionList): 113 | def compute_similarities(self, new_clip_ft): 114 | ''' 115 | The input feature should be of shape (D, ), a one-row vector 116 | This is mostly for backward compatibility 117 | ''' 118 | # if it is a numpy array, make it a tensor 119 | new_clip_ft = to_tensor(new_clip_ft) 120 | 121 | # assuming cosine similarity for features 122 | clip_fts = self.get_stacked_values_torch('clip_ft') 123 | 124 | similarities = F.cosine_similarity(new_clip_ft.unsqueeze(0), clip_fts) 125 | # return similarities.squeeze() 126 | return similarities 127 | 128 | def to_serializable(self): 129 | s_obj_list = [] 130 | for obj in self: 131 | s_obj_dict = copy.deepcopy(obj) 132 | 133 | s_obj_dict['clip_ft'] = to_numpy(s_obj_dict['clip_ft']) 134 | s_obj_dict['text_ft'] = to_numpy(s_obj_dict['text_ft']) 135 | 136 | s_obj_dict['pcd_np'] = np.asarray(s_obj_dict['pcd'].points) 137 | s_obj_dict['bbox_np'] = np.asarray(s_obj_dict['bbox'].get_box_points()) 138 | s_obj_dict['pcd_color_np'] = np.asarray(s_obj_dict['pcd'].colors) 139 | 140 | del s_obj_dict['pcd'] 141 | del s_obj_dict['bbox'] 142 | 143 | s_obj_list.append(s_obj_dict) 144 | 145 | return s_obj_list 146 | 147 | def load_serializable(self, s_obj_list): 148 | assert len(self) == 0, 'MapObjectList should be empty when loading' 149 | for s_obj_dict in s_obj_list: 150 | try: 151 | new_obj = copy.deepcopy(s_obj_dict) 152 | 153 | new_obj['clip_ft'] = to_tensor(new_obj['clip_ft']) 154 | new_obj['text_ft'] = to_tensor(new_obj['text_ft']) 155 | 156 | new_obj['pcd'] = o3d.geometry.PointCloud() 157 | new_obj['pcd'].points = o3d.utility.Vector3dVector(new_obj['pcd_np']) 158 | new_obj['bbox'] = o3d.geometry.OrientedBoundingBox.create_from_points( 159 | o3d.utility.Vector3dVector(new_obj['bbox_np'])) 160 | new_obj['bbox'].color = new_obj['pcd_color_np'][0] 161 | new_obj['pcd'].colors = o3d.utility.Vector3dVector(new_obj['pcd_color_np']) 162 | 163 | del new_obj['pcd_np'] 164 | del new_obj['bbox_np'] 165 | del new_obj['pcd_color_np'] 166 | 167 | self.append(new_obj) 168 | except: 169 | continue -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Open-Vocabulary Functional 3D Scene Graphs for Real-World Indoor Spaces 2 | 3 | [**Project Page**](https://openfungraph.github.io/) 4 | 5 | ![Splash Figure Top](./assets/teaser_top.jpg) 6 | ![Splash Figure](./assets/teaser.png) 7 | 8 | ## Setup 9 | 10 | ### Install the required libraries 11 | 12 | ```bash 13 | conda create -n openfungraph python=3.10 14 | conda activate openfungraph 15 | 16 | ##### Install Pytorch according to your own setup ##### 17 | # For example, if you have a GPU with CUDA 11.8 18 | # Note that this version is compatible with the LLaVA repo 19 | # Here we install cudatoolkit via Conda for installation of Grounded-SAM 20 | conda install pytorch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 pytorch-cuda=11.8 cudatoolkit=11.8 -c pytorch -c nvidia 21 | 22 | # Install the Faiss library (CPU version should be fine) 23 | conda install -c pytorch faiss-cpu=1.7.4 mkl=2021 blas=1.0=mkl 24 | 25 | # Install Pytorch3D by 26 | https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md 27 | # We recommend installing from a local clone to avoid confliction 28 | 29 | # Install the required libraries 30 | pip install tyro open_clip_torch wandb h5py openai hydra-core distinctipy pyviz3d line_profiler 31 | 32 | # Install the gradslam package and its dependencies 33 | git clone https://github.com/krrish94/chamferdist.git 34 | cd chamferdist 35 | pip install . 36 | cd .. 37 | git clone https://github.com/gradslam/gradslam.git 38 | cd gradslam 39 | git checkout conceptfusion 40 | pip install . 41 | ``` 42 | 43 | ### Install [Grounded-SAM](https://github.com/IDEA-Research/Grounded-Segment-Anything) package 44 | 45 | Follow the instructions on the original [repo](https://github.com/IDEA-Research/Grounded-Segment-Anything#install-without-docker). 46 | 47 | First checkout the package by 48 | 49 | ```bash 50 | git clone git@github.com:IDEA-Research/Grounded-Segment-Anything.git 51 | ``` 52 | 53 | Then, install the package Following the commands listed in the original GitHub repo. You can skip the `Install osx` step and the "optional dependencies". 54 | 55 | During this process, you will need to set the `CUDA_HOME` to be where the CUDA toolkit is installed. 56 | The CUDA tookit can be set up system-wide or within a conda environment. 57 | We tested it within a conda environment, i.e. installing [cudatoolkit-dev](https://anaconda.org/conda-forge/cudatoolkit-dev) using conda by former commands. 58 | 59 | ```bash 60 | # and you need to replace `export CUDA_HOME=/path/to/cuda-11.3/` by 61 | export CUDA_HOME=/path/to/anaconda3/envs/openfungraph/ 62 | ``` 63 | 64 | You also need to download `ram_swin_large_14m.pth`, `groundingdino_swint_ogc.pth`, `sam_vit_h_4b8939.pth` following the instruction [here](https://github.com/IDEA-Research/Grounded-Segment-Anything#label-grounded-sam-with-ram-or-tag2text-for-automatic-labeling). 65 | 66 | After installation, set the path to Grounded-SAM as an environment variable. 67 | 68 | ```bash 69 | export GSA_PATH=/path/to/Grounded-Segment-Anything 70 | ``` 71 | 72 | ### Set up LLaVA 73 | 74 | Follow the instructions on the [LLaVA repo](https://github.com/haotian-liu/LLaVA) to set it up. We have tested with model checkpoint `LLaVA-7B-v1.6`. 75 | 76 | ### Install this repo 77 | 78 | ```bash 79 | cd OpenFunGraph 80 | pip install -e . 81 | ``` 82 | 83 | ## Prepare dataset 84 | 85 | Download the [customized SceneFun3D dataset](https://huggingface.co/datasets/OpenFunGraph/SceneFun3D_Graph) and the newly recorded [FunGraph3D dataset](https://huggingface.co/datasets/OpenFunGraph/FunGraph3D). 86 | 87 | In their top repo, file structure is introduced. 88 | Note that related path should be set as ``env_vars.bash.template``. 89 | 90 | ```bash 91 | export FUNGRAPH3D_ROOT= 92 | export FUNGRAPH3D_CONFIG_PATH=${FG_FOLDER}/openfungraph/dataset/dataconfigs/fungraph3d/fungraph3d.yaml 93 | export SCENEFUN3D_ROOT= # for SceneFun3D, it should be with dev / test 94 | export SCENEFUN3D_CONFIG_PATH=${FG_FOLDER}/openfungraph/dataset/dataconfigs/scenefun3d/scenefun3d.yaml 95 | ``` 96 | 97 | OpenFunGraph can also be easily run on other dataset. 98 | See `dataset/datasets_common.py` for how to write your own dataloader. 99 | 100 | ## Run OpenFunGraph 101 | 102 | The env variables needed can be found in `env_vars.bash.template`. 103 | When following the setup guide below, you should change the variables accordingly for easy setup. 104 | 105 | The following commands should be run in the `openfungraph` folder. 106 | 107 | ```bash 108 | cd openfungraph 109 | ``` 110 | 111 | ### Functional Scene Graph Node Detection 112 | 113 | ```bash 114 | export SCENE_NAME= 115 | 116 | bash scenegraph/detection_scenefun3d.sh (or *_fungraph3d.sh) 117 | ``` 118 | 119 | The above commands will save the 2D node detection and segmentation results. 120 | 121 | You can ignore the `There's a wrong phrase happen, this is because of our post-process merged wrong tokens, which will be modified in the future. We will assign it with a random label at this time.` message. 122 | 123 | ### 3D Functional Scene Graph Construction 124 | 125 | Ensure that the `openai` package is installed and that your APIKEY is set. We recommend using GPT-4. 126 | ```bash 127 | export OPENAI_API_KEY= 128 | ``` 129 | 130 | ```bash 131 | CUDA_VISIBLE_DEVICES=0 python scenegraph/build_fungraph_whole_openai.py --dataset_root ${SCENEFUN3D_ROOT} ``or`` ${FUNGRAPH3D_ROOT} --scene_name ${SCENE_NAME} --mapfile ``/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.3_bbox0.9_simsum1.2_dbscan.1_post.pkl.gz`` --part_file ``/part/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.15_bbox0.1_simsum1.2_dbscan.1_parts_post.pkl.gz`` 132 | ``` 133 | 134 | ### Visualize and Evaluation 135 | 136 | After running the algorithm, you can get three modified key assets of object-level nodes ``/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.3_bbox0.9_simsum1.2_dbscan.1_post.pkl.gz`` (the name could be varied depending on what parameters you choose), sub-object-level elements ``/part/pcd_saves/full_pcd_ram_withbg_allclasses_overlap_maskconf0.15_bbox0.1_simsum1.2_dbscan.1_parts_post.pkl.gz``, and the finel graph edges ``/cfslam_funcgraph_edges.pkl`` (or with confidence). 137 | 138 | Visualize them by 139 | ```bash 140 | python scripts/pyviz3d_interactable_results.py --inter_result_path --part_result_path --edge_file --pc_path (--pose_path (only for SCENEFUN3D) /*_transform.npy) 141 | ``` 142 | 143 | Evaluation scripts: 144 | For node evaluation 145 | ```bash 146 | python eval/eval_node.py --dataset --root_path --scene --video