├── figures ├── teaser.gif ├── Fig 1. Teaser_figure.jpg └── Fig 6. Learning_results_version.jpg ├── Hardware_instruction ├── CAD_model │ ├── base.f3d │ ├── Assembly.png │ ├── fingertip.f3d │ ├── base_cover.f3d │ ├── base_finger.f3d │ ├── base_finger_2.f3d │ ├── fingertip_arm_1.f3d │ ├── fingertip_arm_2.f3d │ ├── Assemble_whole_hand.f3z │ ├── LX-224_servo_motor.f3d │ ├── assembly_fingertip.f3z │ ├── assembly_one_finger.f3z │ ├── base_audio_jack_cover.f3d │ └── fingertip_counterweight_mount.f3d └── hardware_document.pdf ├── supplementary └── Object Re-identification Confusion Matrices.zip ├── ARNet_shape_reconstruction ├── dataset │ ├── data_split_1 │ │ ├── real_test_gt.npy │ │ ├── syn_train_gt.npy │ │ ├── real_train_gt.npy │ │ ├── real_valid_gt.npy │ │ ├── real_test_tapping.npy │ │ ├── syn_train_tapping.npy │ │ ├── test_object_list.npy │ │ ├── valid_object_list.npy │ │ ├── real_train_tapping.npy │ │ └── real_valid_tapping.npy │ ├── data_split_2 │ │ ├── real_test_gt.npy │ │ ├── syn_train_gt.npy │ │ ├── real_train_gt.npy │ │ ├── real_valid_gt.npy │ │ ├── real_test_tapping.npy │ │ ├── syn_train_tapping.npy │ │ ├── test_object_list.npy │ │ ├── valid_object_list.npy │ │ ├── real_train_tapping.npy │ │ └── real_valid_tapping.npy │ ├── data_split_3 │ │ ├── real_test_gt.npy │ │ ├── syn_train_gt.npy │ │ ├── real_train_gt.npy │ │ ├── real_valid_gt.npy │ │ ├── real_test_tapping.npy │ │ ├── syn_train_tapping.npy │ │ ├── test_object_list.npy │ │ ├── valid_object_list.npy │ │ ├── real_train_tapping.npy │ │ └── real_valid_tapping.npy │ ├── __pycache__ │ │ ├── AR_recon_dataset.cpython-38.pyc │ │ ├── data_preparation_real.cpython-38.pyc │ │ └── data_preparation_synthetic.cpython-38.pyc │ └── data_preparation.py ├── requirements.txt ├── configs │ ├── config_data_split_1.yaml │ ├── config_data_split_2.yaml │ └── config_data_split_3.yaml ├── compute_L1_cd.py ├── README.md ├── test.py ├── main.py ├── visualization │ └── visualization.py ├── compute_baseline.py └── models │ └── ARnet_recon.py ├── ARNet_material_classification ├── results │ ├── data_split_1 │ │ ├── test │ │ │ ├── label_gt_test.npy │ │ │ └── label_prediction_test.npy │ │ ├── train │ │ │ ├── label_gt_train.npy │ │ │ └── label_prediction_train.npy │ │ └── valid │ │ │ ├── label_gt_valid.npy │ │ │ └── label_prediction_valid.npy │ ├── data_split_2 │ │ ├── test │ │ │ ├── label_gt_test.npy │ │ │ └── label_prediction_test.npy │ │ ├── train │ │ │ ├── label_gt_train.npy │ │ │ └── label_prediction_train.npy │ │ └── valid │ │ │ ├── label_gt_valid.npy │ │ │ └── label_prediction_valid.npy │ ├── data_split_3 │ │ ├── test │ │ │ ├── label_gt_test.npy │ │ │ └── label_prediction_test.npy │ │ ├── train │ │ │ ├── label_gt_train.npy │ │ │ └── label_prediction_train.npy │ │ └── valid │ │ │ ├── label_gt_valid.npy │ │ │ └── label_prediction_valid.npy │ └── object_folder │ │ ├── train │ │ ├── label_gt_train.npy │ │ └── label_prediction_train.npy │ │ └── valid │ │ ├── label_gt_valid.npy │ │ └── label_prediction_valid.npy ├── images │ ├── data_split_1_test_confusion_matrix.png │ ├── data_split_1_train_confusion_matrix.png │ ├── data_split_1_valid_confusion_matrix.png │ ├── data_split_2_test_confusion_matrix.png │ ├── data_split_2_train_confusion_matrix.png │ ├── data_split_2_valid_confusion_matrix.png │ ├── data_split_3_test_confusion_matrix.png │ ├── data_split_3_train_confusion_matrix.png │ ├── data_split_3_valid_confusion_matrix.png │ ├── object_folder_train_confusion_matrix.png │ ├── object_folder_valid_confusion_matrix.png │ ├── data_split_1_test_refine_confusion_matrix.png │ ├── data_split_1_valid_refine_confusion_matrix.png │ ├── data_split_2_test_refine_confusion_matrix.png │ ├── data_split_2_valid_refine_confusion_matrix.png │ ├── data_split_3_test_refine_confusion_matrix.png │ └── data_split_3_valid_refine_confusion_matrix.png ├── requirements.txt ├── configs │ ├── config_objectfolder.yaml │ ├── config_data_split_1.yaml │ ├── config_data_split_3.yaml │ └── config_data_split_2.yaml ├── dataset │ ├── material_simple_categories.json │ ├── data_preparation_objectfolder.py │ ├── data_preparation.py │ └── AR_material_dataset.py ├── README.md ├── test.py ├── main.py ├── compute_baseline.py ├── models │ └── ARNet_material.py └── refine_material_predication.py ├── ARNet_object_classification ├── images │ ├── data_15_dataset_82_objects_points_test_confusion_matrix.png │ ├── data_15_dataset_82_objects_points_test_confusion_matrix.xlsx │ ├── data_15_dataset_82_objects_points_valid_confusion_matrix.png │ └── data_15_dataset_82_objects_points_valid_confusion_matrix.xlsx ├── README.md ├── test.py ├── show_dataset_statistic.py ├── dataset │ └── AR_object_dataset.py ├── main.py ├── configs │ ├── config_data_split_1_audio.yaml │ ├── config_data_split_1_points.yaml │ ├── config_data_split_2_audio.yaml │ ├── config_data_split_2_points.yaml │ ├── config_data_split_3_audio.yaml │ ├── config_data_split_3_points.yaml │ ├── config_data_split_1_audio+points.yaml │ ├── config_data_split_2_audio+points.yaml │ └── config_data_split_3_audio+points.yaml ├── main_sweep.py ├── baseline.py ├── models │ └── ARNet_object.py └── refine_material_predication.py ├── README.md └── LICENSE /figures/teaser.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/figures/teaser.gif -------------------------------------------------------------------------------- /figures/Fig 1. Teaser_figure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/figures/Fig 1. Teaser_figure.jpg -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/base.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/base.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/Assembly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/Assembly.png -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/fingertip.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/fingertip.f3d -------------------------------------------------------------------------------- /Hardware_instruction/hardware_document.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/hardware_document.pdf -------------------------------------------------------------------------------- /figures/Fig 6. Learning_results_version.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/figures/Fig 6. Learning_results_version.jpg -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/base_cover.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/base_cover.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/base_finger.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/base_finger.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/base_finger_2.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/base_finger_2.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/fingertip_arm_1.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/fingertip_arm_1.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/fingertip_arm_2.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/fingertip_arm_2.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/Assemble_whole_hand.f3z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/Assemble_whole_hand.f3z -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/LX-224_servo_motor.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/LX-224_servo_motor.f3d -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/assembly_fingertip.f3z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/assembly_fingertip.f3z -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/assembly_one_finger.f3z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/assembly_one_finger.f3z -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/base_audio_jack_cover.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/base_audio_jack_cover.f3d -------------------------------------------------------------------------------- /supplementary/Object Re-identification Confusion Matrices.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/supplementary/Object Re-identification Confusion Matrices.zip -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/real_test_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/real_test_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/syn_train_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/syn_train_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/real_test_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/real_test_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/syn_train_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/syn_train_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/real_test_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/real_test_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/syn_train_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/syn_train_gt.npy -------------------------------------------------------------------------------- /Hardware_instruction/CAD_model/fingertip_counterweight_mount.f3d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/Hardware_instruction/CAD_model/fingertip_counterweight_mount.f3d -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/real_train_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/real_train_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/real_valid_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/real_valid_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/real_train_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/real_train_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/real_valid_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/real_valid_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/real_train_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/real_train_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/real_valid_gt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/real_valid_gt.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/real_test_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/real_test_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/syn_train_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/syn_train_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/test_object_list.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/test_object_list.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/valid_object_list.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/valid_object_list.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/real_test_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/real_test_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/syn_train_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/syn_train_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/test_object_list.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/test_object_list.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/valid_object_list.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/valid_object_list.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/real_test_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/real_test_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/syn_train_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/syn_train_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/test_object_list.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/test_object_list.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/valid_object_list.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/valid_object_list.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/real_train_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/real_train_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_1/real_valid_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_1/real_valid_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/real_train_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/real_train_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_2/real_valid_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_2/real_valid_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/real_train_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/real_train_tapping.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/data_split_3/real_valid_tapping.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/data_split_3/real_valid_tapping.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_1/test/label_gt_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_1/test/label_gt_test.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_2/test/label_gt_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_2/test/label_gt_test.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_3/test/label_gt_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_3/test/label_gt_test.npy -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_1_test_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_1_test_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_1_train_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_1_train_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_1_valid_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_1_valid_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_2_test_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_2_test_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_2_train_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_2_train_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_2_valid_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_2_valid_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_3_test_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_3_test_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_3_train_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_3_train_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_3_valid_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_3_valid_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_1/train/label_gt_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_1/train/label_gt_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_1/valid/label_gt_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_1/valid/label_gt_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_2/train/label_gt_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_2/train/label_gt_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_2/valid/label_gt_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_2/valid/label_gt_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_3/train/label_gt_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_3/train/label_gt_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_3/valid/label_gt_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_3/valid/label_gt_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/object_folder/train/label_gt_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/object_folder/train/label_gt_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/object_folder/valid/label_gt_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/object_folder/valid/label_gt_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/images/object_folder_train_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/object_folder_train_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/object_folder_valid_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/object_folder_valid_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/__pycache__/AR_recon_dataset.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/__pycache__/AR_recon_dataset.cpython-38.pyc -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_1/test/label_prediction_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_1/test/label_prediction_test.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_2/test/label_prediction_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_2/test/label_prediction_test.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_3/test/label_prediction_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_3/test/label_prediction_test.npy -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_1_test_refine_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_1_test_refine_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_1_valid_refine_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_1_valid_refine_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_2_test_refine_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_2_test_refine_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_2_valid_refine_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_2_valid_refine_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_3_test_refine_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_3_test_refine_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/images/data_split_3_valid_refine_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/images/data_split_3_valid_refine_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_1/train/label_prediction_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_1/train/label_prediction_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_1/valid/label_prediction_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_1/valid/label_prediction_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_2/train/label_prediction_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_2/train/label_prediction_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_2/valid/label_prediction_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_2/valid/label_prediction_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_3/train/label_prediction_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_3/train/label_prediction_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/data_split_3/valid/label_prediction_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/data_split_3/valid/label_prediction_valid.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/object_folder/train/label_prediction_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/object_folder/train/label_prediction_train.npy -------------------------------------------------------------------------------- /ARNet_material_classification/results/object_folder/valid/label_prediction_valid.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_material_classification/results/object_folder/valid/label_prediction_valid.npy -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/__pycache__/data_preparation_real.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/__pycache__/data_preparation_real.cpython-38.pyc -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/dataset/__pycache__/data_preparation_synthetic.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_shape_reconstruction/dataset/__pycache__/data_preparation_synthetic.cpython-38.pyc -------------------------------------------------------------------------------- /ARNet_object_classification/images/data_15_dataset_82_objects_points_test_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_object_classification/images/data_15_dataset_82_objects_points_test_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_object_classification/images/data_15_dataset_82_objects_points_test_confusion_matrix.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_object_classification/images/data_15_dataset_82_objects_points_test_confusion_matrix.xlsx -------------------------------------------------------------------------------- /ARNet_object_classification/images/data_15_dataset_82_objects_points_valid_confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_object_classification/images/data_15_dataset_82_objects_points_valid_confusion_matrix.png -------------------------------------------------------------------------------- /ARNet_object_classification/images/data_15_dataset_82_objects_points_valid_confusion_matrix.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/generalroboticslab/SonicSense/HEAD/ARNet_object_classification/images/data_15_dataset_82_objects_points_valid_confusion_matrix.xlsx -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/requirements.txt: -------------------------------------------------------------------------------- 1 | chamferdist==1.0.0 2 | extensions==0.4 3 | matplotlib==3.7.1 4 | munch==4.0.0 5 | numpy==1.24.2 6 | open3d==0.13.0 7 | pytorch_lightning==2.0.1 8 | PyYAML==6.0.1 9 | tensorboardX==2.6.2 10 | torch 11 | torchvision 12 | tensorboard 13 | -------------------------------------------------------------------------------- /ARNet_material_classification/requirements.txt: -------------------------------------------------------------------------------- 1 | librosa==0.10.0.post2 2 | matplotlib==3.7.1 3 | munch==4.0.0 4 | natsort==8.3.1 5 | numpy==1.24.2 6 | open3d==0.13.0 7 | pandas 8 | pytorch_lightning==2.0.1 9 | PyYAML==6.0.1 10 | scikit_learn 11 | scipy 12 | seaborn==0.12.2 13 | torch 14 | torchmetrics==0.11.4 15 | torchvision 16 | gdown 17 | tensorboard 18 | wandb 19 | openpyxl -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/configs/config_data_split_1.yaml: -------------------------------------------------------------------------------- 1 | exp_name: 'data_split_1' 2 | options: 'bid+batchsum+pointmean' 3 | log_dir: 'log/lightning_logs' 4 | train_ckpt_path: 5 | test_ckpt_path: checkpoint/model_used_for_testing/data_split_1bid+batchsum+pointmean-epoch=729-val_loss=0.00004.ckpt 6 | lr: 0.000005 7 | step_size: 500 8 | decay_rate: 0.7 9 | epochs: 1000 10 | batch_size: 300 11 | num_workers: 8 12 | log_frequency: 10 13 | save_frequency: 10 14 | init_dataset: False 15 | num_of_coarse: 2000 16 | latent_dimension: 1024 17 | dataset_for_testing: 18 | -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/configs/config_data_split_2.yaml: -------------------------------------------------------------------------------- 1 | exp_name: 'data_split_2' 2 | options: 'bid+batchsum+pointmean' 3 | log_dir: 'log/lightning_logs' 4 | train_ckpt_path: 5 | test_ckpt_path: checkpoint/model_used_for_testing/data_split_2bid+batchsum+pointmean-epoch=286-val_loss=0.00006.ckpt 6 | lr: 0.000005 7 | step_size: 500 8 | decay_rate: 0.7 9 | epochs: 1000 10 | batch_size: 300 11 | num_workers: 8 12 | log_frequency: 10 13 | save_frequency: 10 14 | init_dataset: False 15 | num_of_coarse: 2000 16 | latent_dimension: 1024 17 | dataset_for_testing: 18 | -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/configs/config_data_split_3.yaml: -------------------------------------------------------------------------------- 1 | exp_name: 'data_split_3' 2 | options: 'bid+batchsum+pointmean' 3 | log_dir: 'log/lightning_logs' 4 | train_ckpt_path: 5 | test_ckpt_path: checkpoint/model_used_for_testing/data_split_3bid+batchsum+pointmean-epoch=781-val_loss=0.00005.ckpt 6 | lr: 0.000005 7 | step_size: 500 8 | decay_rate: 0.7 9 | epochs: 1000 10 | batch_size: 300 11 | num_workers: 8 12 | log_frequency: 10 13 | save_frequency: 10 14 | init_dataset: False 15 | num_of_coarse: 2000 16 | latent_dimension: 1024 17 | dataset_for_testing: 18 | -------------------------------------------------------------------------------- /ARNet_material_classification/configs/config_objectfolder.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_objectfolder' 2 | exp_name: 'data_split_1' 3 | label_name: ['ceramic','glass','metal','plastic','wood'] 4 | log_dir: 'log/lightning_logs' 5 | ckpt_path: checkpoint/model_used_for_the_results/ARNet_material-object_folder-bc40-lr0.008201187626485543-dropout0.4064263916116527-fc_layer150-best_model-epoch=184-val_loss=0.81-val_acc=0.80.ckpt 6 | lr: 0.008201187626485543 7 | dropout: 0.4064263916116527 8 | batch_size: 40 9 | num_workers: 8 10 | augmentation_technique: None 11 | gaussian_scale: 5 12 | num_label: 5 13 | display_audio: False 14 | init_dataset: False 15 | testing_split: 16 | label_mapping: 17 | 0: 18 | 0 19 | 1: 20 | 1 21 | 2: 22 | 2 23 | 3: 24 | 3 25 | 4: 26 | 4 27 | 5: 28 | 5 29 | 6: 30 | 6 31 | 8: 32 | 7 33 | 9: 34 | 8 35 | objectfolder_label_mapping: 36 | 0: 37 | [1,2,3,4,8,9,10,61,62,63,64,65,73,74,75,76,77,78,79] 38 | 1: 39 | [6,7,22,51,59,60,82,91,92,93,94] 40 | 2: 41 | [17,18,19,20,21,37,38,39,40,41,42,43,44,45,46,47,52,53,54,55,66,67,69,83,84,85,86,87,88,89,90,98] 42 | 3: 43 | [23,24,25,48,49,50,56,57,58,97,99,100] 44 | 4: 45 | [5,11,12,13,14,15,16,27,28,29,30,31,32,33,34,68,80,81] -------------------------------------------------------------------------------- /ARNet_material_classification/configs/config_data_split_1.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_material' 2 | exp_name: 'data_split_1' 3 | label_name: ['ceramic','fabric','foam','glass','metal','paper','plastic','rubber','wood'] 4 | log_dir: 'log/lightning_logs' 5 | ckpt_path: checkpoint/model_used_for_the_results/ARNet_material-data_split_1-bc32-lr0.00902858976192661-dropout0.5210466881162531-fc_layer150-best_model-epoch=170-val_loss=1.49-val_acc=0.61.ckpt 6 | lr: 0.00902858976192661 7 | dropout: 0.5210466881162531 8 | batch_size: 32 9 | num_workers: 8 10 | augmentation_technique: None 11 | gaussian_scale: 5 12 | num_label: 9 13 | display_audio: False 14 | init_dataset: False 15 | testing_split: 16 | label_mapping: 17 | 0: 18 | 0 19 | 1: 20 | 1 21 | 2: 22 | 2 23 | 3: 24 | 3 25 | 4: 26 | 4 27 | 5: 28 | 5 29 | 6: 30 | 6 31 | 8: 32 | 7 33 | 9: 34 | 8 35 | objectfolder_label_mapping: 36 | 0: 37 | [1,2,3,4,8,9,10,61,62,63,64,65,73,74,75,76,77,78,79] 38 | 1: 39 | [6,7,22,51,59,60,82,91,92,93,94] 40 | 2: 41 | [17,18,19,20,21,37,38,39,40,41,42,43,44,45,46,47,52,53,54,55,66,67,69,83,84,85,86,87,88,89,90,98] 42 | 3: 43 | [23,24,25,48,49,50,56,57,58,97,99,100] 44 | 4: 45 | [5,11,12,13,14,15,16,27,28,29,30,31,32,33,34,68,80,81] -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/compute_L1_cd.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import numpy as np 4 | 5 | split='data_split_1' 6 | loss_list=[] 7 | file_list=[] 8 | for file_name in os.listdir(f'reconstruction_results/{split}'): 9 | reconstruction_results=np.load(f'reconstruction_results/{split}/{file_name}',allow_pickle=True)[0] 10 | gt=np.load(f'data/ARDataset/ground_truth_5000_correct_unit/{file_name}',allow_pickle=True) 11 | distanct_1=[] 12 | for i in reconstruction_results: 13 | min_dis=float('inf') 14 | for j in gt: 15 | distance = distance=math.dist(i,j) 16 | if distance 23 | ``` 24 | 25 | ## Evaluation 26 | Choose your ckpt file and change it in test_ckpt_path in config.yaml, then run the following command: 27 | ``` 28 | CUDA_VISIBLE_DEVICES=6 python test.py 29 | ``` 30 | * choice of datasplit : data_split_1/data_split_2/data_split_3 31 | * choice of input type : audio+points/audio/points 32 | The confusion matrices and numerical results are saved under `images` folder. -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/README.md: -------------------------------------------------------------------------------- 1 | # ARNet_shape_reconstruction 2 | 3 | 4 | ## Installation 5 | The code has been tested on Ubuntu 20.04 with CUDA 12.0. 6 | ``` 7 | virtualenv -p /usr/bin/python venv_recon 8 | source venv_recon/bin/activate 9 | cd ARNet_shape_reconstruction 10 | pip install -r requirements.txt 11 | ``` 12 | ## Chamfer Distance Loss 13 | Please find the chamferdist/chamfer.py file under venv_recon/lib, and replace the chamfer.py in [this](https://github.com/krrish94/chamferdist?tab=readme-ov-file) python library to our chamfer.py file under the folder. 14 | 15 | ## Data Preparation 16 | 17 | Download the dataset from the [Google Drive](https://drive.google.com/file/d/1kb4rS1cDhwCSQ4bM6IWvFHymnbocp2rh/view?usp=drive_link) and unzip under the folder. 18 | 19 | ## About Configs 20 | - `exp_name` - used to specify data split 21 | - `ckpt_path` - used to specify saved model for testing 22 | 23 | ## Training 24 | 25 | Run the following command: 26 | ``` 27 | CUDA_VISIBLE_DEVICES=6 python main.py data_split_<1-3> 28 | ``` 29 | 30 | ## Evaluation 31 | Run the following command for evaluation of the trained models: 32 | ``` 33 | CUDA_VISIBLE_DEVICES=6 python test.py data_split_<1-3> 34 | ``` 35 | The visual reconstruction results are saved under `image` folder.The numerical reconstruction results will be saved in .npy file under `reconstruction_results` folder. 36 | -------------------------------------------------------------------------------- /ARNet_object_classification/test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import yaml 3 | import torch 4 | from torch import utils 5 | from munch import munchify 6 | import pytorch_lightning as pl 7 | from models.ARNet_object import ARNet_object 8 | from dataset.AR_object_dataset import AR_object_dataset 9 | 10 | 11 | def load_config(filepath): 12 | with open(filepath, 'r') as stream: 13 | try: 14 | trainer_params = yaml.safe_load(stream) 15 | return trainer_params 16 | except yaml.YAMLError as exc: 17 | print(exc) 18 | 19 | def main(): 20 | choice_of_config = sys.argv[1] 21 | choice_of_input = sys.argv[2] 22 | config_path = f'configs/config_{choice_of_config}_{choice_of_input}.yaml' 23 | params = load_config(filepath=config_path) 24 | params = munchify(params) 25 | pl.seed_everything(1) 26 | for choice_of_dataset in ['valid','test']: 27 | params.testing_split = choice_of_dataset.replace('original_','') 28 | dataset = AR_object_dataset(choice_of_dataset, params) 29 | dataloader = utils.data.DataLoader(dataset, batch_size=10000, shuffle=False, num_workers=params.num_workers) 30 | 31 | model = ARNet_object(params) 32 | ckpt=torch.load(params.ckpt_path) 33 | model.load_state_dict(ckpt['state_dict']) 34 | model.eval() 35 | model.freeze() 36 | trainer=pl.Trainer(accelerator="gpu", devices=1,max_epochs=1,num_nodes=1) 37 | trainer.test(model=model,dataloaders=dataloader,verbose=True) 38 | 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import yaml 3 | import torch 4 | from torch import utils 5 | from munch import munchify 6 | import pytorch_lightning as pl 7 | from models.ARnet_recon import ARnet_recon 8 | from pytorch_lightning import loggers as pl_loggers 9 | from dataset.AR_recon_dataset import AR_recon_dataset_s2r 10 | 11 | def load_config(filepath): 12 | with open(filepath, 'r') as stream: 13 | try: 14 | trainer_params = yaml.safe_load(stream) 15 | return trainer_params 16 | except yaml.YAMLError as exc: 17 | print(exc) 18 | 19 | def main(): 20 | choice_of_config = sys.argv[1] 21 | config_path = f'configs/config_{choice_of_config}.yaml' 22 | params = load_config(filepath=config_path) 23 | params = munchify(params) 24 | 25 | pl.seed_everything(1) 26 | for choice_of_dataset in ['valid', 'test']: 27 | dataset = AR_recon_dataset_s2r(choice_of_dataset, 0, params) 28 | dataloader = utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=params.num_workers) 29 | 30 | logger = pl_loggers.TensorBoardLogger(save_dir=params.log_dir) 31 | model = ARnet_recon(params=params) 32 | ckpt=torch.load(params.test_ckpt_path) 33 | model.load_state_dict(ckpt['state_dict']) 34 | model.eval() 35 | model.freeze() 36 | 37 | trainer=pl.Trainer(max_epochs=params.epochs,accelerator="gpu",log_every_n_steps=1,logger=logger,default_root_dir='checkpoint/',benchmark=None) 38 | trainer.test(model=model, dataloaders=dataloader,verbose=True) 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /ARNet_material_classification/README.md: -------------------------------------------------------------------------------- 1 | # ARNet_material_classification 2 | 3 | ## Installation 4 | The code has been tested on Ubuntu 20.04 with CUDA 12.0. 5 | ``` 6 | virtualenv -p /usr/bin/python venv_classification 7 | source venv_classification/bin/activate 8 | cd ARNet_material_classification 9 | pip install -r requirements.txt 10 | ``` 11 | 12 | ## Data Preparation 13 | 14 | Download the dataset from the [Google Drive](https://drive.google.com/file/d/121ZZw-_Bd2QLxrFwHMaK20bsRWV-05OF/view?usp=drive_link) and unzip under the folder. 15 | 16 | ## About Configs 17 | - `exp_name` - used to specify data split 18 | - `ckpt_path` - used to specify saved model for testing 19 | - `init_dataset` - if it is True, the program with randomly rebalance the dataset, which will create a different dataset. Set it to False to reproduce the results. 20 | 21 | ## Training 22 | 23 | Run the following command for training: 24 | ``` 25 | CUDA_VISIBLE_DEVICES=6 python main.py data_split_<1-3> 26 | ``` 27 | During training, the training and validation dataset are balanced, so the best model is selected based on the best validation accuracy. 28 | ## Evaluation 29 | Run the following command for testing: 30 | ``` 31 | CUDA_VISIBLE_DEVICES=6 python test.py data_split_<1-3> 32 | ``` 33 | During testing, the testing dataset is unbalanced, so F1 score is used to evaluate the performance and the parameters of refinement method is searched on unbalanced validation dataset. 34 | 35 | To test the model performance of the model trained on objectfolder dataset, run the following command: 36 | ``` 37 | CUDA_VISIBLE_DEVICES=6 python test_object_folder.py object_folder 38 | ``` 39 | The confusion matrices will be saved under `images` folder, and numerical results are saved under `results` folder. 40 | 41 | 42 | -------------------------------------------------------------------------------- /ARNet_material_classification/test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import yaml 3 | import torch 4 | from torch import utils 5 | from munch import munchify 6 | import pytorch_lightning as pl 7 | from models.ARNet_material import ARNet_material 8 | from refine_material_predication import get_overall_accuracy 9 | from dataset.AR_material_dataset import AR_material_dataset, AR_material_objectfolder_dataset 10 | 11 | def load_config(filepath): 12 | with open(filepath, 'r') as stream: 13 | try: 14 | trainer_params = yaml.safe_load(stream) 15 | return trainer_params 16 | except yaml.YAMLError as exc: 17 | print(exc) 18 | 19 | def main(): 20 | choice_of_config = sys.argv[1] 21 | config_path = f'configs/config_{choice_of_config}.yaml' 22 | params = load_config(filepath=config_path) 23 | params = munchify(params) 24 | pl.seed_everything(1) 25 | 26 | for choice_of_dataset in ['original_valid', 'original_test']: 27 | params.testing_split = choice_of_dataset.replace('original_','') 28 | dataset = AR_material_dataset(choice_of_dataset, params) 29 | dataloader = utils.data.DataLoader(dataset, batch_size=10000, shuffle=False, num_workers=params.num_workers) 30 | 31 | model = ARNet_material(params) 32 | ckpt=torch.load(params.ckpt_path) 33 | model.load_state_dict(ckpt['state_dict']) 34 | model.eval() 35 | model.freeze() 36 | 37 | trainer=pl.Trainer(accelerator="gpu", devices=1,max_epochs=1,num_nodes=1) 38 | trainer.test(model=model,dataloaders=dataloader,verbose=True) 39 | 40 | get_refined_results(params,config_path) 41 | 42 | def get_refined_results(params,config_path): 43 | get_overall_accuracy(params,config_path).get_best_test_acc() 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /ARNet_object_classification/show_dataset_statistic.py: -------------------------------------------------------------------------------- 1 | from munch import munchify 2 | import yaml 3 | import numpy as np 4 | import json 5 | 6 | def load_config(filepath): 7 | with open(filepath, 'r') as stream: 8 | try: 9 | trainer_params = yaml.safe_load(stream) 10 | return trainer_params 11 | except yaml.YAMLError as exc: 12 | print(exc) 13 | 14 | def read_simplified_label_from_npy(path): 15 | path, idx = path 16 | if 'objectfolder' in path: 17 | label=1 18 | else: 19 | idx=idx[1]+(idx[0]-1)*4 20 | data = [np.load(f'{path}', allow_pickle=True)[idx]] 21 | label=data[0] 22 | for key in simplified_label_mapping: 23 | if label in simplified_label_mapping[key]: 24 | simplified_label= list(simplified_label_mapping.keys()).index(key) 25 | return np.array(int(simplified_label)) 26 | 27 | def _load_data(): 28 | material_label_paths=np.load(f'data/ARdataset/{exp}/{split}_label_list.npy', allow_pickle=True) 29 | audio_paths=np.load(f'data/ARdataset/{exp}/{split}_audio_list.npy', allow_pickle=True) 30 | labels=[0]*9 31 | 32 | #put the label filepath and audio filepath in its own class and count the number 33 | for i in range(len(material_label_paths)): 34 | label = current_label_list[int(read_simplified_label_from_npy(material_label_paths[i]))] 35 | labels[label]+=1 36 | print(labels) 37 | print(f'number of {split} data =',len(material_label_paths) ) 38 | for i in range(len(labels)): 39 | print(f'{split}_label_{i} % =',labels[i]/len(material_label_paths)) 40 | print(split,'==========================total data:',len(material_label_paths)) 41 | 42 | 43 | with open('dataset/material_simple_categories.json') as f: 44 | simplified_label_mapping = json.load(f) 45 | 46 | 47 | # exp='data_split_3' 48 | # split='train' 49 | # params = load_config(filepath='configs/config.yaml') 50 | # params = munchify(params) 51 | # current_label_list=params.label_mapping 52 | # _load_data() 53 | val=[56.67,54.84,57.23] 54 | test=[56.06,53.65,52.88] 55 | re_val=[72.35,81.95,78.24] 56 | re_test=[81.35,78.12,69.79] 57 | 58 | print(np.mean(re_val),np.std(re_val)) -------------------------------------------------------------------------------- /ARNet_material_classification/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import yaml 3 | from torch import utils 4 | from datetime import date 5 | from munch import munchify 6 | import pytorch_lightning as pl 7 | from models.ARNet_material import ARNet_material 8 | from dataset.data_preparation import data_preparation 9 | from pytorch_lightning.callbacks import ModelCheckpoint 10 | from dataset.AR_material_dataset import AR_material_dataset 11 | 12 | def load_config(filepath): 13 | with open(filepath, 'r') as stream: 14 | try: 15 | trainer_params = yaml.safe_load(stream) 16 | return trainer_params 17 | except yaml.YAMLError as exc: 18 | print(exc) 19 | 20 | def main(): 21 | config = params 22 | pl.seed_everything(1,workers=True) 23 | model = ARNet_material(params,config) 24 | train_dataset = AR_material_dataset( 'train',params) 25 | val_dataset = AR_material_dataset('valid',params) 26 | train_dataloader = utils.data.DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=params.num_workers) 27 | val_dataloader = utils.data.DataLoader(val_dataset, batch_size=config.batch_size, shuffle=False, num_workers=params.num_workers) 28 | 29 | checkpoint_callback = ModelCheckpoint(mode='max', 30 | save_top_k=1, 31 | monitor='val_acc', 32 | dirpath=f'checkpoint', 33 | filename=f'{today}/{params.model}-{params.exp_name}-bc{config.batch_size}-lr{config.lr}-dropout{config.dropout}-fc_layer{150}'+'-best_model-{epoch:02d}-{val_loss:.2f}-{val_acc:.2f}') 34 | trainer=pl.Trainer(num_nodes=1, 35 | max_epochs=300, 36 | accelerator="gpu", 37 | logger=True, 38 | log_every_n_steps=1, 39 | default_root_dir='checkpoint/', 40 | deterministic=True, 41 | callbacks=[checkpoint_callback]) 42 | 43 | trainer.fit(model=model, train_dataloaders=train_dataloader,val_dataloaders=val_dataloader) 44 | 45 | if __name__ == '__main__': 46 | choice_of_config = sys.argv[1] 47 | params = load_config(filepath=f'configs/config_{choice_of_config}.yaml') 48 | params = munchify(params) 49 | today = date.today() 50 | today = str(today.strftime("%m/%d/%y")).replace('/','',2) 51 | if params.init_dataset == True: 52 | data_preparation(params).get_even_dataset() 53 | main() 54 | -------------------------------------------------------------------------------- /ARNet_object_classification/dataset/AR_object_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import torch 4 | import numpy as np 5 | from random import * 6 | from natsort import natsorted 7 | import pytorch_lightning as pl 8 | sys.path.append('.') 9 | 10 | class AR_object_dataset(pl.LightningDataModule): 11 | def __init__(self, split,params): 12 | assert split in ['train', 'valid', 'test'], "split error value!" 13 | self.params=params 14 | self.split = split 15 | self.current_label_list=params.label_mapping 16 | self.contact_points_paths,self.audio_paths, self.labels = self._load_data() 17 | print(split,len(self.contact_points_paths),len(self.audio_paths)) 18 | self.total_num_of_data=len(self.contact_points_paths) 19 | 20 | def __getitem__(self, index): 21 | labels =np.array(self.labels[index]) 22 | contact_points_path=self.contact_points_paths[index] 23 | audio_path=self.audio_paths[index] 24 | contact_points= self.read_contact_points_from_npy(contact_points_path) 25 | audio = self.read_audio_from_npy(audio_path) 26 | self.mean, self.std = np.asarray(audio).mean(),np.asarray(audio).std() 27 | audio=(audio-self.mean)/self.std 28 | return torch.from_numpy(contact_points), torch.from_numpy(audio), torch.from_numpy(labels) 29 | 30 | def __len__(self): 31 | return len(self.contact_points_paths) 32 | 33 | def _load_data(self): 34 | contact_points_paths = [] 35 | audio_paths = [] 36 | label = [] 37 | for file_name in natsorted(os.listdir(f'./{self.params.exp_name}/contact_points/{self.split}')): 38 | contact_points_paths.append(f'./{self.params.exp_name}/contact_points/{self.split}/{file_name}') 39 | for label_name in self.params.label_mapping: 40 | if f'_{label_name}.npy' in file_name: 41 | label.append(self.params.label_mapping[label_name]) 42 | for path in contact_points_paths: 43 | path = path.replace('contact_points','audio') 44 | audio_paths.append(path) 45 | return contact_points_paths, audio_paths, label 46 | 47 | def read_contact_points_from_npy(self, path): 48 | raw_contact_points = np.load(f'{path}', allow_pickle=True) 49 | contact_points = [] 50 | for i in raw_contact_points: 51 | contact_points.append(i[0:3]) 52 | return np.array(contact_points, np.float32) 53 | 54 | def read_audio_from_npy(self, path): 55 | audio_data = np.load(f'{path}', allow_pickle=True) 56 | return np.array(audio_data, np.float32) 57 | -------------------------------------------------------------------------------- /ARNet_object_classification/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import yaml 3 | import wandb 4 | from torch import utils 5 | from datetime import date 6 | from munch import munchify 7 | import pytorch_lightning as pl 8 | from models.ARNet_object import ARNet_object 9 | from pytorch_lightning import loggers as pl_loggers 10 | from pytorch_lightning.callbacks import ModelCheckpoint 11 | from dataset.AR_object_dataset import AR_object_dataset 12 | 13 | 14 | def load_config(filepath): 15 | with open(filepath, 'r') as stream: 16 | try: 17 | trainer_params = yaml.safe_load(stream) 18 | return trainer_params 19 | except yaml.YAMLError as exc: 20 | print(exc) 21 | 22 | def main(): 23 | pl.seed_everything(1) 24 | model = ARNet_object(params) 25 | 26 | train_dataset = AR_object_dataset( 'train',params) 27 | val_dataset = AR_object_dataset('valid',params) 28 | 29 | train_dataloader = utils.data.DataLoader(train_dataset, batch_size=params.batch_size, shuffle=True, num_workers=params.num_workers) 30 | val_dataloader = utils.data.DataLoader(val_dataset, batch_size=params.batch_size, shuffle=False, num_workers=params.num_workers) 31 | 32 | logger = pl_loggers.TensorBoardLogger(save_dir=params.log_dir,name=f'{today}/{params.model}-{params.exp_name}-{params.model_inputs}-bc{params.batch_size}-lr{params.lr}-dropout{params.dropout}') 33 | 34 | checkpoint_callback = ModelCheckpoint(mode='max', 35 | save_top_k=1, 36 | monitor='val_acc', 37 | dirpath=f'checkpoint', 38 | filename=f'{today}/{params.model}-{params.exp_name}-{params.model_inputs}bc{params.batch_size}-lr{params.lr}-dropout{params.dropout}-fc_layer{150}'+'-best_model-{epoch:02d}-{train_acc:.2f}-{val_acc:.2f}') 39 | trainer=pl.Trainer(num_nodes=1, 40 | max_epochs=params.epoches, 41 | accelerator="gpu", 42 | log_every_n_steps=1, 43 | logger=logger, 44 | deterministic=True, 45 | default_root_dir='checkpoint/', 46 | callbacks=[checkpoint_callback]) 47 | 48 | trainer.fit(model=model, train_dataloaders=train_dataloader,val_dataloaders=val_dataloader) 49 | 50 | if __name__ == '__main__': 51 | choice_of_config = sys.argv[1] 52 | choice_of_input = sys.argv[2] 53 | config_path = f'configs/config_{choice_of_config}_{choice_of_input}.yaml' 54 | params = load_config(filepath=config_path) 55 | params = munchify(params) 56 | today = date.today() 57 | today = str(today.strftime("%m/%d/%y")).replace('/','',2) 58 | main() 59 | -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import yaml 3 | from datetime import date 4 | from munch import munchify 5 | import pytorch_lightning as pl 6 | from models.ARnet_recon import ARnet_recon 7 | from torch.utils.data.dataloader import DataLoader 8 | from pytorch_lightning import loggers as pl_loggers 9 | from pytorch_lightning.callbacks import ModelCheckpoint 10 | from visualization.visualization import plot_pcd_one_view 11 | from dataset.data_preparation import data_preparation 12 | 13 | def load_config(filepath): 14 | with open(filepath, 'r') as stream: 15 | try: 16 | trainer_params = yaml.safe_load(stream) 17 | return trainer_params 18 | except yaml.YAMLError as exc: 19 | print(exc) 20 | 21 | def main(): 22 | choice_of_config = sys.argv[1] 23 | config_path = f'configs/config_{choice_of_config}.yaml' 24 | params = load_config(filepath=config_path) 25 | params = munchify(params) 26 | 27 | prepare_dataset = data_preparation() 28 | if params.init_dataset == True: 29 | prepare_dataset.prepare_real_dataset(choice_of_config) 30 | prepare_dataset.prepare_synthetic_dataset(choice_of_config) 31 | 32 | pl.seed_everything(1) 33 | today = date.today() 34 | today = str(today.strftime("%m/%d/%y")).replace('/','',2) 35 | logger = pl_loggers.TensorBoardLogger(save_dir=params.log_dir,name=params.exp_name) 36 | 37 | model = ARnet_recon(params=params) 38 | 39 | checkpoint_epoch_callback = ModelCheckpoint(dirpath=f'checkpoint', 40 | save_top_k=1, 41 | every_n_epochs=100, 42 | filename=f'{today}/{params.exp_name}'+'-{epoch:02d}-{val_loss}') 43 | 44 | checkpoint_val_acc_callback = ModelCheckpoint(mode='min', 45 | save_top_k=1, 46 | monitor='val_loss', 47 | dirpath=f'checkpoint', 48 | filename=f'{today}/{params.exp_name}'+'-best_model-{epoch:02d}-{val_loss}') 49 | 50 | trainer=pl.Trainer(logger=logger, 51 | accelerator="gpu", 52 | deterministic=True, 53 | log_every_n_steps=1, 54 | max_epochs=params.epochs, 55 | default_root_dir='checkpoint/', 56 | callbacks=[checkpoint_epoch_callback,checkpoint_val_acc_callback], 57 | reload_dataloaders_every_n_epochs=1) 58 | 59 | trainer.fit(model=model,ckpt_path=params.train_ckpt_path) 60 | 61 | if __name__ == '__main__': 62 | main() 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SonicSense: Object Perception from In-Hand Acoustic Vibration 2 | 3 | [Jiaxun Liu](https://www.jiaxunliu.com/), [Boyuan Chen](http://boyuanchen.com/) 4 |
5 | Duke University 6 |
7 | 8 | ### [Project Website](http://generalroboticslab.com/SonicSense) | [Video](https://www.youtube.com/watch?v=MvSYdLMsvx4) | [Paper](https://arxiv.org/abs/2406.17932) 9 | 10 | ## Overview 11 | We introduce SonicSense, a holistic design of hardware and software to enable rich robot object perception through in-hand acoustic vibration sensing. While previous studies have shown promising results with acoustic sensing for object perception, current solutions are constrained to a handful of objects with simple geometries and homogeneous materials, single-finger sensing, and mixing training and testing on the same objects. SonicSense enables container inventory status differentiation, heterogeneous material prediction, 3D shape reconstruction, and object re-identification from a diverse set of 83 real-world objects. Our system employs a simple but effective heuristic exploration policy to interact with the objects as well as end-to-end learning-based algorithms to fuse vibration signals to infer object properties. Our framework underscores the significance of in-hand acoustic vibration sensing in advancing robot tactile perception. 12 | 13 |

14 | teaser 15 |

16 | 17 | ## Code Structure 18 | 19 | We provide detailed instructions on running our code for [material classification](https://github.com/generalroboticslab/SonicSense/tree/main/ARNet_material_classification), [shape reconstruction](https://github.com/generalroboticslab/SonicSense/tree/main/ARNet_shape_reconstruction) and [object re-identification](https://github.com/generalroboticslab/SonicSense/tree/main/ARNet_object_classification) under each subdirectory. Please refer to specific README files under each directory. 20 | 21 | The full CAD model and instruction of our hardware design are under [Hardware_instruction](https://github.com/generalroboticslab/SonicSense/tree/main/Hardware_instruction) subdirectory. 22 | 23 | ## Citation 24 | 25 | If you find our paper or codebase helpful, please consider citing: 26 | 27 | ``` 28 | @inproceedings{ 29 | liu2024sonicsense, 30 | title={SonicSense: Object Perception from In-Hand Acoustic Vibration}, 31 | author={Jiaxun Liu and Boyuan Chen}, 32 | booktitle={8th Annual Conference on Robot Learning}, 33 | year={2024}, 34 | url={https://openreview.net/forum?id=CpXiqz6qf4} 35 | } 36 | ``` 37 | 38 | ## License 39 | 40 | This repository is released under the Apache License 2.0. See [LICENSE](LICENSE) for additional details. 41 | 42 | ## Acknowledgement 43 | [Point Cloud Renderer](https://github.com/zekunhao1995/PointFlowRenderer), [PyLX-16A](https://github.com/ethanlipson/PyLX-16A) 44 | 45 | 46 | `This work is supported by ARL STRONG program under awards W911NF2320182 and W911NF2220113, by DARPA FoundSci program under award HR00112490372, and DARPA TIAMAT program under award HR00112490419.` 47 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_1_audio.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_1' 3 | model_inputs: 'audio' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split1_audio-bc200-lr3.651467320443295e-05-dropout0.2348302574633227-fc_layer150-best_model-epoch=414-train_acc=1.00-val_acc=0.87.ckpt 6 | lr: 3.651467320443295e-05 7 | dropout: 0.2348302574633227 8 | batch_size: 200 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_1_points.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_1' 3 | model_inputs: 'points' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split1_points-bc200-lr3.651467320443295e-05-dropout0.2348302574633227-fc_layer150-best_model-epoch=171-train_acc=0.98-val_acc=0.53.ckpt 6 | lr: 3.651467320443295e-05 7 | dropout: 0.2348302574633227 8 | batch_size: 200 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_2_audio.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_2' 3 | model_inputs: 'audio' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split2_audio-bc200-lr8.822350445001824e-05-dropout0.2240786067413556-fc_layer150-best_model-epoch=342-train_acc=1.00-val_acc=0.88.ckpt 6 | lr: 8.822350445001824e-05 7 | dropout: 0.2240786067413556 8 | batch_size: 200 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_2_points.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_2' 3 | model_inputs: 'points' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split2_points-bc200-lr8.822350445001824e-05-dropout0.2240786067413556-fc_layer150-best_model-epoch=235-train_acc=1.00-val_acc=0.51.ckpt 6 | lr: 8.822350445001824e-05 7 | dropout: 0.2240786067413556 8 | batch_size: 200 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_3_audio.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_3' 3 | model_inputs: 'audio' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split3_audio-bc400-lr7.591005979309437e-05-dropout0.3252774501746758-fc_layer150-best_model-epoch=454-train_acc=1.00-val_acc=0.81.ckpt 6 | lr: 7.591005979309437e-05 7 | dropout: 0.3252774501746758 8 | batch_size: 400 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_3_points.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_3' 3 | model_inputs: 'points' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split3_points-bc400-lr7.591005979309437e-05-dropout0.3252774501746758-fc_layer150-best_model-epoch=446-train_acc=1.00-val_acc=0.54.ckpt 6 | lr: 7.591005979309437e-05 7 | dropout: 0.3252774501746758 8 | batch_size: 400 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_1_audio+points.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_1' 3 | model_inputs: 'audio+points' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split1_audio+points-bc200-lr3.651467320443295e-05-dropout0.2348302574633227-fc_layer150-best_model-epoch=486-train_acc=1.00-val_acc=0.97.ckpt 6 | lr: 3.651467320443295e-05 7 | dropout: 0.2348302574633227 8 | batch_size: 200 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_2_audio+points.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_2' 3 | model_inputs: 'audio+points' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split2_audio+points-bc200-lr8.822350445001824e-05-dropout0.2240786067413556-fc_layer150-best_model-epoch=411-train_acc=1.00-val_acc=0.94.ckpt 6 | lr: 8.822350445001824e-05 7 | dropout: 0.2240786067413556 8 | batch_size: 200 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_object_classification/configs/config_data_split_3_audio+points.yaml: -------------------------------------------------------------------------------- 1 | model: 'ARNet_object' 2 | exp_name: 'data_split_3' 3 | model_inputs: 'audio+points' 4 | label_name: ['cergym_bottleamic','meyerscleanday','mustard','M_mug','pepsi','rubber_hammer','viniger','tissue_box','large_glass_cup','black_metal_container_cylinder','metal_bottle'] 5 | ckpt_path: checkpoint/model_used_for_the_results/split3_audio+points-bc400-lr7.591005979309437e-05-dropout0.3252774501746758-fc_layer150-best_model-epoch=453-train_acc=1.00-val_acc=0.92.ckpt 6 | lr: 7.591005979309437e-05 7 | dropout: 0.3252774501746758 8 | batch_size: 400 9 | epoches: 500 10 | log_dir: 'log/lightning_logs' 11 | num_workers: 8 12 | num_label: 82 13 | init_dataset: False 14 | number_of_tapping_points: 15 15 | points_latent_dimension: 150 16 | label_mapping: 17 | 4_ceramic_vase: 0 18 | 6_chinese_ceramic_bowl_small: 1 19 | 7_chip_can: 2 20 | 10_coca_colar: 3 21 | 11_coffee_container_paper: 4 22 | 12_cone_plastic: 5 23 | 13_cone_wood: 6 24 | 14_cuboid_ceramic: 7 25 | 15_cuboid_foam: 8 26 | 16_cuboid_metal: 9 27 | 17_cuboid_plastic: 10 28 | 18_cuboid_wood: 11 29 | 20_cylinder_plastic: 12 30 | 21_cylinder_rubber: 13 31 | 22_cylinder_wood: 14 32 | 23_dawn: 15 33 | 25_duke_mug: 16 34 | 26_duke_science_mug: 17 35 | 29_glass_locker_square: 18 36 | 31_gym_bottle: 19 37 | 34_meyerscleanday: 20 38 | 36_mustard: 21 39 | 37_M_mug: 22 40 | 40_oat_milk: 23 41 | 41_olive_oil_spray: 24 42 | 43_paper_cup_with_cover: 25 43 | 44_paper_tape: 26 44 | 45_pepper_powder: 27 45 | 46_pepsi: 28 46 | 47_pickles: 29 47 | 50_poptarts: 30 48 | 53_quadrangular_pyramid_wood: 31 49 | 54_quadrangular_pyramid_plastic: 32 50 | 57_rubber_duck: 33 51 | 58_rubber_hammer: 34 52 | 65_tea_can: 35 53 | 71_triangular_prism_plastic: 36 54 | 72_triangular_prism_wood: 37 55 | 73_triangular_pyramid_plastic: 38 56 | 74_viniger: 39 57 | 77_wooden_hammer: 40 58 | 82_ceramic_mug_no_handle: 41 59 | 84_ceramic_vase_small: 42 60 | 85_cone_foam: 43 61 | 86_cuboid_foam: 44 62 | 87_cuboid_glass_container: 45 63 | 88_cuboid_glass_container_with_wood_lid: 46 64 | 89_cylinder_glass_container_with_wood_lid: 47 65 | 90_cylinder_rubber: 48 66 | 92_glass_cup: 49 67 | 93_small_ceramic_container_with_lid: 50 68 | 94_small_ceramic_container_without_lid: 51 69 | 97_triangular_pyramid_foam: 52 70 | 98_triangular_prism_foam: 53 71 | 106_tissue_box: 54 72 | 107_cuboid_rubber: 55 73 | 108_triangular_pyramid_rubber: 56 74 | 109_cone_rubber: 57 75 | 110_quadrangular_pyramid_rubber: 58 76 | 111_quadrangular_pyramid_foam: 59 77 | 112_cylinder_foam: 60 78 | 114_wood_jar_2: 61 79 | 115_wood_jar_3: 62 80 | 116_hard_paper_box: 63 81 | 119_wood_scoop: 64 82 | 121_triangular_pyramid_fabric: 65 83 | 122_cone_fabric: 66 84 | 123_cuboid_fabric: 67 85 | 124_triangular_prism_fabric: 68 86 | 125_wine_glass: 69 87 | 127_wood_frame: 70 88 | 128_cylinder_fabric: 71 89 | 129_quadrangular_pyramid_fabric: 72 90 | 130_small_glass_cup: 73 91 | 131_large_glass_cup: 74 92 | 133_domino_suger: 75 93 | 135_triangular_prism_rubber: 76 94 | black_metal_container_cylinder: 77 95 | metal_bottle: 78 96 | silvery_metal_container_square: 79 97 | white_bottle: 80 98 | white_paint: 81 99 | 100 | -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/visualization/visualization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import open3d as o3d 3 | import matplotlib.pyplot as plt 4 | from mpl_toolkits.mplot3d import Axes3D 5 | 6 | def o3d_visualize_pc(pc): 7 | point_cloud = o3d.geometry.PointCloud() 8 | point_cloud.points = o3d.utility.Vector3dVector(pc) 9 | o3d.visualization.draw_geometries([point_cloud]) 10 | 11 | def plot_pcd_one_view(filename, pcds, titles, suptitle='', sizes=None, cmap='Reds', zdir='y', 12 | xlim=(-0.5, 0.5), ylim=(-0.5, 0.5), zlim=(-0.5, 0.5)): 13 | if sizes is None: 14 | sizes = [0.5 for i in range(len(pcds))] 15 | fig = plt.figure(figsize=(len(pcds) * 3 * 1.4, 3 * 1.4)) 16 | elev = 30 17 | azim = -45 18 | for j, (pcd, size) in enumerate(zip(pcds, sizes)): 19 | color = pcd[:, 0] 20 | ax = fig.add_subplot(1, len(pcds), j + 1, projection='3d') 21 | ax.view_init(elev, azim) 22 | if len(pcd[0])==3: 23 | ax.scatter(pcd[:, 0], pcd[:, 1], pcd[:, 2], zdir=zdir, c=color, s=size, cmap=cmap, vmin=-1.0, vmax=0.5) 24 | else: 25 | cmap = plt.cm.Spectral 26 | norm = plt.Normalize(vmin=0, vmax=1) 27 | ax.scatter(pcd[:, 0], pcd[:, 1], pcd[:, 2], zdir=zdir, s=size,c = cmap(norm(pcd[:, 3]))) 28 | ax.set_title(titles[j]) 29 | ax.set_axis_off() 30 | ax.set_xlim(xlim) 31 | ax.set_ylim(ylim) 32 | ax.set_zlim(zlim) 33 | plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.9, wspace=0.1, hspace=0.1) 34 | plt.suptitle(suptitle) 35 | fig.savefig(filename) 36 | plt.close(fig) 37 | 38 | def save_image(filename,pcds,titles): 39 | fig = plt.figure() 40 | num_plot=len(titles) 41 | for i in range(num_plot): 42 | pcd=pcds[i] 43 | ax = fig.add_subplot(1,num_plot,i+1,projection='3d') 44 | if i ==0: 45 | valid_points=[] 46 | for point in pcd: 47 | if sum(point)!=0: 48 | valid_points.append(list(point)) 49 | pcd = np.asarray(valid_points) 50 | ax.scatter(pcd[:, 0], pcd[:, 1], pcd[:, 2], s=0.5,color='red') 51 | ax.set_title(titles[i]) 52 | ax.set_axis_off() 53 | ax.set_box_aspect((np.ptp(pcd[:, 0]), np.ptp(pcd[:, 1]), np.ptp(pcd[:, 2]))) 54 | plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.9, wspace=0.1, hspace=0.1) 55 | plt.axis('equal') 56 | fig.savefig(filename) 57 | plt.close(fig) 58 | 59 | def save_overlap_image(filename,pcds,titles): 60 | fig = plt.figure() 61 | ax = fig.add_subplot(1,1,1,projection='3d') 62 | num_plot=len(titles) 63 | colors=['red','purple','gray'] 64 | s_list=[6,3,0.2] 65 | for i in range(num_plot): 66 | pcd=pcds[i] 67 | if i ==0: 68 | valid_points=[] 69 | for point in pcd: 70 | if sum(point)!=0: 71 | valid_points.append(list(point)) 72 | pcd = np.asarray(valid_points) 73 | ax.scatter(pcd[:, 0], pcd[:, 1], pcd[:, 2], s=s_list[i],color=colors[i]) 74 | ax.set_title(titles[i]) 75 | ax.set_axis_off() 76 | ax.set_box_aspect((np.ptp(pcd[:, 0]), np.ptp(pcd[:, 1]), np.ptp(pcd[:, 2]))) 77 | plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.9, wspace=0.1, hspace=0.1) 78 | plt.axis('equal') 79 | filename=filename.replace('.png','_overlap.png') 80 | fig.savefig(f'{filename}') 81 | plt.close(fig) 82 | 83 | def plot_pcd_one_view_new(filename, pcds, titles): 84 | save_image(filename,pcds,titles) 85 | save_overlap_image(filename,pcds,titles) 86 | -------------------------------------------------------------------------------- /ARNet_object_classification/main_sweep.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import wandb 3 | from torch import utils 4 | from datetime import date 5 | from munch import munchify 6 | import pytorch_lightning as pl 7 | from models.ARNet_object import ARNet_object 8 | from pytorch_lightning.loggers import WandbLogger 9 | from dataset.data_preparation import data_preparation 10 | from pytorch_lightning.callbacks import ModelCheckpoint 11 | from dataset.AR_object_dataset import AR_object_dataset 12 | from models.ARNet_object_larger import ARNet_object_larger 13 | from models.ARNet_1024 import ARNet_object_1024 14 | 15 | def load_config(filepath): 16 | with open(filepath, 'r') as stream: 17 | try: 18 | trainer_params = yaml.safe_load(stream) 19 | return trainer_params 20 | except yaml.YAMLError as exc: 21 | print(exc) 22 | 23 | def main(): 24 | exp_name=params.exp_name 25 | sweep_config = { 26 | 'method': 'random', 27 | 'name': exp_name, 28 | 'metric': { 29 | 'goal': 'maximize', 30 | 'name': 'val_acc' 31 | }, 32 | 'parameters': { 33 | 'batch_size': {'distribution': 'q_log_uniform_values', 34 | 'q': 200, 35 | 'min': 200, 36 | 'max': 400,}, 37 | 'lr': {'distribution': 'uniform','max': 0.0001, 'min': 0.00001}, 38 | 'dropout': { 39 | 'distribution': 'uniform','max': 0.4, 'min': 0.2 40 | }, 41 | } 42 | } 43 | print(sweep_config,'=================') 44 | sweep_id = wandb.sweep(sweep_config, project="object_classification") 45 | wandb.agent(sweep_id, function=train, count=10) 46 | 47 | 48 | def train(config=None): 49 | with wandb.init(config=config): 50 | config = wandb.config 51 | config.lr = round(config.lr,6) 52 | config.dropout = round(config.dropout,4) 53 | pl.seed_everything(1) 54 | model = ARNet_object(params,config) 55 | 56 | wandb_logger = WandbLogger(log_model=True) 57 | 58 | train_dataset = AR_object_dataset( 'train',params) 59 | val_dataset = AR_object_dataset('valid',params) 60 | 61 | train_dataloader = utils.data.DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=params.num_workers) 62 | val_dataloader = utils.data.DataLoader(val_dataset, batch_size=config.batch_size, shuffle=False, num_workers=params.num_workers) 63 | checkpoint_epoch_callback = ModelCheckpoint(dirpath=f'checkpoint', 64 | save_top_k=1, 65 | every_n_epochs=10, 66 | filename=f'{today}/{params.model}-{params.exp_name}-bc{config.batch_size}-lr{config.lr}-dropout{config.dropout}-fc_layer{150}'+'-best_model-{epoch:02d}-{train_acc:.2f}-{val_acc:.2f}') 67 | checkpoint_callback = ModelCheckpoint(mode='max', 68 | save_top_k=1, 69 | monitor='val_acc', 70 | dirpath=f'checkpoint', 71 | filename=f'{today}/{params.model}-{params.exp_name}-bc{config.batch_size}-lr{config.lr}-dropout{config.dropout}-fc_layer{150}'+'-best_model-{epoch:02d}-{train_acc:.2f}-{val_acc:.2f}') 72 | trainer=pl.Trainer(num_nodes=1, 73 | max_epochs=params.epoches, 74 | accelerator="gpu", 75 | log_every_n_steps=1, 76 | logger=wandb_logger, 77 | default_root_dir='checkpoint/', 78 | callbacks=[checkpoint_callback,checkpoint_epoch_callback]) 79 | 80 | trainer.fit(model=model, train_dataloaders=train_dataloader,val_dataloaders=val_dataloader) 81 | 82 | if __name__ == '__main__': 83 | params = load_config(filepath='configs/config.yaml') 84 | params = munchify(params) 85 | today = date.today() 86 | today = str(today.strftime("%m/%d/%y")).replace('/','',2) 87 | if params.init_dataset == True: 88 | data_preparation(params).get_even_dataset() 89 | 90 | main() 91 | -------------------------------------------------------------------------------- /ARNet_shape_reconstruction/compute_baseline.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import random 4 | import numpy as np 5 | 6 | #random baseline 7 | for split in ['data_split_3']: 8 | test_gt_list = [] 9 | train_gt_list = [] 10 | for shape in os.listdir(f'data/ARDataset/{split}/tapping_points_test'): 11 | for object_name in os.listdir(f'data/ARDataset/{split}/tapping_points_test/{shape}'): 12 | test_gt_list.append(f'{object_name}') 13 | for shape in os.listdir(f'data/ARDataset/{split}/tapping_points_train'): 14 | for object_name in os.listdir(f'data/ARDataset/{split}/tapping_points_train/{shape}'): 15 | train_gt_list.append(f'{object_name}') 16 | for i in range(len(test_gt_list)): 17 | test_gt_list[i] = f'data/ARDataset/ground_truth_5000_correct_unit/'+test_gt_list[i] 18 | for i in range(len(train_gt_list)): 19 | train_gt_list[i] = f'data/ARDataset/ground_truth_5000_correct_unit/'+train_gt_list[i] 20 | for shape in os.listdir(f'data/ARDataset_synthetic_data/syn_ground_truth_pcd'): 21 | for object_name in os.listdir(f'data/ARDataset_synthetic_data/syn_ground_truth_pcd/{shape}'): 22 | train_gt_list.append(f'data/ARDataset_synthetic_data/syn_ground_truth_pcd/{shape}/{object_name}') 23 | loss = [] 24 | for object_name in test_gt_list: 25 | test_gt = np.load(object_name, allow_pickle = True) 26 | train_object_name = random.choice(train_gt_list) 27 | train_gt = np.load(train_object_name,allow_pickle = True) 28 | print(split, object_name,train_object_name) 29 | 30 | distanct_1=[] 31 | for i in test_gt: 32 | min_dis=float('inf') 33 | for j in train_gt: 34 | distance = distance=math.dist(i,j) 35 | if distance Tensor: 75 | x = self.conv1(x) 76 | x = self.bn1(x) 77 | x = self.relu(x) 78 | x = self.maxpool(x) 79 | x = self.conv2(x) 80 | x = self.bn2(x) 81 | x = self.dropout1(x) 82 | x = self.relu2(x) 83 | x = self.maxpool2(x) 84 | x = self.conv3(x) 85 | x = self.bn3(x) 86 | x = self.relu3(x) 87 | x = torch.flatten(x, 1) 88 | x = self.dropout2(x) 89 | x = self.fc1(x) 90 | x = self.dropout2(x) 91 | x = self.fc2(x) 92 | return x 93 | 94 | def configure_optimizers(self): 95 | optimizer = optim.SGD(self.parameters(), lr=self.learning_rate) 96 | # optimizer = optim.Adam(self.parameters(), lr=self.params.lr,betas=(0.9, 0.999)) 97 | scheduler = Optim.lr_scheduler.StepLR(optimizer, step_size=200, gamma=0.1) 98 | return [optimizer],[scheduler] 99 | 100 | def training_step(self,batch,batch_idx): 101 | image, labels = batch 102 | image = image.cuda() 103 | labels = labels.cuda() 104 | outputs = self.forward(image) 105 | criterion = nn.CrossEntropyLoss() 106 | train_loss = criterion(outputs, labels) 107 | self.train_acc(outputs, labels) 108 | self.log('train_loss', train_loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 109 | self.log('train_acc', self.train_acc, on_epoch=True,prog_bar=True) 110 | return train_loss 111 | 112 | def validation_step(self,batch,batch_idx): 113 | image, labels = batch 114 | image = image.cuda() 115 | labels = labels.cuda() 116 | outputs = self.forward(image) 117 | criterion = nn.CrossEntropyLoss() 118 | val_loss = criterion(outputs, labels) 119 | self.valid_acc(outputs, labels) 120 | self.log('val_loss', val_loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 121 | self.log('val_acc', self.valid_acc, on_epoch=True,prog_bar=True) 122 | return val_loss 123 | 124 | def test_step(self, batch,batch_idx): 125 | image, labels = batch 126 | image = image.cuda() 127 | labels = labels.cuda() 128 | outputs = self.forward(image) 129 | self.test_acc(outputs, labels) 130 | #confusion matrix 131 | self.test_step_y_hats.append(outputs) 132 | self.test_step_ys.append(labels) 133 | y_hat = torch.cat(self.test_step_y_hats) 134 | y = torch.cat(self.test_step_ys) 135 | metric = MulticlassConfusionMatrix(num_classes=self.num_classes).cuda() 136 | cm=metric(y_hat,y) 137 | confusion_matrix_computed = cm.detach().cpu().numpy().astype(int) 138 | uniformed_confusion_matrix=[] 139 | for idx,i in enumerate(confusion_matrix_computed): 140 | uniformed_confusion_matrix.append([val/sum(i) for val in i]) 141 | final_acc_list=[] 142 | for idx in range(len(uniformed_confusion_matrix)): 143 | final_acc_list.append(uniformed_confusion_matrix[idx][idx]) 144 | final_acc=sum(final_acc_list)/len(final_acc_list) 145 | print('final acc among class = ',final_acc) 146 | df_cm = pd.DataFrame(uniformed_confusion_matrix,index=self.label_name,columns=self.label_name) 147 | plt.figure(figsize = (10,8)) 148 | fig_ = sns.heatmap(df_cm, annot=True, cmap='Reds').get_figure() 149 | plt.xlabel('Predicted labels') 150 | plt.ylabel('True lables') 151 | plt.savefig(f'images/{self.params.exp_name}_{self.params.testing_split}_confusion_matrix', dpi=300) 152 | plt.close(fig_) 153 | self.loggers[0].experiment.add_figure("Test confusion matrix", fig_, self.current_epoch) 154 | #save the evaluation results 155 | label_prediction=[i.detach().cpu().numpy() for i in self.test_step_y_hats] 156 | label_gt=[i.detach().cpu().numpy() for i in self.test_step_ys] 157 | np.save(f'results/{self.params.exp_name}/{self.params.testing_split}/label_prediction_{self.params.testing_split}.npy',label_prediction[0]) 158 | np.save(f'results/{self.params.exp_name}/{self.params.testing_split}/label_gt_{self.params.testing_split}.npy',label_gt[0]) 159 | #compute metric 160 | recall_metric = MulticlassRecall(num_classes=self.num_classes, average='none').cuda() 161 | precision_metric = MulticlassPrecision(num_classes=self.num_classes, average='none').cuda() 162 | F1_score_metric = MulticlassF1Score(num_classes=self.num_classes,average='none').cuda() 163 | F1_score=F1_score_metric(y_hat,y) 164 | F1_score_average_metric = MulticlassF1Score(num_classes=self.num_classes,average='macro').cuda() 165 | F1_score_average=F1_score_average_metric(y_hat,y) 166 | recall = recall_metric(y_hat,y) 167 | precision = precision_metric(y_hat,y) 168 | print('torch recall', recall) 169 | print('torch precision ',precision) 170 | print('torch F1',F1_score) 171 | print('torch F1 average',F1_score_average) 172 | self.log('F1_score_average', F1_score_average,on_step=False, on_epoch=True,prog_bar=True) 173 | 174 | return {'preds' : outputs, 'targets' : labels} 175 | 176 | 177 | 178 | -------------------------------------------------------------------------------- /ARNet_object_classification/models/ARNet_object.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import statistics 3 | import numpy as np 4 | import torchmetrics 5 | import pandas as pd 6 | import torch.nn as nn 7 | import seaborn as sns 8 | import torch.optim as Optim 9 | import pytorch_lightning as pl 10 | import matplotlib.pyplot as plt 11 | from torch import optim, nn, Tensor 12 | from sklearn.metrics import confusion_matrix 13 | from torchmetrics.classification import MulticlassConfusionMatrix 14 | 15 | class ARNet_object(pl.LightningModule): 16 | def __init__(self,params, config=None): 17 | super(ARNet_object, self).__init__() 18 | self.params = params 19 | img_channels = params.number_of_tapping_points 20 | 21 | # self.label_name=params.label_name 22 | self.label_name = [i for i in range(params.num_label)] 23 | self.num_classes=params.num_label 24 | self.points_latent_dimension = params.points_latent_dimension 25 | if config == None: 26 | self.dropout=params.dropout 27 | self.learning_rate=params.lr 28 | else: 29 | self.dropout=config.dropout 30 | self.learning_rate=config.lr 31 | 32 | self.train_acc = torchmetrics.Accuracy(task="multiclass", num_classes=self.num_classes) 33 | self.valid_acc = torchmetrics.Accuracy(task="multiclass", num_classes=self.num_classes) 34 | self.test_acc = torchmetrics.Accuracy(task="multiclass", num_classes=self.num_classes) 35 | self.test_step_y_hats = [] 36 | self.test_step_ys = [] 37 | self.val_step_y_hats = [] 38 | self.val_step_ys = [] 39 | #tapping audio encoder 40 | self.dropout= nn.Dropout(p=self.dropout) 41 | 42 | self.conv1 = nn.Conv2d( 43 | in_channels=img_channels, 44 | out_channels=16, 45 | kernel_size=6, 46 | stride=2, 47 | padding=0, 48 | bias=False 49 | ) 50 | self.bn1 = nn.BatchNorm2d(16) 51 | self.relu = nn.ReLU(inplace=True) 52 | self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0) 53 | self.conv2 = nn.Conv2d( 54 | in_channels=16, 55 | out_channels=32, 56 | kernel_size=5, 57 | stride=1, 58 | padding=0, 59 | bias=False 60 | ) 61 | self.bn2 = nn.BatchNorm2d(32) 62 | self.relu2 = nn.ReLU(inplace=True) 63 | self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2) 64 | self.conv3 = nn.Conv2d( 65 | in_channels=32, 66 | out_channels=150, 67 | kernel_size=5, 68 | stride=1, 69 | padding=0, 70 | bias=False 71 | ) 72 | self.bn3 = nn.BatchNorm2d(150) 73 | self.relu3 = nn.ReLU(inplace=True) 74 | 75 | #tapping points encoder 76 | self.first_conv = nn.Sequential( 77 | nn.Conv1d(3, 64, 1), 78 | nn.BatchNorm1d(64), 79 | nn.ReLU(inplace=True), 80 | nn.Conv1d(64, 64, 1) 81 | ) 82 | 83 | self.second_conv = nn.Sequential( 84 | nn.Conv1d(128, 128, 1), 85 | nn.BatchNorm1d(128), 86 | nn.ReLU(inplace=True), 87 | nn.Conv1d(128, self.points_latent_dimension, 1) 88 | ) 89 | #mlp 90 | if self.params.model_inputs == 'audio+points': 91 | feature_dimension=300 92 | else: 93 | feature_dimension=150 94 | 95 | self.fc1 = nn.Linear(feature_dimension, 170) 96 | self.fc2 = nn.Linear(170, 170) 97 | self.fc3 = nn.Linear(170, self.num_classes) 98 | 99 | def forward(self, points ,audio): 100 | #tapping audio encoder 101 | audio = self.conv1(audio) 102 | audio = self.bn1(audio) 103 | audio = self.relu(audio) 104 | audio = self.maxpool(audio) 105 | audio = self.conv2(audio) 106 | audio = self.bn2(audio) 107 | audio = self.dropout(audio) 108 | audio = self.relu2(audio) 109 | audio = self.maxpool2(audio) 110 | audio = self.conv3(audio) 111 | audio = self.bn3(audio) 112 | audio = self.relu3(audio) 113 | audio_feature = torch.flatten(audio, 1) 114 | #tapping points encoder 115 | B = np.shape(points)[0] 116 | N = np.shape(points)[1] 117 | points_feature = self.first_conv(points.transpose(2, 1)) # (B, 256, N) 118 | points_feature_global = torch.max(points_feature, dim=2, keepdim=True)[0] # (B, 256, 1) 119 | points_feature = torch.cat([points_feature_global.expand(-1, -1, N), points_feature], dim=1) # (B, 512, N) 120 | points_feature = self.second_conv(points_feature) # (B, 1024, N) 121 | points_feature_global = torch.max(points_feature,dim=2,keepdim=False)[0] 122 | if self.params.model_inputs == 'audio+points': 123 | feature = torch.cat((audio_feature, points_feature_global), 1) 124 | elif self.params.model_inputs == 'audio': 125 | feature = audio_feature 126 | elif self.params.model_inputs == 'points': 127 | feature = points_feature_global 128 | # mlp 129 | x = self.dropout(feature) 130 | x = self.fc1(x) 131 | x = self.dropout(x) 132 | x = self.fc2(x) 133 | x = self.dropout(x) 134 | x = self.fc3(x) 135 | return x 136 | 137 | def configure_optimizers(self): 138 | optimizer = optim.Adam(self.parameters(), lr=self.learning_rate,betas=(0.9, 0.999)) 139 | return [optimizer] 140 | 141 | def training_step(self,batch,batch_idx): 142 | contact_points, audio, labels = batch 143 | contact_points = contact_points.cuda() 144 | audio = audio.cuda() 145 | labels = labels.cuda() 146 | outputs = self.forward(contact_points, audio) 147 | criterion = nn.CrossEntropyLoss() 148 | train_loss = criterion(outputs, labels) 149 | self.train_acc(outputs, labels) 150 | self.log('train_loss', train_loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 151 | self.log('train_acc', self.train_acc, on_epoch=True,prog_bar=True) 152 | return train_loss 153 | 154 | def validation_step(self,batch,batch_idx): 155 | contact_points, audio, labels = batch 156 | contact_points = contact_points.cuda() 157 | audio = audio.cuda() 158 | labels = labels.cuda() 159 | outputs = self.forward(contact_points, audio) 160 | criterion = nn.CrossEntropyLoss() 161 | val_loss = criterion(outputs, labels) 162 | self.valid_acc(outputs, labels) 163 | self.log('val_loss', val_loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 164 | self.log('val_acc', self.valid_acc, on_epoch=True,prog_bar=True) 165 | return val_loss 166 | 167 | def test_step(self, batch,batch_idx): 168 | contact_points, audio, labels = batch 169 | contact_points = contact_points.cuda() 170 | audio = audio.cuda() 171 | labels = labels.cuda() 172 | outputs = self.forward(contact_points, audio) 173 | self.test_acc(outputs, labels) 174 | self.log('test_acc', self.test_acc,on_step=False, on_epoch=True,prog_bar=True) 175 | #confusion matrix 176 | self.test_step_y_hats.append(outputs) 177 | self.test_step_ys.append(labels) 178 | y_hat = torch.cat(self.test_step_y_hats) 179 | y = torch.cat(self.test_step_ys) 180 | metric = MulticlassConfusionMatrix(num_classes=self.num_classes).cuda() 181 | cm=metric(y_hat,y) 182 | confusion_matrix_computed = cm.detach().cpu().numpy().astype(int) 183 | uniformed_confusion_matrix=[] 184 | for idx,i in enumerate(confusion_matrix_computed): 185 | uniformed_confusion_matrix.append([val/sum(i) for val in i]) 186 | final_acc_list=[] 187 | for idx in range(len(uniformed_confusion_matrix)): 188 | final_acc_list.append(uniformed_confusion_matrix[idx][idx]) 189 | final_acc=sum(final_acc_list)/len(final_acc_list) 190 | print('final acc = ',final_acc) 191 | df_cm = pd.DataFrame(uniformed_confusion_matrix,index=self.label_name,columns=self.label_name) 192 | plt.figure(figsize = (10,8)) 193 | fig_ = sns.heatmap(df_cm, annot=True, cmap='Reds').get_figure() 194 | plt.xlabel('Predicted labels') 195 | plt.ylabel('True lables') 196 | plt.savefig(f'images/{self.params.exp_name}_{self.params.testing_split}_confusion_matrix', dpi=300) 197 | plt.close(fig_) 198 | self.loggers[0].experiment.add_figure("Test confusion matrix", fig_, self.current_epoch) 199 | df = pd.DataFrame(uniformed_confusion_matrix) 200 | df.to_excel(excel_writer = f"images/{self.params.exp_name}_{self.params.testing_split}_confusion_matrix.xlsx") 201 | return {'preds' : outputs, 'targets' : labels} 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /ARNet_material_classification/dataset/data_preparation_objectfolder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import json 5 | import yaml 6 | import random 7 | import librosa 8 | import numpy as np 9 | import open3d as o3d 10 | import matplotlib.pyplot as plt 11 | import pytorch_lightning as pl 12 | sys.path.append('.') 13 | 14 | class data_preparation(object): 15 | def __init__(self,params): 16 | pl.seed_everything(1) 17 | self.split = None 18 | self.exp=params.exp_name 19 | self.current_label_list=params.label_mapping 20 | for folder in ['images','results',f'results/{params.exp_name}',f'results/{params.exp_name}/test',f'results/{params.exp_name}/valid']: 21 | try: 22 | os.makedirs(folder) 23 | except: 24 | pass 25 | 26 | with open('dataset/material_simple_categories.json') as f: 27 | self.simplified_label_mapping = json.load(f) 28 | 29 | def get_even_dataset(self): 30 | #0,3,4,6,8 corresponding to orignial material label 31 | all_object_list=[] 32 | for key in params.objectfolder_label_mapping: 33 | for i in params.objectfolder_label_mapping[key]: 34 | all_object_list.append(i) 35 | print('total num of objects',len(all_object_list)) 36 | 37 | audio_list = [] 38 | label_list = [] 39 | 40 | for key in params.objectfolder_label_mapping: 41 | for i in params.objectfolder_label_mapping[key]: 42 | for file_name in os.listdir(f'objectfolder_data/extracted_audio_npy/{i}/audio'): 43 | 44 | audio_list.append(f'objectfolder_data/extracted_audio_npy/{i}/audio/{file_name}') 45 | label_list.append(key) 46 | print(len(audio_list),len(label_list)) 47 | labels_statistic = [label_list.count(0),label_list.count(1),label_list.count(2),label_list.count(3),label_list.count(4)] 48 | max_number = np.max(labels_statistic) 49 | print('label statistic:', labels_statistic) 50 | num_data_for_validation=[int(0.1*i) for i in labels_statistic] 51 | 52 | audio_list_splitbyclass = [[] for _ in range(5)] 53 | label_list_splitbyclass = [[] for _ in range(5)] 54 | 55 | for idx, label in enumerate(label_list): 56 | audio_list_splitbyclass[label].append(audio_list[idx]) 57 | label_list_splitbyclass[label].append(label) 58 | valid_audio_list=[[] for _ in range(5)] 59 | valid_label_list=[[] for _ in range(5)] 60 | 61 | for idx,i in enumerate(audio_list_splitbyclass): 62 | valid_audio_list[idx] = i[0:num_data_for_validation[idx]] 63 | valid_label_list[idx] = label_list_splitbyclass[idx][0:num_data_for_validation[idx]] 64 | train_audio_list=[[] for _ in range(5)] 65 | train_label_list=[[] for _ in range(5)] 66 | 67 | for idx,i in enumerate(audio_list_splitbyclass): 68 | train_audio_list[idx] = i[num_data_for_validation[idx]::] 69 | train_label_list[idx] = label_list_splitbyclass[idx][num_data_for_validation[idx]::] 70 | 71 | 72 | max_valid = np.max([len(valid_audio_list[0]),len(valid_audio_list[1]),len(valid_audio_list[2]),len(valid_audio_list[3]),len(valid_audio_list[4])]) 73 | max_train = np.max([len(train_audio_list[0]),len(train_audio_list[1]),len(train_audio_list[2]),len(train_audio_list[3]),len(train_audio_list[4])]) 74 | 75 | 76 | for idx,i in enumerate(valid_audio_list): 77 | # print(i) 78 | while len(i) != max_valid: 79 | audio_file = np.random.choice(i) 80 | index=audio_list.index(audio_file) 81 | valid_audio_list[idx].append(audio_file) 82 | valid_label_list[idx].append(label_list[index]) 83 | 84 | 85 | for idx,i in enumerate(train_audio_list): 86 | # print(i) 87 | while len(i) != max_train: 88 | audio_file = np.random.choice(i) 89 | index=audio_list.index(audio_file) 90 | train_audio_list[idx].append(audio_file) 91 | train_label_list[idx].append(label_list[index]) 92 | 93 | final_valid_list_label = [] 94 | final_train_list_label = [] 95 | final_valid_list_audio = [] 96 | final_train_list_audio = [] 97 | 98 | for idx,i in enumerate(valid_label_list): 99 | for idx_2,j in enumerate(i): 100 | final_valid_list_label.append(j) 101 | final_valid_list_audio.append(valid_audio_list[idx][idx_2]) 102 | for idx,i in enumerate(train_label_list): 103 | for idx_2,j in enumerate(i): 104 | final_train_list_label.append(j) 105 | final_train_list_audio.append(train_audio_list[idx][idx_2]) 106 | 107 | print(len(final_train_list_audio),len(final_valid_list_audio)) 108 | for i in final_valid_list_audio: 109 | if i in final_train_list_audio: 110 | 111 | print('error========',i) 112 | 113 | np.save('data/ARdataset/object_folder/train_audio_list',np.array(final_train_list_audio, dtype=object)) 114 | np.save('data/ARdataset/object_folder/train_label_list',np.array(final_train_list_label, dtype=object)) 115 | np.save('data/ARdataset/object_folder/valid_audio_list',np.array(final_valid_list_audio, dtype=object)) 116 | np.save('data/ARdataset/object_folder/valid_label_list',np.array(final_valid_list_label, dtype=object)) 117 | 118 | 119 | def get_index_from_audiofilename(self,filename): 120 | number=re.findall(r'\d+', filename) 121 | return [int(x) for x in number] 122 | 123 | def read_point_cloud_from_txt(self, path): 124 | data = [] 125 | with open(f'{path}', "r") as f: 126 | for line in f: 127 | ls = line.strip().split() 128 | ls = [float(i) for i in ls] 129 | data.append(ls) 130 | points = [] 131 | for i in range(len(data)): 132 | if data[i][4] == 1: 133 | points.append(data[i][0:3]) 134 | else: 135 | points.append([0, 0, 0]) 136 | pcd = o3d.geometry.PointCloud() 137 | pcd.points = o3d.utility.Vector3dVector(np.array(points, dtype=float)) 138 | pcd.scale(0.01, [0, 0, 0]) 139 | 140 | def read_label_from_npy(self, path): 141 | path, idx = path 142 | if 'objectfolder' in path: 143 | label=1 144 | else: 145 | idx=idx[1]+(idx[0]-1)*4 146 | data = [np.load(f'{path}', allow_pickle=True)[idx]] 147 | label=data[0] 148 | return np.array(label) 149 | 150 | def read_simplified_label_from_npy(self, path): 151 | path, idx = path 152 | if 'objectfolder' in path: 153 | label=1 154 | else: 155 | idx=idx[1]+(idx[0]-1)*4 156 | data = [np.load(f'{path}', allow_pickle=True)[idx]] 157 | label=data[0] 158 | for key in self.simplified_label_mapping: 159 | if label in self.simplified_label_mapping[key]: 160 | simplified_label= list(self.simplified_label_mapping.keys()).index(key) 161 | return np.array(int(simplified_label)) 162 | 163 | def read_audio_from_npy(self, path): 164 | # path = path[0] 165 | audio_data = np.load(f'{path}', allow_pickle=True) 166 | return np.array([audio_data], np.float32) 167 | 168 | def show_guassian_noise(self): 169 | audio_paths=np.load(f'data/ARdataset//{self.split}_audio_list.npy', allow_pickle=True) 170 | for idx,i in enumerate(audio_paths): 171 | audio_path=i[0].replace('.wav','.npy') 172 | audio = self.read_audio_from_npy([audio_path]) 173 | librosa.display.specshow(audio[0], x_axis='time', y_axis='mel',sr=44100,cmap='inferno') 174 | plt.colorbar(cmap='inferno',format='%+2.f') 175 | plt.savefig(f'images/origianl_{idx}',dpi=300) 176 | plt.close() 177 | gaussian = np.random.normal(-27.56679, 2, (64, 64)) 178 | gaussian = np.array([gaussian], np.float32) 179 | audio = audio + gaussian 180 | librosa.display.specshow(audio[0], x_axis='time', y_axis='mel',sr=44100,cmap='inferno') 181 | plt.colorbar(cmap='inferno',format='%+2.f') 182 | plt.savefig(f'images/{idx}',dpi=300) 183 | plt.close() 184 | input() 185 | 186 | if __name__ == '__main__': 187 | 188 | def load_config(filepath): 189 | with open(filepath, 'r') as stream: 190 | try: 191 | trainer_params = yaml.safe_load(stream) 192 | return trainer_params 193 | except yaml.YAMLError as exc: 194 | print(exc) 195 | from munch import munchify 196 | params = load_config(filepath='configs/config.yaml') 197 | params = munchify(params) 198 | print(params.objectfolder_label_mapping[0][0]) 199 | data=data_preparation(params) 200 | data.get_even_dataset() 201 | 202 | -------------------------------------------------------------------------------- /ARNet_material_classification/dataset/data_preparation.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import json 5 | import yaml 6 | import random 7 | import librosa 8 | import numpy as np 9 | import open3d as o3d 10 | import matplotlib.pyplot as plt 11 | import pytorch_lightning as pl 12 | sys.path.append('.') 13 | 14 | class data_preparation(object): 15 | def __init__(self,params): 16 | pl.seed_everything(1) 17 | self.split = None 18 | self.exp=params.exp_name 19 | self.current_label_list=params.label_mapping 20 | for folder in ['images','results',f'results/{params.exp_name}',f'results/{params.exp_name}/test',f'results/{params.exp_name}/valid']: 21 | try: 22 | os.makedirs(folder) 23 | except: 24 | pass 25 | 26 | with open('dataset/material_simple_categories.json') as f: 27 | self.simplified_label_mapping = json.load(f) 28 | 29 | def get_even_dataset(self): 30 | 31 | label_data_path_origin = 'data/ARdataset/material_label' 32 | audio_data_path_origin = 'data/ARdataset/audio' 33 | for self.split in ['train','valid','test']: 34 | list_of_object_name=np.load(f'data/ARdataset/{self.exp}/{self.split}_object_list.npy', allow_pickle=True) 35 | #collect all the data path in one list 36 | material_label_paths = [] 37 | audio_paths = [] 38 | for path in list_of_object_name: 39 | # check if current path is a file 40 | if os.path.isfile(os.path.join(label_data_path_origin, path)): 41 | audio_file_name = path.replace('.npy', '') 42 | list_of_file=os.listdir(f'{audio_data_path_origin}/{audio_file_name}') 43 | try: 44 | list_of_file.remove('.DS_Store') 45 | except: 46 | pass 47 | for i in range(len(list_of_file)): 48 | filename=list_of_file[i] 49 | if 'augmented' in filename: 50 | filename=filename.replace('augmented1_','') 51 | filename=filename.replace('augmented2_','') 52 | 53 | data_idx=self.get_index_from_audiofilename(filename) 54 | material_label_paths.append([f'{label_data_path_origin}/{path}', data_idx]) 55 | audio_paths.append([f'{audio_data_path_origin}/{audio_file_name}/{list_of_file[i]}']) 56 | 57 | #store file name according to index 58 | labels=[0 for _ in range(43)] 59 | material_path=[[] for _ in range(43)] 60 | audio_path=[[] for _ in range(43)] 61 | num_43=0 62 | print(len(material_label_paths)) 63 | non_43_material_label_path=[] 64 | non_43_audio_path=[] 65 | for i in range(len(material_label_paths)): 66 | label = self.read_label_from_npy(material_label_paths[i]) 67 | if label==43: 68 | num_43+=1 69 | else: 70 | material_path[label].append(material_label_paths[i]) 71 | audio_path[label].append(audio_paths[i]) 72 | labels[label]+=1 73 | non_43_material_label_path.append(material_label_paths[i]) 74 | non_43_audio_path.append(audio_paths[i]) 75 | print('original material distribution:',labels) 76 | print('number of 43:',num_43) 77 | idx=[] 78 | for i in self.current_label_list: 79 | idx.append(i) 80 | k=len(self.current_label_list) 81 | print('material idx =', idx) 82 | num_data=[0 for _ in range(k)] 83 | material_labels_for_learning=[[] for _ in range(k)] 84 | audio_path_for_learning=[[] for _ in range(k)] 85 | for i in range(len(non_43_material_label_path)): 86 | label = self.read_simplified_label_from_npy(non_43_material_label_path[i]) 87 | if label in idx: 88 | material_labels_for_learning[int(np.where(idx == label)[0])].append(non_43_material_label_path[i]) 89 | audio_path_for_learning[int(np.where(idx == label)[0])].append(non_43_audio_path[i]) 90 | num_data[int(np.where(idx == label)[0])]+=1 91 | print(num_data) 92 | #split dataset 93 | #random shuffle data to randomly split data in testing, validation and training set 94 | #save original data list 95 | list_of_test_data_label=[] 96 | list_of_test_data_audio=[] 97 | for i in range(k): 98 | for idx,j in enumerate(material_labels_for_learning[i]): 99 | list_of_test_data_label.append(j) 100 | list_of_test_data_audio.append(audio_path_for_learning[i][idx]) 101 | 102 | # save data in npy file 103 | name=[f'{self.split}_label_list',f'{self.split}_audio_list'] 104 | data=[list_of_test_data_label 105 | ,list_of_test_data_audio] 106 | for i in range(len(name)): 107 | print(name[i]) 108 | dnpy = np.array(data[i], dtype=object) 109 | np.save(f'data/ARdataset/{self.exp}/original_{name[i]}', dnpy) 110 | 111 | if self.split=='train' or self.split=='valid' or self.split=='test': 112 | #duplicate data to even the dataset 113 | if self.split == 'train': 114 | test_maximum=np.max(num_data) 115 | else: 116 | test_maximum=np.max(num_data) 117 | for i in range(k): 118 | while num_data[i]!=test_maximum and num_data[i]!=0 : 119 | path=audio_path_for_learning[i][random.randint(0, len(audio_path_for_learning[i])-1)] 120 | 121 | audio_path_for_learning[i].append(path) 122 | num_data[i]+=1 123 | 124 | print(f'{self.split} data statistc:', num_data) 125 | 126 | # recreate list of data for finding corresponding audio data 127 | list_of_test_data_label=[] 128 | list_of_test_data_audio=[] 129 | 130 | for i in range(k): 131 | for j in range(len(audio_path_for_learning[i])): 132 | path=audio_path_for_learning[i][j] 133 | index=audio_paths.index(path) 134 | label_path=material_label_paths[index] 135 | 136 | label = int(self.read_simplified_label_from_npy(label_path)) 137 | if label not in self.current_label_list: 138 | pass 139 | else: 140 | list_of_test_data_label.append(label_path) 141 | list_of_test_data_audio.append(path) 142 | print(len(list_of_test_data_audio),len(list_of_test_data_label)) 143 | 144 | # save data in npy file 145 | name=[f'{self.split}_label_list',f'{self.split}_audio_list'] 146 | data=[list_of_test_data_label 147 | ,list_of_test_data_audio] 148 | for i in range(len(name)): 149 | dnpy = np.array(data[i], dtype=object) 150 | np.save(f'data/ARdataset/{self.exp}/{name[i]}', dnpy) 151 | 152 | def get_index_from_audiofilename(self,filename): 153 | number=re.findall(r'\d+', filename) 154 | return [int(x) for x in number] 155 | 156 | def read_point_cloud_from_txt(self, path): 157 | data = [] 158 | with open(f'{path}', "r") as f: 159 | for line in f: 160 | ls = line.strip().split() 161 | ls = [float(i) for i in ls] 162 | data.append(ls) 163 | points = [] 164 | for i in range(len(data)): 165 | if data[i][4] == 1: 166 | points.append(data[i][0:3]) 167 | else: 168 | points.append([0, 0, 0]) 169 | pcd = o3d.geometry.PointCloud() 170 | pcd.points = o3d.utility.Vector3dVector(np.array(points, dtype=float)) 171 | pcd.scale(0.01, [0, 0, 0]) 172 | 173 | def read_label_from_npy(self, path): 174 | path, idx = path 175 | if 'objectfolder' in path: 176 | label=1 177 | else: 178 | idx=idx[1]+(idx[0]-1)*4 179 | data = [np.load(f'{path}', allow_pickle=True)[idx]] 180 | label=data[0] 181 | return np.array(label) 182 | 183 | def read_simplified_label_from_npy(self, path): 184 | path, idx = path 185 | if 'objectfolder' in path: 186 | label=1 187 | else: 188 | idx=idx[1]+(idx[0]-1)*4 189 | data = [np.load(f'{path}', allow_pickle=True)[idx]] 190 | label=data[0] 191 | for key in self.simplified_label_mapping: 192 | if label in self.simplified_label_mapping[key]: 193 | simplified_label= list(self.simplified_label_mapping.keys()).index(key) 194 | return np.array(int(simplified_label)) 195 | 196 | def read_audio_from_npy(self, path): 197 | path = path[0] 198 | audio_data = np.load(f'{path}', allow_pickle=True) 199 | return np.array([audio_data], np.float32) 200 | 201 | def show_guassian_noise(self): 202 | audio_paths=np.load(f'data/ARdataset//{self.split}_audio_list.npy', allow_pickle=True) 203 | for idx,i in enumerate(audio_paths): 204 | audio_path=i[0].replace('.wav','.npy') 205 | audio = self.read_audio_from_npy([audio_path]) 206 | librosa.display.specshow(audio[0], x_axis='time', y_axis='mel',sr=44100,cmap='inferno') 207 | plt.colorbar(cmap='inferno',format='%+2.f') 208 | plt.savefig(f'images/origianl_{idx}',dpi=300) 209 | plt.close() 210 | gaussian = np.random.normal(-27.56679, 2, (64, 64)) 211 | gaussian = np.array([gaussian], np.float32) 212 | audio = audio + gaussian 213 | librosa.display.specshow(audio[0], x_axis='time', y_axis='mel',sr=44100,cmap='inferno') 214 | plt.colorbar(cmap='inferno',format='%+2.f') 215 | plt.savefig(f'images/{idx}',dpi=300) 216 | plt.close() 217 | input() 218 | 219 | if __name__ == '__main__': 220 | 221 | def load_config(filepath): 222 | with open(filepath, 'r') as stream: 223 | try: 224 | trainer_params = yaml.safe_load(stream) 225 | return trainer_params 226 | except yaml.YAMLError as exc: 227 | print(exc) 228 | from munch import munchify 229 | params = load_config(filepath='configs/config.yaml') 230 | params = munchify(params) 231 | data=data_preparation(params) 232 | data.get_even_dataset() 233 | 234 | -------------------------------------------------------------------------------- /ARNet_object_classification/refine_material_predication.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import copy 4 | import yaml 5 | import torch 6 | import pickle 7 | import natsort 8 | import statistics 9 | import numpy as np 10 | import pandas as pd 11 | import open3d as o3d 12 | import seaborn as sns 13 | from munch import munchify 14 | import matplotlib.pyplot as plt 15 | from torchmetrics.classification import MulticlassConfusionMatrix 16 | 17 | class get_overall_accuracy(object): 18 | def __init__(self,params): 19 | self.num_class = params.num_label 20 | self.params = params 21 | 22 | def calculate_accuracy(self,split,k_neighbor,num_loop,mim_occurence): 23 | filepath = 'configs/config.yaml' 24 | with open(filepath, 'r') as stream: 25 | params = yaml.safe_load(stream) 26 | params = munchify(params) 27 | 28 | correct_label_list = params.label_mapping 29 | d_swap = {v: k for k, v in correct_label_list.items()} 30 | correct_label_list = d_swap 31 | npy_files = natsort.natsorted((np.load(f'data/ARdataset/{params.exp_name}/{split}_object_list.npy', allow_pickle=True))) 32 | file_name = np.load(f'data/ARdataset/{params.exp_name}/original_{split}_label_list.npy', allow_pickle=True) 33 | data = np.load(f'results/{params.exp_name}/{split}/label_prediction_{split}.npy', allow_pickle=True) 34 | gtdata = np.load(f'results/{params.exp_name}/{split}/label_gt_{split}.npy', allow_pickle=True) 35 | list_accuracy = [] 36 | correct_prediction = [] 37 | y_hat=[] 38 | y=[] 39 | num_correct_prediction = [0 for _ in range(len(params.label_name))] 40 | total_num_prediction = [0 for _ in range(len(params.label_name))] 41 | for file in npy_files: 42 | object_name = file.replace('.npy', '') 43 | raw_contact_points = self.read_raw_contact_from_txt( 44 | os.path.join('data/ARdataset/contact_position', file.replace('.npy', '.txt'))) 45 | object_name_prediction = [] 46 | label_prediction = [] 47 | # not all of the contact point have valid audio data included in training, so here we extract the contact points with valid material prediction 48 | contact_points_idx_prediction = [] 49 | ground_truth_label = [] 50 | for idx, i in enumerate(file_name): 51 | # print( object_name) 52 | if object_name in i[0]: 53 | 54 | object_name_prediction.append(i) 55 | contact_points_idx_prediction.append((i[1][0] - 1) * 4 + i[1][1]) 56 | label_prediction.append( 57 | correct_label_list[int(np.where(data[idx] == np.max(data[idx]))[0])]) 58 | ground_truth_label.append(correct_label_list[int(gtdata[idx])]) 59 | 60 | 61 | predicted_contact_points = [] 62 | for idx, value in enumerate(contact_points_idx_prediction): 63 | predicted_contact_points.append(raw_contact_points[value]) 64 | # get statistic of occurence of labels and filter out labels with low occurance 65 | label_prediction,maxlabel = self.filter_out_labels_with_low_occurence(k_neighbor,mim_occurence, label_prediction,predicted_contact_points) 66 | 67 | # assign labels according to voting with nearest neighbors 68 | for i in range(num_loop): 69 | voted_label_list = [] 70 | for idx, value in enumerate(predicted_contact_points): 71 | voted_lable = self.get_labels_of_k_neighbor_points(k_neighbor, value,maxlabel, label_prediction,predicted_contact_points) 72 | voted_label_list.append(voted_lable) 73 | label_prediction = voted_label_list.copy() 74 | 75 | #save confusion matrix 76 | for idx in range(len(label_prediction)): 77 | y_hat.append(params.label_mapping[int(label_prediction[idx])]) 78 | y.append(params.label_mapping[int(ground_truth_label[idx])]) 79 | 80 | 81 | # check number of correct prediction 82 | for idx, value in enumerate(label_prediction): 83 | pred_index = params.label_mapping[value] 84 | gt_index = params.label_mapping[ground_truth_label[idx]] 85 | if value == ground_truth_label[idx]: 86 | num_correct_prediction[pred_index] += 1 87 | total_num_prediction[gt_index]+=1 88 | 89 | #save confusion matrix 90 | if split == 'test': 91 | self.save_confusion_matrix(torch.from_numpy(np.array(y_hat)),torch.from_numpy(np.array(y))) 92 | 93 | acc=[] 94 | for idx, i in enumerate(num_correct_prediction): 95 | acc.append(num_correct_prediction[idx]/total_num_prediction[idx]) 96 | # calculate overall prediction accuracy 97 | accuracy_balanced = sum(acc) / len(acc) 98 | 99 | accuracy = sum(num_correct_prediction) / sum(total_num_prediction) 100 | list_accuracy.append(accuracy) 101 | return accuracy,accuracy_balanced 102 | def save_confusion_matrix(self,y_hat,y): 103 | print(y_hat.size(),y.size()) 104 | print(len(y_hat),len(y)) 105 | # for i in range(len(y_hat)): 106 | # print(y_hat[i],y[i]) 107 | # input() 108 | 109 | metric = MulticlassConfusionMatrix(num_classes=self.num_class) 110 | cm=metric(y_hat,y) 111 | confusion_matrix_computed = cm.detach().cpu().numpy().astype(int) 112 | 113 | uniformed_confusion_matrix=[] 114 | for idx,i in enumerate(confusion_matrix_computed): 115 | uniformed_confusion_matrix.append([val/sum(i) for val in i]) 116 | final_acc_list=[] 117 | for idx in range(len(uniformed_confusion_matrix)): 118 | final_acc_list.append(uniformed_confusion_matrix[idx][idx]) 119 | final_acc=sum(final_acc_list)/len(final_acc_list) 120 | print('final acc = ',final_acc) 121 | 122 | df_cm = pd.DataFrame(uniformed_confusion_matrix,index=self.params.label_name,columns=self.params.label_name) 123 | plt.figure(figsize = (10,8)) 124 | fig_ = sns.heatmap(df_cm, annot=True, cmap='Reds').get_figure() 125 | plt.xlabel('Predicted labels') 126 | plt.ylabel('True lables') 127 | plt.savefig(f'images/{self.params.exp_name}_{self.params.testing_split}_refine_confusion_matrix', dpi=300) 128 | plt.close(fig_) 129 | 130 | def read_raw_contact_from_txt(self,path): 131 | # extract [x,y,z] data from {contact_points}.txt file 132 | data = [] 133 | with open(f'{path}', "r") as f: 134 | for line in f: 135 | ls = line.strip().split() 136 | ls = [float(i) for i in ls] 137 | data.append(ls) 138 | points = [] 139 | for i in range(len(data)): 140 | points.append(data[i][0:3]) 141 | #turn unit into meter and apply transformation for aligning contact points with 3D point cloud model 142 | pcd = o3d.geometry.PointCloud() 143 | pcd.points = o3d.utility.Vector3dVector(np.array(points, dtype=float)) 144 | pcd.scale(0.01, [0, 0, 0]) 145 | transform_path = 'data/ARdataset/transformation_matrix/tapping_position_transformation_matrix.pkl' 146 | with open(transform_path, 'rb') as f: 147 | T = pickle.load(f)[0] 148 | pcd_t = copy.deepcopy(pcd).transform(T) 149 | transform_path = 'data/ARdataset/transformation_matrix/tapping_position_transformation_matrix_1.pkl' 150 | with open(transform_path, 'rb') as f: 151 | T = pickle.load(f)[0] 152 | pcd_t = copy.deepcopy(pcd_t).transform(T) 153 | return np.asarray(pcd_t.points) 154 | 155 | def get_labels_of_k_neighbor_points(self,k, value, maxlabel,label_list,predicted_contact_points): 156 | neighbors_labels = [] 157 | neighbors_points = [] 158 | contact_points_list = predicted_contact_points.copy() 159 | contact_points_list = [list(i) for i in contact_points_list] 160 | for i in range(k): 161 | min = float('inf') 162 | for idx, points in enumerate(contact_points_list): 163 | #compute Euclidean Distance of two points 164 | distance = math.dist(value, points) 165 | if distance < min: 166 | min = distance 167 | key = idx 168 | closest_point = points 169 | neighbors_points.append(contact_points_list[key]) 170 | contact_points_list.remove(closest_point) 171 | contact_points_list = predicted_contact_points.copy() 172 | contact_points_list = [list(i) for i in contact_points_list] 173 | 174 | for idx, points in enumerate(contact_points_list): 175 | for value in neighbors_points: 176 | if list(value) == list(points): 177 | neighbors_labels.append(label_list[idx]) 178 | 179 | if len(statistics.multimode(neighbors_labels)) != 1: 180 | voted_label = maxlabel 181 | else: 182 | voted_label = statistics.multimode(neighbors_labels)[0] 183 | return voted_label 184 | 185 | 186 | def filter_out_labels_with_low_occurence(self,k_neighbor,mim_occurence,list_of_label,predicted_contact_points): 187 | labels = np.unique(list_of_label) 188 | labels_statistic=[] 189 | for i in labels: 190 | labels_statistic.append(list_of_label.count(i)) 191 | maxidx=labels_statistic.index(max(labels_statistic)) 192 | maxlabel=labels[maxidx] 193 | for i in range(len(labels_statistic)): 194 | if labels_statistic[i]