├── README.md ├── __pycache__ └── preprocessing_val_language.cpython-38.pyc ├── assets ├── augmentation.png ├── cabinet operations_c.gif ├── f1.png ├── opendoor_c 00_00_00-00_00_30.gif ├── penholder_c 00_00_00-00_00_30.gif ├── stacking_c 00_00_00-00_00_30.gif └── two-stage.png ├── calvin_agent ├── .gitignore ├── __init__.py ├── datasets │ ├── __init__.py │ ├── base_dataset.py │ ├── calvin_data_module.py │ ├── disk_dataset.py │ ├── random.py │ ├── shm_dataset.py │ └── utils │ │ ├── __init__.py │ │ ├── episode_utils.py │ │ └── shared_memory_utils.py ├── evaluation │ ├── __init__.py │ ├── evaluate_policy.py │ ├── evaluate_policy_singlestep.py │ ├── multistep_sequences.py │ └── utils.py ├── inference │ ├── __init__.py │ ├── rollouts_interactive.py │ ├── rollouts_training.py │ └── test_policy_interactive.py ├── models │ ├── __init__.py │ ├── calvin_base_model.py │ ├── decoders │ │ ├── __init__.py │ │ ├── action_decoder.py │ │ └── logistic_policy_network.py │ ├── encoders │ │ ├── __init__.py │ │ ├── goal_encoders.py │ │ └── language_network.py │ ├── mcil.py │ ├── perceptual_encoders │ │ ├── __init__.py │ │ ├── concat_encoders.py │ │ ├── proprio_encoder.py │ │ ├── tactile_encoder.py │ │ ├── vision_network.py │ │ └── vision_network_gripper.py │ └── plan_encoders │ │ ├── __init__.py │ │ ├── plan_proposal_net.py │ │ └── plan_recognition_net.py ├── rollout │ ├── __init__.py │ ├── rollout.py │ ├── rollout_long_horizon.py │ └── rollout_video.py ├── training.py ├── utils │ ├── __init__.py │ ├── automatic_lang_annotator_mp.py │ ├── compute_proprioception_statistics.py │ ├── create_splits.py │ ├── data_visualization.py │ ├── dataset_task_statistics.py │ ├── kl_callbacks.py │ ├── language_annotator.py │ ├── relabel_with_new_lang_model.py │ ├── transforms.py │ ├── utils.py │ ├── visualizations.py │ └── visualize_annotations.py ├── visualization │ ├── __init__.py │ └── tsne_plot.py └── wrappers │ └── calvin_env_wrapper.py ├── calvin_env ├── .flake8 ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── calvin_env │ ├── __init__.py │ ├── camera │ │ ├── __init__.py │ │ ├── camera.py │ │ ├── gripper_camera.py │ │ ├── static_camera.py │ │ └── tactile_sensor.py │ ├── datarenderer.py │ ├── envs │ │ ├── __init__.py │ │ ├── play_lmp_wrapper.py │ │ ├── play_table_env.py │ │ └── tasks.py │ ├── io_utils │ │ ├── data_recorder.py │ │ └── vr_input.py │ ├── robot │ │ ├── IKfast.py │ │ ├── __init__.py │ │ ├── mixed_ik.py │ │ └── robot.py │ ├── scene │ │ ├── __init__.py │ │ ├── objects │ │ │ ├── __init__.py │ │ │ ├── base_object.py │ │ │ ├── button.py │ │ │ ├── door.py │ │ │ ├── fixed_object.py │ │ │ ├── light.py │ │ │ ├── movable_object.py │ │ │ └── switch.py │ │ └── play_table_scene.py │ ├── scripts │ │ ├── check_tasks.py │ │ ├── convert_gripper_actions.py │ │ ├── dataset_to_euler.py │ │ ├── record_video_icra.py │ │ ├── render_low_freq.py │ │ ├── reset_env_rendered_episode.py │ │ └── unnormalize_depth.py │ ├── utils │ │ ├── __init__.py │ │ └── utils.py │ └── vrdatacollector.py ├── conf │ ├── cameras │ │ ├── cameras │ │ │ ├── gripper.yaml │ │ │ ├── opposing.yaml │ │ │ ├── static.yaml │ │ │ └── tactile.yaml │ │ ├── no_cameras.yaml │ │ ├── static_and_gripper.yaml │ │ ├── static_and_tactile.yaml │ │ └── static_gripper_tactile.yaml │ ├── config_data_collection.yaml │ ├── config_rendering.yaml │ ├── digit_sensor │ │ └── config_digit.yml │ ├── recorder │ │ └── recorder.yaml │ ├── robot │ │ ├── panda.yaml │ │ ├── panda_digit.yaml │ │ └── panda_longer_finger.yaml │ ├── scene │ │ ├── basic_playtable.yaml │ │ ├── basic_tabletop.yaml │ │ ├── calvin_scene_A.yaml │ │ ├── calvin_scene_B.yaml │ │ ├── calvin_scene_C.yaml │ │ ├── calvin_scene_D.yaml │ │ ├── calvin_scene_D_eval.yaml │ │ ├── empty_playtable.yaml │ │ ├── tabletop_1.yaml │ │ ├── tabletop_2.yaml │ │ └── tabletop_3.yaml │ ├── tasks │ │ ├── new_playtable_tasks.yaml │ │ └── play_table_tasks.yaml │ └── vr_input │ │ ├── vr_controller │ │ ├── oculus.yaml │ │ └── vive.yaml │ │ └── vr_input.yaml ├── data │ ├── blocks │ │ ├── block_blue.urdf │ │ ├── block_blue_big.urdf │ │ ├── block_blue_middle.urdf │ │ ├── block_blue_small.urdf │ │ ├── block_blue_unseen.urdf │ │ ├── block_pink.urdf │ │ ├── block_pink_big.urdf │ │ ├── block_pink_middle.urdf │ │ ├── block_pink_small.urdf │ │ ├── block_pink_unseen.urdf │ │ ├── block_red.urdf │ │ ├── block_red_big.urdf │ │ ├── block_red_middle.urdf │ │ ├── block_red_small.urdf │ │ ├── block_red_unseen.urdf │ │ └── block_square.stl │ ├── calvin_table_A │ │ ├── meshes │ │ │ ├── base_link.STL │ │ │ ├── base_link.mtl │ │ │ ├── base_link.obj │ │ │ ├── button_link.STL │ │ │ ├── drawer_link.STL │ │ │ ├── drawer_link.mtl │ │ │ ├── drawer_link.obj │ │ │ ├── drawer_link_vhacd2.obj │ │ │ ├── led_link.STL │ │ │ ├── light_link.STL │ │ │ ├── plank_link.STL │ │ │ ├── plank_link.mtl │ │ │ ├── plank_link.obj │ │ │ ├── plank_link_vhacd2.obj │ │ │ ├── slide_link.STL │ │ │ ├── slide_link.mtl │ │ │ ├── slide_link.obj │ │ │ ├── slide_link_vhacd2.obj │ │ │ ├── switch_link.STL │ │ │ ├── switch_link.mtl │ │ │ ├── switch_link.obj │ │ │ └── switch_link_vhacd2.obj │ │ ├── textures │ │ │ ├── dark_wood.png │ │ │ ├── dark_wood__black_handle.png │ │ │ ├── dark_wood__gray_handle.png │ │ │ ├── light_wood.png │ │ │ ├── light_wood__black_handle.png │ │ │ ├── light_wood__gray_handle.png │ │ │ ├── wood.png │ │ │ ├── wood__black_handle.png │ │ │ └── wood__gray_handle.png │ │ └── urdf │ │ │ └── calvin_table_A.urdf │ ├── calvin_table_B │ │ ├── meshes │ │ │ ├── base_link.STL │ │ │ ├── base_link.mtl │ │ │ ├── base_link.obj │ │ │ ├── base_link_vhacd2.obj │ │ │ ├── button_link.STL │ │ │ ├── drawer_link.STL │ │ │ ├── drawer_link.mtl │ │ │ ├── drawer_link.obj │ │ │ ├── drawer_link_vhacd2.obj │ │ │ ├── led_link.STL │ │ │ ├── light_link.STL │ │ │ ├── plank_link.STL │ │ │ ├── plank_link.mtl │ │ │ ├── plank_link.obj │ │ │ ├── plank_link_vhacd2.obj │ │ │ ├── slide_link.STL │ │ │ ├── slide_link.mtl │ │ │ ├── slide_link.obj │ │ │ ├── slide_link_vhacd2.obj │ │ │ ├── switch_link.STL │ │ │ ├── switch_link.mtl │ │ │ ├── switch_link.obj │ │ │ └── switch_link_vhacd2.obj │ │ ├── textures │ │ │ ├── dark_wood.png │ │ │ ├── dark_wood__black_handle.png │ │ │ ├── dark_wood__gray_handle.png │ │ │ ├── light_wood.png │ │ │ ├── light_wood__black_handle.png │ │ │ ├── light_wood__gray_handle.png │ │ │ ├── wood.png │ │ │ ├── wood__black_handle.png │ │ │ └── wood__gray_handle.png │ │ └── urdf │ │ │ └── calvin_table_B.urdf │ ├── calvin_table_C │ │ ├── meshes │ │ │ ├── base_link.STL │ │ │ ├── base_link.mtl │ │ │ ├── base_link.obj │ │ │ ├── button_link.STL │ │ │ ├── drawer_link.STL │ │ │ ├── drawer_link.mtl │ │ │ ├── drawer_link.obj │ │ │ ├── drawer_link_vhacd2.obj │ │ │ ├── led_link.STL │ │ │ ├── light_link.STL │ │ │ ├── plank_link.STL │ │ │ ├── plank_link.mtl │ │ │ ├── plank_link.obj │ │ │ ├── plank_link_vhacd2.obj │ │ │ ├── slide_link.STL │ │ │ ├── slide_link.mtl │ │ │ ├── slide_link.obj │ │ │ ├── slide_link_vhacd2.obj │ │ │ ├── switch_link.STL │ │ │ ├── switch_link.mtl │ │ │ ├── switch_link.obj │ │ │ └── switch_link_vhacd2.obj │ │ ├── textures │ │ │ ├── dark_wood.png │ │ │ ├── dark_wood__black_handle.png │ │ │ ├── dark_wood__gray_handle.png │ │ │ ├── light_wood.png │ │ │ ├── light_wood__black_handle.png │ │ │ ├── light_wood__gray_handle.png │ │ │ ├── wood.png │ │ │ ├── wood__black_handle.png │ │ │ └── wood__gray_handle.png │ │ └── urdf │ │ │ └── calvin_table_C.urdf │ ├── calvin_table_D │ │ ├── meshes │ │ │ ├── base_link.STL │ │ │ ├── base_link.mtl │ │ │ ├── base_link.obj │ │ │ ├── button_link.STL │ │ │ ├── drawer_link.STL │ │ │ ├── drawer_link.mtl │ │ │ ├── drawer_link.obj │ │ │ ├── drawer_link_vhacd2.obj │ │ │ ├── led_link.STL │ │ │ ├── light_link.STL │ │ │ ├── plank_link.STL │ │ │ ├── plank_link.mtl │ │ │ ├── plank_link.obj │ │ │ ├── plank_link_vhacd2.obj │ │ │ ├── slide_link.STL │ │ │ ├── slide_link.mtl │ │ │ ├── slide_link.obj │ │ │ ├── slide_link_vhacd2.obj │ │ │ ├── switch_link.STL │ │ │ ├── switch_link.mtl │ │ │ ├── switch_link.obj │ │ │ └── switch_link_vhacd2.obj │ │ ├── textures │ │ │ ├── dark_wood.png │ │ │ ├── dark_wood__black_handle.png │ │ │ ├── dark_wood__gray_handle.png │ │ │ ├── light_wood.png │ │ │ ├── light_wood__black_handle.png │ │ │ ├── light_wood__gray_handle.png │ │ │ ├── wood.png │ │ │ ├── wood__black_handle.png │ │ │ └── wood__gray_handle.png │ │ └── urdf │ │ │ └── calvin_table_D.urdf │ ├── franka_panda │ │ ├── LICENSE.txt │ │ ├── meshes │ │ │ ├── collision │ │ │ │ ├── finger.obj │ │ │ │ ├── hand.obj │ │ │ │ ├── link0.obj │ │ │ │ ├── link1.obj │ │ │ │ ├── link2.obj │ │ │ │ ├── link3.obj │ │ │ │ ├── link4.obj │ │ │ │ ├── link5.obj │ │ │ │ ├── link6.mtl │ │ │ │ ├── link6.obj │ │ │ │ ├── link7.obj │ │ │ │ ├── longer_finger.mtl │ │ │ │ ├── longer_finger.obj │ │ │ │ ├── longer_finger_v2.mtl │ │ │ │ └── longer_finger_v2.obj │ │ │ └── visual │ │ │ │ ├── Assem1.SLDASM │ │ │ │ ├── FRANKA_Finger.SLDPRT │ │ │ │ ├── colors.png │ │ │ │ ├── digit.STL │ │ │ │ ├── digit_gel_only.STL │ │ │ │ ├── finger.SLDPRT │ │ │ │ ├── finger.mtl │ │ │ │ ├── finger.obj │ │ │ │ ├── hand.mtl │ │ │ │ ├── hand.obj │ │ │ │ ├── link1.mtl │ │ │ │ ├── link1.obj │ │ │ │ ├── link2.mtl │ │ │ │ ├── link2.obj │ │ │ │ ├── link3.mtl │ │ │ │ ├── link3.obj │ │ │ │ ├── link4.mtl │ │ │ │ ├── link4.obj │ │ │ │ ├── link5.mtl │ │ │ │ ├── link5.obj │ │ │ │ ├── link6.mtl │ │ │ │ ├── link6.obj │ │ │ │ ├── longer_finger.STL │ │ │ │ ├── longer_finger.mtl │ │ │ │ ├── longer_finger.obj │ │ │ │ ├── longer_finger_v2.STL │ │ │ │ ├── longer_finger_v2.mtl │ │ │ │ ├── longer_finger_v2.obj │ │ │ │ ├── visualShapeBench.json_0.json │ │ │ │ ├── ~$Assem1.SLDASM │ │ │ │ ├── ~$FRANKA_Finger.SLDPRT │ │ │ │ └── ~$finger.SLDPRT │ │ ├── panda.urdf │ │ ├── panda_digit.urdf │ │ └── panda_longer_finger.urdf │ └── plane │ │ ├── checker_blue.png │ │ ├── plane.mtl │ │ ├── plane.obj │ │ └── plane.urdf ├── egl_check │ ├── EGL_options.cpp │ ├── EGL_options.h │ ├── EGL_options.o_ │ ├── README.md │ ├── build.sh │ ├── glad │ │ ├── EGL │ │ │ └── eglplatform.h │ │ ├── KHR │ │ │ └── khrplatform.h │ │ ├── egl.c │ │ ├── gl.c │ │ ├── glad │ │ │ ├── egl.h │ │ │ ├── gl.h │ │ │ └── glx.h │ │ ├── glx.c │ │ └── linmath.h │ └── list_egl_options.py ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt └── setup.py ├── calvin_models ├── Calvin.egg-info │ ├── PKG-INFO │ ├── SOURCES.txt │ ├── dependency_links.txt │ ├── not-zip-safe │ ├── requires.txt │ └── top_level.txt ├── calvin_agent │ ├── .gitignore │ ├── __init__.py │ ├── datasets │ │ ├── __init__.py │ │ ├── base_dataset.py │ │ ├── calvin_data_module.py │ │ ├── disk_dataset.py │ │ ├── random.py │ │ ├── shm_dataset.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── episode_utils.py │ │ │ └── shared_memory_utils.py │ ├── evaluation │ │ ├── __init__.py │ │ ├── evaluate_policy.py │ │ ├── evaluate_policy_singlestep.py │ │ ├── multistep_sequences.py │ │ └── utils.py │ ├── inference │ │ ├── __init__.py │ │ ├── rollouts_interactive.py │ │ ├── rollouts_training.py │ │ └── test_policy_interactive.py │ ├── models │ │ ├── __init__.py │ │ ├── calvin_base_model.py │ │ ├── decoders │ │ │ ├── __init__.py │ │ │ ├── action_decoder.py │ │ │ └── logistic_policy_network.py │ │ ├── encoders │ │ │ ├── __init__.py │ │ │ ├── goal_encoders.py │ │ │ └── language_network.py │ │ ├── mcil.py │ │ ├── perceptual_encoders │ │ │ ├── __init__.py │ │ │ ├── concat_encoders.py │ │ │ ├── proprio_encoder.py │ │ │ ├── tactile_encoder.py │ │ │ ├── vision_network.py │ │ │ └── vision_network_gripper.py │ │ └── plan_encoders │ │ │ ├── __init__.py │ │ │ ├── plan_proposal_net.py │ │ │ └── plan_recognition_net.py │ ├── rollout │ │ ├── __init__.py │ │ ├── rollout.py │ │ ├── rollout_long_horizon.py │ │ └── rollout_video.py │ ├── training.py │ ├── utils │ │ ├── __init__.py │ │ ├── automatic_lang_annotator_mp.py │ │ ├── compute_proprioception_statistics.py │ │ ├── create_splits.py │ │ ├── data_visualization.py │ │ ├── dataset_task_statistics.py │ │ ├── kl_callbacks.py │ │ ├── language_annotator.py │ │ ├── relabel_with_new_lang_model.py │ │ ├── transforms.py │ │ ├── utils.py │ │ ├── visualizations.py │ │ └── visualize_annotations.py │ ├── visualization │ │ ├── __init__.py │ │ └── tsne_plot.py │ └── wrappers │ │ └── calvin_env_wrapper.py ├── conf │ ├── annotations │ │ ├── new_playtable.yaml │ │ └── new_playtable_validation.yaml │ ├── callbacks │ │ ├── checkpoint │ │ │ ├── lh_sr.yaml │ │ │ ├── save_all.yaml │ │ │ ├── task_sr.yaml │ │ │ └── val_action.yaml │ │ ├── default.yaml │ │ ├── kl_schedule │ │ │ ├── constant.yaml │ │ │ ├── linear.yaml │ │ │ └── sigmoid.yaml │ │ ├── rollout │ │ │ ├── default.yaml │ │ │ └── tasks │ │ │ │ └── new_playtable_tasks.yaml │ │ ├── rollout_lh │ │ │ └── default.yaml │ │ └── tsne_plot │ │ │ └── default.yaml │ ├── config.yaml │ ├── datamodule │ │ ├── datasets │ │ │ ├── lang_dataset │ │ │ │ ├── lang.yaml │ │ │ │ └── lang_shm.yaml │ │ │ ├── vision_dataset │ │ │ │ ├── vision.yaml │ │ │ │ └── vision_shm.yaml │ │ │ ├── vision_lang.yaml │ │ │ ├── vision_lang_shm.yaml │ │ │ └── vision_only.yaml │ │ ├── default.yaml │ │ ├── observation_space │ │ │ ├── all_mods_abs_act.yaml │ │ │ ├── all_mods_rel_act.yaml │ │ │ ├── lang_rgb_static_abs_act.yaml │ │ │ ├── lang_rgb_static_gripper_abs_act.yaml │ │ │ ├── lang_rgb_static_gripper_rel_act.yaml │ │ │ ├── lang_rgb_static_rel_act.yaml │ │ │ ├── lang_rgb_static_robot_scene_abs_act.yaml │ │ │ ├── lang_rgb_static_tactile_abs_act.yaml │ │ │ ├── lang_rgb_static_tactile_rel_act.yaml │ │ │ ├── lang_rgbd_both_abs_act.yaml │ │ │ ├── lang_rgbd_both_rel_act.yaml │ │ │ ├── lang_rgbd_static_gripper_rel_act.yaml │ │ │ ├── lang_rgbd_static_robot_abs_act.yaml │ │ │ └── state_only.yaml │ │ ├── proprioception_dims │ │ │ ├── none.yaml │ │ │ ├── robot_full.yaml │ │ │ ├── robot_no_joints.yaml │ │ │ └── robot_scene.yaml │ │ ├── random.yaml │ │ └── transforms │ │ │ └── play_basic.yaml │ ├── inference │ │ └── config_inference.yaml │ ├── lang_ann.yaml │ ├── logger │ │ ├── tb_logger.yaml │ │ └── wandb.yaml │ ├── loss │ │ └── default.yaml │ ├── model │ │ ├── action_decoder │ │ │ └── logistic.yaml │ │ ├── default.yaml │ │ ├── language_goal │ │ │ ├── default.yaml │ │ │ └── none.yaml │ │ ├── optimizer │ │ │ ├── adam.yaml │ │ │ ├── adamw.yaml │ │ │ └── sgd.yaml │ │ ├── perceptual_encoder │ │ │ ├── RGBD_both.yaml │ │ │ ├── default.yaml │ │ │ ├── depth_gripper │ │ │ │ ├── default.yaml │ │ │ │ └── none.yaml │ │ │ ├── depth_static │ │ │ │ ├── default.yaml │ │ │ │ └── none.yaml │ │ │ ├── gripper_cam.yaml │ │ │ ├── proprio │ │ │ │ └── identity.yaml │ │ │ ├── static_RGBD.yaml │ │ │ ├── static_RGB_tactile.yaml │ │ │ ├── tactile │ │ │ │ ├── default.yaml │ │ │ │ └── none.yaml │ │ │ ├── vision_gripper │ │ │ │ ├── default.yaml │ │ │ │ └── none.yaml │ │ │ └── vision_static │ │ │ │ └── default.yaml │ │ ├── plan_proposal │ │ │ └── default.yaml │ │ ├── plan_recognition │ │ │ └── default.yaml │ │ ├── sbert.yaml │ │ └── visual_goal │ │ │ └── default.yaml │ ├── trainer │ │ └── play_trainer.yaml │ └── training │ │ └── default_training.yaml ├── requirements.txt └── setup.py ├── config_path.json ├── evaluation_calvin.py ├── my_models └── DDP_training │ ├── ModalityFusioner.py │ ├── __pycache__ │ ├── ModalityFusioner.cpython-38.pyc │ ├── data_enhence.cpython-38.pyc │ ├── data_random.cpython-38.pyc │ ├── diffusion_modules.cpython-310.pyc │ ├── diffusion_modules.cpython-38.pyc │ ├── my_utils.cpython-38.pyc │ ├── positional_encoding.cpython-38.pyc │ ├── pretrain_data_random.cpython-38.pyc │ └── transformer_agent.cpython-38.pyc │ ├── config │ └── read_json.py │ ├── data_enhence.py │ ├── data_random.py │ ├── diffusion_modules.py │ ├── log │ ├── 2025-02-02-21-38-47 │ │ └── events.out.tfevents.1738503527.wibot-ub-cyp.2248120.0 │ ├── 2025-02-10-09-37-41 │ │ └── events.out.tfevents.1739151461.wibot-ub-cyp.71224.0 │ ├── 2025-02-10-09-55-22 │ │ └── events.out.tfevents.1739152522.wibot-ub-cyp.127921.0 │ ├── 2025-02-10-10-01-13 │ │ └── events.out.tfevents.1739152873.wibot-ub-cyp.146316.0 │ └── 2025-02-10-10-07-09 │ │ └── events.out.tfevents.1739153229.wibot-ub-cyp.166804.0 │ ├── my_utils.py │ ├── positional_encoding.py │ ├── pretrain_data_random.py │ ├── src │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ └── helpers.cpython-38.pyc │ ├── factory.py │ ├── flamingo.py │ ├── flamingo_lm.py │ ├── helpers.py │ └── utils.py │ ├── training.py │ └── transformer_agent.py ├── preprocessing_val_language.py ├── requirements.txt └── sparate_action_data.py /__pycache__/preprocessing_val_language.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/__pycache__/preprocessing_val_language.cpython-38.pyc -------------------------------------------------------------------------------- /assets/augmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/augmentation.png -------------------------------------------------------------------------------- /assets/cabinet operations_c.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/cabinet operations_c.gif -------------------------------------------------------------------------------- /assets/f1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/f1.png -------------------------------------------------------------------------------- /assets/opendoor_c 00_00_00-00_00_30.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/opendoor_c 00_00_00-00_00_30.gif -------------------------------------------------------------------------------- /assets/penholder_c 00_00_00-00_00_30.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/penholder_c 00_00_00-00_00_30.gif -------------------------------------------------------------------------------- /assets/stacking_c 00_00_00-00_00_30.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/stacking_c 00_00_00-00_00_30.gif -------------------------------------------------------------------------------- /assets/two-stage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/assets/two-stage.png -------------------------------------------------------------------------------- /calvin_agent/.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | play_data/ 3 | __pycache__/ 4 | results/ 5 | runs/ 6 | -------------------------------------------------------------------------------- /calvin_agent/__init__.py: -------------------------------------------------------------------------------- 1 | """'CALVIN - A benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks 2 | :copyright: 2021 by Oier Mees 3 | :license: MIT, see LICENSE for more details. 4 | """ 5 | 6 | __version__ = "0.0.1" 7 | __project__ = "Calvin" 8 | __author__ = "Oier Mees" 9 | __license__ = "MIT" 10 | __email__ = "meeso@informatik.uni-freiburg.de" 11 | -------------------------------------------------------------------------------- /calvin_agent/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/datasets/__init__.py -------------------------------------------------------------------------------- /calvin_agent/datasets/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/datasets/utils/__init__.py -------------------------------------------------------------------------------- /calvin_agent/evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/evaluation/__init__.py -------------------------------------------------------------------------------- /calvin_agent/inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/inference/__init__.py -------------------------------------------------------------------------------- /calvin_agent/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/models/__init__.py -------------------------------------------------------------------------------- /calvin_agent/models/calvin_base_model.py: -------------------------------------------------------------------------------- 1 | class CalvinBaseModel: 2 | """ 3 | Base class for all models that can be evaluated on the CALVIN challenge. 4 | If you want to evaluate your own model, implement the class methods. 5 | """ 6 | 7 | def reset(self): 8 | """ 9 | Call this at the beginning of a new rollout when doing inference. 10 | """ 11 | raise NotImplementedError 12 | 13 | def step(self, obs, goal): 14 | """ 15 | Do one step of inference with the model. 16 | 17 | Args: 18 | obs (dict): Observation from environment. 19 | goal (dict): Goal as visual observation or embedded language instruction. 20 | 21 | Returns: 22 | Predicted action. 23 | """ 24 | -------------------------------------------------------------------------------- /calvin_agent/models/decoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/models/decoders/__init__.py -------------------------------------------------------------------------------- /calvin_agent/models/decoders/action_decoder.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import torch 4 | from torch import nn 5 | 6 | 7 | class ActionDecoder(nn.Module): 8 | def act(self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor) -> torch.Tensor: 9 | raise NotImplementedError 10 | 11 | def loss( 12 | self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor, actions: torch.Tensor 13 | ) -> torch.Tensor: 14 | raise NotImplementedError 15 | 16 | def loss_and_act( 17 | self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor, actions: torch.Tensor 18 | ) -> Tuple[torch.Tensor, torch.Tensor]: 19 | raise NotImplementedError 20 | 21 | def clear_hidden_state(self) -> None: 22 | raise NotImplementedError 23 | 24 | def _sample(self, *args, **kwargs): 25 | raise NotImplementedError 26 | 27 | def forward( 28 | self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor 29 | ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /calvin_agent/models/encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/models/encoders/__init__.py -------------------------------------------------------------------------------- /calvin_agent/models/encoders/goal_encoders.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | 8 | class VisualGoalEncoder(nn.Module): 9 | def __init__( 10 | self, 11 | hidden_size: int, 12 | latent_goal_features: int, 13 | in_features: int, 14 | l2_normalize_goal_embeddings: bool, 15 | activation_function: str, 16 | ): 17 | super().__init__() 18 | self.l2_normalize_output = l2_normalize_goal_embeddings 19 | self.act_fn = getattr(nn, activation_function)() 20 | self.mlp = nn.Sequential( 21 | nn.Linear(in_features=in_features, out_features=hidden_size), 22 | self.act_fn, 23 | nn.Linear(in_features=hidden_size, out_features=hidden_size), 24 | self.act_fn, 25 | nn.Linear(in_features=hidden_size, out_features=latent_goal_features), 26 | ) 27 | 28 | def forward(self, x: torch.Tensor) -> torch.Tensor: 29 | x = self.mlp(x) 30 | if self.l2_normalize_output: 31 | x = F.normalize(x, p=2, dim=1) 32 | return x 33 | 34 | 35 | class LanguageGoalEncoder(nn.Module): 36 | def __init__( 37 | self, 38 | language_features: int, 39 | hidden_size: int, 40 | latent_goal_features: int, 41 | word_dropout_p: float, 42 | l2_normalize_goal_embeddings: bool, 43 | activation_function: str, 44 | ): 45 | super().__init__() 46 | self.l2_normalize_output = l2_normalize_goal_embeddings 47 | self.act_fn = getattr(nn, activation_function)() 48 | self.mlp = nn.Sequential( 49 | nn.Dropout(word_dropout_p), 50 | nn.Linear(in_features=language_features, out_features=hidden_size), 51 | self.act_fn, 52 | nn.Linear(in_features=hidden_size, out_features=hidden_size), 53 | self.act_fn, 54 | nn.Linear(in_features=hidden_size, out_features=latent_goal_features), 55 | ) 56 | 57 | def forward(self, x: torch.Tensor) -> torch.Tensor: 58 | x = self.mlp(x) 59 | if self.l2_normalize_output: 60 | x = F.normalize(x, p=2, dim=1) 61 | return x 62 | -------------------------------------------------------------------------------- /calvin_agent/models/encoders/language_network.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from sentence_transformers import SentenceTransformer 4 | import torch 5 | import torch.nn as nn 6 | 7 | 8 | class SBert(nn.Module): 9 | def __init__(self, nlp_model: str): 10 | # choose model from https://www.sbert.net/docs/pretrained_models.html 11 | super().__init__() 12 | assert isinstance(nlp_model, str) 13 | self.model = SentenceTransformer(nlp_model) 14 | 15 | def forward(self, x: List) -> torch.Tensor: 16 | emb = self.model.encode(x, convert_to_tensor=True) 17 | return torch.unsqueeze(emb, 1) -------------------------------------------------------------------------------- /calvin_agent/models/perceptual_encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/models/perceptual_encoders/__init__.py -------------------------------------------------------------------------------- /calvin_agent/models/perceptual_encoders/proprio_encoder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torch import nn 3 | 4 | 5 | class IdentityEncoder(nn.Module): 6 | def __init__(self, proprioception_dims): 7 | super(IdentityEncoder, self).__init__() 8 | # remove a dimension if we convert robot orientation quaternion to euler angles 9 | self.n_state_obs = int(np.sum(np.diff([list(x) for x in [list(y) for y in proprioception_dims.keep_indices]]))) 10 | self.identity = nn.Identity() 11 | 12 | @property 13 | def out_features(self): 14 | return self.n_state_obs 15 | 16 | def forward(self, x): 17 | return self.identity(x) 18 | -------------------------------------------------------------------------------- /calvin_agent/models/perceptual_encoders/tactile_encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | import torchvision.models as models 5 | 6 | 7 | class TactileEncoder(nn.Module): 8 | def __init__(self, visual_features: int, freeze_tactile_backbone: bool = True): 9 | super(TactileEncoder, self).__init__() 10 | # Load pre-trained resnet-18 11 | net = models.resnet18(pretrained=True) 12 | # Remove the last fc layer, and rebuild 13 | modules = list(net.children())[:-1] 14 | self.net = nn.Sequential(*modules) 15 | if freeze_tactile_backbone: 16 | for param in self.net.parameters(): 17 | param.requires_grad = False 18 | self.fc1 = nn.Linear(1024, 512) 19 | self.fc2 = nn.Linear(512, visual_features) 20 | 21 | def forward(self, x: torch.Tensor) -> torch.Tensor: 22 | x_l = self.net(x[:, :3, :, :]).squeeze() 23 | x_r = self.net(x[:, 3:, :, :]).squeeze() 24 | x = torch.cat((x_l, x_r), dim=-1) 25 | # Add fc layer for final prediction 26 | output = F.relu(self.fc1(x)) # batch, 512 27 | output = self.fc2(output) # batch, 64 28 | return output 29 | -------------------------------------------------------------------------------- /calvin_agent/models/perceptual_encoders/vision_network_gripper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | 8 | def nature_cnn(act_fn, num_c): 9 | return nn.Sequential( 10 | nn.Conv2d(num_c, 32, 8, stride=4), 11 | act_fn, 12 | nn.Conv2d(32, 64, 4, stride=2), 13 | act_fn, 14 | nn.Conv2d(64, 64, 3, stride=1), 15 | act_fn, 16 | nn.Flatten(start_dim=1), 17 | nn.Linear(64 * 7 * 7, 128), 18 | act_fn, 19 | ) 20 | 21 | 22 | class VisionNetwork(nn.Module): 23 | def __init__( 24 | self, 25 | conv_encoder: str, 26 | activation_function: str, 27 | dropout_vis_fc: float, 28 | l2_normalize_output: bool, 29 | visual_features: int, 30 | num_c: int, 31 | ): 32 | super(VisionNetwork, self).__init__() 33 | self.l2_normalize_output = l2_normalize_output 34 | self.act_fn = getattr(nn, activation_function)() 35 | # model 36 | # this calls the method with the name conv_encoder 37 | self.conv_model = eval(conv_encoder) 38 | self.conv_model = self.conv_model(self.act_fn, num_c) 39 | self.fc1 = nn.Sequential( 40 | nn.Linear(in_features=128, out_features=512), self.act_fn, nn.Dropout(dropout_vis_fc) 41 | ) # shape: [N, 512] 42 | self.fc2 = nn.Linear(in_features=512, out_features=visual_features) # shape: [N, 64] 43 | 44 | def forward(self, x: torch.Tensor) -> torch.Tensor: 45 | x = self.conv_model(x) 46 | x = self.fc1(x) 47 | x = self.fc2(x) 48 | if self.l2_normalize_output: 49 | x = F.normalize(x, p=2, dim=1) 50 | return x # shape: [N, 64] 51 | -------------------------------------------------------------------------------- /calvin_agent/models/plan_encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/models/plan_encoders/__init__.py -------------------------------------------------------------------------------- /calvin_agent/models/plan_encoders/plan_proposal_net.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from typing import Tuple 4 | 5 | import torch 6 | from torch.distributions import Independent, Normal 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class PlanProposalNetwork(nn.Module): 12 | def __init__( 13 | self, 14 | perceptual_features: int, 15 | latent_goal_features: int, 16 | plan_features: int, 17 | activation_function: str, 18 | min_std: float, 19 | ): 20 | super(PlanProposalNetwork, self).__init__() 21 | self.perceptual_features = perceptual_features 22 | self.latent_goal_features = latent_goal_features 23 | self.plan_features = plan_features 24 | self.min_std = min_std 25 | self.in_features = self.perceptual_features + self.latent_goal_features 26 | self.act_fn = getattr(nn, activation_function)() 27 | self.fc_model = nn.Sequential( 28 | nn.Linear(in_features=self.in_features, out_features=2048), # shape: [N, 136] 29 | self.act_fn, 30 | nn.Linear(in_features=2048, out_features=2048), 31 | self.act_fn, 32 | nn.Linear(in_features=2048, out_features=2048), 33 | self.act_fn, 34 | nn.Linear(in_features=2048, out_features=2048), 35 | self.act_fn, 36 | ) 37 | self.mean_fc = nn.Linear(in_features=2048, out_features=self.plan_features) # shape: [N, 2048] 38 | self.variance_fc = nn.Linear(in_features=2048, out_features=self.plan_features) # shape: [N, 2048] 39 | 40 | def forward(self, initial_percep_emb: torch.Tensor, latent_goal: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: 41 | x = torch.cat([initial_percep_emb, latent_goal], dim=-1) 42 | x = self.fc_model(x) 43 | mean = self.mean_fc(x) 44 | var = self.variance_fc(x) 45 | std = F.softplus(var) + self.min_std 46 | return mean, std # shape: [N, 256] 47 | 48 | def __call__(self, *args, **kwargs): 49 | mean, std = super().__call__(*args, **kwargs) 50 | pp_dist = Independent(Normal(mean, std), 1) 51 | return pp_dist 52 | -------------------------------------------------------------------------------- /calvin_agent/models/plan_encoders/plan_recognition_net.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from typing import Tuple 4 | 5 | import torch 6 | from torch.distributions import Independent, Normal 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class PlanRecognitionNetwork(nn.Module): 12 | def __init__( 13 | self, 14 | in_features: int, 15 | plan_features: int, 16 | action_space: int, 17 | birnn_dropout_p: float, 18 | min_std: float, 19 | ): 20 | super(PlanRecognitionNetwork, self).__init__() 21 | self.plan_features = plan_features 22 | self.action_space = action_space 23 | self.min_std = min_std 24 | self.in_features = in_features 25 | self.birnn_model = nn.RNN( 26 | input_size=self.in_features, 27 | hidden_size=2048, 28 | nonlinearity="relu", 29 | num_layers=2, 30 | bidirectional=True, 31 | batch_first=True, 32 | dropout=birnn_dropout_p, 33 | ) # shape: [N, seq_len, 64+8] 34 | self.mean_fc = nn.Linear(in_features=4096, out_features=self.plan_features) # shape: [N, seq_len, 4096] 35 | self.variance_fc = nn.Linear(in_features=4096, out_features=self.plan_features) # shape: [N, seq_len, 4096] 36 | 37 | def forward(self, perceptual_emb: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: 38 | x, hn = self.birnn_model(perceptual_emb) 39 | x = x[:, -1] # we just need only last unit output 40 | mean = self.mean_fc(x) 41 | var = self.variance_fc(x) 42 | std = F.softplus(var) + self.min_std 43 | return mean, std # shape: [N, 256] 44 | 45 | def __call__(self, *args, **kwargs): 46 | mean, std = super().__call__(*args, **kwargs) 47 | pr_dist = Independent(Normal(mean, std), 1) 48 | return pr_dist 49 | -------------------------------------------------------------------------------- /calvin_agent/rollout/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/rollout/__init__.py -------------------------------------------------------------------------------- /calvin_agent/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/utils/__init__.py -------------------------------------------------------------------------------- /calvin_agent/utils/data_visualization.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import hydra 4 | from omegaconf import DictConfig 5 | from pytorch_lightning import seed_everything 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | from matplotlib.animation import ArtistAnimation 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | 13 | 14 | def visualize(data): 15 | seq_img = data[1][0][0].numpy() 16 | title = data[4][0] 17 | s, c, h, w = seq_img.shape 18 | seq_img = np.transpose(seq_img, (0, 2, 3, 1)) 19 | imgs = [] 20 | fig = plt.figure() 21 | for j in range(s): 22 | # imgRGB = seq_img[j].astype(int) 23 | imgRGB = seq_img[j] 24 | imgRGB = (imgRGB - imgRGB.min()) / (imgRGB.max() - imgRGB.min()) 25 | img = plt.imshow(imgRGB, animated=True) 26 | imgs.append([img]) 27 | ArtistAnimation(fig, imgs, interval=50) 28 | plt.title(title) 29 | plt.show() 30 | 31 | 32 | @hydra.main(config_path="../../conf", config_name="default.yaml") 33 | def train(cfg: DictConfig) -> None: 34 | # sets seeds for numpy, torch, python.random and PYTHONHASHSEED. 35 | seed_everything(cfg.seed) 36 | data_module = hydra.utils.instantiate(cfg.dataset, num_workers=0) 37 | data_module.setup() 38 | train = data_module.train_dataloader() 39 | dataset = train["lang"] 40 | logger.info(f"Dataset Size: {len(dataset)}") 41 | for i, lang in enumerate(dataset): 42 | logger.info(f"Element : {i}") 43 | visualize(lang) 44 | 45 | 46 | if __name__ == "__main__": 47 | train() 48 | -------------------------------------------------------------------------------- /calvin_agent/utils/relabel_with_new_lang_model.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Dict 3 | 4 | import hydra 5 | import numpy as np 6 | from omegaconf import DictConfig 7 | 8 | """This script allows for re-annotating video sequences of PlayData. 9 | Parameters: 10 | · +path=/path/to/current/auto_lang_ann.npy 11 | · +name_folder=name_to_new_annotations 12 | New annotations sampling from 'annotations=expert' defined in expert.yaml 13 | NLP model selection: 14 | · model.nlp_model=mini -> 'paraphrase-MiniLM-L6-v2' 15 | · model.nlp_model=multi -> 'paraphrase-multilingual-mpnet-base-v2' 16 | · model.nlp_model=mpnet -> 'paraphrase-mpnet-base-v2' 17 | """ 18 | 19 | 20 | @hydra.main(config_path="../../conf", config_name="lang_ann.yaml") 21 | def main(cfg: DictConfig) -> None: 22 | print("Loading data") 23 | path = Path(cfg.path) 24 | data = np.load(path, allow_pickle=True).reshape(-1)[0] 25 | if "training" in cfg.path: 26 | print("using training instructions...") 27 | task_ann = cfg.train_instructions 28 | else: 29 | print("using validation instructions...") 30 | task_ann = cfg.val_instructions 31 | if cfg.reannotate: 32 | print("Re-annotating sequences...") 33 | data["language"]["ann"] = [ 34 | task_ann[task][np.random.randint(len(task_ann[task]))] for task in data["language"]["task"] 35 | ] 36 | print("Loading Language Model") 37 | model = hydra.utils.instantiate(cfg.model) 38 | print(f"Computing Embeddings with Model --> {cfg.model}") 39 | data["language"]["emb"] = model(data["language"]["ann"]).cpu().numpy() 40 | print("Saving data") 41 | save_path = path.parent / ".." / cfg.name_folder 42 | save_path.mkdir(exist_ok=True) 43 | np.save(save_path / "auto_lang_ann.npy", data) 44 | 45 | if "validation" in cfg.path: 46 | embeddings: Dict = {} 47 | for task, ann in cfg.val_instructions.items(): 48 | embeddings[task] = {} 49 | language_embedding = model(list(ann)) 50 | embeddings[task]["emb"] = language_embedding.cpu().numpy() 51 | embeddings[task]["ann"] = ann 52 | np.save(save_path / "embeddings", embeddings) # type:ignore 53 | print("Done saving val language embeddings for Rollouts !") 54 | 55 | 56 | if __name__ == "__main__": 57 | main() 58 | -------------------------------------------------------------------------------- /calvin_agent/utils/visualizations.py: -------------------------------------------------------------------------------- 1 | # Force matplotlib to not use any Xwindows backend. 2 | import matplotlib 3 | import numpy as np 4 | from pytorch_lightning.loggers import WandbLogger 5 | import torch 6 | import wandb 7 | 8 | matplotlib.use("Agg") 9 | import matplotlib.pyplot as plt 10 | 11 | 12 | def visualize_temporal_consistency(max_batched_length_per_demo, gpus, sampled_plans, all_idx, step, logger, prefix=""): 13 | """compute t-SNE plot of embeddings os a task to visualize temporal consistency""" 14 | labels = [] 15 | for demo in max_batched_length_per_demo: 16 | labels = np.concatenate((labels, np.arange(demo) / float(demo)), axis=0) 17 | # because with ddp, data doesn't come ordered anymore 18 | labels = labels[torch.flatten(all_idx).cpu()] 19 | colors = [plt.cm.Spectral(y_i) for y_i in labels] 20 | assert sampled_plans.shape[0] == len(labels), "plt X shape {}, label len {}".format( 21 | sampled_plans.shape[0], len(labels) 22 | ) 23 | 24 | from MulticoreTSNE import MulticoreTSNE as TSNE 25 | 26 | x_tsne = TSNE(perplexity=40, n_jobs=8).fit_transform(sampled_plans.cpu()) 27 | 28 | plt.close("all") 29 | fig, ax = plt.subplots() 30 | _ = ax.scatter(x_tsne[:, 0], x_tsne[:, 1], c=colors, cmap=plt.cm.Spectral) 31 | fig.suptitle("Temporal Consistency of Latent space") 32 | ax.axis("off") 33 | if isinstance(logger, WandbLogger): 34 | logger.experiment.log({prefix + "latent_embedding": wandb.Image(fig)}) 35 | else: 36 | logger.experiment.add_figure(prefix + "latent_embedding", fig, global_step=step) 37 | -------------------------------------------------------------------------------- /calvin_agent/visualization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_agent/visualization/__init__.py -------------------------------------------------------------------------------- /calvin_env/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git 3 | # Default is 79 in PEP 8 4 | max-line-length = 120 5 | select = E,F,W,C 6 | ignore=W503, # line break before binary operator, need for black 7 | E203, # whitespace before ':'. Opposite convention enforced by black 8 | E731, # do not assign a lambda expression, use a def 9 | E722, 10 | F401, 11 | F841, 12 | E402, # module level import not at top of file 13 | E741, # ambiguous variable name 14 | E501, # line too long. Handled by black 15 | C406, # Unnecessary list literal - rewrite as a dict literal 16 | -------------------------------------------------------------------------------- /calvin_env/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tacto"] 2 | path = tacto 3 | url = https://github.com/lukashermann/tacto.git 4 | -------------------------------------------------------------------------------- /calvin_env/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.8 3 | repos: 4 | - repo: https://github.com/psf/black 5 | rev: 22.1.0 6 | hooks: 7 | - id: black 8 | language_version: python3.8 9 | 10 | - repo: https://gitlab.com/pycqa/flake8 11 | rev: 3.8.4 12 | hooks: 13 | - id: flake8 14 | additional_dependencies: [-e, "git+git://github.com/pycqa/pyflakes.git@c72d6cf#egg=pyflakes"] 15 | 16 | - repo: https://github.com/pre-commit/mirrors-isort 17 | rev: v5.6.4 18 | hooks: 19 | - id: isort 20 | 21 | - repo: https://github.com/pre-commit/mirrors-mypy 22 | rev: v0.790 23 | hooks: 24 | - id: mypy 25 | args: [--ignore-missing-imports, --warn-no-return, --warn-redundant-casts, --disallow-incomplete-defs] 26 | 27 | - repo: https://github.com/pre-commit/pre-commit-hooks 28 | rev: v3.4.0 29 | hooks: 30 | - id: check-yaml 31 | - id: trailing-whitespace 32 | - id: end-of-file-fixer 33 | -------------------------------------------------------------------------------- /calvin_env/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Oier Mees 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /calvin_env/README.md: -------------------------------------------------------------------------------- 1 | # calvin_env 2 | 3 | 4 | 5 | ## Installation 6 | ```bash 7 | git clone --recursive https://github.com/mees/calvin_env.git 8 | cd calvin_env/tacto 9 | pip install -e . 10 | cd .. 11 | pip install -e . 12 | ``` 13 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/__init__.py: -------------------------------------------------------------------------------- 1 | """'VR Data Collection and Rendering 2 | :copyright: 2019 by Oier Mees, Lukas Hermann, Wolfram Burgard 3 | :license: GPLv3, see LICENSE for more details. 4 | """ 5 | 6 | __version__ = "0.0.1" 7 | __project__ = "calvin_env" 8 | __author__ = "Oier Mees, Lukas Hermann" 9 | __license__ = "GPLv3" 10 | __email__ = "meeso@informatik.uni-freiburg.de, hermannl@informatik.uni-freiburg.de," 11 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/camera/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/calvin_env/camera/__init__.py -------------------------------------------------------------------------------- /calvin_env/calvin_env/camera/gripper_camera.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pybullet as p 3 | 4 | from calvin_env.camera.camera import Camera 5 | 6 | 7 | class GripperCamera(Camera): 8 | def __init__(self, fov, aspect, nearval, farval, width, height, robot_id, cid, name, objects=None): 9 | self.cid = cid 10 | self.robot_uid = robot_id 11 | links = { 12 | p.getJointInfo(self.robot_uid, i, physicsClientId=self.cid)[12].decode("utf-8"): i 13 | for i in range(p.getNumJoints(self.robot_uid, physicsClientId=self.cid)) 14 | } 15 | self.gripper_cam_link = links["gripper_cam"] 16 | self.fov = fov 17 | self.aspect = aspect 18 | self.nearval = nearval 19 | self.farval = farval 20 | self.width = width 21 | self.height = height 22 | 23 | self.name = name 24 | 25 | def render(self): 26 | camera_ls = p.getLinkState( 27 | bodyUniqueId=self.robot_uid, linkIndex=self.gripper_cam_link, physicsClientId=self.cid 28 | ) 29 | camera_pos, camera_orn = camera_ls[:2] 30 | cam_rot = p.getMatrixFromQuaternion(camera_orn) 31 | cam_rot = np.array(cam_rot).reshape(3, 3) 32 | cam_rot_y, cam_rot_z = cam_rot[:, 1], cam_rot[:, 2] 33 | # camera: eye position, target position, up vector 34 | self.view_matrix = p.computeViewMatrix(camera_pos, camera_pos + cam_rot_y, -cam_rot_z) 35 | self.projection_matrix = p.computeProjectionMatrixFOV( 36 | fov=self.fov, aspect=self.aspect, nearVal=self.nearval, farVal=self.farval 37 | ) 38 | image = p.getCameraImage( 39 | width=self.width, 40 | height=self.height, 41 | viewMatrix=self.view_matrix, 42 | projectionMatrix=self.projection_matrix, 43 | physicsClientId=self.cid, 44 | ) 45 | rgb_img, depth_img = self.process_rgbd(image, self.nearval, self.farval) 46 | return rgb_img, depth_img 47 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/camera/tactile_sensor.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | 5 | from calvin_env.camera.camera import Camera 6 | import tacto 7 | 8 | REPO_BASE = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) 9 | 10 | 11 | class TactileSensor(Camera): 12 | def __init__( 13 | self, width, height, digit_link_ids, visualize_gui, cid, name, config_path, robot_id=None, objects=None 14 | ): 15 | """ 16 | Initialize the camera 17 | Args: 18 | argument_group: initialize the camera and add needed arguments to argparse 19 | 20 | Returns: 21 | None 22 | """ 23 | self.cid = cid 24 | self.name = name 25 | self.robot_uid = robot_id 26 | self.digits = tacto.Sensor( 27 | width=width, height=height, visualize_gui=visualize_gui, config_path=os.path.join(REPO_BASE, config_path) 28 | ) 29 | self.digits.add_camera(robot_id, digit_link_ids) # env.robot.digit_links() 30 | for obj in objects: 31 | # self.digits.add_body(obj) 32 | self.digits.add_object(obj.file.as_posix(), obj.uid, obj.global_scaling) 33 | self.visualize_gui = visualize_gui 34 | 35 | def render(self): 36 | rgb, depth = self.digits.render() 37 | if self.visualize_gui: 38 | self.digits.updateGUI(rgb, depth) 39 | rgb = np.concatenate(rgb, axis=2) 40 | depth = np.stack(depth, axis=2) 41 | return rgb, depth 42 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/envs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/calvin_env/envs/__init__.py -------------------------------------------------------------------------------- /calvin_env/calvin_env/robot/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/calvin_env/robot/__init__.py -------------------------------------------------------------------------------- /calvin_env/calvin_env/scene/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/calvin_env/scene/__init__.py -------------------------------------------------------------------------------- /calvin_env/calvin_env/scene/objects/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/calvin_env/scene/objects/__init__.py -------------------------------------------------------------------------------- /calvin_env/calvin_env/scene/objects/base_object.py: -------------------------------------------------------------------------------- 1 | class BaseObject: 2 | def __init__(self, name, obj_cfg, p, cid, data_path, global_scaling): 3 | self.p = p 4 | self.cid = cid 5 | self.name = name 6 | self.file = data_path / obj_cfg["file"] 7 | self.global_scaling = global_scaling 8 | 9 | def reset(self, state): 10 | pass 11 | 12 | def get_info(self): 13 | pass 14 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/scene/objects/door.py: -------------------------------------------------------------------------------- 1 | MAX_FORCE = 4 2 | 3 | 4 | class Door: 5 | def __init__(self, name, cfg, uid, p, cid): 6 | self.name = name 7 | self.p = p 8 | self.cid = cid 9 | # get joint_index by name (to prevent index errors when additional joints are added) 10 | joint_index = next( 11 | i 12 | for i in range(self.p.getNumJoints(uid, physicsClientId=self.cid)) 13 | if self.p.getJointInfo(uid, i, physicsClientId=self.cid)[1].decode("utf-8") == name 14 | ) 15 | self.joint_index = joint_index 16 | self.uid = uid 17 | self.initial_state = cfg["initial_state"] 18 | self.p.setJointMotorControl2( 19 | self.uid, 20 | self.joint_index, 21 | controlMode=p.VELOCITY_CONTROL, 22 | force=MAX_FORCE, 23 | physicsClientId=self.cid, 24 | ) 25 | 26 | def reset(self, state=None): 27 | _state = self.initial_state if state is None else state 28 | self.p.resetJointState( 29 | self.uid, 30 | self.joint_index, 31 | _state, 32 | physicsClientId=self.cid, 33 | ) 34 | 35 | def get_state(self): 36 | joint_state = self.p.getJointState(self.uid, self.joint_index, physicsClientId=self.cid) 37 | return float(joint_state[0]) 38 | 39 | def get_info(self): 40 | return {"current_state": self.get_state()} 41 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/scene/objects/fixed_object.py: -------------------------------------------------------------------------------- 1 | from calvin_env.scene.objects.base_object import BaseObject 2 | 3 | 4 | class FixedObject(BaseObject): 5 | def __init__(self, name, obj_cfg, p, cid, data_path, global_scaling): 6 | super().__init__(name, obj_cfg, p, cid, data_path, global_scaling) 7 | self.initial_pos = obj_cfg["initial_pos"] 8 | self.initial_orn = self.p.getQuaternionFromEuler(obj_cfg["initial_orn"]) 9 | 10 | self.uid = self.p.loadURDF( 11 | self.file.as_posix(), 12 | self.initial_pos, 13 | self.initial_orn, 14 | globalScaling=global_scaling, 15 | physicsClientId=self.cid, 16 | ) 17 | self.info_dict = {"uid": self.uid} 18 | self.num_joints = self.p.getNumJoints(self.uid, physicsClientId=self.cid) 19 | if self.num_joints > 0: 20 | # save link names and ids in dictionary 21 | links = { 22 | self.p.getJointInfo(self.uid, i, physicsClientId=self.cid)[12].decode("utf-8"): i 23 | for i in range(self.num_joints) 24 | } 25 | links["base_link"] = -1 26 | self.info_dict["links"] = links 27 | 28 | def reset(self, state=None): 29 | pass 30 | 31 | def get_info(self): 32 | obj_info = {**self.info_dict, "contacts": self.p.getContactPoints(bodyA=self.uid, physicsClientId=self.cid)} 33 | return obj_info 34 | 35 | def serialize(self): 36 | joints = ( 37 | self.p.getJointStates(self.uid, list(range(self.num_joints)), physicsClientId=self.cid) 38 | if self.num_joints > 0 39 | else () 40 | ) 41 | return {"uid": self.uid, "info": self.p.getBodyInfo(self.uid, physicsClientId=self.cid), "joints": joints} 42 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/scene/objects/light.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class LightState(Enum): 5 | ON = 1 6 | OFF = 0 7 | 8 | 9 | class Light: 10 | def __init__(self, name, cfg, uid, p, cid): 11 | self.name = name 12 | self.uid = uid 13 | self.p = p 14 | self.cid = cid 15 | self.link = cfg["link"] 16 | self.link_id = next( 17 | i 18 | for i in range(self.p.getNumJoints(uid, physicsClientId=self.cid)) 19 | if self.p.getJointInfo(uid, i, physicsClientId=self.cid)[12].decode("utf-8") == self.link 20 | ) 21 | self.color_on = cfg["color"] 22 | self.color_off = [1, 1, 1, 1] 23 | self.state = LightState.OFF 24 | 25 | def reset(self, state=None): 26 | if state is None: 27 | self.turn_off() 28 | else: 29 | if state == LightState.ON.value: 30 | self.turn_on() 31 | elif state == LightState.OFF.value: 32 | self.turn_off() 33 | else: 34 | print("Light state can be only 0 or 1.") 35 | raise ValueError 36 | 37 | def get_state(self): 38 | return self.state.value 39 | 40 | def get_info(self): 41 | return {"logical_state": self.get_state()} 42 | 43 | def turn_on(self): 44 | self.state = LightState.ON 45 | self.p.changeVisualShape(self.uid, self.link_id, rgbaColor=self.color_on, physicsClientId=self.cid) 46 | 47 | def turn_off(self): 48 | self.state = LightState.OFF 49 | self.p.changeVisualShape(self.uid, self.link_id, rgbaColor=self.color_off, physicsClientId=self.cid) 50 | 51 | def serialize(self): 52 | return self.get_info() 53 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/scripts/convert_gripper_actions.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import sys 3 | 4 | import numpy as np 5 | from tqdm import tqdm 6 | 7 | path = Path(sys.argv[-1]) 8 | 9 | for subdir in ["training", "validation"]: 10 | for file in tqdm((path / subdir).glob("*.npz")): 11 | data = np.load(file) 12 | if data["rel_actions"][-1] == 0: 13 | data = dict(data) 14 | data["rel_actions"][-1] = -1 15 | data["actions"][-1] = -1 16 | np.savez(file, **data) 17 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/scripts/dataset_to_euler.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from shutil import copyfile, copytree 3 | 4 | import numpy as np 5 | import pybullet as p 6 | from tqdm import tqdm 7 | 8 | load_path = Path("/home/hermannl/phd/data/banana_dataset_01_29/validation") 9 | 10 | save_path = Path("/home/hermannl/phd/data/banana_dataset_01_29_euler/validation") 11 | save_path.mkdir(parents=True, exist_ok=True) 12 | 13 | for file in tqdm(load_path.glob("*.npz")): 14 | data = np.load(file) 15 | robot_obs = data["robot_obs"] 16 | robot_obs_euler = np.concatenate([robot_obs[:3], p.getEulerFromQuaternion(robot_obs[3:7]), robot_obs[7:]]) 17 | scene_obs = data["scene_obs"] 18 | scene_obs_euler = scene_obs[:3] 19 | for i in range(6): 20 | scene_obs_euler = np.append(scene_obs_euler, scene_obs[3 + i * 7 : 3 + i * 7 + 3]) 21 | scene_obs_euler = np.append(scene_obs_euler, p.getEulerFromQuaternion(scene_obs[3 + i * 7 + 3 : 3 + i * 7 + 7])) 22 | actions = data["actions"] 23 | actions_euler = np.concatenate([actions[:3], p.getEulerFromQuaternion(actions[3:7]), actions[7:]]) 24 | data_euler = dict(data.items()) 25 | data_euler["robot_obs"] = robot_obs_euler 26 | data_euler["scene_obs"] = scene_obs_euler 27 | data_euler["actions"] = actions_euler 28 | np.savez(save_path / file.name, **data_euler) 29 | 30 | for file in set(load_path.glob("*")) - set(load_path.glob("*.npz")): 31 | if file.is_dir(): 32 | copytree(file, save_path / file.name) 33 | else: 34 | copyfile(file, save_path / file.name) 35 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/scripts/unnormalize_depth.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from shutil import copyfile, copytree 3 | 4 | import numpy as np 5 | from tqdm import tqdm 6 | 7 | load_path = Path("/home/meeso/expert_demos_03_10/training") 8 | 9 | save_path = Path("/home/meeso/expert_demos_03_10/training_unnormalized_depth") 10 | save_path.mkdir(parents=True, exist_ok=True) 11 | 12 | for file in tqdm(load_path.glob("*.npz")): 13 | data = np.load(file) 14 | corrected_data = dict(data.items()) 15 | corrected_data["depth_static"] = data["depth_static"] * 2.0 16 | corrected_data["depth_gripper"] = data["depth_gripper"] * 2.0 17 | np.savez(save_path / file.name, **corrected_data) 18 | 19 | 20 | for file in set(load_path.glob("*")) - set(load_path.glob("*.npz")): 21 | if file.is_dir(): 22 | copytree(file, save_path / file.name) 23 | else: 24 | copyfile(file, save_path / file.name) 25 | -------------------------------------------------------------------------------- /calvin_env/calvin_env/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/calvin_env/utils/__init__.py -------------------------------------------------------------------------------- /calvin_env/calvin_env/vrdatacollector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | from copy import deepcopy 3 | import logging 4 | import os 5 | import sys 6 | 7 | import hydra 8 | import pybullet as p 9 | import quaternion # noqa 10 | 11 | from calvin_env.io_utils.data_recorder import DataRecorder 12 | from calvin_env.io_utils.vr_input import VrInput 13 | 14 | # A logger for this file 15 | log = logging.getLogger(__name__) 16 | 17 | 18 | @hydra.main(config_path="../conf", config_name="config_data_collection") 19 | def main(cfg): 20 | # Load Scene 21 | env = hydra.utils.instantiate(cfg.env) 22 | vr_input = hydra.utils.instantiate(cfg.vr_input) 23 | 24 | data_recorder = None 25 | if cfg.recorder.record: 26 | data_recorder = DataRecorder(env, cfg.recorder.record_fps, cfg.recorder.enable_tts) 27 | 28 | log.info("Initialization done!") 29 | log.info("Entering Loop") 30 | 31 | record = False 32 | 33 | while 1: 34 | # get input events 35 | action = vr_input.get_vr_action() 36 | obs, _, _, info = env.step(action) 37 | done = False 38 | if vr_input.reset_button_pressed: 39 | done = True 40 | if vr_input.start_button_pressed: 41 | record = True 42 | if vr_input.reset_button_hold: 43 | data_recorder.delete_episode() 44 | if record and cfg.recorder.record: 45 | data_recorder.step(vr_input.prev_vr_events, obs, done, info) 46 | if done: 47 | record = False 48 | env.reset() 49 | 50 | 51 | if __name__ == "__main__": 52 | main() 53 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/cameras/gripper.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.camera.gripper_camera.GripperCamera 2 | name: gripper 3 | fov: 75 4 | aspect: 1 5 | nearval: 0.01 6 | farval: 2 7 | width: 84 8 | height: 84 9 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/cameras/opposing.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.camera.static_camera.StaticCamera 2 | name: opposing 3 | fov: 75 4 | aspect: 1 5 | nearval: 0.01 6 | farval: 2 7 | width: 200 8 | height: 200 9 | look_at: [ 0.4, 0.5, 0.6 ] 10 | look_from: [ 0.4, 1.5, 0.9 ] 11 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/cameras/static.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.camera.static_camera.StaticCamera 2 | name: static 3 | fov: 10 4 | aspect: 1 5 | nearval: 0.01 6 | farval: 10 7 | width: 200 8 | height: 200 9 | look_at: [ -0.026242351159453392, -0.0302329882979393, 0.3920000493526459] 10 | look_from: [ 2.871459009488717, -2.166602199425597, 2.555159848480571] 11 | up_vector: [ 0.4041403970338857, 0.22629790978217404, 0.8862616969685161] 12 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/cameras/tactile.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.camera.tactile_sensor.TactileSensor 2 | name: tactile 3 | width: 120 4 | height: 160 5 | digit_link_ids: [10, 12] # ${robot.digit_link_ids} 6 | visualize_gui: true 7 | config_path: conf/digit_sensor/config_digit.yml 8 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/no_cameras.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/conf/cameras/no_cameras.yaml -------------------------------------------------------------------------------- /calvin_env/conf/cameras/static_and_gripper.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - cameras@static: static 3 | - cameras@gripper: gripper 4 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/static_and_tactile.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - cameras@static: static 3 | - cameras@tactile: tactile 4 | -------------------------------------------------------------------------------- /calvin_env/conf/cameras/static_gripper_tactile.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - cameras@static: static 3 | - cameras@gripper: gripper 4 | - cameras@tactile: tactile 5 | -------------------------------------------------------------------------------- /calvin_env/conf/config_data_collection.yaml: -------------------------------------------------------------------------------- 1 | seed: 0 2 | use_vr: true 3 | data_path: data 4 | save_dir: /tmp 5 | record: true 6 | 7 | hydra: 8 | run: 9 | dir: ${save_dir}/${now:%Y-%m-%d}/${now:%H-%M-%S} 10 | 11 | defaults: 12 | - cameras: no_cameras 13 | - vr_input: vr_input 14 | - env: play_table_env 15 | - scene: calvin_scene_D 16 | - robot: panda_longer_finger 17 | - tasks: new_playtable_tasks 18 | - recorder: recorder 19 | - override hydra/job_logging: colorlog 20 | - override hydra/hydra_logging: colorlog 21 | -------------------------------------------------------------------------------- /calvin_env/conf/config_rendering.yaml: -------------------------------------------------------------------------------- 1 | load_dir: ??? 2 | data_path: data 3 | save_dir: ??? 4 | show_gui: false 5 | processes: 1 6 | set_static_cam: false 7 | 8 | env: 9 | cameras: ${cameras} 10 | show_gui: ${show_gui} 11 | use_vr: false 12 | 13 | hydra: 14 | run: 15 | dir: ${save_dir}/${now:%Y-%m-%d}/${now:%H-%M-%S} 16 | 17 | defaults: 18 | - cameras: static_and_tactile 19 | - override hydra/job_logging: colorlog 20 | - override hydra/hydra_logging: colorlog 21 | -------------------------------------------------------------------------------- /calvin_env/conf/recorder/recorder.yaml: -------------------------------------------------------------------------------- 1 | record: ${record} 2 | record_fps: 30.0 3 | show_fps: false 4 | enable_tts: true 5 | -------------------------------------------------------------------------------- /calvin_env/conf/robot/panda.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.robot.robot.Robot 2 | filename: franka_panda/panda.urdf 3 | base_position: ${scene.robot_base_position} 4 | base_orientation: ${scene.robot_base_orientation} 5 | initial_joint_positions: ${scene.robot_initial_joint_positions} 6 | max_joint_force: 200.0 7 | gripper_force: 200 8 | arm_joint_ids: [0, 1, 2, 3, 4, 5, 6] 9 | lower_joint_limits: [-2.8973, -1.7628, -2.8973, -3.0718, -2.8973, -0.0175, -2.8973] 10 | upper_joint_limits: [2.8973, 1.7628, 2.8973, -0.0698, 2.8973, 3.7525, 2.8973] 11 | gripper_joint_ids: [9, 10] 12 | gripper_joint_limits: [0, 0.04] 13 | tcp_link_id: 13 14 | end_effector_link_id: 7 15 | gripper_cam_link: 12 16 | use_nullspace: true 17 | max_velocity: 2 18 | use_ik_fast: false 19 | magic_scaling_factor_pos: 1 # 1.6 20 | magic_scaling_factor_orn: 1 # 2.2 21 | use_target_pose: true 22 | euler_obs: true 23 | -------------------------------------------------------------------------------- /calvin_env/conf/robot/panda_digit.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - panda 3 | 4 | filename: franka_panda/panda_digit.urdf 5 | gripper_joint_ids: [9, 11] 6 | tcp_link_id: 15 7 | -------------------------------------------------------------------------------- /calvin_env/conf/robot/panda_longer_finger.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - panda 3 | 4 | filename: franka_panda/panda_longer_finger.urdf 5 | gripper_joint_ids: [9, 11] 6 | tcp_link_id: 15 7 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/basic_playtable.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [0.3, 0.15, 0.6] 7 | robot_base_orientation: [0, 0, 1.5707963] 8 | robot_initial_joint_positions: [-0.017792060227770554, -0.7601235411041661, 0.019782607023391807, -2.342050140544315, 0.029840531355804868, 1.5411935298621688, 0.7534486589746342] 9 | surfaces: [] 10 | objects: 11 | fixed_objects: 12 | table: 13 | file: own_objects/modified_table_new_handles/playtable_modified.urdf 14 | initial_pos: [0.7, 1.0, 0] 15 | initial_orn: [0, 0, 3.141592653589793] 16 | joints: 17 | top_left_door_joint: 18 | initial_state: 0 # revolute 19 | slide_door_joint: 20 | initial_state: 0 # prismatic 21 | drawer: 22 | file: own_objects/drawer/drawer.urdf 23 | initial_pos: [0.475, 1.0, -0.004] 24 | initial_orn: [0, 0, 3.141592653589793] 25 | joints: 26 | drawer_joint: 27 | initial_state: 0 # prismatic 28 | movable_objects: 29 | thuna: 30 | file: ais_objects/thuna/thuna.urdf 31 | initial_pos: [-0.1, 0.6, 0.54] 32 | initial_orn: [0, 0, 0] 33 | bowl: 34 | file: 024_bowl/google_16k/textured.urdf 35 | initial_pos: [-0.1, 0.92, 0.63] 36 | initial_orn: [0, 0, 0] 37 | banana: 38 | file: blocks/block_red.urdf 39 | initial_pos: [0, 0.65, 0.61] 40 | initial_orn: [0, 0, 0] 41 | salt: 42 | file: ais_objects/salt/salt.urdf 43 | initial_pos: [0.35, 0.9, 0.75] 44 | initial_orn: [0, 0, 0] 45 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/basic_tabletop.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [0.3, 0.15, 0.6] 7 | robot_base_orientation: [0, 0, 1.5707963] 8 | robot_initial_joint_positions: [-0.017792060227770554, -0.7601235411041661, 0.019782607023391807, -2.342050140544315, 0.029840531355804868, 1.5411935298621688, 0.7534486589746342] 9 | surfaces: [] 10 | objects: 11 | fixed_objects: 12 | table: 13 | file: table/hightable.urdf 14 | initial_pos: [ 0.3, 0.7, 0.02 ] 15 | initial_orn: [0, 0, 0] 16 | movable_objects: 17 | plate: 18 | file: 029_plate/google_16k/textured.urdf 19 | initial_pos: [ 0.35, 0.72, 0.61 ] 20 | initial_orn: [0, 0, 0] 21 | thuna: 22 | file: ais_objects/thuna/thuna.urdf 23 | initial_pos: [ -0.1, 0.9, 0.65 ] 24 | initial_orn: [0, 0, 0] 25 | banana: 26 | file: 011_banana/demo/banana_vhacd.urdf 27 | initial_pos: [ 0.5, 0.9, 0.61 ] 28 | initial_orn: [0, 0, 0] 29 | bowl: 30 | file: 024_bowl/google_16k/textured.urdf 31 | initial_pos: [-0.1, 0.72, 0.75] 32 | initial_orn: [0, 0, 0] 33 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/calvin_scene_A.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [-0.34, -0.46, 0.24] 7 | robot_base_orientation: [0, 0, 0] 8 | robot_initial_joint_positions: [-1.21779206, 1.03987646, 2.11978261, -2.34205014, -0.87015947, 1.64119353, 0.55344866] 9 | surfaces: 10 | table: [[-0.2, -0.15, 0.46], [0.35, -0.03, 0.46]] 11 | slider_left: [[-0.32, 0.05, 0.46], [-0.16, 0.12, 0.46]] 12 | slider_right: [[-0.05, 0.05, 0.46], [0.13, 0.12, 0.46]] 13 | objects: 14 | fixed_objects: 15 | table: 16 | file: calvin_table_A/urdf/calvin_table_A.urdf 17 | initial_pos: [0, 0, 0] 18 | initial_orn: [0, 0, 0] 19 | joints: 20 | base__slide: 21 | initial_state: 0 # Prismatic 22 | base__drawer: 23 | initial_state: 0 # Prismatic 24 | buttons: 25 | base__button: 26 | initial_state: 0 # Prismatic 27 | effect: led 28 | switches: 29 | base__switch: 30 | initial_state: 0 # Revolute 31 | effect: lightbulb 32 | lights: 33 | lightbulb: 34 | link: light_link 35 | color: [1, 1, 0, 1] # yellow 36 | led: 37 | link: led_link 38 | color: [0, 1, 0, 1] # green 39 | movable_objects: 40 | block_pink: 41 | file: blocks/block_pink_small.urdf 42 | initial_pos: any 43 | initial_orn: any 44 | block_blue: 45 | file: blocks/block_blue_big.urdf 46 | initial_pos: any 47 | initial_orn: any 48 | block_red: 49 | file: blocks/block_red_middle.urdf 50 | initial_pos: any 51 | initial_orn: any 52 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/calvin_scene_B.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [-0.34, -0.46, 0.24] 7 | robot_base_orientation: [0, 0, 0] 8 | robot_initial_joint_positions: [-1.21779206, 1.03987646, 2.11978261, -2.34205014, -0.87015947, 1.64119353, 0.55344866] 9 | surfaces: 10 | table: [[-0.35, -0.15, 0.46], [0.15, -0.03, 0.46]] 11 | slider_left: [[-0.12, 0.05, 0.46], [0.06, 0.12, 0.46]] 12 | slider_right: [[0.15, 0.05, 0.46], [0.33, 0.12, 0.46]] 13 | objects: 14 | fixed_objects: 15 | table: 16 | file: calvin_table_B/urdf/calvin_table_B.urdf 17 | initial_pos: [0, 0, 0] 18 | initial_orn: [0, 0, 0] 19 | joints: 20 | base__slide: 21 | initial_state: 0 # Prismatic 22 | base__drawer: 23 | initial_state: 0 # Prismatic 24 | buttons: 25 | base__button: 26 | initial_state: 0 # Prismatic 27 | effect: led 28 | switches: 29 | base__switch: 30 | initial_state: 0 # Revolute 31 | effect: lightbulb 32 | lights: 33 | lightbulb: 34 | link: light_link 35 | color: [1, 1, 0, 1] # yellow 36 | led: 37 | link: led_link 38 | color: [0, 1, 0, 1] # green 39 | movable_objects: 40 | block_red: 41 | file: blocks/block_red_small.urdf 42 | initial_pos: any 43 | initial_orn: any 44 | block_blue: 45 | file: blocks/block_blue_big.urdf 46 | initial_pos: any 47 | initial_orn: any 48 | block_pink: 49 | file: blocks/block_pink_middle.urdf 50 | initial_pos: any 51 | initial_orn: any 52 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/calvin_scene_C.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [-0.34, -0.46, 0.24] 7 | robot_base_orientation: [0, 0, 0] 8 | robot_initial_joint_positions: [-1.21779206, 1.03987646, 2.11978261, -2.34205014, -0.87015947, 1.64119353, 0.55344866] 9 | surfaces: 10 | table: [[0.0, -0.15, 0.46], [0.35, -0.03, 0.46]] 11 | slider_left: [[-0.12, 0.05, 0.46], [0.06, 0.12, 0.46]] 12 | slider_right: [[0.15, 0.05, 0.46], [0.3, 0.12, 0.46]] 13 | objects: 14 | fixed_objects: 15 | table: 16 | file: calvin_table_C/urdf/calvin_table_C.urdf 17 | initial_pos: [0, 0, 0] 18 | initial_orn: [0, 0, 0] 19 | joints: 20 | base__slide: 21 | initial_state: 0 # Prismatic 22 | base__drawer: 23 | initial_state: 0 # Prismatic 24 | buttons: 25 | base__button: 26 | initial_state: 0 # Prismatic 27 | effect: led 28 | switches: 29 | base__switch: 30 | initial_state: 0 # Revolute 31 | effect: lightbulb 32 | lights: 33 | lightbulb: 34 | link: light_link 35 | color: [1, 1, 0, 1] # yellow 36 | led: 37 | link: led_link 38 | color: [0, 1, 0, 1] # green 39 | movable_objects: 40 | block_blue: 41 | file: blocks/block_blue_small.urdf 42 | initial_pos: any 43 | initial_orn: any 44 | block_red: 45 | file: blocks/block_red_big.urdf 46 | initial_pos: any 47 | initial_orn: any 48 | 49 | block_pink: 50 | file: blocks/block_pink_middle.urdf 51 | initial_pos: any 52 | initial_orn: any 53 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/calvin_scene_D.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [-0.34, -0.46, 0.24] 7 | robot_base_orientation: [0, 0, 0] 8 | robot_initial_joint_positions: [-1.21779206, 1.03987646, 2.11978261, -2.34205014, -0.87015947, 1.64119353, 0.55344866] 9 | surfaces: 10 | table: [[0.0, -0.15, 0.46], [0.35, -0.03, 0.46]] 11 | slider_left: [[-0.32, 0.05, 0.46], [-0.16, 0.12, 0.46]] 12 | slider_right: [[-0.05, 0.05, 0.46], [0.13, 0.12, 0.46]] 13 | objects: 14 | fixed_objects: 15 | table: 16 | file: calvin_table_D/urdf/calvin_table_D.urdf 17 | initial_pos: [0, 0, 0] 18 | initial_orn: [0, 0, 0] 19 | joints: 20 | base__slide: 21 | initial_state: 0 # Prismatic 22 | base__drawer: 23 | initial_state: 0 # Prismatic 24 | buttons: 25 | base__button: 26 | initial_state: 0 # Prismatic 27 | effect: led 28 | switches: 29 | base__switch: 30 | initial_state: 0 # Revolute 31 | effect: lightbulb 32 | lights: 33 | lightbulb: 34 | link: light_link 35 | color: [1, 1, 0, 1] # yellow 36 | led: 37 | link: led_link 38 | color: [0, 1, 0, 1] # green 39 | movable_objects: 40 | block_red: 41 | file: blocks/block_red_middle.urdf 42 | initial_pos: any 43 | initial_orn: any 44 | block_blue: 45 | file: blocks/block_blue_small.urdf 46 | initial_pos: any 47 | initial_orn: any 48 | block_pink: 49 | file: blocks/block_pink_big.urdf 50 | initial_pos: any 51 | initial_orn: any 52 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/calvin_scene_D_eval.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [-0.34, -0.46, 0.24] 7 | robot_base_orientation: [0, 0, 0] 8 | robot_initial_joint_positions: [-1.21779206, 1.03987646, 2.11978261, -2.34205014, -0.87015947, 1.64119353, 0.55344866] 9 | surfaces: 10 | table: [[0.0, -0.15, 0.46], [0.35, -0.03, 0.46]] 11 | slider_left: [[-0.32, 0.05, 0.46], [-0.16, 0.12, 0.46]] 12 | slider_right: [[-0.05, 0.05, 0.46], [0.13, 0.12, 0.46]] 13 | objects: 14 | fixed_objects: 15 | table: 16 | file: calvin_table_D/urdf/calvin_table_D.urdf 17 | initial_pos: [0, 0, 0] 18 | initial_orn: [0, 0, 0] 19 | joints: 20 | base__slide: 21 | initial_state: 0 # Prismatic 22 | base__drawer: 23 | initial_state: 0 # Prismatic 24 | buttons: 25 | base__button: 26 | initial_state: 0 # Prismatic 27 | effect: led 28 | switches: 29 | base__switch: 30 | initial_state: 0 # Revolute 31 | effect: lightbulb 32 | lights: 33 | lightbulb: 34 | link: light_link 35 | color: [1, 1, 0, 1] # yellow 36 | led: 37 | link: led_link 38 | color: [0, 1, 0, 1] # green 39 | movable_objects: 40 | block_red: 41 | file: blocks/block_red_middle.urdf 42 | initial_pos: [0.05, -0.12, 0.46] 43 | initial_orn: [0, 0, 1.57] 44 | block_blue: 45 | file: blocks/block_blue_small.urdf 46 | initial_pos: [0.23, -0.12, 0.46] 47 | initial_orn: [0, 0, 0] 48 | block_pink: 49 | file: blocks/block_pink_big.urdf 50 | initial_pos: [0.10, 0.08, 0.46] 51 | initial_orn: [0, 0, 1.57] 52 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/empty_playtable.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [0.3, 0.15, 0.6] 7 | robot_base_orientation: [0, 0, 1.5707963] 8 | robot_initial_joint_positions: [-0.017792060227770554, -0.7601235411041661, 0.019782607023391807, -2.342050140544315, 0.029840531355804868, 1.5411935298621688, 0.7534486589746342] 9 | surfaces: [] 10 | objects: 11 | fixed_objects: 12 | table: 13 | file: own_objects/modified_table_new_handles/playtable_modified.urdf 14 | initial_pos: [0.7, 1.0, 0] 15 | initial_orn: [0, 0, 3.141592653589793] 16 | joints: 17 | top_left_door_joint: 18 | initial_state: 0 # revolute 19 | slide_door_joint: 20 | initial_state: 0 # prismatic 21 | drawer: 22 | file: own_objects/drawer/drawer.urdf 23 | initial_pos: [0.475, 1.0, -0.004] 24 | initial_orn: [0, 0, 3.141592653589793] 25 | joints: 26 | drawer_joint: 27 | initial_state: 0 # prismatic 28 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/tabletop_1.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [0.3, 0.15, 0.6] 7 | robot_base_orientation: [0, 0, 1.5707963] 8 | robot_initial_joint_positions: [-0.017792060227770554, -0.7601235411041661, 0.019782607023391807, -2.342050140544315, 0.029840531355804868, 1.5411935298621688, 0.7534486589746342] 9 | surfaces: [] 10 | objects: 11 | fixed_objects: 12 | table: 13 | file: table/hightable.urdf 14 | initial_pos: [0.3, 0.7, 0.02] 15 | initial_orn: [0, 0, 0] 16 | bin: 17 | file: ais_objects/bin_10_30_50/bin_10_30_50.urdf 18 | initial_pos: [0.7, 0.75, 0.6] 19 | initial_orn: [1.57, 0, 0] 20 | movable_objects: 21 | frying_pan: 22 | file: tabletop/frying_pan/frying_pan.urdf 23 | initial_pos: [0.15, 0.9, 0.63] 24 | initial_orn: [0, 0, 0] 25 | knife: 26 | file: tabletop/kitchen_knife/kitchen_knife.urdf 27 | initial_pos: [0.35, 0.92, 0.61] 28 | initial_orn: [0, 0, 0] 29 | bowl: 30 | file: 024_bowl/google_16k/textured.urdf 31 | initial_pos: [0.2, 0.45, 0.62] 32 | initial_orn: [0, 0, 0] 33 | frying_pan_2: 34 | file: tabletop/frying_pan_2/frying_pan.urdf 35 | initial_pos: [-0.1, 0.6, 0.62] 36 | initial_orn: [0, 0, -1.57] 37 | whisk: 38 | file: tabletop/whisk/whisk.urdf 39 | initial_pos: [0.13, 0.7, 0.62] 40 | initial_orn: [0, 0, 0] 41 | spatula: 42 | file: tabletop/spatula/spatula.urdf 43 | initial_pos: [0.4, 0.5, 0.61] 44 | initial_orn: [0, 0, 0] 45 | teapot_2: 46 | file: tabletop/teapot_2/teapot.urdf 47 | initial_pos: [-0.1, 0.85, 0.63] 48 | initial_orn: [0, 0, 0] 49 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/tabletop_2.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [0.3, 0.15, 0.6] 7 | robot_base_orientation: [0, 0, 1.5707963] 8 | robot_initial_joint_positions: [-0.017792060227770554, -0.7601235411041661, 0.019782607023391807, -2.342050140544315, 0.029840531355804868, 1.5411935298621688, 0.7534486589746342] 9 | surfaces: [] 10 | objects: 11 | fixed_objects: 12 | table: 13 | file: table/hightable.urdf 14 | initial_pos: [0.3, 0.7, 0.02] 15 | initial_orn: [0, 0, 0] 16 | bin: 17 | file: ais_objects/bin_10_30_50/bin_10_30_50.urdf 18 | initial_pos: [0.7, 0.75, 0.6] 19 | initial_orn: [1.57, 0, 0] 20 | movable_objects: 21 | drill: 22 | file: 035_power_drill/google_16k/textured.urdf 23 | initial_pos: [0.13, 0.7, 0.62] 24 | initial_orn: [0, 0, 0] 25 | marker: 26 | file: 040_large_marker/google_16k/textured.urdf 27 | initial_pos: [0.35, 0.92, 0.61] 28 | initial_orn: [0, 0, 0] 29 | spatula: 30 | file: tabletop/plaster_spatula/plaster_spatula.urdf 31 | initial_pos: [0.2, 0.45, 0.62] 32 | initial_orn: [0, 0, 0] 33 | hammer: 34 | file: 048_hammer/google_16k/textured.urdf 35 | initial_pos: [0.15, 0.9, 0.63] 36 | initial_orn: [0, 0, 0] 37 | screwdriver: 38 | file: 044_flat_screwdriver/google_16k/textured.urdf 39 | initial_pos: [-0.1, 0.6, 0.62] 40 | initial_orn: [0, 0, 0] 41 | utility_knife: 42 | file: tabletop/utility_knife/utility_knife.urdf 43 | initial_pos: [0.4, 0.5, 0.61] 44 | initial_orn: [0, 0, 0] 45 | flashlight: 46 | file: tabletop/flashlight/flashlight.urdf 47 | initial_pos: [-0.1, 0.85, 0.63] 48 | initial_orn: [0, 0, 0] 49 | -------------------------------------------------------------------------------- /calvin_env/conf/scene/tabletop_3.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.scene.play_table_scene.PlayTableScene 2 | _recursive_: false 3 | data_path: ${data_path} 4 | global_scaling: 0.8 5 | euler_obs: ${robot.euler_obs} 6 | robot_base_position: [0.3, 0.15, 0.6] 7 | robot_base_orientation: [0, 0, 1.5707963] 8 | robot_initial_joint_positions: [-1.4653239989567401, 1.4817260599394233, 1.525733453119315, -2.435192704551518, -1.809600862016179, 1.8558817172819435, -1.23183583862092] 9 | surfaces: [] 10 | objects: 11 | fixed_objects: 12 | table: 13 | file: table/hightable.urdf 14 | initial_pos: [0.3, 0.7, 0.02] 15 | initial_orn: [0, 0, 0] 16 | bin: 17 | file: ais_objects/bin_10_30_50/bin_10_30_50.urdf 18 | initial_pos: [0.7, 0.75, 0.6] 19 | initial_orn: [1.57, 0, 0] 20 | movable_objects: 21 | hammer: 22 | file: tabletop/hammer/hammer.urdf 23 | initial_pos: [0.18, 0.67, 0.62] 24 | initial_orn: [0, 0, 1.57] 25 | hammer_2: 26 | file: tabletop/hammer_2/hammer.urdf 27 | initial_pos: [0.4, 0.8, 0.61] 28 | initial_orn: [0, 0, 0] 29 | power_drill: 30 | file: tabletop/power_drill/power_drill.urdf 31 | initial_pos: [0.2, 0.45, 0.62] 32 | initial_orn: [3.14, 0, 0] 33 | skillet: 34 | file: tabletop/skillet/textured.urdf 35 | initial_pos: [0.1, 0.9, 0.63] 36 | initial_orn: [0, 0, 1] 37 | teapot: 38 | file: tabletop/teapot/teapot.urdf 39 | initial_pos: [-0.1, 0.6, 0.62] 40 | initial_orn: [0, 0, 0] 41 | teapot_lid: 42 | file: tabletop/teapot_lid/teapot_lid.urdf 43 | initial_pos: [0.4, 0.5, 0.61] 44 | initial_orn: [0, 0, 0] 45 | salt: 46 | file: ais_objects/salt/salt.urdf 47 | initial_pos: [-0.15, 0.85, 0.63] 48 | initial_orn: [1.57, -1.57, 0] 49 | -------------------------------------------------------------------------------- /calvin_env/conf/vr_input/vr_controller/oculus.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | POSITION: 1 3 | ORIENTATION: 2 4 | ANALOG: 3 5 | BUTTONS: 6 6 | BUTTON_A: 7 7 | BUTTON_B: 1 8 | vr_controller_id: 4 9 | gripper_orientation_offset: [0, 3, 0.7853981633974483] 10 | gripper_position_offset: [0, 0.7, -0.2] 11 | -------------------------------------------------------------------------------- /calvin_env/conf/vr_input/vr_controller/vive.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | POSITION: 1 3 | ORIENTATION: 2 4 | ANALOG: 3 5 | BUTTONS: 6 6 | BUTTON_A: 2 7 | BUTTON_B: 1 8 | vr_controller_id: 3 9 | gripper_orientation_offset: [0, 3, 3.14] 10 | gripper_position_offset: [-0.2, 0.3, 0] 11 | -------------------------------------------------------------------------------- /calvin_env/conf/vr_input/vr_input.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_env.io_utils.vr_input.VrInput 2 | limit_angle: [90, 0, 0, -1] 3 | visualize_vr_pos: true 4 | reset_button_queue_len: 60 5 | 6 | defaults: 7 | - vr_controller: vive 8 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_blue.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_blue_big.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_blue_middle.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_blue_small.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_blue_unseen.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_pink.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_pink_big.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_pink_middle.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_pink_small.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_pink_unseen.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_red.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_red_big.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_red_middle.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_red_small.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_red_unseen.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /calvin_env/data/blocks/block_square.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/blocks/block_square.stl -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/base_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/base_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/base_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/button_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/button_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/drawer_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/drawer_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/drawer_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/led_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/led_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/light_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/light_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/plank_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/plank_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/plank_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/plank_link.obj: -------------------------------------------------------------------------------- 1 | # Blender v2.93.1 OBJ File: '' 2 | # www.blender.org 3 | mtllib plank_link.mtl 4 | o plank_link 5 | v 0.325000 0.025000 -0.087500 6 | v 0.325000 0.025000 0.087500 7 | v 0.325000 0.000000 -0.087500 8 | v 0.325000 0.000000 0.087500 9 | v -0.325000 0.025000 -0.087500 10 | v -0.325000 0.000000 -0.087500 11 | v -0.325000 0.025000 0.087500 12 | v -0.325000 0.000000 0.087500 13 | vt 0.346154 0.000000 14 | vt 0.346154 0.269231 15 | vt 0.307692 0.000000 16 | vt 0.307692 0.269231 17 | vt 0.615385 0.000000 18 | vt 0.615385 1.000000 19 | vt 0.576923 0.000000 20 | vt 0.576923 1.000000 21 | vt 0.269231 0.269231 22 | vt 0.269231 0.000000 23 | vt 0.307692 0.269231 24 | vt 0.307692 0.000000 25 | vt 0.576923 0.000000 26 | vt 0.576923 1.000000 27 | vt 0.538462 0.000000 28 | vt 0.538462 1.000000 29 | vt 0.269231 1.000000 30 | vt 0.000000 1.000000 31 | vt 0.269231 0.000000 32 | vt 0.000000 0.000000 33 | vt 0.269231 0.000000 34 | vt 0.538462 0.000000 35 | vt 0.269231 1.000000 36 | vt 0.538462 1.000000 37 | vn 1.0000 0.0000 0.0000 38 | vn 0.0000 0.0000 -1.0000 39 | vn -1.0000 0.0000 0.0000 40 | vn 0.0000 0.0000 1.0000 41 | vn 0.0000 1.0000 0.0000 42 | vn 0.0000 -1.0000 0.0000 43 | usemtl Material 44 | s off 45 | f 1/1/1 2/2/1 3/3/1 46 | f 3/3/1 2/2/1 4/4/1 47 | f 5/5/2 1/6/2 6/7/2 48 | f 6/7/2 1/6/2 3/8/2 49 | f 7/9/3 5/10/3 8/11/3 50 | f 8/11/3 5/10/3 6/12/3 51 | f 2/13/4 7/14/4 4/15/4 52 | f 4/15/4 7/14/4 8/16/4 53 | f 5/17/5 7/18/5 1/19/5 54 | f 1/19/5 7/18/5 2/20/5 55 | f 8/21/6 6/22/6 4/23/6 56 | f 4/23/6 6/22/6 3/24/6 57 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/plank_link_vhacd2.obj: -------------------------------------------------------------------------------- 1 | o convex_0 2 | v 0.325725 0.025391 0.088783 3 | v -0.325722 -0.000725 -0.088225 4 | v -0.325722 -0.000725 0.088783 5 | v 0.325725 -0.000725 -0.088225 6 | v -0.325722 0.025391 -0.088225 7 | v -0.325722 0.025391 0.088783 8 | v 0.325725 -0.000725 0.088783 9 | v 0.325725 0.025391 -0.088225 10 | f 4 5 8 11 | f 3 2 4 12 | f 2 3 5 13 | f 4 2 5 14 | f 3 1 6 15 | f 1 5 6 16 | f 5 3 6 17 | f 1 3 7 18 | f 3 4 7 19 | f 4 1 7 20 | f 1 4 8 21 | f 5 1 8 22 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/slide_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/slide_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/slide_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/switch_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/meshes/switch_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/meshes/switch_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl None 5 | Ns 500 6 | Ka 0.8 0.8 0.8 7 | Kd 0.8 0.8 0.8 8 | Ks 0.8 0.8 0.8 9 | d 1 10 | illum 2 11 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/dark_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/dark_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/dark_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/dark_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/dark_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/dark_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/light_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/light_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/light_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/light_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/light_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/light_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_A/textures/wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_A/textures/wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/base_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/base_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/base_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/light_wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/button_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/button_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/drawer_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/drawer_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/drawer_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/light_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/led_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/led_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/light_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/light_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/plank_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/plank_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/plank_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/light_wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/plank_link.obj: -------------------------------------------------------------------------------- 1 | # Blender v2.93.1 OBJ File: '' 2 | # www.blender.org 3 | mtllib plank_link.mtl 4 | o plank_link 5 | v 0.325000 0.025000 -0.087500 6 | v 0.325000 0.025000 0.087500 7 | v 0.325000 0.000000 -0.087500 8 | v 0.325000 0.000000 0.087500 9 | v -0.325000 0.025000 -0.087500 10 | v -0.325000 0.000000 -0.087500 11 | v -0.325000 0.025000 0.087500 12 | v -0.325000 0.000000 0.087500 13 | vt 0.346154 0.000000 14 | vt 0.346154 0.269231 15 | vt 0.307692 0.000000 16 | vt 0.307692 0.269231 17 | vt 0.615385 0.000000 18 | vt 0.615385 1.000000 19 | vt 0.576923 0.000000 20 | vt 0.576923 1.000000 21 | vt 0.269231 0.269231 22 | vt 0.269231 0.000000 23 | vt 0.307692 0.269231 24 | vt 0.307692 0.000000 25 | vt 0.576923 0.000000 26 | vt 0.576923 1.000000 27 | vt 0.538462 0.000000 28 | vt 0.538462 1.000000 29 | vt 0.269231 1.000000 30 | vt 0.000000 1.000000 31 | vt 0.269231 0.000000 32 | vt 0.000000 0.000000 33 | vt 0.269231 0.000000 34 | vt 0.538462 0.000000 35 | vt 0.269231 1.000000 36 | vt 0.538462 1.000000 37 | vn 1.0000 0.0000 0.0000 38 | vn 0.0000 0.0000 -1.0000 39 | vn -1.0000 0.0000 0.0000 40 | vn 0.0000 0.0000 1.0000 41 | vn 0.0000 1.0000 0.0000 42 | vn 0.0000 -1.0000 0.0000 43 | usemtl Material 44 | s off 45 | f 1/1/1 2/2/1 3/3/1 46 | f 3/3/1 2/2/1 4/4/1 47 | f 5/5/2 1/6/2 6/7/2 48 | f 6/7/2 1/6/2 3/8/2 49 | f 7/9/3 5/10/3 8/11/3 50 | f 8/11/3 5/10/3 6/12/3 51 | f 2/13/4 7/14/4 4/15/4 52 | f 4/15/4 7/14/4 8/16/4 53 | f 5/17/5 7/18/5 1/19/5 54 | f 1/19/5 7/18/5 2/20/5 55 | f 8/21/6 6/22/6 4/23/6 56 | f 4/23/6 6/22/6 3/24/6 57 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/plank_link_vhacd2.obj: -------------------------------------------------------------------------------- 1 | o convex_0 2 | v 0.325725 0.025391 0.088783 3 | v -0.325722 -0.000725 -0.088225 4 | v -0.325722 -0.000725 0.088783 5 | v 0.325725 -0.000725 -0.088225 6 | v -0.325722 0.025391 -0.088225 7 | v -0.325722 0.025391 0.088783 8 | v 0.325725 -0.000725 0.088783 9 | v 0.325725 0.025391 -0.088225 10 | f 4 5 8 11 | f 3 2 4 12 | f 2 3 5 13 | f 4 2 5 14 | f 3 1 6 15 | f 1 5 6 16 | f 5 3 6 17 | f 1 3 7 18 | f 3 4 7 19 | f 4 1 7 20 | f 1 4 8 21 | f 5 1 8 22 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/slide_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/slide_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/slide_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/light_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/switch_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/meshes/switch_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/meshes/switch_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl None 5 | Ns 500 6 | Ka 0.8 0.8 0.8 7 | Kd 0.8 0.8 0.8 8 | Ks 0.8 0.8 0.8 9 | d 1 10 | illum 2 11 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/dark_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/dark_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/dark_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/dark_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/dark_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/dark_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/light_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/light_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/light_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/light_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/light_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/light_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_B/textures/wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_B/textures/wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/base_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/base_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/base_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/button_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/button_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/drawer_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/drawer_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/drawer_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/led_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/led_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/light_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/light_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/plank_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/plank_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/plank_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/plank_link.obj: -------------------------------------------------------------------------------- 1 | # Blender v2.93.1 OBJ File: '' 2 | # www.blender.org 3 | mtllib plank_link.mtl 4 | o plank_link 5 | v 0.325000 0.025000 -0.087500 6 | v 0.325000 0.025000 0.087500 7 | v 0.325000 0.000000 -0.087500 8 | v 0.325000 0.000000 0.087500 9 | v -0.325000 0.025000 -0.087500 10 | v -0.325000 0.000000 -0.087500 11 | v -0.325000 0.025000 0.087500 12 | v -0.325000 0.000000 0.087500 13 | vt 0.346154 0.000000 14 | vt 0.346154 0.269231 15 | vt 0.307692 0.000000 16 | vt 0.307692 0.269231 17 | vt 0.615385 0.000000 18 | vt 0.615385 1.000000 19 | vt 0.576923 0.000000 20 | vt 0.576923 1.000000 21 | vt 0.269231 0.269231 22 | vt 0.269231 0.000000 23 | vt 0.307692 0.269231 24 | vt 0.307692 0.000000 25 | vt 0.576923 0.000000 26 | vt 0.576923 1.000000 27 | vt 0.538462 0.000000 28 | vt 0.538462 1.000000 29 | vt 0.269231 1.000000 30 | vt 0.000000 1.000000 31 | vt 0.269231 0.000000 32 | vt 0.000000 0.000000 33 | vt 0.269231 0.000000 34 | vt 0.538462 0.000000 35 | vt 0.269231 1.000000 36 | vt 0.538462 1.000000 37 | vn 1.0000 0.0000 0.0000 38 | vn 0.0000 0.0000 -1.0000 39 | vn -1.0000 0.0000 0.0000 40 | vn 0.0000 0.0000 1.0000 41 | vn 0.0000 1.0000 0.0000 42 | vn 0.0000 -1.0000 0.0000 43 | usemtl Material 44 | s off 45 | f 1/1/1 2/2/1 3/3/1 46 | f 3/3/1 2/2/1 4/4/1 47 | f 5/5/2 1/6/2 6/7/2 48 | f 6/7/2 1/6/2 3/8/2 49 | f 7/9/3 5/10/3 8/11/3 50 | f 8/11/3 5/10/3 6/12/3 51 | f 2/13/4 7/14/4 4/15/4 52 | f 4/15/4 7/14/4 8/16/4 53 | f 5/17/5 7/18/5 1/19/5 54 | f 1/19/5 7/18/5 2/20/5 55 | f 8/21/6 6/22/6 4/23/6 56 | f 4/23/6 6/22/6 3/24/6 57 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/plank_link_vhacd2.obj: -------------------------------------------------------------------------------- 1 | o convex_0 2 | v 0.325725 0.025391 0.088783 3 | v -0.325722 -0.000725 -0.088225 4 | v -0.325722 -0.000725 0.088783 5 | v 0.325725 -0.000725 -0.088225 6 | v -0.325722 0.025391 -0.088225 7 | v -0.325722 0.025391 0.088783 8 | v 0.325725 -0.000725 0.088783 9 | v 0.325725 0.025391 -0.088225 10 | f 4 5 8 11 | f 3 2 4 12 | f 2 3 5 13 | f 4 2 5 14 | f 3 1 6 15 | f 1 5 6 16 | f 5 3 6 17 | f 1 3 7 18 | f 3 4 7 19 | f 4 1 7 20 | f 1 4 8 21 | f 5 1 8 22 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/slide_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/slide_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/slide_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/switch_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/meshes/switch_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/meshes/switch_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl None 5 | Ns 500 6 | Ka 0.8 0.8 0.8 7 | Kd 0.8 0.8 0.8 8 | Ks 0.8 0.8 0.8 9 | d 1 10 | illum 2 11 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/dark_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/dark_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/dark_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/dark_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/dark_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/dark_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/light_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/light_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/light_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/light_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/light_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/light_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_C/textures/wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_C/textures/wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/base_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/base_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/base_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/button_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/button_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/drawer_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/drawer_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/drawer_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/led_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/led_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/light_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/light_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/plank_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/plank_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/plank_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/plank_link.obj: -------------------------------------------------------------------------------- 1 | # Blender v2.93.1 OBJ File: '' 2 | # www.blender.org 3 | mtllib plank_link.mtl 4 | o plank_link 5 | v 0.325000 0.025000 -0.087500 6 | v 0.325000 0.025000 0.087500 7 | v 0.325000 0.000000 -0.087500 8 | v 0.325000 0.000000 0.087500 9 | v -0.325000 0.025000 -0.087500 10 | v -0.325000 0.000000 -0.087500 11 | v -0.325000 0.025000 0.087500 12 | v -0.325000 0.000000 0.087500 13 | vt 0.346154 0.000000 14 | vt 0.346154 0.269231 15 | vt 0.307692 0.000000 16 | vt 0.307692 0.269231 17 | vt 0.615385 0.000000 18 | vt 0.615385 1.000000 19 | vt 0.576923 0.000000 20 | vt 0.576923 1.000000 21 | vt 0.269231 0.269231 22 | vt 0.269231 0.000000 23 | vt 0.307692 0.269231 24 | vt 0.307692 0.000000 25 | vt 0.576923 0.000000 26 | vt 0.576923 1.000000 27 | vt 0.538462 0.000000 28 | vt 0.538462 1.000000 29 | vt 0.269231 1.000000 30 | vt 0.000000 1.000000 31 | vt 0.269231 0.000000 32 | vt 0.000000 0.000000 33 | vt 0.269231 0.000000 34 | vt 0.538462 0.000000 35 | vt 0.269231 1.000000 36 | vt 0.538462 1.000000 37 | vn 1.0000 0.0000 0.0000 38 | vn 0.0000 0.0000 -1.0000 39 | vn -1.0000 0.0000 0.0000 40 | vn 0.0000 0.0000 1.0000 41 | vn 0.0000 1.0000 0.0000 42 | vn 0.0000 -1.0000 0.0000 43 | usemtl Material 44 | s off 45 | f 1/1/1 2/2/1 3/3/1 46 | f 3/3/1 2/2/1 4/4/1 47 | f 5/5/2 1/6/2 6/7/2 48 | f 6/7/2 1/6/2 3/8/2 49 | f 7/9/3 5/10/3 8/11/3 50 | f 8/11/3 5/10/3 6/12/3 51 | f 2/13/4 7/14/4 4/15/4 52 | f 4/15/4 7/14/4 8/16/4 53 | f 5/17/5 7/18/5 1/19/5 54 | f 1/19/5 7/18/5 2/20/5 55 | f 8/21/6 6/22/6 4/23/6 56 | f 4/23/6 6/22/6 3/24/6 57 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/plank_link_vhacd2.obj: -------------------------------------------------------------------------------- 1 | o convex_0 2 | v 0.325725 0.025391 0.088783 3 | v -0.325722 -0.000725 -0.088225 4 | v -0.325722 -0.000725 0.088783 5 | v 0.325725 -0.000725 -0.088225 6 | v -0.325722 0.025391 -0.088225 7 | v -0.325722 0.025391 0.088783 8 | v 0.325725 -0.000725 0.088783 9 | v 0.325725 0.025391 -0.088225 10 | f 4 5 8 11 | f 3 2 4 12 | f 2 3 5 13 | f 4 2 5 14 | f 3 1 6 15 | f 1 5 6 16 | f 5 3 6 17 | f 1 3 7 18 | f 3 4 7 19 | f 4 1 7 20 | f 1 4 8 21 | f 5 1 8 22 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/slide_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/slide_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/slide_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | map_Kd ../textures/dark_wood__gray_handle.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/switch_link.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/meshes/switch_link.STL -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/meshes/switch_link.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl None 5 | Ns 500 6 | Ka 0.8 0.8 0.8 7 | Kd 0.8 0.8 0.8 8 | Ks 0.8 0.8 0.8 9 | d 1 10 | illum 2 11 | -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/dark_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/dark_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/dark_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/dark_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/dark_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/dark_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/light_wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/light_wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/light_wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/light_wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/light_wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/light_wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/wood.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/wood.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/wood__black_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/wood__black_handle.png -------------------------------------------------------------------------------- /calvin_env/data/calvin_table_D/textures/wood__gray_handle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/calvin_table_D/textures/wood__gray_handle.png -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/collision/longer_finger.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl DefaultMaterial 5 | Ns 225.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/collision/longer_finger_v2.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl DefaultMaterial 5 | Ns 225.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/Assem1.SLDASM: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/Assem1.SLDASM -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/FRANKA_Finger.SLDPRT: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/FRANKA_Finger.SLDPRT -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/colors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/colors.png -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/digit.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/digit.STL -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/digit_gel_only.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/digit_gel_only.STL -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/finger.SLDPRT: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/finger.SLDPRT -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/finger.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'link2.blend' 2 | # Material Count: 2 3 | 4 | newmtl Part__Feature001_006.001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.901961 0.921569 0.929412 8 | Ks 0.250000 0.250000 0.250000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | map_Kd colors.png 14 | 15 | newmtl Part__Feature_007.001 16 | Ns -1.960784 17 | Ka 1.000000 1.000000 1.000000 18 | Kd 0.250980 0.250980 0.250980 19 | Ks 0.250000 0.250000 0.250000 20 | Ke 0.000000 0.000000 0.000000 21 | Ni 1.000000 22 | d 1.000000 23 | illum 2 24 | map_Kd colors.png 25 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/hand.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'link2.blend' 2 | # Material Count: 5 3 | 4 | newmtl Part__Feature001_008_005.001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.250980 0.250980 0.250980 8 | Ks 0.007812 0.007812 0.007812 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | map_Kd colors.png 14 | 15 | newmtl Part__Feature002_005_005.001 16 | Ns -1.960784 17 | Ka 1.000000 1.000000 1.000000 18 | Kd 0.901961 0.921569 0.929412 19 | Ks 0.015625 0.015625 0.015625 20 | Ke 0.000000 0.000000 0.000000 21 | Ni 1.000000 22 | d 1.000000 23 | illum 2 24 | 25 | newmtl Part__Feature005_001_005.001 26 | Ns -1.960784 27 | Ka 1.000000 1.000000 1.000000 28 | Kd 1.000000 1.000000 1.000000 29 | Ks 0.015625 0.015625 0.015625 30 | Ke 0.000000 0.000000 0.000000 31 | Ni 1.000000 32 | d 1.000000 33 | illum 2 34 | map_Kd colors.png 35 | 36 | newmtl Part__Feature005_001_005_001.001 37 | Ns -1.960784 38 | Ka 1.000000 1.000000 1.000000 39 | Kd 0.901961 0.921569 0.929412 40 | Ks 0.015625 0.015625 0.015625 41 | Ke 0.000000 0.000000 0.000000 42 | Ni 1.000000 43 | d 1.000000 44 | illum 2 45 | map_Kd colors.png 46 | 47 | newmtl Part__Feature_009_005.001 48 | Ns -1.960784 49 | Ka 1.000000 1.000000 1.000000 50 | Kd 0.250980 0.250980 0.250980 51 | Ks 0.015625 0.015625 0.015625 52 | Ke 0.000000 0.000000 0.000000 53 | Ni 1.000000 54 | d 1.000000 55 | illum 2 56 | map_Kd colors.png 57 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/link1.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'link2.blend' 2 | # Material Count: 1 3 | 4 | newmtl Part__Feature_001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.062500 0.062500 0.062500 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | map_Kd colors.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/link2.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'link3.blend' 2 | # Material Count: 1 3 | 4 | newmtl Part__Feature024 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.125000 0.125000 0.125000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | map_Kd colors.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/link3.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'link4.blend' 2 | # Material Count: 4 3 | 4 | newmtl Part__Feature001_010_001_002 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.007812 0.007812 0.007812 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | map_Kd colors.png 14 | 15 | newmtl Part__Feature002_007_001_002 16 | Ns -1.960784 17 | Ka 1.000000 1.000000 1.000000 18 | Kd 1.000000 1.000000 1.000000 19 | Ks 0.007812 0.007812 0.007812 20 | Ke 0.000000 0.000000 0.000000 21 | Ni 1.000000 22 | d 1.000000 23 | illum 2 24 | map_Kd colors.png 25 | 26 | newmtl Part__Feature003_004_001_002 27 | Ns -1.960784 28 | Ka 1.000000 1.000000 1.000000 29 | Kd 1.000000 1.000000 1.000000 30 | Ks 0.007812 0.007812 0.007812 31 | Ke 0.000000 0.000000 0.000000 32 | Ni 1.000000 33 | d 1.000000 34 | illum 2 35 | map_Kd colors.png 36 | 37 | newmtl Part__Feature_001_001_001_002 38 | Ns -1.960784 39 | Ka 1.000000 1.000000 1.000000 40 | Kd 0.250980 0.250980 0.250980 41 | Ks 0.007812 0.007812 0.007812 42 | Ke 0.000000 0.000000 0.000000 43 | Ni 1.000000 44 | d 1.000000 45 | illum 2 46 | map_Kd colors.png 47 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/link4.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'link4.blend' 2 | # Material Count: 4 3 | 4 | newmtl Part__Feature001_001_003_001.001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.007812 0.007812 0.007812 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Feature002_001_003_001.001 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.250980 0.250980 0.250980 18 | Ks 0.007812 0.007812 0.007812 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | map_Kd colors.png 24 | 25 | newmtl Part__Feature003_001_003_001.001 26 | Ns -1.960784 27 | Ka 1.000000 1.000000 1.000000 28 | Kd 1.000000 1.000000 1.000000 29 | Ks 0.007812 0.007812 0.007812 30 | Ke 0.000000 0.000000 0.000000 31 | Ni 1.000000 32 | d 1.000000 33 | illum 2 34 | map_Kd colors.png 35 | 36 | newmtl Part__Feature_002_003_001.001 37 | Ns -1.960784 38 | Ka 1.000000 1.000000 1.000000 39 | Kd 1.000000 1.000000 1.000000 40 | Ks 0.007812 0.007812 0.007812 41 | Ke 0.000000 0.000000 0.000000 42 | Ni 1.000000 43 | d 1.000000 44 | illum 2 45 | map_Kd colors.png 46 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/link5.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 3 3 | 4 | newmtl Part__Feature_002_004_003.002 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.015625 0.015625 0.015625 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | map_Kd colors.png 14 | 15 | newmtl Shell001_001_001_003.002 16 | Ns -1.960784 17 | Ka 1.000000 1.000000 1.000000 18 | Kd 0.250000 0.250000 0.250000 19 | Ks 0.015625 0.015625 0.015625 20 | Ke 0.000000 0.000000 0.000000 21 | Ni 1.000000 22 | d 1.000000 23 | illum 2 24 | map_Kd colors.png 25 | 26 | newmtl Shell_001_001_003.002 27 | Ns -1.960784 28 | Ka 1.000000 1.000000 1.000000 29 | Kd 1.000000 1.000000 1.000000 30 | Ks 0.015625 0.015625 0.015625 31 | Ke 0.000000 0.000000 0.000000 32 | Ni 1.000000 33 | d 1.000000 34 | illum 2 35 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/longer_finger.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/longer_finger.STL -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/longer_finger.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Material 5 | Ns 323.999994 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.694038 0.720149 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/longer_finger_v2.STL: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/franka_panda/meshes/visual/longer_finger_v2.STL -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/longer_finger_v2.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl None 5 | Ns 500.000001 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.800000 0.800000 0.800000 8 | Ks 0.800000 0.800000 0.800000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/~$Assem1.SLDASM: -------------------------------------------------------------------------------- 1 | Erick -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/~$FRANKA_Finger.SLDPRT: -------------------------------------------------------------------------------- 1 | Erick -------------------------------------------------------------------------------- /calvin_env/data/franka_panda/meshes/visual/~$finger.SLDPRT: -------------------------------------------------------------------------------- 1 | Erick -------------------------------------------------------------------------------- /calvin_env/data/plane/checker_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/data/plane/checker_blue.png -------------------------------------------------------------------------------- /calvin_env/data/plane/plane.mtl: -------------------------------------------------------------------------------- 1 | newmtl Material 2 | Ns 10.0000 3 | Ni 1.5000 4 | d 1.0000 5 | Tr 0.0000 6 | Tf 1.0000 1.0000 1.0000 7 | illum 2 8 | Ka 0.0000 0.0000 0.0000 9 | Kd 0.5880 0.5880 0.5880 10 | Ks 0.0000 0.0000 0.0000 11 | Ke 0.0000 0.0000 0.0000 12 | map_Ka cube.tga 13 | map_Kd checker_blue.png 14 | -------------------------------------------------------------------------------- /calvin_env/data/plane/plane.obj: -------------------------------------------------------------------------------- 1 | # Blender v2.66 (sub 1) OBJ File: '' 2 | # www.blender.org 3 | mtllib plane.mtl 4 | o Plane 5 | v 15.000000 -15.000000 0.000000 6 | v 15.000000 15.000000 0.000000 7 | v -15.000000 15.000000 0.000000 8 | v -15.000000 -15.000000 0.000000 9 | 10 | vt 15.000000 0.000000 11 | vt 15.000000 15.000000 12 | vt 0.000000 15.000000 13 | vt 0.000000 0.000000 14 | 15 | usemtl Material 16 | s off 17 | f 1/1 2/2 3/3 18 | f 1/1 3/3 4/4 19 | -------------------------------------------------------------------------------- /calvin_env/data/plane/plane.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /calvin_env/egl_check/EGL_options.h: -------------------------------------------------------------------------------- 1 | struct b3gWindowConstructionInfo 2 | { 3 | int m_width; 4 | int m_height; 5 | bool m_fullscreen; 6 | int m_colorBitsPerPixel; 7 | void* m_windowHandle; 8 | const char* m_title; 9 | int m_openglVersion; 10 | int m_renderDevice; 11 | 12 | b3gWindowConstructionInfo(int width = 1024, int height = 768) 13 | : m_width(width), 14 | m_height(height), 15 | m_fullscreen(false), 16 | m_colorBitsPerPixel(32), 17 | m_windowHandle(0), 18 | m_title("title"), 19 | m_openglVersion(3), 20 | m_renderDevice(-1) 21 | { 22 | } 23 | }; 24 | 25 | class EGLOpenGLWindow 26 | { 27 | struct EGLInternalData2* m_data; 28 | bool m_OpenGLInitialized; 29 | bool m_requestedExit; 30 | 31 | public: 32 | EGLOpenGLWindow(); 33 | virtual ~EGLOpenGLWindow(); 34 | 35 | virtual void createDefaultWindow(int width, int height, const char* title) 36 | { 37 | b3gWindowConstructionInfo ci(width, height); 38 | ci.m_title = title; 39 | createWindow(ci); 40 | } 41 | 42 | virtual void createWindow(const b3gWindowConstructionInfo& ci); 43 | 44 | virtual void closeWindow(); 45 | 46 | virtual void runMainLoop(); 47 | virtual float getTimeInSeconds(); 48 | 49 | virtual bool requestedExit() const; 50 | virtual void setRequestExit(); 51 | 52 | virtual void startRendering(); 53 | 54 | virtual void endRendering(); 55 | 56 | 57 | virtual void setWindowTitle(const char* title); 58 | 59 | virtual float getRetinaScale() const; 60 | virtual void setAllowRetina(bool allow); 61 | 62 | virtual int getWidth() const; 63 | virtual int getHeight() const; 64 | 65 | virtual int fileOpenDialog(char* fileName, int maxFileNameLength); 66 | }; 67 | -------------------------------------------------------------------------------- /calvin_env/egl_check/EGL_options.o_: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_env/egl_check/EGL_options.o_ -------------------------------------------------------------------------------- /calvin_env/egl_check/README.md: -------------------------------------------------------------------------------- 1 | # Which EGL device am I using? 2 | 3 | This file provides to options to see which EGL device is being used: 4 | 5 | ``` 6 | bash build.sh # compile c++ code 7 | python list_egl_devices.py # run c++ code multiple times 8 | python egl_python.py # see what the pyOpenGL default option is 9 | ``` 10 | -------------------------------------------------------------------------------- /calvin_env/egl_check/build.sh: -------------------------------------------------------------------------------- 1 | g++ glad/egl.c glad/gl.c EGL_options.cpp -I glad/ -l dl -fPIC -o EGL_options.o 2 | -------------------------------------------------------------------------------- /calvin_env/egl_check/list_egl_options.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | if not os.path.isfile("./EGL_options.o"): 5 | subprocess.call(["bash", "./build.sh"]) 6 | 7 | print("----------Default-------------") 8 | p = subprocess.Popen(["./EGL_options.o"], stderr=subprocess.PIPE) 9 | p.wait() 10 | out, err = p.communicate() 11 | print(err) 12 | 13 | N = int(err.decode("utf-8").split(" of ")[1].split(".")[0]) 14 | 15 | print("number of EGL devices: {}".format(N)) 16 | 17 | for i in range(N): 18 | print(f"----------Option #{i + 1} (id={i})-------------") 19 | my_env = os.environ.copy() 20 | my_env["EGL_VISIBLE_DEVICE"] = "{}".format(i) 21 | p = subprocess.Popen(["./EGL_options.o"], env=my_env) 22 | p.wait() 23 | print() 24 | -------------------------------------------------------------------------------- /calvin_env/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | # https://github.com/psf/black 3 | line-length = 120 4 | target-version = ["py38"] 5 | exclude = "(.eggs|.git|.hg|.mypy_cache|.nox|.tox|.venv|.svn|_build|buck-out|build|dist)" 6 | 7 | [tool.isort] 8 | profile = "black" 9 | line_length = 120 10 | force_sort_within_sections = "True" 11 | order_by_type = "False" 12 | -------------------------------------------------------------------------------- /calvin_env/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | black 2 | flake8 3 | isort 4 | pre-commit 5 | mypy 6 | pytest 7 | pytest-cov 8 | -------------------------------------------------------------------------------- /calvin_env/requirements.txt: -------------------------------------------------------------------------------- 1 | cloudpickle 2 | gitpython 3 | gym 4 | hydra-core 5 | hydra-colorlog 6 | matplotlib 7 | numba 8 | numpy 9 | numpy-quaternion 10 | omegaconf 11 | opencv-python 12 | pandas 13 | pybullet 14 | scipy 15 | -------------------------------------------------------------------------------- /calvin_env/setup.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | 3 | """Setup calvin_env installation.""" 4 | 5 | from os import path as op 6 | import re 7 | 8 | from setuptools import find_packages, setup 9 | 10 | 11 | def _read(f): 12 | return open(op.join(op.dirname(__file__), f)).read() if op.exists(f) else "" 13 | 14 | 15 | _meta = _read("calvin_env/__init__.py") 16 | 17 | 18 | def find_meta(_meta, string): 19 | l_match = re.search(r"^" + string + r'\s*=\s*"(.*)"', _meta, re.M) 20 | if l_match: 21 | return l_match.group(1) 22 | raise RuntimeError(f"Unable to find {string} string.") 23 | 24 | 25 | install_requires = [ 26 | l for l in _read("requirements.txt").split("\n") if l and not l.startswith("#") and not l.startswith("-") 27 | ] 28 | 29 | meta = dict( 30 | name=find_meta(_meta, "__project__"), 31 | version=find_meta(_meta, "__version__"), 32 | license=find_meta(_meta, "__license__"), 33 | description="VR Data Collection and Rendering", 34 | platforms="Any", 35 | zip_safe=False, 36 | keywords="calvin_env".split(), 37 | author=find_meta(_meta, "__author__"), 38 | author_email=find_meta(_meta, "__email__"), 39 | url=" https://github.com/mees/calvin_env", 40 | packages=find_packages(exclude=["tests"]), 41 | install_requires=install_requires, 42 | ) 43 | 44 | if __name__ == "__main__": 45 | print("find_package", find_packages(exclude=["tests"])) 46 | setup(**meta) 47 | -------------------------------------------------------------------------------- /calvin_models/Calvin.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: Calvin 3 | Version: 0.0.1 4 | Summary: CALVIN - A benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks 5 | Home-page: https://github.com/mees/calvin 6 | Author: Oier Mees 7 | Author-email: meeso@informatik.uni-freiburg.de 8 | License: MIT 9 | Keywords: pytorch,Lfp 10 | Platform: Any 11 | 12 | UNKNOWN 13 | 14 | -------------------------------------------------------------------------------- /calvin_models/Calvin.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /calvin_models/Calvin.egg-info/not-zip-safe: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /calvin_models/Calvin.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | cmake 2 | wheel 3 | numpy>1.2 4 | hydra-core==1.1.1 5 | hydra-colorlog 6 | matplotlib 7 | opencv-python 8 | omegaconf 9 | plotly 10 | pyhash 11 | pytorch-lightning==1.8.6 12 | lightning_lite 13 | torch==1.13.1 14 | torchvision 15 | MulticoreTSNE 16 | gitpython 17 | scipy 18 | sentence-transformers 19 | setuptools==57.5.0 20 | gym 21 | moviepy 22 | tqdm 23 | termcolor 24 | wandb 25 | -------------------------------------------------------------------------------- /calvin_models/Calvin.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | calvin_agent 2 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | play_data/ 3 | __pycache__/ 4 | results/ 5 | runs/ 6 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/__init__.py: -------------------------------------------------------------------------------- 1 | """'CALVIN - A benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks 2 | :copyright: 2021 by Oier Mees 3 | :license: MIT, see LICENSE for more details. 4 | """ 5 | 6 | __version__ = "0.0.1" 7 | __project__ = "Calvin" 8 | __author__ = "Oier Mees" 9 | __license__ = "MIT" 10 | __email__ = "meeso@informatik.uni-freiburg.de" 11 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/datasets/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/datasets/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/datasets/utils/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/evaluation/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/inference/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/models/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/calvin_base_model.py: -------------------------------------------------------------------------------- 1 | class CalvinBaseModel: 2 | """ 3 | Base class for all models that can be evaluated on the CALVIN challenge. 4 | If you want to evaluate your own model, implement the class methods. 5 | """ 6 | 7 | def reset(self): 8 | """ 9 | Call this at the beginning of a new rollout when doing inference. 10 | """ 11 | raise NotImplementedError 12 | 13 | def step(self, obs, goal): 14 | """ 15 | Do one step of inference with the model. 16 | 17 | Args: 18 | obs (dict): Observation from environment. 19 | goal (dict): Goal as visual observation or embedded language instruction. 20 | 21 | Returns: 22 | Predicted action. 23 | """ 24 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/decoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/models/decoders/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/decoders/action_decoder.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import torch 4 | from torch import nn 5 | 6 | 7 | class ActionDecoder(nn.Module): 8 | def act(self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor) -> torch.Tensor: 9 | raise NotImplementedError 10 | 11 | def loss( 12 | self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor, actions: torch.Tensor 13 | ) -> torch.Tensor: 14 | raise NotImplementedError 15 | 16 | def loss_and_act( 17 | self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor, actions: torch.Tensor 18 | ) -> Tuple[torch.Tensor, torch.Tensor]: 19 | raise NotImplementedError 20 | 21 | def clear_hidden_state(self) -> None: 22 | raise NotImplementedError 23 | 24 | def _sample(self, *args, **kwargs): 25 | raise NotImplementedError 26 | 27 | def forward( 28 | self, latent_plan: torch.Tensor, perceptual_emb: torch.Tensor, latent_goal: torch.Tensor 29 | ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/models/encoders/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/encoders/goal_encoders.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | 8 | class VisualGoalEncoder(nn.Module): 9 | def __init__( 10 | self, 11 | hidden_size: int, 12 | latent_goal_features: int, 13 | in_features: int, 14 | l2_normalize_goal_embeddings: bool, 15 | activation_function: str, 16 | ): 17 | super().__init__() 18 | self.l2_normalize_output = l2_normalize_goal_embeddings 19 | self.act_fn = getattr(nn, activation_function)() 20 | self.mlp = nn.Sequential( 21 | nn.Linear(in_features=in_features, out_features=hidden_size), 22 | self.act_fn, 23 | nn.Linear(in_features=hidden_size, out_features=hidden_size), 24 | self.act_fn, 25 | nn.Linear(in_features=hidden_size, out_features=latent_goal_features), 26 | ) 27 | 28 | def forward(self, x: torch.Tensor) -> torch.Tensor: 29 | x = self.mlp(x) 30 | if self.l2_normalize_output: 31 | x = F.normalize(x, p=2, dim=1) 32 | return x 33 | 34 | 35 | class LanguageGoalEncoder(nn.Module): 36 | def __init__( 37 | self, 38 | language_features: int, 39 | hidden_size: int, 40 | latent_goal_features: int, 41 | word_dropout_p: float, 42 | l2_normalize_goal_embeddings: bool, 43 | activation_function: str, 44 | ): 45 | super().__init__() 46 | self.l2_normalize_output = l2_normalize_goal_embeddings 47 | self.act_fn = getattr(nn, activation_function)() 48 | self.mlp = nn.Sequential( 49 | nn.Dropout(word_dropout_p), 50 | nn.Linear(in_features=language_features, out_features=hidden_size), 51 | self.act_fn, 52 | nn.Linear(in_features=hidden_size, out_features=hidden_size), 53 | self.act_fn, 54 | nn.Linear(in_features=hidden_size, out_features=latent_goal_features), 55 | ) 56 | 57 | def forward(self, x: torch.Tensor) -> torch.Tensor: 58 | x = self.mlp(x) 59 | if self.l2_normalize_output: 60 | x = F.normalize(x, p=2, dim=1) 61 | return x 62 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/encoders/language_network.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from sentence_transformers import SentenceTransformer 4 | import torch 5 | import torch.nn as nn 6 | 7 | 8 | class SBert(nn.Module): 9 | def __init__(self, nlp_model: str): 10 | # choose model from https://www.sbert.net/docs/pretrained_models.html 11 | super().__init__() 12 | assert isinstance(nlp_model, str) 13 | self.model = SentenceTransformer(nlp_model) 14 | 15 | def forward(self, x: List) -> torch.Tensor: 16 | emb = self.model.encode(x, convert_to_tensor=True) 17 | return torch.unsqueeze(emb, 1) -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/perceptual_encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/models/perceptual_encoders/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/perceptual_encoders/proprio_encoder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torch import nn 3 | 4 | 5 | class IdentityEncoder(nn.Module): 6 | def __init__(self, proprioception_dims): 7 | super(IdentityEncoder, self).__init__() 8 | # remove a dimension if we convert robot orientation quaternion to euler angles 9 | self.n_state_obs = int(np.sum(np.diff([list(x) for x in [list(y) for y in proprioception_dims.keep_indices]]))) 10 | self.identity = nn.Identity() 11 | 12 | @property 13 | def out_features(self): 14 | return self.n_state_obs 15 | 16 | def forward(self, x): 17 | return self.identity(x) 18 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/perceptual_encoders/tactile_encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | import torchvision.models as models 5 | 6 | 7 | class TactileEncoder(nn.Module): 8 | def __init__(self, visual_features: int, freeze_tactile_backbone: bool = True): 9 | super(TactileEncoder, self).__init__() 10 | # Load pre-trained resnet-18 11 | net = models.resnet18(pretrained=True) 12 | # Remove the last fc layer, and rebuild 13 | modules = list(net.children())[:-1] 14 | self.net = nn.Sequential(*modules) 15 | if freeze_tactile_backbone: 16 | for param in self.net.parameters(): 17 | param.requires_grad = False 18 | self.fc1 = nn.Linear(1024, 512) 19 | self.fc2 = nn.Linear(512, visual_features) 20 | 21 | def forward(self, x: torch.Tensor) -> torch.Tensor: 22 | x_l = self.net(x[:, :3, :, :]).squeeze() 23 | x_r = self.net(x[:, 3:, :, :]).squeeze() 24 | x = torch.cat((x_l, x_r), dim=-1) 25 | # Add fc layer for final prediction 26 | output = F.relu(self.fc1(x)) # batch, 512 27 | output = self.fc2(output) # batch, 64 28 | return output 29 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/perceptual_encoders/vision_network_gripper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | 8 | def nature_cnn(act_fn, num_c): 9 | return nn.Sequential( 10 | nn.Conv2d(num_c, 32, 8, stride=4), 11 | act_fn, 12 | nn.Conv2d(32, 64, 4, stride=2), 13 | act_fn, 14 | nn.Conv2d(64, 64, 3, stride=1), 15 | act_fn, 16 | nn.Flatten(start_dim=1), 17 | nn.Linear(64 * 7 * 7, 128), 18 | act_fn, 19 | ) 20 | 21 | 22 | class VisionNetwork(nn.Module): 23 | def __init__( 24 | self, 25 | conv_encoder: str, 26 | activation_function: str, 27 | dropout_vis_fc: float, 28 | l2_normalize_output: bool, 29 | visual_features: int, 30 | num_c: int, 31 | ): 32 | super(VisionNetwork, self).__init__() 33 | self.l2_normalize_output = l2_normalize_output 34 | self.act_fn = getattr(nn, activation_function)() 35 | # model 36 | # this calls the method with the name conv_encoder 37 | self.conv_model = eval(conv_encoder) 38 | self.conv_model = self.conv_model(self.act_fn, num_c) 39 | self.fc1 = nn.Sequential( 40 | nn.Linear(in_features=128, out_features=512), self.act_fn, nn.Dropout(dropout_vis_fc) 41 | ) # shape: [N, 512] 42 | self.fc2 = nn.Linear(in_features=512, out_features=visual_features) # shape: [N, 64] 43 | 44 | def forward(self, x: torch.Tensor) -> torch.Tensor: 45 | x = self.conv_model(x) 46 | x = self.fc1(x) 47 | x = self.fc2(x) 48 | if self.l2_normalize_output: 49 | x = F.normalize(x, p=2, dim=1) 50 | return x # shape: [N, 64] 51 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/plan_encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/models/plan_encoders/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/plan_encoders/plan_proposal_net.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from typing import Tuple 4 | 5 | import torch 6 | from torch.distributions import Independent, Normal 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class PlanProposalNetwork(nn.Module): 12 | def __init__( 13 | self, 14 | perceptual_features: int, 15 | latent_goal_features: int, 16 | plan_features: int, 17 | activation_function: str, 18 | min_std: float, 19 | ): 20 | super(PlanProposalNetwork, self).__init__() 21 | self.perceptual_features = perceptual_features 22 | self.latent_goal_features = latent_goal_features 23 | self.plan_features = plan_features 24 | self.min_std = min_std 25 | self.in_features = self.perceptual_features + self.latent_goal_features 26 | self.act_fn = getattr(nn, activation_function)() 27 | self.fc_model = nn.Sequential( 28 | nn.Linear(in_features=self.in_features, out_features=2048), # shape: [N, 136] 29 | self.act_fn, 30 | nn.Linear(in_features=2048, out_features=2048), 31 | self.act_fn, 32 | nn.Linear(in_features=2048, out_features=2048), 33 | self.act_fn, 34 | nn.Linear(in_features=2048, out_features=2048), 35 | self.act_fn, 36 | ) 37 | self.mean_fc = nn.Linear(in_features=2048, out_features=self.plan_features) # shape: [N, 2048] 38 | self.variance_fc = nn.Linear(in_features=2048, out_features=self.plan_features) # shape: [N, 2048] 39 | 40 | def forward(self, initial_percep_emb: torch.Tensor, latent_goal: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: 41 | x = torch.cat([initial_percep_emb, latent_goal], dim=-1) 42 | x = self.fc_model(x) 43 | mean = self.mean_fc(x) 44 | var = self.variance_fc(x) 45 | std = F.softplus(var) + self.min_std 46 | return mean, std # shape: [N, 256] 47 | 48 | def __call__(self, *args, **kwargs): 49 | mean, std = super().__call__(*args, **kwargs) 50 | pp_dist = Independent(Normal(mean, std), 1) 51 | return pp_dist 52 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/models/plan_encoders/plan_recognition_net.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from typing import Tuple 4 | 5 | import torch 6 | from torch.distributions import Independent, Normal 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class PlanRecognitionNetwork(nn.Module): 12 | def __init__( 13 | self, 14 | in_features: int, 15 | plan_features: int, 16 | action_space: int, 17 | birnn_dropout_p: float, 18 | min_std: float, 19 | ): 20 | super(PlanRecognitionNetwork, self).__init__() 21 | self.plan_features = plan_features 22 | self.action_space = action_space 23 | self.min_std = min_std 24 | self.in_features = in_features 25 | self.birnn_model = nn.RNN( 26 | input_size=self.in_features, 27 | hidden_size=2048, 28 | nonlinearity="relu", 29 | num_layers=2, 30 | bidirectional=True, 31 | batch_first=True, 32 | dropout=birnn_dropout_p, 33 | ) # shape: [N, seq_len, 64+8] 34 | self.mean_fc = nn.Linear(in_features=4096, out_features=self.plan_features) # shape: [N, seq_len, 4096] 35 | self.variance_fc = nn.Linear(in_features=4096, out_features=self.plan_features) # shape: [N, seq_len, 4096] 36 | 37 | def forward(self, perceptual_emb: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: 38 | x, hn = self.birnn_model(perceptual_emb) 39 | x = x[:, -1] # we just need only last unit output 40 | mean = self.mean_fc(x) 41 | var = self.variance_fc(x) 42 | std = F.softplus(var) + self.min_std 43 | return mean, std # shape: [N, 256] 44 | 45 | def __call__(self, *args, **kwargs): 46 | mean, std = super().__call__(*args, **kwargs) 47 | pr_dist = Independent(Normal(mean, std), 1) 48 | return pr_dist 49 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/rollout/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/rollout/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/utils/__init__.py -------------------------------------------------------------------------------- /calvin_models/calvin_agent/utils/data_visualization.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import hydra 4 | from omegaconf import DictConfig 5 | from pytorch_lightning import seed_everything 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | from matplotlib.animation import ArtistAnimation 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | 13 | 14 | def visualize(data): 15 | seq_img = data[1][0][0].numpy() 16 | title = data[4][0] 17 | s, c, h, w = seq_img.shape 18 | seq_img = np.transpose(seq_img, (0, 2, 3, 1)) 19 | imgs = [] 20 | fig = plt.figure() 21 | for j in range(s): 22 | # imgRGB = seq_img[j].astype(int) 23 | imgRGB = seq_img[j] 24 | imgRGB = (imgRGB - imgRGB.min()) / (imgRGB.max() - imgRGB.min()) 25 | img = plt.imshow(imgRGB, animated=True) 26 | imgs.append([img]) 27 | ArtistAnimation(fig, imgs, interval=50) 28 | plt.title(title) 29 | plt.show() 30 | 31 | 32 | @hydra.main(config_path="../../conf", config_name="default.yaml") 33 | def train(cfg: DictConfig) -> None: 34 | # sets seeds for numpy, torch, python.random and PYTHONHASHSEED. 35 | seed_everything(cfg.seed) 36 | data_module = hydra.utils.instantiate(cfg.dataset, num_workers=0) 37 | data_module.setup() 38 | train = data_module.train_dataloader() 39 | dataset = train["lang"] 40 | logger.info(f"Dataset Size: {len(dataset)}") 41 | for i, lang in enumerate(dataset): 42 | logger.info(f"Element : {i}") 43 | visualize(lang) 44 | 45 | 46 | if __name__ == "__main__": 47 | train() 48 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/utils/visualizations.py: -------------------------------------------------------------------------------- 1 | # Force matplotlib to not use any Xwindows backend. 2 | import matplotlib 3 | import numpy as np 4 | from pytorch_lightning.loggers import WandbLogger 5 | import torch 6 | import wandb 7 | 8 | matplotlib.use("Agg") 9 | import matplotlib.pyplot as plt 10 | 11 | 12 | def visualize_temporal_consistency(max_batched_length_per_demo, gpus, sampled_plans, all_idx, step, logger, prefix=""): 13 | """compute t-SNE plot of embeddings os a task to visualize temporal consistency""" 14 | labels = [] 15 | for demo in max_batched_length_per_demo: 16 | labels = np.concatenate((labels, np.arange(demo) / float(demo)), axis=0) 17 | # because with ddp, data doesn't come ordered anymore 18 | labels = labels[torch.flatten(all_idx).cpu()] 19 | colors = [plt.cm.Spectral(y_i) for y_i in labels] 20 | assert sampled_plans.shape[0] == len(labels), "plt X shape {}, label len {}".format( 21 | sampled_plans.shape[0], len(labels) 22 | ) 23 | 24 | from MulticoreTSNE import MulticoreTSNE as TSNE 25 | 26 | x_tsne = TSNE(perplexity=40, n_jobs=8).fit_transform(sampled_plans.cpu()) 27 | 28 | plt.close("all") 29 | fig, ax = plt.subplots() 30 | _ = ax.scatter(x_tsne[:, 0], x_tsne[:, 1], c=colors, cmap=plt.cm.Spectral) 31 | fig.suptitle("Temporal Consistency of Latent space") 32 | ax.axis("off") 33 | if isinstance(logger, WandbLogger): 34 | logger.experiment.log({prefix + "latent_embedding": wandb.Image(fig)}) 35 | else: 36 | logger.experiment.add_figure(prefix + "latent_embedding", fig, global_step=step) 37 | -------------------------------------------------------------------------------- /calvin_models/calvin_agent/visualization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/calvin_agent/visualization/__init__.py -------------------------------------------------------------------------------- /calvin_models/conf/annotations/new_playtable_validation.yaml: -------------------------------------------------------------------------------- 1 | # rotation 2 | rotate_red_block_right: ["take the red block and rotate it to the right"] 3 | rotate_red_block_left: ["take the red block and rotate it to the left"] 4 | rotate_blue_block_right: ["take the blue block and rotate it to the right"] 5 | rotate_blue_block_left: ["take the blue block and rotate it to the left"] 6 | rotate_pink_block_right: ["take the pink block and rotate it to the right"] 7 | rotate_pink_block_left: ["take the pink block and rotate it to the left"] 8 | 9 | # sliding 10 | push_red_block_right: ["go push the red block right"] 11 | push_red_block_left: ["go push the red block left"] 12 | push_blue_block_right: ["go push the blue block right"] 13 | push_blue_block_left: ["go push the blue block left"] 14 | push_pink_block_right: ["go push the pink block right"] 15 | push_pink_block_left: ["go push the pink block left"] 16 | 17 | # open/close 18 | move_slider_left: [ "push the sliding door to the left side"] 19 | move_slider_right: [ "push the sliding door to the right side"] 20 | open_drawer: ["pull the handle to open the drawer"] 21 | close_drawer: ["push the handle to close the drawer"] 22 | 23 | # lifting 24 | lift_red_block_table: ["grasp and lift the red block"] 25 | lift_blue_block_table: ["grasp and lift the blue block"] 26 | lift_pink_block_table: ["grasp and lift the pink block"] 27 | 28 | lift_red_block_slider: [ "lift the red block from the sliding cabinet"] 29 | lift_blue_block_slider: [ "lift the blue block from the sliding cabinet"] 30 | lift_pink_block_slider: [ "lift the pink block from the sliding cabinet"] 31 | 32 | lift_red_block_drawer: ["Take the red block from the drawer"] 33 | lift_blue_block_drawer: ["Take the blue block from the drawer"] 34 | lift_pink_block_drawer: ["Take the pink block from the drawer"] 35 | 36 | place_in_slider: [ "store the grasped block in the sliding cabinet"] 37 | place_in_drawer: [ "store the grasped block in the drawer"] 38 | 39 | push_into_drawer: ["slide the block that it falls into the drawer"] 40 | 41 | stack_block: ["stack the grasped block"] 42 | unstack_block: ["remove the stacked block"] 43 | 44 | turn_on_lightbulb: ["use the switch to turn on the light bulb"] 45 | turn_off_lightbulb: ["use the switch to turn off the light bulb"] 46 | turn_on_led: ["press the button to turn on the led light"] 47 | turn_off_led: ["press the button to turn off the led light"] 48 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/checkpoint/lh_sr.yaml: -------------------------------------------------------------------------------- 1 | _target_: pytorch_lightning.callbacks.ModelCheckpoint 2 | save_top_k: 3 3 | verbose: True 4 | monitor: eval_lh/avg_seq_len 5 | mode: max 6 | dirpath: saved_models 7 | filename: '{epoch}' #put back in when PL fixes this _{val/accuracy:.4f}' 8 | every_n_epochs: ${callbacks.rollout_lh.rollout_freq} 9 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/checkpoint/save_all.yaml: -------------------------------------------------------------------------------- 1 | _target_: pytorch_lightning.callbacks.ModelCheckpoint 2 | save_top_k: -1 3 | verbose: True 4 | dirpath: saved_models 5 | filename: '{epoch}' #put back in when PL fixes this _{val/accuracy:.4f}' 6 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/checkpoint/task_sr.yaml: -------------------------------------------------------------------------------- 1 | _target_: pytorch_lightning.callbacks.ModelCheckpoint 2 | save_top_k: 3 3 | verbose: True 4 | monitor: tasks/average_sr 5 | mode: max 6 | dirpath: saved_models 7 | filename: '{epoch}' #put back in when PL fixes this _{val/accuracy:.4f}' 8 | every_n_epochs: ${callbacks.rollout.rollout_freq} 9 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/checkpoint/val_action.yaml: -------------------------------------------------------------------------------- 1 | _target_: pytorch_lightning.callbacks.ModelCheckpoint 2 | save_top_k: 3 3 | verbose: True 4 | monitor: val_act/action_loss_pp 5 | mode: min 6 | dirpath: saved_models 7 | filename: '{epoch}' #put back in when PL fixes this _{val/accuracy:.4f}' 8 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/default.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - rollout: default 3 | - rollout_lh: default 4 | - checkpoint: save_all 5 | - tsne_plot: default 6 | - kl_schedule: constant 7 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/kl_schedule/constant.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.utils.kl_callbacks.KLConstantSchedule 2 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/kl_schedule/linear.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.utils.kl_callbacks.KLLinearSchedule 2 | start_epoch: 10 3 | end_epoch: 50 4 | max_kl_beta: ${loss.kl_beta} 5 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/kl_schedule/sigmoid.yaml: -------------------------------------------------------------------------------- 1 | # @package _group_ 2 | _target_: calvin_agent.utils.kl_callbacks.KLSigmoidSchedule 3 | start_epoch: 10 4 | end_epoch: 50 5 | max_kl_beta: ${loss.kl_beta} 6 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/rollout/default.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - /callbacks/rollout/tasks@tasks: new_playtable_tasks 3 | - /annotations@val_annotations: new_playtable_validation 4 | _target_: calvin_agent.rollout.rollout.Rollout 5 | _recursive_: false 6 | env_cfg: 7 | _target_: calvin_agent.wrappers.calvin_env_wrapper.CalvinEnvWrapper 8 | skip_epochs: 1 9 | rollout_freq: 1 10 | video: true 11 | num_rollouts_per_task: 10 12 | check_percentage_of_batch: 1 # which percentage of sequences do we want to check for possible tasks 13 | ep_len: 120 14 | empty_cache: true 15 | log_video_to_file: false 16 | save_dir: ./videos 17 | add_goal_thumbnail: true 18 | min_window_size: ${datamodule.datasets.vision_dataset.min_window_size} 19 | max_window_size: ${datamodule.datasets.vision_dataset.max_window_size} 20 | id_selection_strategy: "select_longest" 21 | lang_folder: ${datamodule.datasets.lang_dataset.lang_folder} 22 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/rollout_lh/default.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - /callbacks/rollout/tasks@tasks: new_playtable_tasks 3 | - /annotations@val_annotations: new_playtable_validation 4 | _target_: calvin_agent.rollout.rollout_long_horizon.RolloutLongHorizon 5 | _recursive_: false 6 | env_cfg: 7 | _target_: calvin_agent.wrappers.calvin_env_wrapper.CalvinEnvWrapper 8 | skip_epochs: 1 9 | rollout_freq: 1 10 | num_videos: 16 11 | num_sequences: 128 12 | replan_freq: 30 13 | ep_len: 360 14 | empty_cache: true 15 | log_video_to_file: false 16 | save_dir: ./videos 17 | lang_folder: ${datamodule.datasets.lang_dataset.lang_folder} 18 | debug: false 19 | -------------------------------------------------------------------------------- /calvin_models/conf/callbacks/tsne_plot/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.visualization.tsne_plot.TSNEPlot 2 | perplexity: 40 3 | n_jobs: 8 4 | plot_percentage: 0.2 5 | opacity: 0.3 6 | marker_size: 5 7 | -------------------------------------------------------------------------------- /calvin_models/conf/config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - callbacks: default 3 | - datamodule: default 4 | - model: default 5 | - loss: default 6 | - training: default_training 7 | - trainer: play_trainer 8 | - logger: wandb 9 | 10 | - override hydra/job_logging: colorlog 11 | - override hydra/hydra_logging: colorlog 12 | - _self_ 13 | 14 | seed: 42 15 | log_dir: ../ 16 | slurm: false 17 | 18 | hydra: 19 | run: 20 | dir: ${log_dir}/runs/${now:%Y-%m-%d}/${now:%H-%M-%S} 21 | sweep: 22 | dir: ${log_dir}/runs/${now:%Y-%m-%d}/${now:%H-%M-%S} 23 | subdir: ${hydra.job.override_dirname} 24 | job: 25 | config: 26 | override_dirname: 27 | exclude_keys: 28 | - log_dir 29 | - datamodule.root_data_dir 30 | - trainer.gpus 31 | - model.tsne_plot 32 | - datamodule.num_workers 33 | - trainer.limit_train_batches 34 | - trainer.limit_val_batches 35 | - model.action_decoder.load_action_bounds 36 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/lang_dataset/lang.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.datasets.disk_dataset.DiskDataset 2 | key: "lang" 3 | save_format: "npz" 4 | batch_size: 32 5 | min_window_size: 16 6 | max_window_size: 32 7 | proprio_state: ${datamodule.proprioception_dims} 8 | obs_space: ${datamodule.observation_space} 9 | skip_frames: 1 10 | pad: true 11 | lang_folder: "lang_annotations" 12 | num_workers: 2 13 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/lang_dataset/lang_shm.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.datasets.shm_dataset.ShmDataset 2 | key: "lang" 3 | batch_size: 32 4 | min_window_size: 16 5 | max_window_size: 32 6 | proprio_state: ${datamodule.proprioception_dims} 7 | obs_space: ${datamodule.observation_space} 8 | pad: true 9 | lang_folder: "lang_annotations" 10 | num_workers: 2 11 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/vision_dataset/vision.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.datasets.disk_dataset.DiskDataset 2 | key: "vis" 3 | save_format: "npz" 4 | batch_size: 32 5 | min_window_size: 16 6 | max_window_size: 32 7 | proprio_state: ${datamodule.proprioception_dims} 8 | obs_space: ${datamodule.observation_space} 9 | pad: true 10 | lang_folder: "lang_annotations" #${datamodule.datasets.lang_dataset.lang_folder} 11 | num_workers: 2 12 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/vision_dataset/vision_shm.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.datasets.shm_dataset.ShmDataset 2 | key: "vis" 3 | batch_size: 32 4 | min_window_size: 16 5 | max_window_size: 32 6 | proprio_state: ${datamodule.proprioception_dims} 7 | obs_space: ${datamodule.observation_space} 8 | pad: true 9 | lang_folder: "lang_annotations" 10 | num_workers: 2 11 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/vision_lang.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - vision_dataset: vision 3 | - lang_dataset: lang 4 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/vision_lang_shm.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - vision_dataset: vision_shm 3 | - lang_dataset: lang_shm 4 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/datasets/vision_only.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - vision_dataset: vision 3 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/default.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - datasets: vision_lang_shm 3 | - transforms: play_basic 4 | - proprioception_dims: robot_no_joints #robot_full 5 | - observation_space: lang_rgb_static_abs_act 6 | _target_: calvin_agent.datasets.calvin_data_module.CalvinDataModule 7 | _recursive_: false 8 | root_data_dir: "dataset/task_D_D" 9 | action_space: 7 10 | action_max: [1., 1., 1., 1., 1., 1., 1.,] 11 | action_min: [-1., -1., -1., -1., -1., -1., -1] 12 | shuffle_val: false 13 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/all_mods_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper', 'rgb_tactile'] 2 | depth_obs: ['depth_static', 'depth_gripper', 'depth_tactile'] 3 | state_obs: ['robot_obs', 'scene_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/all_mods_rel_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper', 'rgb_tactile'] 2 | depth_obs: ['depth_static', 'depth_gripper', 'depth_tactile'] 3 | state_obs: ['robot_obs', 'scene_obs'] 4 | actions: ['rel_actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_gripper_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_gripper_rel_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['rel_actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_rel_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['rel_actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_robot_scene_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs', 'scene_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_tactile_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_tactile'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgb_static_tactile_rel_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_tactile'] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['rel_actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgbd_both_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper'] 2 | depth_obs: ['depth_static', 'depth_gripper'] 3 | state_obs: ['robot_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgbd_both_rel_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper'] 2 | depth_obs: ['depth_static', 'depth_gripper'] 3 | state_obs: ['robot_obs'] 4 | actions: ['rel_actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgbd_static_gripper_rel_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static', 'rgb_gripper'] 2 | depth_obs: ['depth_gripper'] 3 | state_obs: ['robot_obs'] 4 | actions: ['rel_actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/lang_rgbd_static_robot_abs_act.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: ['rgb_static'] 2 | depth_obs: ['depth_static'] 3 | state_obs: ['robot_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/observation_space/state_only.yaml: -------------------------------------------------------------------------------- 1 | rgb_obs: [] 2 | depth_obs: [] 3 | state_obs: ['robot_obs'] 4 | actions: ['actions'] 5 | language: ['language'] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/proprioception_dims/none.yaml: -------------------------------------------------------------------------------- 1 | n_state_obs: 0 2 | keep_indices: [[0, 0]] 3 | robot_orientation_idx: [3, 6] 4 | normalize: False 5 | normalize_robot_orientation: False 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/proprioception_dims/robot_full.yaml: -------------------------------------------------------------------------------- 1 | n_state_obs: 15 2 | keep_indices: [[0, 15]] 3 | robot_orientation_idx: [3, 6] 4 | normalize: True 5 | normalize_robot_orientation: True 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/proprioception_dims/robot_no_joints.yaml: -------------------------------------------------------------------------------- 1 | n_state_obs: 8 2 | keep_indices: [[0, 7], [14,15]] 3 | robot_orientation_idx: [3, 6] 4 | normalize: True 5 | normalize_robot_orientation: True 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/proprioception_dims/robot_scene.yaml: -------------------------------------------------------------------------------- 1 | n_state_obs: 54 2 | keep_indices: [[0, 54]] 3 | robot_orientation_idx: [3, 6] 4 | normalize: True 5 | normalize_robot_orientation: True 6 | -------------------------------------------------------------------------------- /calvin_models/conf/datamodule/random.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.datasets.random.RandomDataModule 2 | batch_size: 16 3 | action_space: 7 4 | action_max: [1, 1, 1, 1, 1, 1, 1] 5 | action_min: [-1, -1, -1, -1, -1, -1, -1] 6 | -------------------------------------------------------------------------------- /calvin_models/conf/inference/config_inference.yaml: -------------------------------------------------------------------------------- 1 | train_folder: ??? # config path to the config.yaml of the training folder (in .hydra) 2 | load_checkpoint: ??? 3 | seed: 42 4 | log_dir: /tmp 5 | visualize: True 6 | ep_len: 120 7 | replan_freq: 30 8 | processes: 1 9 | 10 | hydra: 11 | run: 12 | dir: ${log_dir}/inference_runs/${now:%Y-%m-%d}/${now:%H-%M-%S} 13 | 14 | defaults: 15 | - override hydra/job_logging: colorlog 16 | - override hydra/hydra_logging: colorlog 17 | -------------------------------------------------------------------------------- /calvin_models/conf/lang_ann.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - callbacks: default 3 | - datamodule: default 4 | - model: sbert 5 | - loss: default 6 | - training: default_training 7 | - trainer: play_trainer 8 | - logger: wandb 9 | - annotations@train_instructions: new_playtable 10 | - annotations@val_instructions: new_playtable_validation 11 | 12 | - override hydra/job_logging: colorlog 13 | - override hydra/hydra_logging: colorlog 14 | - override datamodule/observation_space: state_only 15 | - override datamodule/datasets: vision_only 16 | # - override datamodule/datasets/vision_dataset: vision_shm 17 | - _self_ 18 | 19 | seed: 42 20 | log_dir: ../../ 21 | slurm: false 22 | eps: 0.01 23 | postprocessing: true 24 | lang_folder: "lang_annotations" 25 | with_text: false 26 | reannotate: false 27 | prior_steps_window: 16 28 | validation_scene: calvin_scene_D 29 | datamodule: 30 | datasets: 31 | vision_dataset: 32 | min_window_size: 64 33 | max_window_size: 64 34 | shuffle_val: true 35 | 36 | hydra: 37 | run: 38 | dir: ${log_dir}/runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_lang_ann 39 | sweep: 40 | dir: ${log_dir}/runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_lang_ann 41 | job: 42 | config: 43 | override_dirname: 44 | exclude_keys: 45 | - log_dir 46 | - datamodule.root_data_dir 47 | - trainer.gpus 48 | - model.tsne_plot 49 | - datamodule.num_workers 50 | - trainer.limit_train_batches 51 | - trainer.limit_val_batches 52 | - model.action_decoder.load_action_bounds 53 | -------------------------------------------------------------------------------- /calvin_models/conf/logger/tb_logger.yaml: -------------------------------------------------------------------------------- 1 | _target_: pytorch_lightning.loggers.TensorBoardLogger 2 | save_dir: . 3 | name: play_lmp 4 | version: "" 5 | -------------------------------------------------------------------------------- /calvin_models/conf/logger/wandb.yaml: -------------------------------------------------------------------------------- 1 | _target_: pytorch_lightning.loggers.WandbLogger 2 | save_dir: . 3 | name: play_lmp 4 | group: play_lmp 5 | log_model: false 6 | project: "multi_play" 7 | id: ??? 8 | -------------------------------------------------------------------------------- /calvin_models/conf/loss/default.yaml: -------------------------------------------------------------------------------- 1 | kl_beta: 0.001 2 | -------------------------------------------------------------------------------- /calvin_models/conf/model/action_decoder/logistic.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.decoders.logistic_policy_network.LogisticPolicyNetwork 2 | n_mixtures: 10 3 | hidden_size: 2048 4 | out_features: ${datamodule.action_space} 5 | log_scale_min: -7.0 6 | act_max_bound: ${datamodule.action_max} 7 | act_min_bound: ${datamodule.action_min} 8 | dataset_dir: ${datamodule.root_data_dir} 9 | policy_rnn_dropout_p: 0.0 10 | load_action_bounds: true 11 | num_classes: 256 12 | perceptual_features: ?? 13 | latent_goal_features: 32 14 | plan_features: 256 15 | -------------------------------------------------------------------------------- /calvin_models/conf/model/default.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - perceptual_encoder: default 3 | - plan_proposal: default 4 | - plan_recognition: default 5 | - visual_goal: default 6 | - language_goal: default 7 | - action_decoder: logistic 8 | - optimizer: adam 9 | 10 | _target_: calvin_agent.models.mcil.MCIL 11 | _recursive_: false 12 | 13 | kl_beta: ${loss.kl_beta} 14 | replan_freq: 30 15 | -------------------------------------------------------------------------------- /calvin_models/conf/model/language_goal/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.encoders.goal_encoders.LanguageGoalEncoder 2 | language_features: 384 3 | hidden_size: 2048 4 | latent_goal_features: 32 5 | word_dropout_p: 0.0 6 | l2_normalize_goal_embeddings: False 7 | activation_function: ReLU #ELU 8 | -------------------------------------------------------------------------------- /calvin_models/conf/model/language_goal/none.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/conf/model/language_goal/none.yaml -------------------------------------------------------------------------------- /calvin_models/conf/model/optimizer/adam.yaml: -------------------------------------------------------------------------------- 1 | _target_: torch.optim.Adam 2 | lr: ${training.lr} 3 | #weight_decay: 1e-6 4 | -------------------------------------------------------------------------------- /calvin_models/conf/model/optimizer/adamw.yaml: -------------------------------------------------------------------------------- 1 | _target_: torch.optim.AdamW 2 | lr: ${training.lr} 3 | weight_decay: 1e-6 4 | #amsgrad: False 5 | -------------------------------------------------------------------------------- /calvin_models/conf/model/optimizer/sgd.yaml: -------------------------------------------------------------------------------- 1 | _target_: torch.optim.SGD 2 | lr: ${training.lr} 3 | momentum: 0.9 4 | #weight_decay: 0.0005 5 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/RGBD_both.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.concat_encoders.ConcatEncoders 2 | _recursive_: false 3 | 4 | defaults: 5 | - vision_static: default 6 | - vision_gripper: default 7 | - depth_static: default 8 | - depth_gripper: default 9 | - proprio: identity 10 | - tactile: none 11 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.concat_encoders.ConcatEncoders 2 | _recursive_: false 3 | 4 | defaults: 5 | - vision_static: default 6 | - vision_gripper: none 7 | - depth_static: none 8 | - depth_gripper: none 9 | - proprio: identity 10 | - tactile: none 11 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/depth_gripper/default.yaml: -------------------------------------------------------------------------------- 1 | num_c: 1 2 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/depth_gripper/none.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/conf/model/perceptual_encoder/depth_gripper/none.yaml -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/depth_static/default.yaml: -------------------------------------------------------------------------------- 1 | num_c: 1 2 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/depth_static/none.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/conf/model/perceptual_encoder/depth_static/none.yaml -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/gripper_cam.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.concat_encoders.ConcatEncoders 2 | _recursive_: false 3 | 4 | defaults: 5 | - vision_static: default 6 | - vision_gripper: default 7 | - depth_static: none 8 | - depth_gripper: none 9 | - proprio: identity 10 | - tactile: none 11 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/proprio/identity.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.proprio_encoder.IdentityEncoder 2 | proprioception_dims: ${datamodule.proprioception_dims} 3 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/static_RGBD.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.concat_encoders.ConcatEncoders 2 | _recursive_: false 3 | 4 | defaults: 5 | - vision_static: default 6 | - vision_gripper: none 7 | - depth_static: default 8 | - depth_gripper: none 9 | - proprio: identity 10 | - tactile: none 11 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/static_RGB_tactile.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.concat_encoders.ConcatEncoders 2 | _recursive_: false 3 | 4 | defaults: 5 | - vision_static: default 6 | - vision_gripper: none 7 | - depth_static: none 8 | - depth_gripper: none 9 | - proprio: identity 10 | - tactile: default 11 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/tactile/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.tactile_encoder.TactileEncoder 2 | visual_features: 64 3 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/tactile/none.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/conf/model/perceptual_encoder/tactile/none.yaml -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/vision_gripper/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.vision_network_gripper.VisionNetwork 2 | activation_function: ReLU #ELU 3 | dropout_vis_fc: 0.0 4 | l2_normalize_output: false 5 | visual_features: 64 6 | conv_encoder: nature_cnn 7 | num_c: 3 8 | -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/vision_gripper/none.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/calvin_models/conf/model/perceptual_encoder/vision_gripper/none.yaml -------------------------------------------------------------------------------- /calvin_models/conf/model/perceptual_encoder/vision_static/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.perceptual_encoders.vision_network.VisionNetwork 2 | input_width: 200 3 | input_height: 200 4 | activation_function: ReLU #ELU 5 | dropout_vis_fc: 0.0 6 | l2_normalize_output: false 7 | visual_features: 64 8 | num_c: 3 9 | -------------------------------------------------------------------------------- /calvin_models/conf/model/plan_proposal/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.plan_encoders.plan_proposal_net.PlanProposalNetwork 2 | perceptual_features: ??? 3 | latent_goal_features: 32 4 | plan_features: 256 5 | activation_function: ReLU #ELU 6 | min_std: 0.0001 7 | -------------------------------------------------------------------------------- /calvin_models/conf/model/plan_recognition/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.plan_encoders.plan_recognition_net.PlanRecognitionNetwork 2 | in_features: ??? 3 | plan_features: 256 4 | action_space: ${datamodule.action_space} 5 | birnn_dropout_p: 0.0 6 | min_std: 0.0001 7 | -------------------------------------------------------------------------------- /calvin_models/conf/model/sbert.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.encoders.language_network.SBert 2 | nlp_model: paraphrase-MiniLM-L6-v2 3 | -------------------------------------------------------------------------------- /calvin_models/conf/model/visual_goal/default.yaml: -------------------------------------------------------------------------------- 1 | _target_: calvin_agent.models.encoders.goal_encoders.VisualGoalEncoder 2 | in_features: ??? 3 | hidden_size: 2048 4 | latent_goal_features: 32 5 | l2_normalize_goal_embeddings: False 6 | activation_function: ReLU #ELU 7 | -------------------------------------------------------------------------------- /calvin_models/conf/trainer/play_trainer.yaml: -------------------------------------------------------------------------------- 1 | devices: 1 2 | accelerator: gpu 3 | precision: 16 4 | val_check_interval: 1.0 5 | max_epochs: 100 6 | -------------------------------------------------------------------------------- /calvin_models/conf/training/default_training.yaml: -------------------------------------------------------------------------------- 1 | lr: 0.0001 2 | -------------------------------------------------------------------------------- /calvin_models/requirements.txt: -------------------------------------------------------------------------------- 1 | cmake 2 | wheel 3 | numpy>1.2 4 | hydra-core==1.1.1 5 | hydra-colorlog 6 | matplotlib 7 | opencv-python 8 | omegaconf 9 | plotly 10 | pyhash 11 | pytorch-lightning==1.8.6 12 | lightning_lite 13 | torch==1.13.1 14 | torchvision 15 | MulticoreTSNE 16 | gitpython 17 | scipy 18 | sentence-transformers 19 | setuptools==57.5.0 20 | gym 21 | moviepy 22 | tqdm 23 | termcolor 24 | wandb 25 | -------------------------------------------------------------------------------- /calvin_models/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Setup Calvin installation.""" 4 | 5 | from os import path as op 6 | import re 7 | 8 | from setuptools import find_packages, setup 9 | 10 | 11 | def _read(f): 12 | return open(op.join(op.dirname(__file__), f)).read() if op.exists(f) else "" 13 | 14 | 15 | _meta = _read("calvin_agent/__init__.py") 16 | 17 | 18 | def find_meta(_meta, string): 19 | l_match = re.search(r"^" + string + r'\s*=\s*"(.*)"', _meta, re.M) 20 | if l_match: 21 | return l_match.group(1) 22 | raise RuntimeError(f"Unable to find {string} string.") 23 | 24 | 25 | install_requires = [ 26 | l for l in _read("requirements.txt").split("\n") if l and not l.startswith("#") and not l.startswith("-") 27 | ] 28 | 29 | meta = dict( 30 | name=find_meta(_meta, "__project__"), 31 | version=find_meta(_meta, "__version__"), 32 | license=find_meta(_meta, "__license__"), 33 | description="CALVIN - A benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks", 34 | platforms=("Any"), 35 | zip_safe=False, 36 | keywords="pytorch Lfp".split(), 37 | author=find_meta(_meta, "__author__"), 38 | author_email=find_meta(_meta, "__email__"), 39 | url=" https://github.com/mees/calvin", 40 | packages=find_packages(exclude=["tests"]), 41 | install_requires=install_requires, 42 | ) 43 | 44 | if __name__ == "__main__": 45 | print("find_package", find_packages(exclude=["tests"])) 46 | setup(**meta) 47 | -------------------------------------------------------------------------------- /config_path.json: -------------------------------------------------------------------------------- 1 | { 2 | "CALVIN_dataset_path" : "/home/wibot/Data/SC/", 3 | "BERT_path": "/home/wibot/SC/roboBert/bert-base-uncased", 4 | "dataset_wo_image_path" : "/home/wibot/SC/roboBert/my_models/DDP_training/" 5 | } -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/ModalityFusioner.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/ModalityFusioner.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/data_enhence.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/data_enhence.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/data_random.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/data_random.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/diffusion_modules.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/diffusion_modules.cpython-310.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/diffusion_modules.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/diffusion_modules.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/my_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/my_utils.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/positional_encoding.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/positional_encoding.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/pretrain_data_random.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/pretrain_data_random.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/__pycache__/transformer_agent.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/__pycache__/transformer_agent.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/config/read_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | loaded_config = dict() 4 | with open('DDP_training.json', 'r') as f: 5 | loaded_config = json.load(f) 6 | 7 | print(loaded_config) -------------------------------------------------------------------------------- /my_models/DDP_training/log/2025-02-02-21-38-47/events.out.tfevents.1738503527.wibot-ub-cyp.2248120.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/log/2025-02-02-21-38-47/events.out.tfevents.1738503527.wibot-ub-cyp.2248120.0 -------------------------------------------------------------------------------- /my_models/DDP_training/log/2025-02-10-09-37-41/events.out.tfevents.1739151461.wibot-ub-cyp.71224.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/log/2025-02-10-09-37-41/events.out.tfevents.1739151461.wibot-ub-cyp.71224.0 -------------------------------------------------------------------------------- /my_models/DDP_training/log/2025-02-10-09-55-22/events.out.tfevents.1739152522.wibot-ub-cyp.127921.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/log/2025-02-10-09-55-22/events.out.tfevents.1739152522.wibot-ub-cyp.127921.0 -------------------------------------------------------------------------------- /my_models/DDP_training/log/2025-02-10-10-01-13/events.out.tfevents.1739152873.wibot-ub-cyp.146316.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/log/2025-02-10-10-01-13/events.out.tfevents.1739152873.wibot-ub-cyp.146316.0 -------------------------------------------------------------------------------- /my_models/DDP_training/log/2025-02-10-10-07-09/events.out.tfevents.1739153229.wibot-ub-cyp.166804.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/log/2025-02-10-10-07-09/events.out.tfevents.1739153229.wibot-ub-cyp.166804.0 -------------------------------------------------------------------------------- /my_models/DDP_training/my_utils.py: -------------------------------------------------------------------------------- 1 | def get_parameter_number(model): 2 | total_num = sum(p.numel() for p in model.parameters()) 3 | trainable_num = sum(p.numel() for p in model.parameters() if p.requires_grad) 4 | return {'Total': total_num, 'Trainable': trainable_num} 5 | 6 | 7 | def lang_to_embed(selected_tasks:dict) -> dict: 8 | import torch 9 | from transformers import BertModel, BertTokenizer 10 | 11 | BERT_PATH = "./bert-base-uncased" 12 | tokenizer = BertTokenizer.from_pretrained(BERT_PATH) 13 | bert = BertModel.from_pretrained(BERT_PATH) 14 | 15 | tasks_lang_x = dict() 16 | tasks_lang_mask = dict() 17 | 18 | for task in selected_tasks.keys(): 19 | task_str = task.replace("_"," ") 20 | tasks_lang_x[task_str] = "no tensor" 21 | 22 | input_str = list(tasks_lang_x.keys()) 23 | tokens_info = tokenizer(input_str, 24 | padding="max_length", 25 | max_length=8, 26 | truncation=True, 27 | return_tensors="pt" 28 | ) 29 | 30 | languages_x = bert(input_ids=tokens_info["input_ids"], attention_mask=tokens_info["attention_mask"])["last_hidden_state"].detach() 31 | tokens_mask = 1 - tokens_info["attention_mask"] 32 | tokens_mask = torch.where(tokens_mask == 1, -torch.inf, tokens_mask).detach() 33 | 34 | for name_id, task_str in enumerate(input_str): 35 | tasks_lang_x[task_str] = languages_x[name_id] 36 | tasks_lang_mask[task_str] = tokens_mask[name_id] 37 | 38 | return tasks_lang_x, tasks_lang_mask 39 | 40 | def remove_prefix(storage, isEval=False): 41 | from collections import OrderedDict 42 | # 创建一个新的有序字典 43 | new_state_dict = OrderedDict() 44 | # 遍历状态字典中的键和值 45 | for k, v in storage.items(): 46 | # 去除键中的"module."前缀 47 | name = k.replace("module.", "") 48 | if isEval == True: 49 | name = name.replace("nets.", "") 50 | # 将新的键和值添加到新的字典中 51 | new_state_dict[name] = v 52 | # 返回新的存储位置和新的字典 53 | return new_state_dict 54 | 55 | -------------------------------------------------------------------------------- /my_models/DDP_training/positional_encoding.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import math 4 | 5 | class PositionalEncoding(nn.Module): 6 | "Implement the PE function." 7 | 8 | def __init__(self, d_model, dropout, max_len=5000): 9 | super(PositionalEncoding, self).__init__() 10 | self.dropout = nn.Dropout(p=dropout) 11 | 12 | # 初始化Shape为(max_len, d_model)的PE (positional encoding) 13 | pe = torch.zeros(max_len, d_model)#.to(0) 14 | # 初始化一个tensor [[0, 1, 2, 3, ...]] 15 | position = torch.arange(0, max_len).unsqueeze(1) 16 | # 这里就是sin和cos括号中的内容，通过e和ln进行了变换 17 | div_term = torch.exp( 18 | torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model) 19 | ) 20 | # 计算PE(pos, 2i) 21 | pe[:, 0::2] = torch.sin(position * div_term) 22 | # 计算PE(pos, 2i+1) 23 | pe[:, 1::2] = torch.cos(position * div_term) 24 | # 为了方便计算，在最外面在unsqueeze出一个batch 25 | pe = pe.unsqueeze(0) 26 | # 如果一个参数不参与梯度下降，但又希望保存model的时候将其保存下来 27 | # 这个时候就可以用register_buffer 28 | self.register_buffer("pe", pe) 29 | 30 | def forward(self, x): 31 | 32 | x = x + self.pe[:, : x.size(1)].requires_grad_(False) 33 | return self.dropout(x) 34 | -------------------------------------------------------------------------------- /my_models/DDP_training/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/src/__init__.py -------------------------------------------------------------------------------- /my_models/DDP_training/src/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/src/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/src/__pycache__/helpers.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PeterWangsicheng/RoboBERT/c992a13d7866d009da7924c11424a05d96b029b3/my_models/DDP_training/src/__pycache__/helpers.cpython-38.pyc -------------------------------------------------------------------------------- /my_models/DDP_training/src/utils.py: -------------------------------------------------------------------------------- 1 | def extend_instance(obj, mixin): 2 | """Apply mixins to a class instance after creation""" 3 | base_cls = obj.__class__ 4 | base_cls_name = obj.__class__.__name__ 5 | obj.__class__ = type( 6 | base_cls_name, (mixin, base_cls), {} 7 | ) # mixin needs to go first for our forward() logic to work 8 | 9 | 10 | def getattr_recursive(obj, att): 11 | """ 12 | Return nested attribute of obj 13 | Example: getattr_recursive(obj, 'a.b.c') is equivalent to obj.a.b.c 14 | """ 15 | if att == "": 16 | return obj 17 | i = att.find(".") 18 | if i < 0: 19 | return getattr(obj, att) 20 | else: 21 | return getattr_recursive(getattr(obj, att[:i]), att[i + 1 :]) 22 | 23 | 24 | def setattr_recursive(obj, att, val): 25 | """ 26 | Set nested attribute of obj 27 | Example: setattr_recursive(obj, 'a.b.c', val) is equivalent to obj.a.b.c = val 28 | """ 29 | if "." in att: 30 | obj = getattr_recursive(obj, ".".join(att.split(".")[:-1])) 31 | setattr(obj, att.split(".")[-1], val) 32 | 33 | 34 | def apply_with_stopping_condition( 35 | module, apply_fn, apply_condition=None, stopping_condition=None, **other_args 36 | ): 37 | if stopping_condition(module): 38 | return 39 | if apply_condition(module): 40 | apply_fn(module, **other_args) 41 | for child in module.children(): 42 | apply_with_stopping_condition( 43 | child, 44 | apply_fn, 45 | apply_condition=apply_condition, 46 | stopping_condition=stopping_condition, 47 | **other_args 48 | ) 49 | -------------------------------------------------------------------------------- /preprocessing_val_language.py: -------------------------------------------------------------------------------- 1 | from omegaconf import OmegaConf 2 | from transformers import BertModel, BertTokenizer 3 | import open_clip 4 | 5 | def preprocessing_val_languages_bert(val_annotations): 6 | import json 7 | 8 | loaded_config = dict() 9 | with open('config_path.json', 'r') as f: 10 | loaded_config = json.load(f) 11 | BERT_PATH = loaded_config["BERT_path"] 12 | 13 | tokenizer = BertTokenizer.from_pretrained(BERT_PATH) 14 | bert = BertModel.from_pretrained(BERT_PATH) 15 | new_val_annotations = dict() 16 | for task, text in val_annotations.items(): 17 | text = [task.replace("_"," "), text[0]] 18 | tokens_info = tokenizer(text, 19 | padding="max_length", 20 | max_length=16, 21 | truncation=True, 22 | return_tensors="pt" 23 | ) 24 | languages_x_dict = bert(input_ids=tokens_info["input_ids"], attention_mask=tokens_info["attention_mask"]) 25 | languages_x = languages_x_dict["last_hidden_state"].detach() 26 | tokens_mask = tokens_info["attention_mask"] 27 | new_val_annotations[task] = {"text": text, "languages_x": languages_x} 28 | 29 | return new_val_annotations 30 | 31 | def preprocessing_val_languages_clip(val_annotations): 32 | # create text encoder 33 | clip_vision_encoder_path: str = "ViT-L-14" 34 | clip_vision_encoder_pretrained: str = "openai" 35 | clip_encoder, _, image_processor = open_clip.create_model_and_transforms( 36 | clip_vision_encoder_path, pretrained=clip_vision_encoder_pretrained 37 | ) 38 | text_encoder = clip_encoder 39 | 40 | tokenizer = open_clip.get_tokenizer('ViT-L-14') 41 | 42 | new_val_annotations = dict() 43 | for task, text in val_annotations.items(): 44 | text = [task.replace("_"," "), text[0]] 45 | tokens_info = tokenizer(text) 46 | languages_x = text_encoder.encode_text(tokens_info).unsqueeze(1).detach() 47 | new_val_annotations[task] = {"text": text, "languages_x": languages_x} 48 | 49 | return new_val_annotations 50 | 51 | 52 | 53 | if __name__ == "__main__": 54 | val_annotations = OmegaConf.load("./calvin_models/conf/annotations/new_playtable_validation.yaml") 55 | print(preprocessing_val_languages(val_annotations)) 56 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ConfigArgParse==1.7 2 | diffusers==0.29.2 3 | einops==0.8.1 4 | einops_exts==0.0.4 5 | GitPython==3.1.43 6 | GitPython==3.1.44 7 | gym==0.26.2 8 | hydra-core==1.3.2 9 | lightning_lite==1.8.6 10 | matplotlib==3.7.5 11 | MulticoreTSNE==0.1 12 | MulticoreTSNE==0.1 13 | numpy==1.23.0 14 | numpy_quaternion==2023.0.4 15 | omegaconf==2.1.2 16 | opencv_python==4.6.0.66 17 | opencv_python_headless==4.5.5.64 18 | Pillow==11.1.0 19 | plotly==5.22.0 20 | pybullet==3.2.6 21 | pyhash==0.9.3 22 | pytorch_lightning==1.8.6 23 | pyttsx3==2.98 24 | scikit_learn==1.3.2 25 | scipy==1.15.1 26 | sentence_transformers==3.0.1 27 | setuptools==57.5.0 28 | tacto==0.0.3 29 | termcolor==2.5.0 30 | torch==2.3.1+cu118 31 | torchvision==0.18.1+cu118 32 | tqdm==4.66.4 33 | transformers==4.37.2 34 | wandb==0.19.6 35 | -------------------------------------------------------------------------------- /sparate_action_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def separate_data(dataset_name="ABCD_D", separate_mode="language"): 4 | import pickle 5 | import numpy as np 6 | import json 7 | 8 | loaded_json = dict() 9 | with open("./config_path.json", "r") as f: 10 | loaded_json = json.load(f) 11 | 12 | dataset_path = loaded_json["CALVIN_dataset_path"] 13 | file_prefix = dataset_path + "/task_" + dataset_name + "/training" 14 | print("extract path is:", file_prefix) 15 | 16 | ep_start_end_ids = None 17 | if separate_mode == "all": 18 | ep_start_end_ids = np.load(file_prefix + "/ep_start_end_ids.npy") 19 | 20 | elif separate_mode == "language": 21 | ep_start_end_ids = np.load( 22 | f"{file_prefix}/lang_annotations/auto_lang_ann.npy", allow_pickle=True 23 | ).item()["info"]["indx"] 24 | 25 | dataset_wo_image = dict() 26 | ep_start_end_ids = ep_start_end_ids 27 | ids_len = len(ep_start_end_ids) 28 | for id, start_end_pair in enumerate(ep_start_end_ids): 29 | start_idx, end_idx = start_end_pair[0], start_end_pair[1] 30 | 31 | for frame_idx in range(start_idx, end_idx+1): 32 | frame = np.load(f"{file_prefix}/episode_{frame_idx:07d}.npz") 33 | rel_actions = frame["rel_actions"] 34 | robot_obs = frame["robot_obs"] 35 | dataset_wo_image[frame_idx] = { 36 | "rel_actions": rel_actions, 37 | #"robot_obs": robot_obs 38 | } 39 | print((id + 1) / ids_len * 100) 40 | 41 | 42 | with open(f'dataset_wo_image_{dataset_name}.pkl', 'wb') as f: 43 | pickle.dump(dataset_wo_image, f) 44 | 45 | print("complete") 46 | 47 | if __name__ == "__main__": 48 | parser = argparse.ArgumentParser(description='configurating the extraction specifications') 49 | parser.add_argument('--dataset_name', type=str, default="ABCD_D") 50 | parser.add_argument('--separate_mode', type=str, default="language", help="only sparate the data with lanuage label") 51 | args = parser.parse_args() 52 | separate_data(dataset_name=args.dataset_name, separate_mode=args.separate_mode) 53 | --------------------------------------------------------------------------------