├── manipgen ├── __init__.py ├── real_world │ ├── __init__.py │ ├── vlm_planning │ │ ├── __init__.py │ │ └── prompts │ │ │ ├── planning_examples.py │ │ │ ├── tagging_prompt.py │ │ │ ├── segmentation_doublechecker_prompt.py │ │ │ └── workspace_prompt.py │ ├── utils │ │ ├── constants.py │ │ ├── vis_utils.py │ │ ├── calibration_utils.py │ │ └── rl_utils.py │ └── scripts │ │ ├── get_camera_intrinsics.py │ │ └── run_manipgen.py ├── assets │ ├── umi │ │ ├── franka_finger_holder.stl │ │ └── soft_gripper_finger.stl │ ├── urdf │ │ └── franka_description │ │ │ └── meshes │ │ │ ├── collision │ │ │ ├── hand.stl │ │ │ ├── finger.stl │ │ │ ├── link0.stl │ │ │ ├── link1.stl │ │ │ ├── link2.stl │ │ │ ├── link3.stl │ │ │ ├── link4.stl │ │ │ ├── link5.stl │ │ │ ├── link6.stl │ │ │ ├── link7.stl │ │ │ ├── stltoobj.bat │ │ │ ├── stltoobj.mlx │ │ │ └── finger.obj │ │ │ ├── visual │ │ │ ├── daetoobj.mlx │ │ │ ├── daetoobj.bat │ │ │ ├── link1.mtl │ │ │ ├── link2.mtl │ │ │ ├── finger.mtl │ │ │ ├── link5.mtl │ │ │ ├── link4.mtl │ │ │ ├── link3.mtl │ │ │ ├── hand.mtl │ │ │ ├── link7.mtl │ │ │ ├── link0.mtl │ │ │ └── link6.mtl │ │ │ └── hande │ │ │ ├── finger_1-collision-simple.obj │ │ │ └── finger_2-collision-simple.obj │ └── industreal │ │ ├── yaml │ │ ├── industreal_asset_info_gears.yaml │ │ ├── industreal_asset_info_franka_table.yaml │ │ └── industreal_asset_info_pegs.yaml │ │ ├── mesh │ │ ├── industreal_franka_table │ │ │ ├── industreal_franka_finger_rubber_coll.mtl │ │ │ ├── industreal_franka_finger_coll_subdiv_6x.mtl │ │ │ └── industreal_franka_finger_rubber_viz.mtl │ │ ├── industreal_pegs │ │ │ ├── industreal_tray_insert_round_peg_12mm.mtl │ │ │ ├── industreal_tray_insert_round_peg_16mm.mtl │ │ │ ├── industreal_tray_insert_round_peg_4mm.mtl │ │ │ ├── industreal_tray_insert_round_peg_8mm.mtl │ │ │ ├── industreal_tray_pick_round_peg_12mm.mtl │ │ │ ├── industreal_tray_pick_round_peg_16mm.mtl │ │ │ ├── industreal_tray_pick_round_peg_4mm.mtl │ │ │ ├── industreal_tray_pick_round_peg_8mm.mtl │ │ │ ├── industreal_tray_insert_rectangular_peg_12mm.mtl │ │ │ ├── industreal_tray_insert_rectangular_peg_16mm.mtl │ │ │ ├── industreal_tray_insert_rectangular_peg_4mm.mtl │ │ │ ├── industreal_tray_insert_rectangular_peg_8mm.mtl │ │ │ ├── industreal_tray_pick_rectangular_peg_12mm.mtl │ │ │ ├── industreal_tray_pick_rectangular_peg_16mm.mtl │ │ │ ├── industreal_tray_pick_rectangular_peg_4mm.mtl │ │ │ └── industreal_tray_pick_rectangular_peg_8mm.mtl │ │ └── industreal_gears │ │ │ ├── industreal_gear_base.mtl │ │ │ ├── industreal_gear_large.mtl │ │ │ ├── industreal_gear_medium.mtl │ │ │ └── industreal_gear_small.mtl │ │ └── urdf │ │ ├── industreal_franka_finray_finger_changelog.txt │ │ ├── industreal_gear_base.urdf │ │ ├── industreal_gear_large.urdf │ │ ├── industreal_gear_small.urdf │ │ ├── industreal_gear_medium.urdf │ │ ├── industreal_round_peg_4mm.urdf │ │ ├── industreal_round_peg_8mm.urdf │ │ ├── industreal_round_peg_12mm.urdf │ │ ├── industreal_round_peg_16mm.urdf │ │ ├── industreal_rectangular_peg_4mm.urdf │ │ ├── industreal_rectangular_peg_8mm.urdf │ │ ├── industreal_rectangular_peg_12mm.urdf │ │ ├── industreal_rectangular_peg_16mm.urdf │ │ ├── industreal_round_hole_4mm.urdf │ │ ├── industreal_round_hole_8mm.urdf │ │ ├── industreal_round_hole_12mm.urdf │ │ ├── industreal_round_hole_16mm.urdf │ │ ├── industreal_rectangular_hole_4mm.urdf │ │ ├── industreal_rectangular_hole_8mm.urdf │ │ ├── industreal_rectangular_hole_12mm.urdf │ │ ├── industreal_rectangular_hole_16mm.urdf │ │ └── industreal_franka_urdf_changelog.txt ├── config │ ├── dagger │ │ ├── pick.yaml │ │ ├── open.yaml │ │ ├── close.yaml │ │ ├── pick_cube.yaml │ │ ├── place.yaml │ │ ├── place_cube.yaml │ │ ├── grasp_handle.yaml │ │ └── robomimic │ │ │ ├── base_pcd.json │ │ │ ├── base_depth.json │ │ │ ├── bc_transformer.json │ │ │ ├── bc_transformer_gmm.json │ │ │ ├── bc_gmm.json │ │ │ ├── bc_rnn.json │ │ │ ├── bc_mlp.json │ │ │ └── bc_rnn_gmm.json │ ├── real │ │ ├── place.yaml │ │ ├── open.yaml │ │ ├── close.yaml │ │ ├── grasp_handle.yaml │ │ ├── pick.yaml │ │ └── perception.yaml │ ├── train │ │ ├── open.yaml │ │ ├── pick.yaml │ │ ├── close.yaml │ │ ├── place.yaml │ │ ├── reach.yaml │ │ ├── place_cube.yaml │ │ ├── grasp_handle.yaml │ │ └── pick_cube.yaml │ ├── task │ │ ├── pick_cube.yaml │ │ ├── reach.yaml │ │ ├── place_cube.yaml │ │ ├── grasp_handle.yaml │ │ ├── open.yaml │ │ ├── close.yaml │ │ ├── pick.yaml │ │ ├── place.yaml │ │ └── IndustRealBase.yaml │ ├── config_test.yaml │ ├── config_train.yaml │ └── config_sample.yaml ├── utils │ ├── policy_testers │ │ ├── reach_test.py │ │ ├── __init__.py │ │ ├── grasp_handle_tester.py │ │ ├── place_cube_tester.py │ │ ├── open_tester.py │ │ ├── pick_cube_tester.py │ │ └── place_tester.py │ ├── initial_state_samplers │ │ └── __init__.py │ └── robomimic_utils.py ├── scripts │ ├── download_checkpoints.sh │ ├── download_datasets.sh │ └── urdf_templates │ │ └── pseudo_articulated_object.urdf ├── envs │ ├── __init__.py │ └── franka_close.py ├── object_lists │ └── articulated │ │ └── 0.txt ├── test_policy.py ├── multitask_dagger_loop.py └── train.py ├── imgs ├── manipgen.jpg ├── mechanical_data.jpeg └── example_wrist_view.gif ├── setup.py ├── requirements.txt ├── install.sh └── .gitmodules /manipgen/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /manipgen/real_world/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /manipgen/real_world/vlm_planning/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /imgs/manipgen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/imgs/manipgen.jpg -------------------------------------------------------------------------------- /imgs/mechanical_data.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/imgs/mechanical_data.jpeg -------------------------------------------------------------------------------- /imgs/example_wrist_view.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/imgs/example_wrist_view.gif -------------------------------------------------------------------------------- /manipgen/assets/umi/franka_finger_holder.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/umi/franka_finger_holder.stl -------------------------------------------------------------------------------- /manipgen/assets/umi/soft_gripper_finger.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/umi/soft_gripper_finger.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/hand.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/hand.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/finger.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/finger.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link0.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link0.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link1.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link1.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link2.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link2.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link3.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link3.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link4.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link4.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link5.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link5.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link6.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link6.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/link7.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mihdalal/manipgen/HEAD/manipgen/assets/urdf/franka_description/meshes/collision/link7.stl -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/daetoobj.mlx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/yaml/industreal_asset_info_gears.yaml: -------------------------------------------------------------------------------- 1 | base: 2 | height: 0.005 3 | density: 2700.0 4 | gears: 5 | height: 0.025 6 | density: 1000.0 7 | grasp_offset: 0.017 8 | shafts: 9 | height: 0.020 -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_franka_table/industreal_franka_finger_rubber_coll.mtl: -------------------------------------------------------------------------------- 1 | newmtl 0.752941_0.752941_0.752941_0.000000_0.000000 2 | Ka 0.0000 0.0000 0.0000 3 | Kd 0.7529 0.7529 0.7529 4 | Ks 0.0000 0.0000 0.0000 5 | Tf 0.0000 0.0000 0.0000 6 | d 1 7 | Ns 0 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name="manipgen", 5 | version="1.0.0", 6 | install_requires=[], 7 | author="Murtaza Dalal", 8 | author_email="mdalal@andrew.cmu.edu", 9 | url="none", 10 | packages=["manipgen"], 11 | ) 12 | 13 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/daetoobj.bat: -------------------------------------------------------------------------------- 1 | SET PATH=%PATH%;C:/Tools/Assimp/bin/x64/ 2 | forfiles /m *.dae /c "cmd /c assimp export @file @fname.obj --verbose --show-log -ptv" 3 | 4 | REM SET PATH=%PATH%;C:/Program Files/VCG/MeshLab/ 5 | REM forfiles /m *.dae /c "cmd /c meshlabserver -i @file -o @fname.obj -m vn vt -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_franka_finray_finger_changelog.txt: -------------------------------------------------------------------------------- 1 | Changes to industreal_franka.urdf: 2 | 1. replace industreal finger with finray finger 3 | 2. offset fingertip and fingertip centered position by 0.0959 along z-axis 4 | 3. offset panda hand position by 0.004 along z-axis 5 | 4. add camera mount visual mesh to urdf 6 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/stltoobj.bat: -------------------------------------------------------------------------------- 1 | REM SET PATH=%PATH%;C:/Tools/Assimp/bin/x64/ 2 | REM forfiles /m *.dae /c "cmd /c assimp export @file @fname.obj --verbose --show-log -ptv" 3 | 4 | SET PATH=%PATH%;C:/Program Files/VCG/MeshLab/ 5 | forfiles /m *.stl /c "cmd /c meshlabserver -i @file -o @fname.obj -m vn -s stltoobj.mlx" -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link1.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Part__Feature_001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.062500 0.062500 0.062500 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link2.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl Part__Feature024 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.125000 0.125000 0.125000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_round_peg_12mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.007 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_round_peg_16mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.008 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_round_peg_4mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.005 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_round_peg_8mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.006 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_round_peg_12mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.015 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_round_peg_16mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.016 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_round_peg_4mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.013 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_round_peg_8mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.014 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_rectangular_peg_12mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.003 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_rectangular_peg_16mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.004 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_rectangular_peg_4mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.001 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_insert_rectangular_peg_8mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.002 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_rectangular_peg_12mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.011 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_rectangular_peg_16mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.012 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_rectangular_peg_4mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.009 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_pegs/industreal_tray_pick_rectangular_peg_8mm.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl Material.010 5 | Ns 250.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.500000 0.500000 0.500000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 2 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.23.5 2 | rtree==1.1.0 3 | imageio==2.33.1 4 | moviepy==1.0.3 5 | meson 6 | ray 7 | submitit 8 | open3d 9 | wandb 10 | torch==2.1.1 11 | torchvision==0.16.1 12 | pyrealsense2 13 | h5py 14 | lxml 15 | geometrout==0.0.3.4 16 | kornia 17 | meshcat 18 | opencv-python 19 | pycocotools 20 | matplotlib 21 | autolab_core 22 | pytorch-kinematics 23 | hydra-core>=1.2 24 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_gears/industreal_gear_base.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl 0.615686_0.811765_0.929412_0.000000_0.000000 5 | Ns 0.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.015996 0.258183 0.015996 8 | Ks 0.000000 0.000000 0.000000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 1 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_gears/industreal_gear_large.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl 0.752941_0.752941_0.752941_0.000000_0.000000 5 | Ns 0.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.000000 0.000000 0.000000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 1 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_franka_table/industreal_franka_finger_coll_subdiv_6x.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 1 3 | 4 | newmtl 0.980392_0.713725_0.003922_0.000000_0.000000 5 | Ns 0.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.980400 0.713700 0.003922 8 | Ks 0.000000 0.000000 0.000000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 1 13 | -------------------------------------------------------------------------------- /manipgen/real_world/utils/constants.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | FRANKA_LOWER_LIMITS = np.array([-2.8973, -1.7628, -2.8973, -3.0718, -2.8973, 0.05, -2.8973]) 5 | FRANKA_UPPER_LIMITS = np.array([2.8973, 1.7628, 2.8973, -0.0698, 2.8973, 3.75, 2.8973]) 6 | FRANKA_VELOCITY_LIMIT = np.array([2.1750, 2.1750, 2.1750, 2.1750, 2.6100, 2.6100, 2.6100]) 7 | FRANKA_ACCELERATION_LIMIT = np.array([15.0, 7.5, 10.0, 12.5, 15.0, 20.0, 20.0]) -------------------------------------------------------------------------------- /manipgen/config/dagger/pick.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/pick_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: True # include segmantation mask of first frame in obs 12 | multitask: False -------------------------------------------------------------------------------- /manipgen/config/dagger/open.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/open_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: False # include segmantation mask of first frame in obs 12 | multitask: False 13 | -------------------------------------------------------------------------------- /manipgen/config/dagger/close.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/close_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: False # include segmantation mask of first frame in obs 12 | multitask: False 13 | -------------------------------------------------------------------------------- /manipgen/config/dagger/pick_cube.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/pick_cube_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: False # include segmantation mask of first frame in obs 12 | multitask: False -------------------------------------------------------------------------------- /manipgen/config/dagger/place.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/place_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: False # include segmantation mask of first frame in obs 12 | multitask: False 13 | -------------------------------------------------------------------------------- /manipgen/config/dagger/place_cube.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/place_cube_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: False # include segmantation mask of first frame in obs 12 | multitask: False -------------------------------------------------------------------------------- /manipgen/config/dagger/grasp_handle.yaml: -------------------------------------------------------------------------------- 1 | dataset_path: datasets/grasp_handle_depth 2 | batch_size: None # defaults to value in student robomimic cfg 3 | lr: None # defaults to value in student robomimic cfg 4 | deterministic_expert: True 5 | buffer_size: 1000 6 | num_learning_iterations: 2000 7 | num_learning_epochs: 5 8 | num_transitions_per_iter: 32 9 | student_cfg_path: config/dagger/robomimic/template.json 10 | visual_obs_type: depth # 'depth' or 'pcd' 11 | use_seg_obs: False # include segmantation mask of first frame in obs 12 | multitask: False 13 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/yaml/industreal_asset_info_franka_table.yaml: -------------------------------------------------------------------------------- 1 | franka_hand_length: 0.0584 # distance from origin of hand to origin of finger 2 | franka_finger_length: 0.053671 # distance from origin of finger to bottom of fingerpad 3 | franka_fingerpad_length: 0.017608 # distance from top of inner surface of fingerpad to bottom of inner surface of fingerpad 4 | franka_gripper_width_max: 0.080 # maximum opening width of gripper 5 | 6 | table_depth: 1.40 # depth of table 7 | table_width: 1.40 # width of table 8 | table_height: 1.04 # height of table 9 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/reach_test.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymtorch 2 | 3 | import torch 4 | 5 | from manipgen.utils.policy_testers.base_tester import BaseTester 6 | 7 | 8 | class FrankaReachTester(BaseTester): 9 | def __init__(self, env, policy, num_steps, task_name): 10 | env.init_states = None 11 | super().__init__(env, policy, num_steps, task_name) 12 | self.num_pre_steps = 0 13 | 14 | def pre_steps(self, obs, keep_runs): 15 | self.env.pre_steps(obs, keep_runs, self.num_pre_steps) 16 | return obs, keep_runs 17 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/finger.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 2 3 | 4 | newmtl Part__Feature001_006 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.901961 0.921569 0.929412 8 | Ks 0.250000 0.250000 0.250000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Feature_007 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.250980 0.250980 0.250980 18 | Ks 0.250000 0.250000 0.250000 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_gear_base.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_gear_large.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_gear_small.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_gear_medium.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_gears/industreal_gear_medium.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl 0.752941_0.752941_0.752941_0.000000_0.000000 5 | Ns 0.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.000000 0.000000 0.000000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 1 13 | 14 | newmtl 0.752941_0.752941_0.752941_0.000000_0.001 15 | Ns 0.000000 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.955973 0.871366 0.791298 18 | Ks 0.000000 0.000000 0.000000 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.450000 21 | d 1.000000 22 | illum 1 23 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_gears/industreal_gear_small.mtl: -------------------------------------------------------------------------------- 1 | # Blender 3.5.1 MTL File: 'None' 2 | # www.blender.org 3 | 4 | newmtl 0.752941_0.752941_0.752941_0.000000_0.000000 5 | Ns 0.000000 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.955973 0.871366 0.791298 8 | Ks 0.000000 0.000000 0.000000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.450000 11 | d 1.000000 12 | illum 1 13 | 14 | newmtl 0.752941_0.752941_0.752941_0.000000_0.002 15 | Ns 0.000000 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.955973 0.871366 0.791298 18 | Ks 0.000000 0.000000 0.000000 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.450000 21 | d 1.000000 22 | illum 1 23 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_peg_4mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_peg_8mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/mesh/industreal_franka_table/industreal_franka_finger_rubber_viz.mtl: -------------------------------------------------------------------------------- 1 | newmtl 0.301961_0.301961_0.301961_0.000000_0.000000 2 | Ka 0.0000 0.0000 0.0000 3 | Kd 0.302 0.302 0.302 4 | Ks 0.0000 0.0000 0.0000 5 | Tf 0.0000 0.0000 0.0000 6 | d 1 7 | Ns 0 8 | newmtl 0.752941_0.752941_0.752941_0.000000_0.000000 9 | Ka 0.0000 0.0000 0.0000 10 | Kd 0.7529 0.7529 0.7529 11 | Ks 0.0000 0.0000 0.0000 12 | Tf 0.0000 0.0000 0.0000 13 | d 1 14 | Ns 0 15 | newmtl 0.917647_0.917647_0.917647_0.000000_0.000000 16 | Ka 0.0000 0.0000 0.0000 17 | Kd 0.9176 0.9176 0.9176 18 | Ks 0.0000 0.0000 0.0000 19 | Tf 0.0000 0.0000 0.0000 20 | d 1 21 | Ns 0 22 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_peg_12mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_peg_16mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_peg_4mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_peg_8mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_peg_12mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_peg_16mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/config/real/place.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - base 3 | - _self_ 4 | 5 | task_instance_config: 6 | motion: 7 | duration: 4.0 8 | 9 | control: 10 | prop_gains: [1000, 1000, 1000, 50, 50, 50] 11 | # NOTE: deriv_gains computed automatically as 2 * sqrt(prop_gains) 12 | mode: 13 | type: leaky_plai # options: nominal, plai, leaky_plai 14 | nominal: 15 | action_scale: null 16 | plai: 17 | action_scale: [0.0005, 0.0005, 0.0005, 0.001, 0.001, 0.001] 18 | leaky_plai: 19 | action_scale: [0.002, 0.002, 0.002, 0.05, 0.05, 0.05] 20 | pos_err_thresh: [0.02, 0.02, 0.02] 21 | rot_err_thresh: 4.0 # degrees 22 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_hole_4mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_hole_8mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_hole_12mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_round_hole_16mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/config/real/open.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - base 3 | - _self_ 4 | 5 | task_instance_config: 6 | motion: 7 | duration: 8.0 8 | 9 | control: 10 | prop_gains: [1500, 1500, 1000, 50, 50, 50] 11 | # NOTE: deriv_gains computed automatically as 2 * sqrt(prop_gains) 12 | mode: 13 | type: leaky_plai # options: nominal, plai, leaky_plai 14 | nominal: 15 | action_scale: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] 16 | plai: 17 | action_scale: [0.001, 0.001, 0.001, 0.004, 0.004, 0.004] 18 | leaky_plai: 19 | action_scale: [0.003, 0.003, 0.003, 0.0005, 0.0005, 0.0005] 20 | pos_err_thresh: [0.02, 0.02, 0.003] 21 | rot_err_thresh: 0.005 # degrees 22 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_hole_4mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_hole_8mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/config/real/close.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - base 3 | - _self_ 4 | 5 | task_instance_config: 6 | motion: 7 | duration: 6.0 8 | 9 | control: 10 | prop_gains: [1000, 1000, 1000, 50, 50, 50] 11 | # NOTE: deriv_gains computed automatically as 2 * sqrt(prop_gains) 12 | mode: 13 | type: leaky_plai # options: nominal, plai, leaky_plai 14 | nominal: 15 | action_scale: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] 16 | plai: 17 | action_scale: [0.001, 0.001, 0.001, 0.004, 0.004, 0.004] 18 | leaky_plai: 19 | action_scale: [0.005, 0.005, 0.003, 0.0005, 0.0005, 0.0005] 20 | pos_err_thresh: [0.03, 0.03, 0.003] 21 | rot_err_thresh: 0.005 # degrees 22 | -------------------------------------------------------------------------------- /manipgen/config/real/grasp_handle.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - base 3 | - _self_ 4 | 5 | task_instance_config: 6 | motion: 7 | duration: 6.0 8 | 9 | control: 10 | prop_gains: [1000, 1000, 1000, 50, 50, 50] 11 | # NOTE: deriv_gains computed automatically as 2 * sqrt(prop_gains) 12 | mode: 13 | type: leaky_plai # options: nominal, plai, leaky_plai 14 | nominal: 15 | action_scale: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01] 16 | plai: 17 | action_scale: [0.001, 0.001, 0.001, 0.004, 0.004, 0.004] 18 | leaky_plai: 19 | action_scale: [0.002, 0.002, 0.002, 0.004, 0.004, 0.004] 20 | pos_err_thresh: [0.02, 0.02, 0.02] 21 | rot_err_thresh: 2.0 # degrees 22 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_hole_12mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_rectangular_hole_16mm.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /manipgen/config/real/pick.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - base 3 | - _self_ 4 | 5 | policy_config: 6 | dagger: 7 | use_seg_obs: True 8 | 9 | task_instance_config: 10 | motion: 11 | duration: 5.0 12 | 13 | control: 14 | prop_gains: [1000, 1000, 1000, 50, 50, 50] 15 | # NOTE: deriv_gains computed automatically as 2 * sqrt(prop_gains) 16 | mode: 17 | type: leaky_plai # options: nominal, plai, leaky_plai 18 | nominal: 19 | action_scale: null 20 | plai: 21 | action_scale: [0.0005, 0.0005, 0.0005, 0.001, 0.001, 0.001] 22 | leaky_plai: 23 | action_scale: [0.002, 0.002, 0.002, 0.05, 0.05, 0.05] 24 | pos_err_thresh: [0.02, 0.02, 0.02] 25 | rot_err_thresh: 4.0 # degrees 26 | -------------------------------------------------------------------------------- /manipgen/scripts/download_checkpoints.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir real_world/checkpoints/ 4 | 5 | wget https://huggingface.co/minliu01/ManipGen/resolve/main/pick.pth -O real_world/checkpoints/pick.pth 6 | wget https://huggingface.co/minliu01/ManipGen/resolve/main/place.pth -O real_world/checkpoints/place.pth 7 | wget https://huggingface.co/minliu01/ManipGen/resolve/main/grasp_handle.pth -O real_world/checkpoints/grasp_handle.pth 8 | wget https://huggingface.co/minliu01/ManipGen/resolve/main/close.pth -O real_world/checkpoints/close.pth 9 | wget https://huggingface.co/minliu01/ManipGen/resolve/main/open.pth -O real_world/checkpoints/open.pth 10 | wget https://huggingface.co/minliu01/ManipGen/resolve/main/neural_mp.pth -O real_world/checkpoints/neural_mp.pth 11 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/hande/finger_1-collision-simple.obj: -------------------------------------------------------------------------------- 1 | # cube.obj 2 | # 3 | o cube 4 | v -0.0105 0.025 0.047 5 | v 0.0105 0.025 0.047 6 | v -0.0105 0.035 0.047 7 | v 0.0105 0.035 0.047 8 | v -0.0105 0.035 0.000 9 | v 0.0105 0.035 0.000 10 | v -0.0105 0.025 0.000 11 | v 0.0105 0.025 0.000 12 | vn 0.000000 0.000000 1.000000 13 | vn 0.000000 1.000000 0.000000 14 | vn 0.000000 0.000000 -1.000000 15 | vn 0.000000 -1.000000 0.000000 16 | vn 1.000000 0.000000 0.000000 17 | vn -1.000000 0.000000 0.000000 18 | f 1//1 2//1 3//1 19 | f 3//1 2//1 4//1 20 | f 3//2 4//2 5//2 21 | f 5//2 4//2 6//2 22 | f 5//3 6//3 7//3 23 | f 7//3 6//3 8//3 24 | f 7//4 8//4 1//4 25 | f 1//4 8//4 2//4 26 | f 2//5 8//5 4//5 27 | f 4//5 8//5 6//5 28 | f 7//6 1//6 5//6 29 | f 5//6 1//6 3//6 -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/hande/finger_2-collision-simple.obj: -------------------------------------------------------------------------------- 1 | # cube.obj 2 | # 3 | o cube 4 | v -0.0105 -0.035 0.047 5 | v 0.0105 -0.035 0.047 6 | v -0.0105 -0.025 0.047 7 | v 0.0105 -0.025 0.047 8 | v -0.0105 -0.025 0.000 9 | v 0.0105 -0.025 0.000 10 | v -0.0105 -0.035 0.000 11 | v 0.0105 -0.035 0.000 12 | vn 0.000000 0.000000 1.000000 13 | vn 0.000000 1.000000 0.000000 14 | vn 0.000000 0.000000 -1.000000 15 | vn 0.000000 -1.000000 0.000000 16 | vn 1.000000 0.000000 0.000000 17 | vn -1.000000 0.000000 0.000000 18 | f 1//1 2//1 3//1 19 | f 3//1 2//1 4//1 20 | f 3//2 4//2 5//2 21 | f 5//2 4//2 6//2 22 | f 5//3 6//3 7//3 23 | f 7//3 6//3 8//3 24 | f 7//4 8//4 1//4 25 | f 1//4 8//4 2//4 26 | f 2//5 8//5 4//5 27 | f 4//5 8//5 6//5 28 | f 7//6 1//6 5//6 29 | f 5//6 1//6 3//6 -------------------------------------------------------------------------------- /manipgen/config/real/perception.yaml: -------------------------------------------------------------------------------- 1 | calibrate_extrinsics: 2 | camera: 3 | image_width: 1920 4 | image_height: 1080 5 | tag: 6 | type: tagStandard52h13 7 | length: 0.050 # [meters] 8 | active_pixel_ratio: 0.6 # For details, see https://github.com/AprilRobotics/apriltag/wiki/AprilTag-User-Guide 9 | robot: 10 | home_joint_angles: [0.02550676, -0.25173378, -0.3518326, -2.5239587, -0.11268669, 2.2990525, 0.5429185] 11 | num_goals: 1024 12 | goal_pos_bounds: 13 | x: [0.4, 0.6] # 0.35 can cause self-collision 14 | y: [-0.15, 0.15] 15 | z: [0.2, 0.45] 16 | pos_to_point_at: [0.5069, 0.0, 0.0] 17 | tag_detection: 18 | num_detections: 16 19 | display_images: True 20 | output: 21 | file_name: extrinsics_industreallib.json 22 | 23 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | wget https://github.com/git-lfs/git-lfs/releases/download/v3.4.1/git-lfs-linux-amd64-v3.4.1.tar.gz 2 | tar -xvf git-lfs-linux-amd64-v3.4.1.tar.gz 3 | ./git-lfs-3.4.1/install.sh 4 | rm -rf git-lfs-linux-amd64-v3.4.1.tar.gz 5 | rm -rf git-lfs-3.4.1 6 | git lfs install 7 | git lfs track "*.plugin" 8 | ./isaacgym/create_conda_env_rlgpu.sh # make sure you have mamba, this will build your conda env very fast 9 | pip install -e isaacgym/python/ 10 | pip install -e IsaacGymEnvs 11 | pip install -e industreallib 12 | pip install -e rl_games 13 | pip install -e Depth-Anything 14 | pip install -e . # install manipgen 15 | pip install -r requirements.txt 16 | pip install torch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 --index-url https://download.pytorch.org/whl/cu121 17 | pip install -e robomimic 18 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link5.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 3 3 | 4 | newmtl Part__Feature_002_004_003 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.015625 0.015625 0.015625 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Shell001_001_001_003 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.250000 0.250000 0.250000 18 | Ks 0.015625 0.015625 0.015625 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Shell_001_001_003 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 1.000000 1.000000 1.000000 28 | Ks 0.015625 0.015625 0.015625 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | -------------------------------------------------------------------------------- /manipgen/envs/__init__.py: -------------------------------------------------------------------------------- 1 | from manipgen.envs.franka_pick_cube import FrankaPickCube 2 | from manipgen.envs.franka_place_cube import FrankaPlaceCube 3 | from manipgen.envs.franka_pick import FrankaPick 4 | from manipgen.envs.franka_place import FrankaPlace 5 | from manipgen.envs.franka_grasp_handle import FrankaGraspHandle 6 | from manipgen.envs.franka_open import FrankaOpen 7 | from manipgen.envs.franka_close import FrankaClose 8 | from manipgen.envs.franka_reach import FrankaReach 9 | 10 | environments = {} 11 | 12 | environments["pick_cube"] = FrankaPickCube 13 | environments["place_cube"] = FrankaPlaceCube 14 | environments["pick"] = FrankaPick 15 | environments["place"] = FrankaPlace 16 | environments["grasp_handle"] = FrankaGraspHandle 17 | environments["open"] = FrankaOpen 18 | environments["close"] = FrankaClose 19 | environments['reach'] = FrankaReach 20 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/__init__.py: -------------------------------------------------------------------------------- 1 | from manipgen.utils.policy_testers.base_tester import BaseTester 2 | from manipgen.utils.policy_testers.reach_test import FrankaReachTester 3 | from manipgen.utils.policy_testers.pick_cube_tester import FrankaPickCubeTester 4 | from manipgen.utils.policy_testers.place_cube_tester import FrankaPlaceCubeTester 5 | from manipgen.utils.policy_testers.place_tester import FrankaPlaceTester 6 | from manipgen.utils.policy_testers.grasp_handle_tester import FrankaGraspHandleTester 7 | from manipgen.utils.policy_testers.open_tester import FrankaOpenTester 8 | 9 | testers = { 10 | "pick_cube": FrankaPickCubeTester, 11 | "place_cube": FrankaPlaceCubeTester, 12 | "pick": FrankaPickCubeTester, 13 | "place": FrankaPlaceTester, 14 | "grasp_handle": FrankaGraspHandleTester, 15 | "open": FrankaOpenTester, 16 | "close": FrankaOpenTester, 17 | "reach": FrankaReachTester, 18 | } 19 | -------------------------------------------------------------------------------- /manipgen/utils/initial_state_samplers/__init__.py: -------------------------------------------------------------------------------- 1 | from manipgen.utils.initial_state_samplers.pick_cube_sampler import ( 2 | FrankaPickCubePoseSampler, 3 | ) 4 | from manipgen.utils.initial_state_samplers.place_cube_sampler import ( 5 | FrankaPlaceCubePoseSampler, 6 | ) 7 | from manipgen.utils.initial_state_samplers.pick_sampler import ( 8 | FrankaPickPoseSampler, 9 | ) 10 | from manipgen.utils.initial_state_samplers.place_sampler import ( 11 | FrankaPlacePoseSampler, 12 | ) 13 | from manipgen.utils.initial_state_samplers.grasp_handle_sampler import ( 14 | FrankaGraspHandlePoseSampler, 15 | ) 16 | from manipgen.utils.initial_state_samplers.open_sampler import ( 17 | FrankaOpenPoseSampler, 18 | ) 19 | 20 | samplers = { 21 | "pick_cube": FrankaPickCubePoseSampler, 22 | "place_cube": FrankaPlaceCubePoseSampler, 23 | "pick": FrankaPickPoseSampler, 24 | "place": FrankaPlacePoseSampler, 25 | "grasp_handle": FrankaGraspHandlePoseSampler, 26 | "open": FrankaOpenPoseSampler, 27 | "close": FrankaOpenPoseSampler, 28 | } 29 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link4.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 4 3 | 4 | newmtl Part__Feature001_001_003_001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.007812 0.007812 0.007812 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Feature002_001_003_001 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.250980 0.250980 0.250980 18 | Ks 0.007812 0.007812 0.007812 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Part__Feature003_001_003_001 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 1.000000 1.000000 1.000000 28 | Ks 0.007812 0.007812 0.007812 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | 34 | newmtl Part__Feature_002_003_001 35 | Ns -1.960784 36 | Ka 1.000000 1.000000 1.000000 37 | Kd 1.000000 1.000000 1.000000 38 | Ks 0.007812 0.007812 0.007812 39 | Ke 0.000000 0.000000 0.000000 40 | Ni 1.000000 41 | d 1.000000 42 | illum 2 43 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link3.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 4 3 | 4 | newmtl Part__Feature001_010_001_002.001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 1.000000 1.000000 8 | Ks 0.007812 0.007812 0.007812 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Feature002_007_001_002.001 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 1.000000 1.000000 1.000000 18 | Ks 0.007812 0.007812 0.007812 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Part__Feature003_004_001_002.001 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 1.000000 1.000000 1.000000 28 | Ks 0.007812 0.007812 0.007812 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | 34 | newmtl Part__Feature_001_001_001_002.001 35 | Ns -1.960784 36 | Ka 1.000000 1.000000 1.000000 37 | Kd 0.250980 0.250980 0.250980 38 | Ks 0.007812 0.007812 0.007812 39 | Ke 0.000000 0.000000 0.000000 40 | Ni 1.000000 41 | d 1.000000 42 | illum 2 43 | -------------------------------------------------------------------------------- /manipgen/real_world/vlm_planning/prompts/planning_examples.py: -------------------------------------------------------------------------------- 1 | """ 2 | ### Plan examples ### 3 | Here are some examples of input tasks and the corresponding plans that you should generate and the type of format we are looking for. 4 | MAKE SURE YOU USE THE SAME FORMAT AS THE EXAMPLES BELOW FOR YOUR OUTPUT PLANS. 5 | Example 1: 6 | Task: Put away all remaining items on the table in the shelf. 7 | Output: 8 | [ 9 | ("spoon", "pick"), 10 | ("shelf", "place"), 11 | ("turquoise pot", "pick"), 12 | ("shelf", "place"), 13 | ("plastic bottle", "pick"), 14 | ("shelf", "place"), 15 | ] 16 | 17 | Example 2: 18 | Task: Clean up the table by putting everything that is not in the bin into the bin. 19 | Output: 20 | [ 21 | ("yellow plastic bottle", "pick"), 22 | ("bin", "place"), 23 | ("spoon", "pick"), 24 | ("bin", "place"), 25 | ("turquoise pot", "pick"), 26 | ("bin", "place"), 27 | ("blue plastic bottle", "pick"), 28 | ("bin", "place"), 29 | ("white toy kettle", "pick"), 30 | ("bin", "place"), 31 | ("white roll of tape", "pick"), 32 | ("bin", "place"), 33 | ] 34 | """ -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/hand.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 5 3 | 4 | newmtl Part__Feature001_008_005 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.250980 0.250980 0.250980 8 | Ks 0.007812 0.007812 0.007812 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Feature002_005_005 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.901961 0.921569 0.929412 18 | Ks 0.015625 0.015625 0.015625 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Part__Feature005_001_005 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 1.000000 1.000000 1.000000 28 | Ks 0.015625 0.015625 0.015625 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | 34 | newmtl Part__Feature005_001_005_001 35 | Ns -1.960784 36 | Ka 1.000000 1.000000 1.000000 37 | Kd 0.901961 0.921569 0.929412 38 | Ks 0.015625 0.015625 0.015625 39 | Ke 0.000000 0.000000 0.000000 40 | Ni 1.000000 41 | d 1.000000 42 | illum 2 43 | 44 | newmtl Part__Feature_009_005 45 | Ns -1.960784 46 | Ka 1.000000 1.000000 1.000000 47 | Kd 0.250980 0.250980 0.250980 48 | Ks 0.015625 0.015625 0.015625 49 | Ke 0.000000 0.000000 0.000000 50 | Ni 1.000000 51 | d 1.000000 52 | illum 2 53 | -------------------------------------------------------------------------------- /manipgen/scripts/download_datasets.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Download ManipGen-UniDexGrasp 4 | if [ ! -d "assets/unidexgrasp" ]; then 5 | wget https://huggingface.co/datasets/minliu01/ManipGen-UniDexGrasp/resolve/main/manipgen-unidexgrasp_part_aa 6 | wget https://huggingface.co/datasets/minliu01/ManipGen-UniDexGrasp/resolve/main/manipgen-unidexgrasp_part_ab 7 | wget https://huggingface.co/datasets/minliu01/ManipGen-UniDexGrasp/resolve/main/manipgen-unidexgrasp_part_ac 8 | wget https://huggingface.co/datasets/minliu01/ManipGen-UniDexGrasp/resolve/main/manipgen-unidexgrasp_part_ad 9 | wget https://huggingface.co/datasets/minliu01/ManipGen-UniDexGrasp/resolve/main/manipgen-unidexgrasp_part_ae 10 | wget https://huggingface.co/datasets/minliu01/ManipGen-UniDexGrasp/resolve/main/manipgen-unidexgrasp_part_af 11 | 12 | cat manipgen-unidexgrasp_part_a* > manipgen-unidexgrasp.tar.gz 13 | tar -xvf manipgen-unidexgrasp.tar.gz -C assets/ 14 | 15 | rm manipgen-unidexgrasp_part_a* 16 | rm manipgen-unidexgrasp.tar.gz 17 | fi 18 | 19 | # Download ManipGen-PartNet 20 | if [ ! -d "assets/partnet" ]; then 21 | wget https://huggingface.co/datasets/minliu01/ManipGen-PartNet/resolve/main/manipgen-partnet.tar.gz 22 | tar -xvf manipgen-partnet.tar.gz -C assets/ 23 | rm manipgen-partnet.tar.gz 24 | fi 25 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "IsaacGymEnvs"] 2 | path = IsaacGymEnvs 3 | url = https://github.com/mihdalal/IsaacGymEnvs 4 | branch = skills_planning 5 | [submodule "rl_games"] 6 | path = rl_games 7 | url = https://github.com/mihdalal/rl_games 8 | branch = skills_planning 9 | [submodule "vkdevicechooser"] 10 | path = vkdevicechooser 11 | url = https://github.com/aejsmith/vkdevicechooser.git 12 | [submodule "Grounded-Segment-Anything"] 13 | path = Grounded-Segment-Anything 14 | url = https://github.com/IDEA-Research/Grounded-Segment-Anything.git 15 | [submodule "industreallib"] 16 | path = industreallib 17 | url = git@github.com:mihdalal/industreallib.git 18 | [submodule "pointnet2_ops"] 19 | path = pointnet2_ops 20 | url = git@github.com:upriyam-cmu/pointnet2_ops.git 21 | [submodule "tracikpy"] 22 | path = tracikpy 23 | url = git@github.com:mjd3/tracikpy.git 24 | [submodule "manipgen_robomimic"] 25 | path = manipgen_robomimic 26 | url = https://github.com/mihdalal/robomimic 27 | branch = manipgen_robomimic 28 | [submodule "robofin"] 29 | path = robofin 30 | url = https://github.com/mihdalal/robofin 31 | branch = manipgen 32 | [submodule "frankapy"] 33 | path = frankapy 34 | url = https://github.com/mihdalal/frankapy.git 35 | branch = skills_planning 36 | [submodule "robomimic"] 37 | path = robomimic 38 | url = git@github.com:mihdalal/robomimic.git 39 | branch = neural_mp_skills_planning 40 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/urdf/industreal_franka_urdf_changelog.txt: -------------------------------------------------------------------------------- 1 | Command to generate original xacro: 2 | xacro $(rospack find franka_description)/robots/panda/panda.urdf.xacro arm:=panda hand:=true gazebo:=true 3 | 4 | Changes: 5 | - Removed dummy sub-links 6 | - Removed Gazebo fields, safety fields, and mimic fields 7 | - For dynamics fields, kept only damping and friction parameters, which were set to 0.0 8 | - Verified that joint limits, joint velocity limits, and torque limits match Franka spec sheet 9 | - Reduced joint effort limits on hand to 70 (max continuous force according to Franka spec sheet) 10 | - Reduced joint velocity limits on hand to 0.050 (max velocity according to spec sheet) 11 | - Removed joint limit on joint 7 to allow continuous rotation for screwing tasks 12 | - Removed link 8 (dummy link), removed joint 8 (connects joint 7 to joint 8), and modified panda_hand_joint 13 | (connected joint 7 to hand instead of joint 8 to hand; updated transform to include offset of original joint 8) 14 | - Renamed panda_hand_tcp joint and panda_hand_tcp link (dummy link); added finite mass and inertia 15 | - Replaced *.dae* with *.obj*, as *.dae* files are actually never used by Gym 16 | - Updated visual geometry (original fingers) and collision geometry (boxes) on fingers to match our custom fingertips; 17 | updated mass, COM, and inertia based on mass measurements and Onshape COM and inertia calculations 18 | - Added friction to gripper joints to mitigate non-mirroring artifact -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/stltoobj.mlx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/base_pcd.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "num_data_workers": 0, 4 | "hdf5_cache_mode": null, 5 | "hdf5_use_swmr": true, 6 | "hdf5_load_next_obs": false, 7 | "hdf5_normalize_obs": false, 8 | "hdf5_filter_key": null, 9 | "hdf5_validation_filter_key": null, 10 | "dataset_keys": [ 11 | "actions" 12 | ], 13 | "goal_mode": null 14 | }, 15 | "observation": { 16 | "modalities": { 17 | "obs": { 18 | "low_dim": [ 19 | "state" 20 | ], 21 | "pcd": [ 22 | "visual" 23 | ] 24 | } 25 | }, 26 | "encoder": { 27 | "low_dim": { 28 | "core_class": null, 29 | "core_kwargs": {}, 30 | "obs_randomizer_class": null, 31 | "obs_randomizer_kwargs": {} 32 | }, 33 | "rgb": { 34 | "core_class": "VisualCore", 35 | "core_kwargs": {}, 36 | "obs_randomizer_class": null, 37 | "obs_randomizer_kwargs": {} 38 | }, 39 | "pcd": { 40 | "core_class": "PCDCore", 41 | "core_kwargs": { 42 | "backbone_kwargs": { 43 | "encoder_size": "transic", 44 | "n_features": 1 45 | } 46 | }, 47 | "obs_randomizer_class": null, 48 | "obs_randomizer_kwargs": {} 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /manipgen/envs/franka_close.py: -------------------------------------------------------------------------------- 1 | from manipgen.envs.franka_open import FrankaOpen 2 | from manipgen.envs.franka_grasp_handle import FrankaGraspHandle 3 | from isaacgymenvs.utils.torch_jit_utils import quat_conjugate, quat_rotate 4 | import torch 5 | import numpy as np 6 | 7 | class FrankaClose(FrankaOpen): 8 | """We only need to change the initial and target object dof positions for close task. 9 | """ 10 | def _set_object_init_and_target_dof_pos(self): 11 | """ Set initial and target object dof positions for close task """ 12 | self.rest_object_dof_pos = self.object_dof_upper_limits.clone() 13 | self.target_object_dof_pos = self.object_dof_lower_limits.clone() 14 | 15 | def reset_idx(self, env_ids: torch.Tensor, validation_set: bool = False, switch_object: bool = False, init_states = None): 16 | FrankaGraspHandle.reset_idx(self, env_ids, validation_set, switch_object, init_states) 17 | 18 | # record position of fingertip center in the handle frame 19 | self.init_fingertip_centered_pos_local[:] = quat_rotate( 20 | quat_conjugate(self.handle_quat), self.fingertip_centered_pos - self.handle_pos 21 | ) 22 | self.prev_fingertip_centered_pos_local = quat_rotate( 23 | quat_conjugate(self.handle_quat), self.fingertip_centered_pos - self.handle_pos 24 | ) 25 | 26 | if not self.sample_mode and self.object_meta["type"] == "door" and self.cfg_env.rl.partial_close_for_door: 27 | self.target_object_dof_pos = self.init_object_dof_pos - np.deg2rad(self.cfg_env.rl.partial_close_degree) 28 | self.target_object_dof_pos = torch.clamp(self.target_object_dof_pos, self.object_dof_lower_limits, self.object_dof_upper_limits) 29 | -------------------------------------------------------------------------------- /manipgen/config/train/open.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_open 38 | full_experiment_name: franka_open 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: True 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:500,${....max_iterations}} 57 | early_stop: 100 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.000 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/config/train/pick.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_pick 38 | full_experiment_name: franka_pick 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: True 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:800,${....max_iterations}} 57 | early_stop: 200 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.000 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/config/train/close.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_close 38 | full_experiment_name: franka_close 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: True 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:300,${....max_iterations}} 57 | early_stop: 100 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.000 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/config/train/place.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_place 38 | full_experiment_name: franka_place 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: True 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:500,${....max_iterations}} 57 | early_stop: 200 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.000 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/config/train/reach.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_reach 38 | full_experiment_name: franka_reach 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: True 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:1000,${....max_iterations}} 57 | early_stop: 1000 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.000 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/config/train/place_cube.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_place_cube 38 | full_experiment_name: franka_place_cube 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: False 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:400,${....max_iterations}} 57 | early_stop: 200 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.0 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/scripts/urdf_templates/pseudo_articulated_object.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /manipgen/config/train/grasp_handle.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | seed: ${...seed} 3 | algo: 4 | name: a2c_continuous_sp 5 | 6 | model: 7 | name: continuous_a2c_logstd 8 | 9 | network: 10 | name: actor_critic 11 | separate: False 12 | 13 | space: 14 | continuous: 15 | mu_activation: None 16 | sigma_activation: None 17 | mu_init: 18 | name: default 19 | sigma_init: 20 | name: const_initializer 21 | val: 0 22 | fixed_sigma: True 23 | mlp: 24 | units: [256, 128, 64] 25 | activation: elu 26 | d2rl: False 27 | 28 | initializer: 29 | name: default 30 | regularizer: 31 | name: None 32 | 33 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 34 | load_path: ${...checkpoint} # path to the checkpoint to load 35 | 36 | config: 37 | name: franka_grasp_handle 38 | full_experiment_name: franka_grasp_handle 39 | env_name: rlgpu 40 | ppo: True 41 | mixed_precision: True 42 | normalize_input: True 43 | normalize_value: True 44 | value_bootstrap: True 45 | num_actors: ${....task.env.numEnvs} 46 | reward_shaper: 47 | scale_value: 1.0 48 | normalize_advantage: True 49 | gamma: 0.99 50 | tau: 0.95 51 | learning_rate: 5e-4 52 | lr_schedule: adaptive 53 | schedule_type: standard 54 | kl_threshold: 0.008 55 | score_to_win: 10000 56 | max_epochs: ${resolve_default:500,${....max_iterations}} 57 | early_stop: 100 58 | save_best_after: 100 59 | save_frequency: 50 60 | print_stats: True 61 | grad_norm: 1.0 62 | entropy_coef: 0.000 63 | truncate_grads: True 64 | e_clip: 0.2 65 | horizon_length: 32 66 | minibatch_size: 16384 67 | mini_epochs: 5 68 | critic_coef: 4 69 | clip_value: True 70 | seq_len: 4 71 | bounds_loss_coef: 0.0001 72 | -------------------------------------------------------------------------------- /manipgen/config/train/pick_cube.yaml: -------------------------------------------------------------------------------- 1 | name: 'pick_cube' 2 | params: 3 | seed: ${...seed} 4 | algo: 5 | name: a2c_continuous_sp 6 | 7 | model: 8 | name: continuous_a2c_logstd 9 | 10 | network: 11 | name: actor_critic 12 | separate: False 13 | 14 | space: 15 | continuous: 16 | mu_activation: None 17 | sigma_activation: None 18 | mu_init: 19 | name: default 20 | sigma_init: 21 | name: const_initializer 22 | val: 0 23 | fixed_sigma: True 24 | mlp: 25 | units: [256, 128, 64] 26 | activation: elu 27 | d2rl: False 28 | 29 | initializer: 30 | name: default 31 | regularizer: 32 | name: None 33 | 34 | load_checkpoint: ${if:${...checkpoint},True,False} # flag which sets whether to load the checkpoint 35 | load_path: ${...checkpoint} # path to the checkpoint to load 36 | 37 | config: 38 | name: franka_pick_cube 39 | full_experiment_name: franka_pick_cube 40 | env_name: rlgpu 41 | ppo: True 42 | mixed_precision: True 43 | normalize_input: True 44 | normalize_value: True 45 | value_bootstrap: True 46 | num_actors: ${....task.env.numEnvs} 47 | reward_shaper: 48 | scale_value: 1.0 49 | normalize_advantage: True 50 | gamma: 0.99 51 | tau: 0.95 52 | learning_rate: 5e-4 53 | lr_schedule: adaptive 54 | schedule_type: standard 55 | kl_threshold: 0.008 56 | score_to_win: 10000 57 | max_epochs: ${resolve_default:300,${....max_iterations}} 58 | early_stop: 200 59 | save_best_after: 100 60 | save_frequency: 50 61 | print_stats: True 62 | grad_norm: 1.0 63 | entropy_coef: 0.0 64 | truncate_grads: True 65 | e_clip: 0.2 66 | horizon_length: 32 67 | minibatch_size: 16384 68 | mini_epochs: 5 69 | critic_coef: 4 70 | clip_value: True 71 | seq_len: 4 72 | bounds_loss_coef: 0.0001 73 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link7.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 8 3 | 4 | newmtl Part__Mirroring001_004_002 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.250980 0.250980 0.250980 8 | Ks 0.015625 0.015625 0.015625 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Mirroring002_004_001 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.250980 0.250980 0.250980 18 | Ks 0.031250 0.031250 0.031250 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Part__Mirroring003_004_001 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 0.250980 0.250980 0.250980 28 | Ks 0.031250 0.031250 0.031250 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | 34 | newmtl Part__Mirroring004_004_002 35 | Ns -1.960784 36 | Ka 1.000000 1.000000 1.000000 37 | Kd 1.000000 1.000000 1.000000 38 | Ks 0.031250 0.031250 0.031250 39 | Ke 0.000000 0.000000 0.000000 40 | Ni 1.000000 41 | d 1.000000 42 | illum 2 43 | 44 | newmtl Part__Mirroring005_004_001 45 | Ns -1.960784 46 | Ka 1.000000 1.000000 1.000000 47 | Kd 0.250980 0.250980 0.250980 48 | Ks 0.031250 0.031250 0.031250 49 | Ke 0.000000 0.000000 0.000000 50 | Ni 1.000000 51 | d 1.000000 52 | illum 2 53 | 54 | newmtl Part__Mirroring006_004_001 55 | Ns -1.960784 56 | Ka 1.000000 1.000000 1.000000 57 | Kd 0.250980 0.250980 0.250980 58 | Ks 0.031250 0.031250 0.031250 59 | Ke 0.000000 0.000000 0.000000 60 | Ni 1.000000 61 | d 1.000000 62 | illum 2 63 | 64 | newmtl Part__Mirroring007_004_001 65 | Ns -1.960784 66 | Ka 1.000000 1.000000 1.000000 67 | Kd 0.250980 0.250980 0.250980 68 | Ks 0.031250 0.031250 0.031250 69 | Ke 0.000000 0.000000 0.000000 70 | Ni 1.000000 71 | d 1.000000 72 | illum 2 73 | 74 | newmtl Part__Mirroring_004_001 75 | Ns -1.960784 76 | Ka 1.000000 1.000000 1.000000 77 | Kd 0.898039 0.917647 0.929412 78 | Ks 0.031250 0.031250 0.031250 79 | Ke 0.000000 0.000000 0.000000 80 | Ni 1.000000 81 | d 1.000000 82 | illum 2 83 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/grasp_handle_tester.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymtorch 2 | 3 | import torch 4 | 5 | from manipgen.utils.rlgames_utils import get_actions 6 | from manipgen.utils.policy_testers.base_tester import BaseTester 7 | 8 | 9 | class FrankaGraspHandleTester(BaseTester): 10 | def __init__(self, env, policy, num_steps, task_name): 11 | super().__init__(env, policy, num_steps, task_name) 12 | 13 | def run_steps(self, run_pre_steps=True, is_deterministic=True): 14 | """ 15 | Args: 16 | run_pre_steps: if true, teleport franka to easy initial poses or take actions with hand-engineering code before running the test 17 | Returns: 18 | keep_runs: a tensor of shape (num_envs,) indicating whether the run is kept. A run might fail in pre_steps, which should not be counted 19 | success: a tensor of shape (num_envs,) indicating whether the run is successful 20 | images: a list of lists of images, each list of images is for one environment 21 | """ 22 | obs = self.env.reset() 23 | keep_runs = torch.ones(self.num_envs, device=self.device).bool() 24 | if run_pre_steps: 25 | assert False, "`pre_steps` is not implemented for this testers. Please specify initial states." 26 | 27 | images = [] 28 | for step in range(self.num_steps): 29 | actions = get_actions(obs, self.policy, is_deterministic=is_deterministic) 30 | obs, reward, done, info = self.env.step(actions) 31 | 32 | if step % 3 == 2: 33 | images.append(info["camera_images"]) 34 | 35 | hit_joint_limits = info["hit_joint_limits"] 36 | keep_runs = keep_runs & ~hit_joint_limits 37 | 38 | teleportation_images = self.env.hardcode_control(get_camera_images=True) 39 | teleportation_images = [teleportation_images[i] for i in range(0, len(teleportation_images), 3)] 40 | images.extend(teleportation_images) 41 | 42 | success = self.env._check_grasp_success() 43 | 44 | self.env.reset_idx(torch.arange(self.num_envs, device=self.device)) 45 | 46 | return keep_runs, success, images -------------------------------------------------------------------------------- /manipgen/real_world/scripts/get_camera_intrinsics.py: -------------------------------------------------------------------------------- 1 | """ 2 | Get device serial number and intrinsics matrix for RealSense cameras. 3 | """ 4 | 5 | import numpy as np 6 | import pyrealsense2 as rs 7 | from argparse import ArgumentParser 8 | 9 | def get_camera_intrinsics(serial_number): 10 | # Initialize the RealSense pipeline 11 | pipeline = rs.pipeline() 12 | config = rs.config() 13 | 14 | # Enable depth and color streams with the specified serial number 15 | config.enable_device(serial_number) 16 | config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30) 17 | config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30) 18 | 19 | # Start the pipeline 20 | pipeline.start(config) 21 | 22 | try: 23 | # Wait for the first frame 24 | frames = pipeline.wait_for_frames() 25 | 26 | # Get the depth and color intrinsics 27 | depth_intrinsics = frames.get_depth_frame().profile.as_video_stream_profile().intrinsics 28 | 29 | # Construct the camera intrinsics matrix 30 | intrinsics_matrix = np.array( 31 | [ 32 | [depth_intrinsics.fx, 0, depth_intrinsics.ppx], 33 | [0, depth_intrinsics.fy, depth_intrinsics.ppy], 34 | [0, 0, 1], 35 | ] 36 | ) 37 | 38 | finally: 39 | # Stop the pipeline and release resources 40 | pipeline.stop() 41 | 42 | return intrinsics_matrix 43 | 44 | 45 | if __name__ == "__main__": 46 | parser = ArgumentParser() 47 | args = parser.parse_args() 48 | device_ls = [] 49 | for cam in rs.context().query_devices(): 50 | device_ls.append(cam.get_info(rs.camera_info(1))) 51 | device_ls.sort() 52 | 53 | for i, device in enumerate(device_ls): 54 | try: 55 | intrinsics = get_camera_intrinsics(device) 56 | # print intrinsics matrix as proper numpy array with commas 57 | print() 58 | print(f"Device {i}: {device}") 59 | print(f"Intrinsics:\n", repr(intrinsics)) 60 | print() 61 | except: 62 | print(f"Failed to get intrinsics for device {i}: {device}") 63 | continue 64 | -------------------------------------------------------------------------------- /manipgen/object_lists/articulated/0.txt: -------------------------------------------------------------------------------- 1 | drawer-00001-0-0 2 | door-00000-0-0 3 | drawer-00000-0-0 4 | door-47227-4-0 5 | drawer-46619-12-0 6 | door-38340-23-0 7 | drawer-12045-7-0 8 | door-45653-16-0 9 | drawer-45194-19-0 10 | door-48000-16-0 11 | drawer-46102-63-0 12 | door-40339-17-0 13 | drawer-44781-23-0 14 | door-40826-13-0 15 | drawer-3472-6-0 16 | door-7261-6-0 17 | drawer-45664-13-0 18 | door-46146-21-0 19 | drawer-46113-34-0 20 | door-47847-13-0 21 | drawer-48859-4-0 22 | door-48254-20-0 23 | drawer-7163-7-0 24 | door-10716-5-0 25 | drawer-47163-4-0 26 | door-7147-5-0 27 | drawer-47183-23-0 28 | door-46202-26-0 29 | drawer-46033-4-0 30 | door-46189-16-0 31 | drawer-46486-10-0 32 | door-11304-5-0 33 | drawer-46808-19-0 34 | door-45428-29-0 35 | drawer-11211-4-0 36 | door-46976-23-0 37 | drawer-48170-10-0 38 | door-45910-19-0 39 | drawer-21238-17-0 40 | door-45213-4-0 41 | drawer-46571-16-0 42 | door-47701-4-0 43 | drawer-7319-6-0 44 | door-47327-21-0 45 | drawer-49016-3-0 46 | door-46754-10-0 47 | drawer-23040-10-0 48 | door-47853-4-0 49 | drawer-45440-10-0 50 | door-10147-5-0 51 | drawer-25648-14-0 52 | door-45913-21-0 53 | drawer-46955-18-0 54 | door-46480-22-0 55 | drawer-47293-4-0 56 | door-46313-19-0 57 | drawer-46017-10-0 58 | door-45703-4-0 59 | drawer-8874-5-0 60 | door-45909-5-0 61 | drawer-47126-13-0 62 | door-11712-7-0 63 | drawer-45803-11-0 64 | door-47808-4-0 65 | drawer-34219-8-0 66 | door-46117-16-0 67 | drawer-45607-16-0 68 | door-46663-20-0 69 | drawer-48499-4-0 70 | door-46486-10-0 71 | drawer-12254-5-0 72 | door-46949-15-0 73 | drawer-46573-10-0 74 | door-8982-4-0 75 | drawer-45855-19-0 76 | door-46699-30-0 77 | drawer-46191-21-0 78 | door-40951-23-0 79 | drawer-45238-25-0 80 | door-47524-12-0 81 | drawer-45423-17-0 82 | door-46524-15-0 83 | drawer-49188-15-0 84 | door-45951-20-0 85 | drawer-10147-5-0 86 | door-10467-5-0 87 | drawer-34834-32-0 88 | door-20979-10-0 89 | drawer-45325-16-0 90 | door-33543-22-0 91 | drawer-47188-25-0 92 | door-45233-10-0 93 | drawer-7356-6-0 94 | door-47669-4-0 95 | drawer-9209-4-0 96 | door-46401-17-0 97 | drawer-10720-5-0 98 | door-24323-12-0 99 | drawer-7355-6-0 100 | door-48618-4-0 101 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/place_cube_tester.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymtorch 2 | 3 | import torch 4 | 5 | from manipgen.utils.policy_testers.base_tester import BaseTester 6 | from manipgen.utils.rlgames_utils import get_actions 7 | 8 | class FrankaPlaceCubeTester(BaseTester): 9 | def __init__(self, env, policy, num_steps, task_name): 10 | super().__init__(env, policy, num_steps, task_name) 11 | 12 | self.num_pre_steps = 200 13 | 14 | def pre_steps(self, obs, keep_runs): 15 | self.env.pre_steps(obs, keep_runs, self.num_pre_steps) 16 | return obs, keep_runs 17 | 18 | def run_steps(self, run_pre_steps=True, is_deterministic=True): 19 | """ 20 | Args: 21 | run_pre_steps: if true, teleport franka to easy initial poses or take actions with hand-engineering code before running the test 22 | Returns: 23 | keep_runs: a tensor of shape (num_envs,) indicating whether the run is kept. A run might fail in pre_steps, which should not be counted 24 | success: a tensor of shape (num_envs,) indicating whether the run is successful 25 | images: a list of lists of images, each list of images is for one environment 26 | """ 27 | obs = self.env.reset() 28 | keep_runs = torch.ones(self.num_envs, device=self.device).bool() 29 | if run_pre_steps: 30 | obs, keep_runs = self.pre_steps(obs, keep_runs) 31 | self.env.progress_buf[:] = 0 32 | 33 | images = [] 34 | for step in range(self.num_steps): 35 | self.env.progress_buf[:] = 0 36 | actions = get_actions(obs, self.policy, is_deterministic=is_deterministic) 37 | obs, reward, done, info = self.env.step(actions) 38 | 39 | if step % 3 == 2: 40 | images.append(info["camera_images"]) 41 | 42 | teleportation_images = self.env.hardcode_control(get_camera_images=True) 43 | teleportation_images = [teleportation_images[i] for i in range(0, len(teleportation_images), 3)] 44 | images.extend(teleportation_images) 45 | 46 | success = self.env._check_place_success() 47 | 48 | self.env.reset_idx(torch.arange(self.num_envs, device=self.device)) 49 | 50 | return keep_runs, success, images 51 | -------------------------------------------------------------------------------- /manipgen/config/task/pick_cube.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'pick_cube' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 22 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | cube_size: 0.0385 25 | 26 | sampler: 27 | # inverse kinematics coefficient 28 | damping: 0.10 29 | 30 | # threshold for accepting IK results 31 | ik_solver_pos_tolerance: 0.05 32 | ik_solver_rot_tolerance: 0.04 33 | 34 | # max number of IK iterations 35 | max_ik_iters: 15 36 | 37 | # max number of backtrack iterations 38 | max_backtrack_iters: 20 39 | 40 | # noise for sampling object position: 41 | # for x and y axis, uniformly sample from [center-object_pos_noise, center+object_pos_noise] 42 | object_pos_noise: 0.2 43 | 44 | # noise for sampling eef position: 45 | # eef position is sampled from a half-sphere with radius target_eef_radius centered at target object position 46 | target_eef_radius: 0.08 47 | 48 | # minimum distance to table of target eef position 49 | target_eef_z_offset: 0.01 50 | 51 | rl: 52 | pos_action_scale: [0.05, 0.05, 0.05] 53 | rot_action_scale: [0.05, 0.05, 0.05] 54 | force_action_scale: [1.0, 1.0, 1.0] 55 | torque_action_scale: [1.0, 1.0, 1.0] 56 | 57 | clamp_rot: True 58 | clamp_rot_thresh: 1.0e-6 59 | 60 | max_episode_length: ${..env.episode_length} 61 | 62 | dist_gripper_reward_temp: -1.0 63 | dist_xy_reward_temp: -3.0 64 | 65 | gripper_keypoint_scale: 0.5 66 | success_bonus: 0.0 67 | 68 | enable_object_in_view_reward: False 69 | object_in_view_reward_temp: -0.8 70 | object_in_view_reward_threshold: 0.27 71 | 72 | randomize: 73 | franka_arm_initial_dof_pos: [0, 0.1963, 0, -2.0, 0, 2.3416, 0.7854] 74 | franka_gripper_initial_state: 1.0 75 | 76 | ctrl: 77 | task_space_impedance: 78 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 79 | task_prop_gains: [300, 300, 300, 50, 50, 50] 80 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 81 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/open_tester.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymtorch 2 | 3 | import torch 4 | 5 | from manipgen.utils.rlgames_utils import get_actions 6 | from manipgen.utils.policy_testers.base_tester import BaseTester 7 | 8 | 9 | class FrankaOpenTester(BaseTester): 10 | def __init__(self, env, policy, num_steps, task_name): 11 | super().__init__(env, policy, num_steps, task_name) 12 | 13 | def pre_steps(self, obs, keep_runs): 14 | assert False, "`pre_steps` is not implemented for this testers. Please specify initial states." 15 | 16 | def run_steps(self, run_pre_steps=True, is_deterministic=True): 17 | """ 18 | Args: 19 | run_pre_steps: if true, teleport franka to easy initial poses or take actions with hand-engineering code before running the test 20 | Returns: 21 | keep_runs: a tensor of shape (num_envs,) indicating whether the run is kept. A run might fail in pre_steps, which should not be counted 22 | success: a tensor of shape (num_envs,) indicating whether the run is successful 23 | images: a list of lists of images, each list of images is for one environment 24 | """ 25 | obs = self.env.reset() 26 | keep_runs = torch.ones(self.num_envs, device=self.device).bool() 27 | if run_pre_steps: 28 | obs, keep_runs = self.pre_steps(obs, keep_runs) 29 | self.env.progress_buf[:] = 0 30 | 31 | # check whether the initial states are valid 32 | self.env.ctrl_target_fingertip_midpoint_pos[:] = self.env.fingertip_centered_pos 33 | self.env.ctrl_target_fingertip_midpoint_quat[:] = self.env.fingertip_centered_quat 34 | self.env.move_gripper_to_target_pose(gripper_dof_pos=-0.04, sim_steps=60) 35 | keep_runs &= self.env.gripper_dof_pos.sum(dim=1) > 0.002 36 | 37 | images = [] 38 | for step in range(self.num_steps): 39 | actions = get_actions(obs, self.policy, is_deterministic=is_deterministic) 40 | obs, reward, done, info = self.env.step(actions) 41 | 42 | if step % 3 == 2: 43 | images.append(info["camera_images"]) 44 | 45 | success = torch.clamp(self.env.relative_dof_completeness, max=1.0) 46 | 47 | self.env.reset_idx(torch.arange(self.num_envs, device=self.device)) 48 | 49 | return keep_runs, success, images 50 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/pick_cube_tester.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymtorch 2 | 3 | import torch 4 | 5 | from manipgen.utils.rlgames_utils import get_actions 6 | from manipgen.utils.policy_testers.base_tester import BaseTester 7 | 8 | 9 | class FrankaPickCubeTester(BaseTester): 10 | def __init__(self, env, policy, num_steps, task_name): 11 | super().__init__(env, policy, num_steps, task_name) 12 | 13 | self.num_pre_steps = 100 14 | 15 | def pre_steps(self, obs, keep_runs): 16 | self.env.pre_steps(obs, keep_runs, self.num_pre_steps) 17 | return obs, keep_runs 18 | 19 | def run_steps(self, run_pre_steps=True, is_deterministic=True): 20 | """ 21 | Args: 22 | run_pre_steps: if true, teleport franka to easy initial poses or take actions with hand-engineering code before running the test 23 | Returns: 24 | keep_runs: a tensor of shape (num_envs,) indicating whether the run is kept. A run might fail in pre_steps, which should not be counted 25 | success: a tensor of shape (num_envs,) indicating whether the run is successful 26 | images: a list of lists of images, each list of images is for one environment 27 | """ 28 | obs = self.env.reset() 29 | keep_runs = torch.ones(self.num_envs, device=self.device).bool() 30 | if run_pre_steps: 31 | obs, keep_runs = self.pre_steps(obs, keep_runs) 32 | self.env.progress_buf[:] = 0 33 | 34 | images = [] 35 | for step in range(self.num_steps): 36 | self.env.progress_buf[:] = 0 37 | actions = get_actions(obs, self.policy, is_deterministic=is_deterministic) 38 | obs, reward, done, info = self.env.step(actions) 39 | 40 | if step % 3 == 2: 41 | images.append(info["camera_images"]) 42 | 43 | hit_joint_limits = info["hit_joint_limits"] 44 | keep_runs = keep_runs & ~hit_joint_limits 45 | 46 | teleportation_images = self.env.hardcode_control(get_camera_images=True) 47 | teleportation_images = [teleportation_images[i] for i in range(0, len(teleportation_images), 3)] 48 | images.extend(teleportation_images) 49 | 50 | success = self.env._check_lift_success().bool() 51 | 52 | self.env.reset_idx(torch.arange(self.num_envs, device=self.device)) 53 | 54 | return keep_runs, success, images -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/base_depth.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "num_data_workers": 0, 4 | "hdf5_cache_mode": null, 5 | "hdf5_use_swmr": true, 6 | "hdf5_load_next_obs": false, 7 | "hdf5_normalize_obs": false, 8 | "hdf5_filter_key": null, 9 | "hdf5_validation_filter_key": null, 10 | "dataset_keys": [ 11 | "actions" 12 | ], 13 | "goal_mode": null 14 | }, 15 | "observation": { 16 | "modalities": { 17 | "obs": { 18 | "low_dim": [ 19 | "state" 20 | ], 21 | "rgb": [], 22 | "depth": [ 23 | "visual" 24 | ] 25 | } 26 | }, 27 | "encoder": { 28 | "low_dim": { 29 | "core_class": null, 30 | "core_kwargs": {}, 31 | "obs_randomizer_class": null, 32 | "obs_randomizer_kwargs": {} 33 | }, 34 | "rgb": { 35 | "core_class": "VisualCore", 36 | "core_kwargs": {}, 37 | "obs_randomizer_class": null, 38 | "obs_randomizer_kwargs": {} 39 | }, 40 | "depth": { 41 | "core_class": "VisualCore", 42 | "core_kwargs": { 43 | "feature_dimension": 64, 44 | "flatten": true, 45 | "backbone_class": "ResNet18Conv", 46 | "backbone_kwargs": { 47 | "pretrained": false, 48 | "input_coord_conv": false 49 | }, 50 | "pool_class": "SpatialSoftmax", 51 | "pool_kwargs": { 52 | "num_kp": 32, 53 | "learnable_temperature": false, 54 | "temperature": 1.0, 55 | "noise_std": 0.0, 56 | "output_variance": false 57 | } 58 | }, 59 | "obs_randomizer_class": "CropRandomizer", 60 | "obs_randomizer_kwargs": { 61 | "crop_height": 76, 62 | "crop_width": 76, 63 | "num_crops": 1, 64 | "pos_enc": false 65 | } 66 | } 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /manipgen/real_world/vlm_planning/prompts/tagging_prompt.py: -------------------------------------------------------------------------------- 1 | SYSTEM_PROMPT =""" 2 | ### System Description ### 3 | You are an expert at identifying objects in images. You will see images of the scene and output a list of every single object that is present in the scene in JSON format. 4 | We provide additional details specifying the task below. 5 | 6 | 7 | ### Task Description ### 8 | Task: You will return a comma-separated list of all the objects that are on the table only. 9 | The table is defined as the flat surface that the robot is on - it has a floral tablecloth. 10 | Also, return objects that are inside shelves, drawers, and microwaves. 11 | Also return sub-objects of objects that are on the table, for example, drawers, etc. 12 | We want the superset of objects in both images, so make sure you get all the objects. 13 | 14 | ### Helpful Hints ### 15 | 1. If possible describe the object with a color, for example, "black bin" or "orange carrot". 16 | 2. DO NOT return sub-objects of the robot. It should be considered as a single object. 17 | 3. DO NOT describe objects with text labels on them, for example, "Coca Cola" or "HARRY'S". 18 | 4. Each object tag SHOULD NOT be more than 3 words, including words separated by "-". 19 | 5. Use information from all input images to make sure you dont miss any objects, and make sure you get the right answer. 20 | 6. Pay attention to objects such as shelves and microwaves, and makes sure to include objects that are inside them. 21 | 22 | ### Formatting ### 23 | 0. The input will only be image(s) of the scene. 24 | 1. Each individual object should be mentioned separately in the list. 25 | 2. For object names, please be descriptive, this helps SAM figure out which object you are referring to. 26 | This means, color, material, texture etc. Any of these can help Grounded SAM understand the object you are referring to. 27 | Give detailed descriptions, give clear 3+ length word descriptions of the object. 28 | For example, . 29 | 3. Formatting: Respond with the following schema: 30 | ```json 31 | { 32 | "output": [object1, object2, ...] 33 | } 34 | ``` 35 | 4. object1, and so on, should be strings that describe the object. 36 | Thanks for helping me control the robot, please follow the instructions carefully. Start your response with an explanation in 50 words, and then provide your output in the JSON schema above. 37 | """ 38 | USER_EXAMPLES = [] 39 | ASSISTANT_EXAMPLES = [] -------------------------------------------------------------------------------- /manipgen/test_policy.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymapi 2 | from isaacgym import gymutil 3 | from isaacgym import gymtorch 4 | 5 | import torch 6 | import time 7 | from pathlib import Path 8 | 9 | import hydra 10 | from omegaconf import DictConfig, OmegaConf 11 | 12 | from isaacgymenvs.utils.reformat import omegaconf_to_dict 13 | from isaacgymenvs.utils.utils import set_seed 14 | from manipgen.envs import environments 15 | from manipgen.utils.policy_testers import testers 16 | from manipgen.utils.rlgames_utils import load_model 17 | 18 | 19 | @hydra.main(version_base="1.1", config_path="./config", config_name="config_test") 20 | def test_policy(cfg: DictConfig): 21 | set_seed(cfg.seed) 22 | 23 | # set up environment 24 | use_init_states = cfg.init_states != "" 25 | init_states = None 26 | if use_init_states: 27 | init_states = torch.load(cfg.init_states) 28 | # reverse the order of the states 29 | for k, v in init_states.items(): 30 | init_states[k] = torch.flip(v, [0]) 31 | 32 | env = environments[cfg.task_name]( 33 | cfg, 34 | init_states=init_states 35 | ) 36 | env.disable_automatic_reset = True 37 | 38 | # set up policy 39 | policy = load_model( 40 | actions_num=env.num_actions, 41 | obs_shape=(env.num_obs,), 42 | device=env.device, 43 | checkpoint_path=cfg.checkpoint, 44 | rl_config=omegaconf_to_dict(cfg["train"]), 45 | ) 46 | 47 | # set up tester 48 | task_name = cfg.task_name 49 | if task_name in ("pick", "place"): 50 | code = cfg.task.env.object_code.replace("/", "-") 51 | scale = int(100 * cfg.task.env.object_scale) 52 | task_name = task_name + "_" + code + "_" + f"{scale:03d}" 53 | elif task_name in ("grasp_handle", "open", "close", "open_nograsp", "close_nograsp"): 54 | task_name = task_name + "_" + cfg.task.env.object_code 55 | tester = testers[cfg.task_name]( 56 | env=env, 57 | policy=policy, 58 | num_steps=cfg.num_steps, 59 | task_name=task_name, 60 | ) 61 | 62 | # generate initial states 63 | tester.run( 64 | num_iterations=cfg.num_iterations, 65 | collect_failed_runs=cfg.collect_failed_runs, 66 | save_success_rate=cfg.save_success_rate, 67 | is_deterministic=cfg.deterministic_policy, 68 | ) 69 | 70 | 71 | if __name__ == "__main__": 72 | test_policy() 73 | -------------------------------------------------------------------------------- /manipgen/config/task/reach.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'reach' 6 | use_init_states: False 7 | headless: True 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 27 13 | numActions: 6 14 | episode_length: 128 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | num_gripper_move_sim_steps: 90 # number of timesteps to reserve for moving gripper before first step of episode 24 | 25 | sampler: 26 | # we don't need a sampler for reaching task 27 | 28 | rl: 29 | pos_action_scale: [0.1, 0.1, 0.1] 30 | rot_action_scale: [0.1, 0.1, 0.1] 31 | force_action_scale: [1.0, 1.0, 1.0] 32 | torque_action_scale: [1.0, 1.0, 1.0] 33 | 34 | unidirectional_rot: True # constrain Franka Z-rot to be unidirectional 35 | unidirectional_force: False # constrain Franka Z-force to be unidirectional (useful for debugging) 36 | 37 | clamp_rot: True 38 | clamp_rot_thresh: 1.0e-6 39 | 40 | num_keypoints: 5 # number of keypoints used in reward 41 | keypoint_scale: 0.5 # length of line of keypoints 42 | 43 | max_episode_length: ${..env.episode_length} 44 | use_keypoints_6D: True 45 | 46 | randomize: 47 | franka_arm_initial_dof_pos: [0.0, 0.0, 0.0, -1.59695, 0.0, 2.5307, 0.0] # initial joint angles after reset (halfway between joint limits) 48 | franka_gripper_initial_state: 1.0 49 | 50 | ctrl_workspace_bounds: [[0.3, 0.5], [-0.25, 0.25], [0.05, 0.35]] # bounds within which to select target position for initial movement of gripper using closed-loop controller ([[x_min, x_max], [y_min, y_max], [z_min, z_max]]) 51 | fingertip_centered_rot_initial: [3.1416, 0.0, 0.0] # target rotation (Euler) for initial movement of gripper 52 | fingertip_centered_rot_noise: [0.4, 0.4, 0.4] # hard 53 | 54 | rl_workspace_bounds: [[0.3, 0.5], [-0.25, 0.25], [0.05, 0.35]] # bounds within which to select target position for subsequent movement of gripper using RL policy ([[x_min, x_max], [y_min, y_max], [z_min, z_max]]) 55 | target_rot_initial: [3.1416, 0.0, 0.0] # target rotation (Euler) for RL policy 56 | target_rot_noise_level: [0.4, 0.4, 0.4] # hard 57 | # NOTE: For consistency with FrankaPy, the fingertip centered frame is located 9 mm above the very tip of the finger 58 | # NOTE: the RL policy currently always targets the Z-down rotation 59 | 60 | ctrl: 61 | task_space_impedance: 62 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 63 | task_prop_gains: [1000, 1000, 1000, 50, 50, 50] 64 | task_deriv_gains: [63.2, 63.2, 63.2, 1.41, 1.41, 1.41] 65 | 66 | debug: 67 | verbose: False 68 | log: False 69 | visualize: False -------------------------------------------------------------------------------- /manipgen/config/task/place_cube.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'place_cube' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 30 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | receptacle_friction: 1.0 25 | 26 | cube_size: 0.0385 27 | receptacle_size: 0.0496 28 | 29 | sampler: 30 | # number of RL steps for pick policy 31 | num_rl_steps: 90 32 | # number of steps for closing gripper 33 | num_close_gripper_steps: 40 34 | # number of steps for lifting the arm 35 | num_lift_up_steps: 40 36 | # number of steps for randomizing franka and object pose 37 | num_randomization_steps: 30 38 | 39 | # for each pick result, we sample multiple randomized franka and object poses 40 | num_randomization_per_policy: 3 41 | # for each randomized franka and object pose, we sample multiple receptacle poses 42 | num_receptacle_pose_per_randomization: 5 43 | 44 | # configuration for pick cube policy 45 | policy_init_states_path: "init_states/franka_pick_cube_init_states.pt" 46 | policy_config_path: "config/train/pick_cube.yaml" 47 | policy_checkpoint_path: ${resolve_default:"checkpoints/pick_cube/franka_pick_cube.pth",${...checkpoint}} 48 | 49 | # noise for sampling receptacle position: 50 | # for x and y axis, uniformly sample from [object_pos - receptacle_pos_noise, object_pos + receptacle_pos_noise] 51 | receptacle_pos_noise: 0.2 52 | 53 | # lower and upper bound for object height 54 | object_height_lower: 0.08 55 | object_height_upper: 0.16 56 | 57 | rl: 58 | pos_action_scale: [0.05, 0.05, 0.05] 59 | rot_action_scale: [0.05, 0.05, 0.05] 60 | force_action_scale: [1.0, 1.0, 1.0] 61 | torque_action_scale: [1.0, 1.0, 1.0] 62 | 63 | clamp_rot: True 64 | clamp_rot_thresh: 1.0e-6 65 | 66 | max_episode_length: ${..env.episode_length} 67 | 68 | dist_object_reward_temp: -1.0 69 | dist_xy_reward_temp: -3.0 70 | 71 | object_keypoint_scale: 0.5 72 | success_xy_threshold: 0.015 73 | success_z_threshold: 0.005 74 | success_bonus: 0.0 75 | 76 | enable_object_in_view_reward: True 77 | object_in_view_reward_temp: -0.8 78 | object_in_view_reward_threshold: 0.30 79 | 80 | randomize: 81 | franka_arm_initial_dof_pos: [0, 0.1963, 0, -2.0, 0, 2.3416, 0.7854] 82 | franka_gripper_initial_state: 0.0 83 | 84 | ctrl: 85 | task_space_impedance: 86 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 87 | task_prop_gains: [300, 300, 300, 50, 50, 50] 88 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 89 | -------------------------------------------------------------------------------- /manipgen/real_world/vlm_planning/prompts/segmentation_doublechecker_prompt.py: -------------------------------------------------------------------------------- 1 | SYSTEM_PROMPT = """ 2 | ### System Description ### 3 | You are an expert at looking at labelled images and picking the correct tag. You will pick the correct phrase that matches which object is segmented in the image 4 | and respond in JSON format. We provide additional details specifying the task below. 5 | 6 | ### Task Description ### 7 | Task: Given an input image, a segmented image, and a tag of the object of interest, you will pick the correct phrase 8 | that matches the object that is segmented in the image. To do so you need to compare the segmented image with the input image 9 | and make sure the phrase you pick matches the phrase associated the with the object that we are asking you to pick. 10 | The input image will contain multiple different segmentation masks with different objects segmented, but the same label 11 | and different probabilities. Only one of the segmentation masks will be correct, your task is to pick the correct one. 12 | We will also give as input the skill that the robot will use to interact with the object. 13 | Make sure that the phrase you pick is the one that matches the object that the robot will interact with and that it is 14 | physically plausible to do so. 15 | 16 | ### Helpful Hints ### 17 | 1. The correct output is not always the one with the highest probability, 18 | make sure to check the segmentation mask and the input image to make sure the object is correctly segmented. 19 | 2. If the object of interest is a handle, make sure to only pick the mask that corresponds to the actual handle not masks that 20 | cover the entire object! 21 | 22 | ### Formatting ### 23 | 1. Input format: The input will be an image, a list of segmented images, one for each predicted phrase, a tag of the object of interest (') and 24 | a list of predicted phrases from Grounded SAM: (' ', ' ', ...) and skill that the robot will use to interact with the object. 25 | Additionally, to help you select the correct object, I will pass a segmented image with the original tagging that we used to pick 26 | the object. Make sure the object you pick matches the object in the segmented image with the original tagging. The original tagging image 27 | will be the one with many different masks on the same image. 28 | Example tag and predicted phrase list: ('pick', 'green cup', 'green cup(0.8)', 'green cup(0.7)', 'green cup(0.6)') 29 | 2. Respond with the following schema: 30 | ```json 31 | { 32 | "output": 33 | } 34 | ``` 35 | Example output 36 | ```json 37 | { 38 | "output": "green cup(0.8)" 39 | } 40 | ``` 41 | Thanks for helping me control the robot, please follow the instructions carefully. Start your response with an explanation in 50 words, and then provide your evaluation in the JSON schema above. 42 | 43 | """ 44 | 45 | USER_EXAMPLES = [] 46 | ASSISTANT_EXAMPLES = [] -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link0.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 12 3 | 4 | newmtl Face636_001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 0.901961 0.921569 0.929412 8 | Ks 0.125000 0.125000 0.125000 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Part__Feature017_001 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 1.000000 1.000000 1.000000 18 | Ks 0.500000 0.500000 0.500000 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Part__Feature018_001 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 1.000000 1.000000 1.000000 28 | Ks 0.500000 0.500000 0.500000 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | 34 | newmtl Part__Feature019_001 35 | Ns -1.960784 36 | Ka 1.000000 1.000000 1.000000 37 | Kd 1.000000 1.000000 1.000000 38 | Ks 0.125000 0.125000 0.125000 39 | Ke 0.000000 0.000000 0.000000 40 | Ni 1.000000 41 | d 1.000000 42 | illum 2 43 | 44 | newmtl Part__Feature022_001 45 | Ns -1.960784 46 | Ka 1.000000 1.000000 1.000000 47 | Kd 0.901961 0.921569 0.929412 48 | Ks 0.125000 0.125000 0.125000 49 | Ke 0.000000 0.000000 0.000000 50 | Ni 1.000000 51 | d 1.000000 52 | illum 2 53 | 54 | newmtl Part__Feature023_001 55 | Ns -1.960784 56 | Ka 1.000000 1.000000 1.000000 57 | Kd 0.250980 0.250980 0.250980 58 | Ks 0.125000 0.125000 0.125000 59 | Ke 0.000000 0.000000 0.000000 60 | Ni 1.000000 61 | d 1.000000 62 | illum 2 63 | 64 | newmtl Shell001_001 65 | Ns -1.960784 66 | Ka 1.000000 1.000000 1.000000 67 | Kd 0.250980 0.250980 0.250980 68 | Ks 0.125000 0.125000 0.125000 69 | Ke 0.000000 0.000000 0.000000 70 | Ni 1.000000 71 | d 1.000000 72 | illum 2 73 | 74 | newmtl Shell002_001 75 | Ns -1.960784 76 | Ka 1.000000 1.000000 1.000000 77 | Kd 0.901961 0.921569 0.929412 78 | Ks 0.125000 0.125000 0.125000 79 | Ke 0.000000 0.000000 0.000000 80 | Ni 1.000000 81 | d 1.000000 82 | illum 2 83 | 84 | newmtl Shell003_001 85 | Ns -1.960784 86 | Ka 1.000000 1.000000 1.000000 87 | Kd 0.901961 0.921569 0.929412 88 | Ks 0.125000 0.125000 0.125000 89 | Ke 0.000000 0.000000 0.000000 90 | Ni 1.000000 91 | d 1.000000 92 | illum 2 93 | 94 | newmtl Shell009_001 95 | Ns -1.960784 96 | Ka 1.000000 1.000000 1.000000 97 | Kd 0.250980 0.250980 0.250980 98 | Ks 0.125000 0.125000 0.125000 99 | Ke 0.000000 0.000000 0.000000 100 | Ni 1.000000 101 | d 1.000000 102 | illum 2 103 | 104 | newmtl Shell010_001 105 | Ns -1.960784 106 | Ka 1.000000 1.000000 1.000000 107 | Kd 0.901961 0.921569 0.929412 108 | Ks 0.125000 0.125000 0.125000 109 | Ke 0.000000 0.000000 0.000000 110 | Ni 1.000000 111 | d 1.000000 112 | illum 2 113 | 114 | newmtl Shell_001 115 | Ns -1.960784 116 | Ka 1.000000 1.000000 1.000000 117 | Kd 0.250980 0.250980 0.250980 118 | Ks 0.125000 0.125000 0.125000 119 | Ke 0.000000 0.000000 0.000000 120 | Ni 1.000000 121 | d 1.000000 122 | illum 2 123 | -------------------------------------------------------------------------------- /manipgen/real_world/vlm_planning/prompts/workspace_prompt.py: -------------------------------------------------------------------------------- 1 | SYSTEM_PROMPT = """ 2 | ### System Description ### 3 | You are an expert robot planner controlling a Franka robot arm that will be used to perform picking or placing tasks. You will see images of the scene and classify whether the robot will perform a pick or place task in the 4 | free-space setting or top-obstructed setting by responding in JSON format. We define the meaning of those settings below. 5 | 6 | ### Task Description ### 7 | Task: You will clasify whether the pick or place task is going to be executed in the free-space setting or top-obstructed setting. 8 | Free-space setting: The robot is going to pick or place objects that are just sitting on the table, or somewhere without obstacles obstructing the robot's ability to grasp/place the object from the top. For example, picking up a cup from the table or an opened drawer. 9 | Tight-space setting: The robot is going to pick or place objects that are inside receptacles with a top, so they have obstacles that obstruct the robot's ability to grasp or place the object. For example, picking up a cup from a rung of the shelf or inside the microwave. 10 | You will be given a single image of the scene along with the current skill and coresponding object the robot is going to execute its skill on. 11 | 12 | ### Helpful Hints ### 13 | 1. Even if there is a microwave or drawer in the scene, you should only consider the object that the robot is going to interact with. 14 | 2. top obstructed are only those that have an obstruction above the object. Receptacles with obstacles on the front, back, left, right but not the top are free-space still. 15 | 3. Objects inside bins/containers are not considered top-obstructed settings because there is nothing blocking the robot from above. 16 | 4. The classification result will be used to help the robot plan its orientation to grasp or place the object. In top-obstructed setting, the robot needs to operate from the side of the receptacle. In free-space setting, the robot can operate from above the object or receptacle. 17 | 5. Commonly seen top-obstructed settings: objects inside microwaves and objects inside shelves. 18 | 6. Commonly seen free-space settings: objects on tables, objects in open drawers. 19 | 20 | ### Formatting ### 21 | 1. Input format: ('', ''). For example, ('cup', 'pick'). 22 | You need to determine whether the setting is free-space or top-obstructed. 23 | 2. Respond with the following schema: 24 | ```json 25 | { 26 | "output": True or False 27 | } 28 | ``` 29 | If the task is going to be executed in the free-space setting, return 30 | ```json 31 | { 32 | "output": True 33 | } 34 | ``` 35 | If the task is going to be executed in the top-obstructed setting, return 36 | ```json 37 | { 38 | "output": False 39 | } 40 | ``` 41 | Thanks for helping me control the robot, please follow the instructions carefully. Start your response with an explanation in 50 words, and then provide your evaluation in the JSON schema above. 42 | 43 | """ 44 | 45 | USER_EXAMPLES = [] 46 | ASSISTANT_EXAMPLES = [] -------------------------------------------------------------------------------- /manipgen/multitask_dagger_loop.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import argparse 4 | if __name__ == "__main__": 5 | # argparse the following args: 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument("--task", type=str, default='pick') 8 | parser.add_argument("--model", type=str, default='mlp') 9 | parser.add_argument("--gpu_id", type=int, default=0) 10 | parser.add_argument("--not_multitask", action='store_true') 11 | parser.add_argument("--num_objects_per_iteration", type=int, default=100) 12 | parser.add_argument("--max_epochs_per_iteration", type=int, default=100) 13 | args = parser.parse_args() 14 | 15 | task = args.task 16 | model = args.model 17 | gpu_id = args.gpu_id 18 | num_objects_per_iteration = args.num_objects_per_iteration 19 | max_epochs_per_iteration = args.max_epochs_per_iteration 20 | gpu_cmd = { 21 | "DISABLE_LAYER_NV_OPTIMUS_1": "1", 22 | "DISABLE_LAYER_AMD_SWITCHABLE_GRAPHICS_1": "1", 23 | "MESA_VK_DEVICE_SELECT": "10de:2230", 24 | "DISPLAY": ":1", 25 | "VK_ICD_FILENAMES": "/etc/vulkan/icd.d/nvidia_icd.json", 26 | "ENABLE_DEVICE_CHOOSER_LAYER": "1", 27 | "VULKAN_DEVICE_INDEX": f"{gpu_id}", 28 | "CUDA_VISIBLE_DEVICES": f"{gpu_id}", 29 | } 30 | iters = len(os.listdir(f"object_lists/{task}/")) - 1 31 | max_epochs = max_epochs_per_iteration 32 | if args.not_multitask: 33 | iters = 1 34 | max_epochs = 1000000 35 | for outer_epoch in range(1000): 36 | for idx in range(iters): 37 | command_list =[ 38 | "python", "dagger.py", f"task={task}", f"exp_name=dagger_{task}_{model}", f"wandb_run_name=dagger_{task}_{model}", 39 | f"checkpoint=multitask/{task}/policies", 40 | f"init_states=multitask/{task}/init_states", 41 | "num_envs=32", "wandb_activate=True", "wandb_project=dagger-multitask", 42 | "dagger.buffer_size=100", "dagger.batch_size=2048", 43 | "dagger.lr=0.0001", "dagger.num_transitions_per_iter=120", "dagger.num_learning_epochs=1", 44 | f"dagger.student_cfg_path=config/dagger/robomimic/bc_{model}.json", 45 | f"object_list=object_lists/unidexgrasp/{idx}.txt", 46 | "dagger.multitask=True", "local_obs=True", "global_obs=False", "dagger.visual_obs_type=depth", 47 | "test_episodes=100", "capture_local_obs=False", "capture_video=False", "test_frequency=1000", 48 | f"dagger.num_learning_iterations={max_epochs}", 49 | "task.sim.physx.max_gpu_contact_pairs=16777216", 50 | "train_dir=runs/multitask/" 51 | ] 52 | # check if runs_dagger/dagger_{task}_{model}/nn/checkpoint_latest.pth exists, if so resume from that path 53 | if os.path.exists(f"runs/multitask/dagger_{task}_{model}/nn/checkpoint_latest.pth"): 54 | command_list.append(f"resume=runs/multitask/dagger_{task}_{model}/nn/checkpoint_latest.pth") 55 | print("Running command:", " ".join(command_list)) 56 | subprocess.run(command_list, env={**os.environ, **gpu_cmd}) 57 | -------------------------------------------------------------------------------- /manipgen/utils/policy_testers/place_tester.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymtorch 2 | 3 | import torch 4 | 5 | from manipgen.utils.policy_testers.base_tester import BaseTester 6 | from manipgen.utils.rlgames_utils import get_actions 7 | 8 | class FrankaPlaceTester(BaseTester): 9 | def __init__(self, env, policy, num_steps, task_name): 10 | super().__init__(env, policy, num_steps, task_name) 11 | 12 | def pre_steps(self, obs, keep_runs): 13 | self.env.pre_steps(obs, keep_runs) 14 | return obs, keep_runs 15 | 16 | def run_steps(self, run_pre_steps=True, is_deterministic=True): 17 | """ 18 | Args: 19 | run_pre_steps: if true, teleport franka to easy initial poses or take actions with hand-engineering code before running the test 20 | Returns: 21 | keep_runs: a tensor of shape (num_envs,) indicating whether the run is kept. A run might fail in pre_steps, which should not be counted 22 | success: a tensor of shape (num_envs,) indicating whether the run is successful 23 | images: a list of lists of images, each list of images is for one environment 24 | """ 25 | obs = self.env.reset() 26 | keep_runs = torch.ones(self.num_envs, device=self.device).bool() 27 | if run_pre_steps: 28 | # obs, keep_runs = self.pre_steps(obs, keep_runs) 29 | # self.env.progress_buf[:] = 0 30 | assert "We do not have a pre_steps for place. Please specify the initial states" 31 | 32 | obs = self.env.reset() 33 | 34 | images = [] 35 | for step in range(self.num_steps): 36 | actions = get_actions(obs, self.policy, is_deterministic=is_deterministic) 37 | obs, reward, done, info = self.env.step(actions) 38 | 39 | if step % 3 == 2: 40 | images.append(info["camera_images"]) 41 | 42 | hit_joint_limits = info["hit_joint_limits"] 43 | keep_runs = keep_runs & ~hit_joint_limits 44 | 45 | teleportation_images = [] 46 | # open gripper 47 | self.env.ctrl_target_fingertip_midpoint_pos[:] = self.env.fingertip_centered_pos 48 | self.env.ctrl_target_fingertip_midpoint_quat[:] = self.env.fingertip_centered_quat 49 | target_gripper_dof_pos = self.env.gripper_dof_pos[:] + 0.02 50 | camera_images = self.env.move_gripper_to_target_pose(gripper_dof_pos=target_gripper_dof_pos, sim_steps=40, get_camera_images=True) 51 | teleportation_images.extend(camera_images) 52 | # pull out the arm 53 | self.env.ctrl_target_fingertip_midpoint_pos[:] -= 0.15 * self.env.gripper_direction 54 | camera_images = self.env.move_gripper_to_target_pose(gripper_dof_pos=target_gripper_dof_pos, sim_steps=30, get_camera_images=True) 55 | teleportation_images.extend(camera_images) 56 | teleportation_images = [teleportation_images[i] for i in range(0, len(teleportation_images), 3)] 57 | images.extend(teleportation_images) 58 | 59 | success = self.env._check_place_success() 60 | 61 | self.env.reset_idx(torch.arange(self.num_envs, device=self.device)) 62 | 63 | return keep_runs, success, images 64 | -------------------------------------------------------------------------------- /manipgen/config/task/grasp_handle.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'grasp_handle' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 73 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | 25 | # partnet 26 | object_code: ${...object_code} 27 | object_list: ${...object_list} 28 | 29 | sampler: 30 | # inverse kinematics coefficient 31 | damping: 0.10 32 | 33 | # threshold for accepting IK results 34 | ik_solver_pos_tolerance: 0.05 35 | ik_solver_rot_tolerance: 0.04 36 | 37 | # max number of IK iterations 38 | max_ik_iters: 15 39 | 40 | # max number of backtrack iterations 41 | max_backtrack_iters: 20 42 | 43 | # noise for sampling eef position: 44 | # eef position is sampled from a half-sphere with radius target_eef_radius centered at target object position 45 | # for vision-based sampling, the radius is set to half of this value 46 | target_eef_radius: 0.08 47 | 48 | rl: 49 | # episode length 50 | max_episode_length: ${..env.episode_length} 51 | 52 | # actions 53 | pos_action_scale: [0.05, 0.05, 0.05] 54 | rot_action_scale: [0.05, 0.05, 0.05] 55 | force_action_scale: [1.0, 1.0, 1.0] 56 | torque_action_scale: [1.0, 1.0, 1.0] 57 | 58 | clamp_rot: True 59 | clamp_rot_thresh: 1.0e-6 60 | 61 | # reward settings 62 | dist_object_gripper_reward_temp: -1.0 63 | dist_object_dof_pos_reward_temp: -100.0 64 | gripper_direction_reward_temp: 0.5 65 | gripper_direction_threshold: 0.8 66 | eef_height_consistency_reward_temp: -40.0 67 | success_bonus: 15.0 68 | success_threshold: 0.005 69 | 70 | num_object_keypoints: 16 71 | gripper_keypoint_dof: 5 72 | gripper_keypoint_scale: 0.5 73 | 74 | enable_object_in_view_reward: False 75 | object_in_view_reward_temp: -0.8 76 | object_in_view_reward_threshold: 0.27 77 | 78 | randomize: 79 | franka_arm_initial_dof_pos: [0, -0.5, 0, -3.0, 0, 3, 0.7854] 80 | franka_gripper_initial_state: 1.0 81 | 82 | object_pos_radius: 0.7 83 | object_pos_radius_noise: 0.05 84 | door_pos_radius_offset: 0.15 85 | object_pos_angle_noise: 1.57079 # pi / 2 86 | object_z_max: 0.80 87 | handle_z_min: 0.15 88 | door_rotation_noise: 0.26180 # pi / 12 89 | drawer_rotation_noise: 0.78540 # pi / 4 90 | 91 | object_dof_damping_lower: 1.0 92 | object_dof_damping_upper: 2.0 93 | object_dof_friction_lower: 0.25 94 | object_dof_friction_upper: 0.50 95 | door_damping_scale: 0.10 96 | door_friction_scale: 0.10 97 | 98 | max_object_init_dof_ratio: 0.9 99 | 100 | ctrl: 101 | all: 102 | gripper_prop_gains: [200, 200] # 500 -> 100 alleviate penetration 103 | gripper_deriv_gains: [1, 1] 104 | task_space_impedance: 105 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 106 | task_prop_gains: [300, 300, 300, 50, 50, 50] 107 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 108 | -------------------------------------------------------------------------------- /manipgen/config/task/open.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'open' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 78 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | 25 | # partnet 26 | object_code: ${...object_code} 27 | object_list: ${...object_list} 28 | 29 | sampler: 30 | # number of RL steps for grasp policy 31 | num_rl_steps: 120 32 | # number of steps for closing gripper 33 | num_close_gripper_steps: 60 34 | # number of steps for randomizing franka and object pose 35 | num_randomization_steps: 20 36 | 37 | # for each grasp result, we sample multiple randomized states 38 | num_randomization_per_policy: 5 39 | 40 | # configuration for grasp handle policy 41 | # resolve the value of policy_init_states_path and policy_checkpoint_path programmatically 42 | # too complicated to be resolved in the config file 43 | policy_init_states_path: "" 44 | policy_config_path: "config/train/grasp_handle.yaml" 45 | policy_checkpoint_path: "" 46 | 47 | rl: 48 | # episode length 49 | max_episode_length: ${..env.episode_length} 50 | 51 | # actions 52 | pos_action_scale: [0.05, 0.05, 0.05] 53 | rot_action_scale: [0.05, 0.05, 0.05] 54 | force_action_scale: [1.0, 1.0, 1.0] 55 | torque_action_scale: [1.0, 1.0, 1.0] 56 | 57 | clamp_rot: True 58 | clamp_rot_thresh: 1.0e-6 59 | 60 | # reward settings 61 | dist_object_gripper_reward_temp: 0.0 62 | dist_object_dof_pos_reward_temp: -2.5 63 | grasp_pos_offset_reward_temp: -30.0 64 | gripper_direction_reward_temp: 0.5 65 | gripper_direction_threshold: 0.8 66 | action_penalty_temp: -0.4 67 | action_penalty_threshold: 0.0 68 | success_threshold: 0.2 69 | 70 | num_object_keypoints: 16 71 | gripper_keypoint_dof: 5 72 | gripper_keypoint_scale: 0.5 73 | 74 | partial_open_for_door: True 75 | partial_open_degree: 30 76 | 77 | randomize: 78 | franka_arm_initial_dof_pos: [0, -0.5, 0, -3.0, 0, 3, 0.7854] 79 | franka_gripper_initial_state: 0.0 80 | 81 | object_pos_radius: 0.7 82 | object_pos_radius_noise: 0.05 83 | door_pos_radius_offset: 0.15 84 | object_pos_angle_noise: 1.57079 # pi / 2 85 | object_z_max: 0.80 86 | handle_z_min: 0.15 87 | door_rotation_noise: 0.26180 # pi / 12 88 | drawer_rotation_noise: 0.78540 # pi / 4 89 | 90 | object_dof_damping_lower: 1.0 91 | object_dof_damping_upper: 2.0 92 | object_dof_friction_lower: 0.25 93 | object_dof_friction_upper: 0.50 94 | door_damping_scale: 0.10 95 | door_friction_scale: 0.10 96 | 97 | max_object_init_dof_ratio: 0.6 98 | 99 | ctrl: 100 | all: 101 | gripper_prop_gains: [200, 200] # 500 -> 100 alleviate penetration 102 | gripper_deriv_gains: [1, 1] 103 | task_space_impedance: 104 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 105 | task_prop_gains: [300, 300, 300, 50, 50, 50] 106 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 107 | -------------------------------------------------------------------------------- /manipgen/config/task/close.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'close' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 78 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | 25 | # partnet 26 | object_code: ${...object_code} 27 | object_list: ${...object_list} 28 | 29 | sampler: 30 | # number of RL steps for grasp policy 31 | num_rl_steps: 120 32 | # number of steps for closing gripper 33 | num_close_gripper_steps: 60 34 | # number of steps for randomizing franka and object pose 35 | num_randomization_steps: 20 36 | 37 | # for each grasp result, we sample multiple randomized states 38 | num_randomization_per_policy: 5 39 | 40 | # configuration for grasp handle policy 41 | # resolve the value of policy_init_states_path and policy_checkpoint_path programmatically 42 | # too complicated to be resolved in the config file 43 | policy_init_states_path: "" 44 | policy_config_path: "config/train/grasp_handle.yaml" 45 | policy_checkpoint_path: "" 46 | 47 | rl: 48 | # episode length 49 | max_episode_length: ${..env.episode_length} 50 | 51 | # actions 52 | pos_action_scale: [0.05, 0.05, 0.05] 53 | rot_action_scale: [0.05, 0.05, 0.05] 54 | force_action_scale: [1.0, 1.0, 1.0] 55 | torque_action_scale: [1.0, 1.0, 1.0] 56 | 57 | clamp_rot: True 58 | clamp_rot_thresh: 1.0e-6 59 | 60 | # reward settings 61 | dist_object_gripper_reward_temp: 0.0 62 | dist_object_dof_pos_reward_temp: -2.5 63 | grasp_pos_offset_reward_temp: -30.0 64 | gripper_direction_reward_temp: 0.5 65 | gripper_direction_threshold: 0.8 66 | action_penalty_temp: -0.4 67 | action_penalty_threshold: 0.0 68 | success_threshold: 0.2 69 | 70 | num_object_keypoints: 16 71 | gripper_keypoint_dof: 5 72 | gripper_keypoint_scale: 0.5 73 | 74 | partial_close_for_door: True 75 | partial_close_degree: 30 76 | 77 | randomize: 78 | franka_arm_initial_dof_pos: [0, -0.5, 0, -3.0, 0, 3, 0.7854] 79 | franka_gripper_initial_state: 0.0 80 | 81 | object_pos_radius: 0.7 82 | object_pos_radius_noise: 0.05 83 | door_pos_radius_offset: 0.15 84 | object_pos_angle_noise: 1.57079 # pi / 2 85 | object_z_max: 0.80 86 | handle_z_min: 0.15 87 | door_rotation_noise: 0.26180 # pi / 12 88 | drawer_rotation_noise: 0.78540 # pi / 4 89 | 90 | object_dof_damping_lower: 1.0 91 | object_dof_damping_upper: 2.0 92 | object_dof_friction_lower: 0.25 93 | object_dof_friction_upper: 0.50 94 | door_damping_scale: 0.10 95 | door_friction_scale: 0.10 96 | 97 | max_object_init_dof_ratio: 0.6 98 | 99 | ctrl: 100 | all: 101 | gripper_prop_gains: [200, 200] # 500 -> 100 alleviate penetration 102 | gripper_deriv_gains: [1, 1] 103 | task_space_impedance: 104 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 105 | task_prop_gains: [300, 300, 300, 50, 50, 50] 106 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 107 | -------------------------------------------------------------------------------- /manipgen/real_world/scripts/run_manipgen.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import hydra 3 | from manipgen.real_world.neural_mp import NeuralMP 4 | from manipgen.real_world.manipgen import ManipGen 5 | from manipgen.real_world.utils.neural_mp_env_wrapper import IndustrealEnvWrapper 6 | from manipgen.real_world.industreal_psl_env import FrankaRealPSLEnv 7 | from manipgen.real_world.vlm_planner import VLMPlanner 8 | 9 | def get_args(): 10 | """Gets arguments from the command line.""" 11 | parser = argparse.ArgumentParser() 12 | 13 | parser.add_argument( 14 | "-d", 15 | "--debug_mode", 16 | action="store_true", 17 | required=False, 18 | help="Enable output for debugging", 19 | ) 20 | parser.add_argument( 21 | "-c", 22 | "--vlm_cam_idx", 23 | required=True, 24 | type=int, 25 | help="Choose one global camera for VLM planning", 26 | ) 27 | parser.add_argument( 28 | "--prompt", 29 | type=str, default="put everything on the table in the black box", 30 | ) 31 | parser.add_argument( 32 | "--pick_checkpoint", 33 | type=str, default="real_world/checkpoints/pick.pth", 34 | ) 35 | parser.add_argument( 36 | "--place_checkpoint", 37 | type=str, default="real_world/checkpoints/place.pth", 38 | ) 39 | parser.add_argument( 40 | "--grasp_handle_checkpoint", 41 | type=str, default="real_world/checkpoints/grasp_handle.pth", 42 | ) 43 | parser.add_argument( 44 | "--open_checkpoint", 45 | type=str, default="real_world/checkpoints/open.pth", 46 | ) 47 | parser.add_argument( 48 | "--close_checkpoint", 49 | type=str, default="real_world/checkpoints/close.pth", 50 | ) 51 | parser.add_argument( 52 | "--neural_mp_checkpoint", 53 | type=str, default="real_world/checkpoints/neural_mp.pth", 54 | ) 55 | 56 | args = parser.parse_args() 57 | 58 | return args 59 | 60 | 61 | if __name__ == "__main__": 62 | args = get_args() 63 | hydra.initialize(config_path="../../config/real/", job_name="run_manipgen") 64 | cfg = hydra.compose(config_name='base') 65 | 66 | # load local policies 67 | env = FrankaRealPSLEnv(args, cfg) 68 | env.load_policy(task="pick", checkpoint_path=args.pick_checkpoint) 69 | env.load_policy(task="place", checkpoint_path=args.place_checkpoint) 70 | env.load_policy(task="grasp_handle", checkpoint_path=args.grasp_handle_checkpoint) 71 | env.load_policy(task="open", checkpoint_path=args.open_checkpoint) 72 | env.load_policy(task="close", checkpoint_path=args.close_checkpoint) 73 | 74 | # motion planning 75 | neural_mp_env = IndustrealEnvWrapper(env) 76 | motion_planner = NeuralMP( 77 | neural_mp_env, 78 | model_path=args.neural_mp_checkpoint, 79 | train_mode=True, 80 | tto=True, 81 | in_hand=False, 82 | max_neural_mp_rollout_length=100, 83 | num_robot_points=2048, 84 | num_obstacle_points=4096, 85 | clamp_joint_limit=True, 86 | visualize=False, 87 | ) 88 | 89 | # VLM planning 90 | vlm_planner = VLMPlanner(env, cam_ids=[args.vlm_cam_idx]) 91 | 92 | manipgen = ManipGen(env, vlm_planner, motion_planner) 93 | manipgen.solve_task(args.prompt, args.debug_mode) 94 | -------------------------------------------------------------------------------- /manipgen/real_world/utils/vis_utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import open3d as o3d 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | 6 | def display_rgb(rgb): 7 | """Visualize RGB image.""" 8 | plt.figure() 9 | plt.imshow(rgb) 10 | plt.tight_layout() 11 | plt.show() 12 | plt.close() 13 | 14 | def display_raw_depth(depth, clamp_threshold=0.3, display=True): 15 | """Convert raw depth image to colorized depth image for visualization.""" 16 | depth = depth.astype(np.float32) / 1e4 17 | depth = depth.clip(0, clamp_threshold) 18 | depth = -depth 19 | min_val = np.min(depth) 20 | max_val = np.max(depth) 21 | depth_range = max_val - min_val 22 | depth_image = (255.0 / depth_range * (depth - min_val)).astype("uint8") 23 | depth_image = cv2.applyColorMap(depth_image, cv2.COLORMAP_INFERNO) 24 | depth_image = depth_image[:, :, ::-1] 25 | if display: 26 | display_rgb(depth_image) 27 | return depth_image 28 | 29 | def display_pcd(points, colors=None): 30 | """Visualize point cloud.""" 31 | pcd = o3d.geometry.PointCloud() 32 | pcd.points = o3d.utility.Vector3dVector(points) 33 | if colors is not None: 34 | pcd.colors = o3d.utility.Vector3dVector(colors) 35 | o3d.visualization.draw_geometries([pcd]) 36 | 37 | def crop_depth_to_policy_view(depth, cfg): 38 | """Crop depth image to local policy view.""" 39 | assert depth.shape == (cfg.raw_depth_h, cfg.raw_depth_w) 40 | 41 | h, w = depth.shape 42 | center = (w // 2, h // 2) 43 | M = cv2.getRotationMatrix2D(center, cfg.process_depth_rotation, 1.0) 44 | depth = cv2.warpAffine(depth, M, (w, h)) 45 | 46 | # crop to policy view (after center cropping) 47 | h_offset = cfg.process_depth_h_offset 48 | h_boarder = (cfg.raw_depth_h - cfg.vis_depth_w) // 2 49 | w_offset = cfg.process_depth_w_offset 50 | w_boarder = (cfg.raw_depth_w - cfg.vis_depth_w) // 2 51 | depth = depth[h_boarder+h_offset:-h_boarder+h_offset, w_boarder+w_offset:-w_boarder+w_offset] 52 | 53 | # resize to 76 x 76 54 | depth = cv2.resize(depth, (cfg.vis_depth_w_down, cfg.vis_depth_w_down), interpolation=cv2.INTER_NEAREST) 55 | 56 | return depth 57 | 58 | def crop_rgb_to_policy_view(rgb, cfg): 59 | """Crop RGB image to local policy view.""" 60 | assert rgb.shape == (cfg.raw_depth_h, cfg.raw_depth_w, 3) 61 | 62 | h, w, _ = rgb.shape 63 | center = (w // 2, h // 2) 64 | M = cv2.getRotationMatrix2D(center, cfg.process_depth_rotation, 1.0) 65 | rgb = cv2.warpAffine(rgb, M, (w, h)) 66 | 67 | # crop to policy view (after center cropping) 68 | h_offset = cfg.process_depth_h_offset 69 | h_boarder = (cfg.raw_depth_h - cfg.vis_depth_w) // 2 70 | w_offset = cfg.process_depth_w_offset 71 | w_boarder = (cfg.raw_depth_w - cfg.vis_depth_w) // 2 72 | rgb = rgb[h_boarder+h_offset:-h_boarder+h_offset, w_boarder+w_offset:-w_boarder+w_offset] 73 | 74 | # resize to 76 x 76 75 | rgb = cv2.resize(rgb, (cfg.vis_depth_w_down, cfg.vis_depth_w_down), interpolation=cv2.INTER_NEAREST) 76 | 77 | return rgb 78 | 79 | def debug_wrist_view(env, cfg): 80 | """Display visual observation for local policies.""" 81 | obs = env.get_obs() 82 | depth = obs['cam1_depth'][0].copy() 83 | depth = crop_depth_to_policy_view(depth, cfg) 84 | display_raw_depth(depth) 85 | -------------------------------------------------------------------------------- /manipgen/config/config_test.yaml: -------------------------------------------------------------------------------- 1 | # choose the task to be executed 2 | task_name: ${task.name} 3 | 4 | # set random seed 5 | seed: 3407 6 | 7 | # number of environments 8 | num_envs: 1024 9 | 10 | # set the maximum number of learning iterations to train for. overrides default per-environment setting 11 | max_iterations: '' 12 | 13 | # device config 14 | device: 'cuda:0' 15 | 16 | # visualization: viewer 17 | render: False 18 | 19 | # visualization: camera - if set, capture `capture_length` frames every `capture_interval` steps. 20 | # `capture_envs` specifies the number of environments to capture videos from 21 | capture_video: False 22 | capture_interval: ${task.env.episode_length} 23 | capture_length: ${task.env.episode_length} 24 | capture_envs: 8 25 | collect_failed_runs: False 26 | capture_obs_camera: False # capture images from obs cameras 27 | capture_depth: False # capture depth images 28 | 29 | # set the initial progress_buf to be random - reduce correlation between samples in the experience buffer 30 | random_time: False 31 | 32 | # ratio of samples to use for validation (not used here) 33 | val_ratio: 0.0 34 | 35 | # save results in specified file 36 | save_success_rate: '' 37 | 38 | # set checkpoint path 39 | checkpoint: '' 40 | 41 | # set whether the policy is deterministic or not 42 | deterministic_policy: True 43 | 44 | # initial states path - if not set, inferred from task 45 | init_states: '' 46 | 47 | # unidexgrasp / partnet 48 | object_code: '' 49 | object_scale: 0.06 # only applicable for unidexgrasp 50 | object_list: '' # format: [[object_code,object_scale],...] for unidexgrasp, [object_code,...] for partnet 51 | 52 | # number of iterations to run the test 53 | num_iterations: 5 54 | 55 | # number of steps per iteration 56 | num_steps: ${task.env.episode_length} 57 | 58 | defaults: 59 | - task: 'pick_cube' 60 | - train: ${task} 61 | - override hydra/job_logging: disabled 62 | - _self_ 63 | 64 | # set the directory where the output files get saved 65 | hydra: 66 | output_subdir: null 67 | run: 68 | dir: . 69 | 70 | local_obs: False 71 | global_obs: False 72 | task: 73 | env: 74 | local_obs: 75 | width: 84 76 | height: 84 77 | render_type: 78 | depth: True 79 | rgb: False 80 | segmentation: False 81 | horizontal_fov: 42 # real world: crop 640 x 480 to 332 x 332 82 | camera_offset: [0.06, 0.0, 0.023] 83 | camera_angle: 0.0 84 | use_back_wrist_camera: False 85 | global_obs: 86 | width: 128 87 | height: 128 88 | locations: [[0.5, -0.5, 1.5], [0.5, 0.5, 1.5], [-0.5, -0.5, 1.5], [-0.5, 0.5, 1.5]] 89 | render_type: 90 | depth: True 91 | rgb: False 92 | segmentation: False 93 | 94 | # set True for samplers 95 | sample_mode: False 96 | 97 | # control gripper 98 | gripper_control: True 99 | 100 | ## Device config 101 | # 'physx' or 'flex' 102 | physics_engine: 'physx' 103 | # whether to use cpu or gpu pipeline 104 | pipeline: 'gpu' 105 | # device for running physics simulation 106 | sim_device: 'cuda:0' 107 | # device to run RL 108 | rl_device: 'cuda:0' 109 | graphics_device_id: 0 110 | 111 | ## PhysX arguments 112 | num_threads: 4 # Number of worker threads per scene used by PhysX - for CPU PhysX only. 113 | solver_type: 1 # 0: pgs, 1: tgs 114 | num_subscenes: 4 # Splits the simulation into N physics scenes and runs each one in a separate thread -------------------------------------------------------------------------------- /manipgen/real_world/utils/calibration_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the NVIDIA Source Code License [see LICENSE for details]. 4 | """IndustRealLib: Perception utilities module. 5 | 6 | This module defines utility functions for perceiving a scene with an Intel 7 | RealSense camera. 8 | """ 9 | 10 | # Standard Library 11 | import json 12 | import os 13 | 14 | # Third Party 15 | import cv2 16 | import numpy as np 17 | from omegaconf import OmegaConf 18 | 19 | 20 | def get_perception_config(file_name, module_name): 21 | """Gets an IndustRealLib perception configuration from a YAML file.""" 22 | config = OmegaConf.load(os.path.join(os.path.dirname(__file__), "../..", "config", "real", file_name))[ 23 | module_name 24 | ] 25 | 26 | return config 27 | 28 | 29 | def label_tag_detection(image, tag_corner_pixels, tag_family): 30 | """Labels a tag detection on an image.""" 31 | image_labeled = image.copy() 32 | 33 | corner_a = (int(tag_corner_pixels[0][0]), int(tag_corner_pixels[0][1])) 34 | corner_b = (int(tag_corner_pixels[1][0]), int(tag_corner_pixels[1][1])) 35 | corner_c = (int(tag_corner_pixels[2][0]), int(tag_corner_pixels[2][1])) 36 | corner_d = (int(tag_corner_pixels[3][0]), int(tag_corner_pixels[3][1])) 37 | 38 | # Draw oriented box on image 39 | cv2.line(img=image_labeled, pt1=corner_a, pt2=corner_b, color=(0, 255, 0), thickness=2) 40 | cv2.line(img=image_labeled, pt1=corner_b, pt2=corner_c, color=(0, 255, 0), thickness=2) 41 | cv2.line(img=image_labeled, pt1=corner_c, pt2=corner_d, color=(0, 255, 0), thickness=2) 42 | cv2.line(img=image_labeled, pt1=corner_d, pt2=corner_a, color=(0, 255, 0), thickness=2) 43 | 44 | # Draw tag family on image 45 | cv2.putText( 46 | img=image_labeled, 47 | text=tag_family.decode("utf-8"), 48 | org=(corner_a[0], corner_c[1] - 10), 49 | fontFace=cv2.FONT_HERSHEY_SIMPLEX, 50 | fontScale=0.5, 51 | color=(255, 0, 0), 52 | thickness=2, 53 | lineType=cv2.LINE_AA, 54 | ) 55 | 56 | return image_labeled 57 | 58 | 59 | def get_tag_pose_in_camera_frame(detector, image, intrinsics, tag_length, tag_active_pixel_ratio): 60 | """Detects an AprilTag in an image. Gets the pose of the tag in the camera frame.""" 61 | gray_image = cv2.cvtColor(src=image.astype(np.uint8), code=cv2.COLOR_BGR2GRAY) 62 | tag_active_length = tag_length * tag_active_pixel_ratio 63 | detection = detector.detect( 64 | img=gray_image, 65 | estimate_tag_pose=True, 66 | camera_params=[intrinsics["fx"], intrinsics["fy"], intrinsics["cx"], intrinsics["cy"]], 67 | tag_size=tag_active_length, 68 | ) 69 | 70 | if detection: 71 | is_detected = True 72 | pos = detection[0].pose_t.copy().squeeze() # (3, ) 73 | ori_mat = detection[0].pose_R.copy() 74 | center_pixel = detection[0].center 75 | corner_pixels = detection[0].corners 76 | family = detection[0].tag_family 77 | 78 | else: 79 | is_detected = False 80 | pos, ori_mat, center_pixel, corner_pixels, family = None, None, None, None, None 81 | 82 | return is_detected, pos, ori_mat, center_pixel, corner_pixels, family 83 | 84 | 85 | def get_extrinsics(file_name): 86 | """Loads the extrinsics from a JSON file.""" 87 | with open(os.path.join(os.path.dirname(__file__), "..", "io", file_name)) as f: 88 | json_obj = f.read() 89 | 90 | extrinsics_dict = json.loads(json_obj) 91 | 92 | return extrinsics_dict -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/bc_transformer.json: -------------------------------------------------------------------------------- 1 | { 2 | "algo_name": "bc", 3 | "train": { 4 | "seq_length": 1, 5 | "pad_seq_length": true, 6 | "frame_stack": 4, 7 | "pad_frame_stack": true, 8 | "batch_size": 100 9 | }, 10 | "algo": { 11 | "optim_params": { 12 | "policy": { 13 | "optimizer_type": "adamw", 14 | "learning_rate": { 15 | "initial": 0.0001, 16 | "decay_factor": 0.1, 17 | "epoch_schedule": [1000], 18 | "scheduler_type": "linear" 19 | }, 20 | "regularization": { 21 | "L2": 0.01 22 | } 23 | } 24 | }, 25 | "loss": { 26 | "l2_weight": 1.0, 27 | "l1_weight": 0.0, 28 | "cos_weight": 0.0 29 | }, 30 | "actor_layer_dims": [], 31 | "gaussian": { 32 | "enabled": false, 33 | "fixed_std": false, 34 | "init_std": 0.1, 35 | "min_std": 0.01, 36 | "std_activation": "softplus", 37 | "low_noise_eval": true 38 | }, 39 | "gmm": { 40 | "enabled": false, 41 | "num_modes": 5, 42 | "min_std": 0.0001, 43 | "std_activation": "softplus", 44 | "low_noise_eval": true 45 | }, 46 | "vae": { 47 | "enabled": false, 48 | "latent_dim": 14, 49 | "latent_clip": null, 50 | "kl_weight": 1.0, 51 | "decoder": { 52 | "is_conditioned": true, 53 | "reconstruction_sum_across_elements": false 54 | }, 55 | "prior": { 56 | "learn": false, 57 | "is_conditioned": false, 58 | "use_gmm": false, 59 | "gmm_num_modes": 10, 60 | "gmm_learn_weights": false, 61 | "use_categorical": false, 62 | "categorical_dim": 10, 63 | "categorical_gumbel_softmax_hard": false, 64 | "categorical_init_temp": 1.0, 65 | "categorical_temp_anneal_step": 0.001, 66 | "categorical_min_temp": 0.3 67 | }, 68 | "encoder_layer_dims": [ 69 | 300, 70 | 400 71 | ], 72 | "decoder_layer_dims": [ 73 | 300, 74 | 400 75 | ], 76 | "prior_layer_dims": [ 77 | 300, 78 | 400 79 | ] 80 | }, 81 | "rnn": { 82 | "enabled": false, 83 | "horizon": 10, 84 | "hidden_dim": 400, 85 | "rnn_type": "LSTM", 86 | "num_layers": 2, 87 | "open_loop": false, 88 | "kwargs": { 89 | "bidirectional": false 90 | } 91 | }, 92 | "transformer": { 93 | "enabled": true, 94 | "context_length": 4, 95 | "embed_dim": 512, 96 | "num_layers": 6, 97 | "num_heads": 8, 98 | "emb_dropout": 0.1, 99 | "attn_dropout": 0.1, 100 | "block_output_dropout": 0.1, 101 | "sinusoidal_embedding": false, 102 | "activation": "gelu", 103 | "supervise_all_steps": false, 104 | "nn_parameter_for_timesteps": true 105 | } 106 | } 107 | } -------------------------------------------------------------------------------- /manipgen/config/task/pick.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'pick' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 105 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | 25 | # unidexgrasp 26 | object_code: ${...object_code} 27 | object_scale: ${...object_scale} 28 | object_list: ${...object_list} 29 | 30 | # clutter and obstacles 31 | enable_clutter_and_obstacle: True 32 | num_clutter_objects: 5 33 | clutter_object_radius: 0.05 34 | clutter_dist_max: 0.40 35 | num_obstacles: 2 36 | obstacle_width: 0.40 37 | obstacle_height: 0.30 38 | obstacle_pos_noise: 0.10 39 | obstacle_rot_noise: 0.25 40 | obstacle_show_up_prob: 0.7 41 | use_unidexgrasp_clutter: False # if False, uses capsules; if True, use unidexgrasp objects; automatically set to True in multitask dagger 42 | num_unidexgrasp_clutter_rest_poses: 16 43 | unidexgrasp_file_list: assets/unidexgrasp/trainset3419.txt 44 | 45 | sampler: 46 | # inverse kinematics coefficient 47 | damping: 0.10 48 | 49 | # threshold for accepting IK results 50 | ik_solver_pos_tolerance: 0.05 51 | ik_solver_rot_tolerance: 0.04 52 | 53 | # max number of IK iterations 54 | max_ik_iters: 15 55 | 56 | # max number of backtrack iterations 57 | max_backtrack_iters: 20 58 | 59 | # noise for sampling object position: 60 | # for x and y axis, uniformly sample from [center-object_pos_noise, center+object_pos_noise] 61 | object_pos_noise: 0.2 62 | 63 | # noise for sampling eef position: 64 | # eef position is sampled from a half-sphere with radius target_eef_radius centered at target object position 65 | # for vision-based sampling, the radius is set to half of this value 66 | target_eef_radius: 0.08 67 | 68 | # minimum distance to table of target eef position 69 | target_eef_z_offset: 0.01 70 | 71 | rl: 72 | # episode length 73 | max_episode_length: ${..env.episode_length} 74 | 75 | # actions 76 | pos_action_scale: [0.05, 0.05, 0.05] 77 | rot_action_scale: [0.05, 0.05, 0.05] 78 | force_action_scale: [1.0, 1.0, 1.0] 79 | torque_action_scale: [1.0, 1.0, 1.0] 80 | 81 | clamp_rot: True 82 | clamp_rot_thresh: 1.0e-6 83 | 84 | # filter initial object poses where the grasp success rate is lower than the threshold 85 | filter_pose_threshold: 0.01 86 | 87 | # reward settings 88 | dist_gripper_reward_temp: -1.0 89 | dist_xy_reward_temp: -3.0 90 | gripper_offset_reward_temp: -20.0 91 | eef_pose_consistency_reward_temp: -5.0 92 | success_bonus: 20.0 93 | 94 | gripper_keypoint_scale: 0.5 95 | gripper_keypoint_dof: 5 96 | num_object_keypoints: 16 97 | 98 | finger_contact_force_threshold: 1.0 99 | finger_contact_reward_temp: -0.01 100 | 101 | enable_object_in_view_reward: False 102 | object_in_view_reward_temp: -0.8 103 | object_in_view_reward_threshold: 0.27 104 | 105 | randomize: 106 | franka_arm_initial_dof_pos: [0, 0.1963, 0, -2.0, 0, 2.3416, 0.7854] 107 | franka_gripper_initial_state: 1.0 108 | 109 | ctrl: 110 | task_space_impedance: 111 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 112 | task_prop_gains: [300, 300, 300, 50, 50, 50] 113 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 114 | -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/bc_transformer_gmm.json: -------------------------------------------------------------------------------- 1 | { 2 | "algo_name": "bc", 3 | "train": { 4 | "seq_length": 1, 5 | "pad_seq_length": true, 6 | "frame_stack": 4, 7 | "pad_frame_stack": true, 8 | "batch_size": 100 9 | }, 10 | "algo": { 11 | "optim_params": { 12 | "policy": { 13 | "optimizer_type": "adamw", 14 | "learning_rate": { 15 | "initial": 0.0001, 16 | "decay_factor": 0.1, 17 | "epoch_schedule": [1000], 18 | "scheduler_type": "linear" 19 | }, 20 | "regularization": { 21 | "L2": 0.01 22 | } 23 | } 24 | }, 25 | "loss": { 26 | "l2_weight": 1.0, 27 | "l1_weight": 0.0, 28 | "cos_weight": 0.0 29 | }, 30 | "actor_layer_dims": [], 31 | "gaussian": { 32 | "enabled": false, 33 | "fixed_std": false, 34 | "init_std": 0.1, 35 | "min_std": 0.01, 36 | "std_activation": "softplus", 37 | "low_noise_eval": true 38 | }, 39 | "gmm": { 40 | "enabled": true, 41 | "num_modes": 5, 42 | "min_std": 0.0001, 43 | "std_activation": "softplus", 44 | "low_noise_eval": true 45 | }, 46 | "vae": { 47 | "enabled": false, 48 | "latent_dim": 14, 49 | "latent_clip": null, 50 | "kl_weight": 1.0, 51 | "decoder": { 52 | "is_conditioned": true, 53 | "reconstruction_sum_across_elements": false 54 | }, 55 | "prior": { 56 | "learn": false, 57 | "is_conditioned": false, 58 | "use_gmm": false, 59 | "gmm_num_modes": 10, 60 | "gmm_learn_weights": false, 61 | "use_categorical": false, 62 | "categorical_dim": 10, 63 | "categorical_gumbel_softmax_hard": false, 64 | "categorical_init_temp": 1.0, 65 | "categorical_temp_anneal_step": 0.001, 66 | "categorical_min_temp": 0.3 67 | }, 68 | "encoder_layer_dims": [ 69 | 300, 70 | 400 71 | ], 72 | "decoder_layer_dims": [ 73 | 300, 74 | 400 75 | ], 76 | "prior_layer_dims": [ 77 | 300, 78 | 400 79 | ] 80 | }, 81 | "rnn": { 82 | "enabled": false, 83 | "horizon": 10, 84 | "hidden_dim": 400, 85 | "rnn_type": "LSTM", 86 | "num_layers": 2, 87 | "open_loop": false, 88 | "kwargs": { 89 | "bidirectional": false 90 | } 91 | }, 92 | "transformer": { 93 | "enabled": true, 94 | "context_length": 4, 95 | "embed_dim": 512, 96 | "num_layers": 6, 97 | "num_heads": 8, 98 | "emb_dropout": 0.1, 99 | "attn_dropout": 0.1, 100 | "block_output_dropout": 0.1, 101 | "sinusoidal_embedding": false, 102 | "activation": "gelu", 103 | "supervise_all_steps": false, 104 | "nn_parameter_for_timesteps": true 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/bc_gmm.json: -------------------------------------------------------------------------------- 1 | { 2 | "algo_name": "bc", 3 | "train": { 4 | "seq_length": 1, 5 | "pad_seq_length": true, 6 | "frame_stack": 0, 7 | "pad_frame_stack": true, 8 | "batch_size": 100 9 | }, 10 | "algo": { 11 | "optim_params": { 12 | "policy": { 13 | "optimizer_type": "adam", 14 | "learning_rate": { 15 | "initial": 0.0001, 16 | "decay_factor": 0.1, 17 | "epoch_schedule": [], 18 | "scheduler_type": "multistep" 19 | }, 20 | "regularization": { 21 | "L2": 0.0 22 | } 23 | } 24 | }, 25 | "loss": { 26 | "l2_weight": 1.0, 27 | "l1_weight": 0.0, 28 | "cos_weight": 0.0 29 | }, 30 | "actor_layer_dims": [ 31 | 4096, 32 | 4096 33 | ], 34 | "gaussian": { 35 | "enabled": false, 36 | "fixed_std": false, 37 | "init_std": 0.1, 38 | "min_std": 0.01, 39 | "std_activation": "softplus", 40 | "low_noise_eval": true 41 | }, 42 | "gmm": { 43 | "enabled": true, 44 | "num_modes": 5, 45 | "min_std": 0.0001, 46 | "std_activation": "softplus", 47 | "low_noise_eval": true 48 | }, 49 | "vae": { 50 | "enabled": false, 51 | "latent_dim": 14, 52 | "latent_clip": null, 53 | "kl_weight": 1.0, 54 | "decoder": { 55 | "is_conditioned": true, 56 | "reconstruction_sum_across_elements": false 57 | }, 58 | "prior": { 59 | "learn": false, 60 | "is_conditioned": false, 61 | "use_gmm": false, 62 | "gmm_num_modes": 10, 63 | "gmm_learn_weights": false, 64 | "use_categorical": false, 65 | "categorical_dim": 10, 66 | "categorical_gumbel_softmax_hard": false, 67 | "categorical_init_temp": 1.0, 68 | "categorical_temp_anneal_step": 0.001, 69 | "categorical_min_temp": 0.3 70 | }, 71 | "encoder_layer_dims": [ 72 | 300, 73 | 400 74 | ], 75 | "decoder_layer_dims": [ 76 | 300, 77 | 400 78 | ], 79 | "prior_layer_dims": [ 80 | 300, 81 | 400 82 | ] 83 | }, 84 | "rnn": { 85 | "enabled": false, 86 | "horizon": 10, 87 | "hidden_dim": 400, 88 | "rnn_type": "LSTM", 89 | "num_layers": 2, 90 | "open_loop": false, 91 | "kwargs": { 92 | "bidirectional": false 93 | } 94 | }, 95 | "transformer": { 96 | "enabled": false, 97 | "context_length": 10, 98 | "embed_dim": 512, 99 | "num_layers": 6, 100 | "num_heads": 8, 101 | "emb_dropout": 0.1, 102 | "attn_dropout": 0.1, 103 | "block_output_dropout": 0.1, 104 | "sinusoidal_embedding": false, 105 | "activation": "gelu", 106 | "supervise_all_steps": false, 107 | "nn_parameter_for_timesteps": true 108 | } 109 | } 110 | } -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/bc_rnn.json: -------------------------------------------------------------------------------- 1 | { 2 | "algo_name": "bc", 3 | "train": { 4 | "seq_length": 8, 5 | "pad_seq_length": true, 6 | "frame_stack": 0, 7 | "pad_frame_stack": true, 8 | "batch_size": 100 9 | }, 10 | "algo": { 11 | "optim_params": { 12 | "policy": { 13 | "optimizer_type": "adam", 14 | "learning_rate": { 15 | "initial": 0.0001, 16 | "decay_factor": 0.1, 17 | "epoch_schedule": [], 18 | "scheduler_type": "multistep" 19 | }, 20 | "regularization": { 21 | "L2": 0.0 22 | } 23 | } 24 | }, 25 | "loss": { 26 | "l2_weight": 1.0, 27 | "l1_weight": 0.0, 28 | "cos_weight": 0.0 29 | }, 30 | "actor_layer_dims": [ 31 | 1024, 32 | 1024 33 | ], 34 | "gaussian": { 35 | "enabled": false, 36 | "fixed_std": false, 37 | "init_std": 0.1, 38 | "min_std": 0.01, 39 | "std_activation": "softplus", 40 | "low_noise_eval": true 41 | }, 42 | "gmm": { 43 | "enabled": false, 44 | "num_modes": 5, 45 | "min_std": 0.0001, 46 | "std_activation": "softplus", 47 | "low_noise_eval": true 48 | }, 49 | "vae": { 50 | "enabled": false, 51 | "latent_dim": 14, 52 | "latent_clip": null, 53 | "kl_weight": 1.0, 54 | "decoder": { 55 | "is_conditioned": true, 56 | "reconstruction_sum_across_elements": false 57 | }, 58 | "prior": { 59 | "learn": false, 60 | "is_conditioned": false, 61 | "use_gmm": false, 62 | "gmm_num_modes": 10, 63 | "gmm_learn_weights": false, 64 | "use_categorical": false, 65 | "categorical_dim": 10, 66 | "categorical_gumbel_softmax_hard": false, 67 | "categorical_init_temp": 1.0, 68 | "categorical_temp_anneal_step": 0.001, 69 | "categorical_min_temp": 0.3 70 | }, 71 | "encoder_layer_dims": [ 72 | 300, 73 | 400 74 | ], 75 | "decoder_layer_dims": [ 76 | 300, 77 | 400 78 | ], 79 | "prior_layer_dims": [ 80 | 300, 81 | 400 82 | ] 83 | }, 84 | "rnn": { 85 | "enabled": true, 86 | "horizon": 8, 87 | "hidden_dim": 1024, 88 | "rnn_type": "LSTM", 89 | "num_layers": 2, 90 | "open_loop": false, 91 | "kwargs": { 92 | "bidirectional": false 93 | } 94 | }, 95 | "transformer": { 96 | "enabled": false, 97 | "context_length": 10, 98 | "embed_dim": 512, 99 | "num_layers": 6, 100 | "num_heads": 8, 101 | "emb_dropout": 0.1, 102 | "attn_dropout": 0.1, 103 | "block_output_dropout": 0.1, 104 | "sinusoidal_embedding": false, 105 | "activation": "gelu", 106 | "supervise_all_steps": false, 107 | "nn_parameter_for_timesteps": true 108 | } 109 | } 110 | } -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/bc_mlp.json: -------------------------------------------------------------------------------- 1 | { 2 | "algo_name": "bc", 3 | "train": { 4 | "seq_length": 1, 5 | "pad_seq_length": true, 6 | "frame_stack": 0, 7 | "pad_frame_stack": true, 8 | "batch_size": 100 9 | }, 10 | "algo": { 11 | "optim_params": { 12 | "policy": { 13 | "optimizer_type": "adam", 14 | "learning_rate": { 15 | "initial": 0.0001, 16 | "decay_factor": 0.1, 17 | "epoch_schedule": [], 18 | "scheduler_type": "multistep" 19 | }, 20 | "regularization": { 21 | "L2": 0.0 22 | } 23 | } 24 | }, 25 | "loss": { 26 | "l2_weight": 1.0, 27 | "l1_weight": 0.0, 28 | "cos_weight": 0.0 29 | }, 30 | "actor_layer_dims": [ 31 | 4096, 32 | 4096 33 | ], 34 | "gaussian": { 35 | "enabled": false, 36 | "fixed_std": false, 37 | "init_std": 0.1, 38 | "min_std": 0.01, 39 | "std_activation": "softplus", 40 | "low_noise_eval": true 41 | }, 42 | "gmm": { 43 | "enabled": false, 44 | "num_modes": 5, 45 | "min_std": 0.0001, 46 | "std_activation": "softplus", 47 | "low_noise_eval": true 48 | }, 49 | "vae": { 50 | "enabled": false, 51 | "latent_dim": 14, 52 | "latent_clip": null, 53 | "kl_weight": 1.0, 54 | "decoder": { 55 | "is_conditioned": true, 56 | "reconstruction_sum_across_elements": false 57 | }, 58 | "prior": { 59 | "learn": false, 60 | "is_conditioned": false, 61 | "use_gmm": false, 62 | "gmm_num_modes": 10, 63 | "gmm_learn_weights": false, 64 | "use_categorical": false, 65 | "categorical_dim": 10, 66 | "categorical_gumbel_softmax_hard": false, 67 | "categorical_init_temp": 1.0, 68 | "categorical_temp_anneal_step": 0.001, 69 | "categorical_min_temp": 0.3 70 | }, 71 | "encoder_layer_dims": [ 72 | 300, 73 | 400 74 | ], 75 | "decoder_layer_dims": [ 76 | 300, 77 | 400 78 | ], 79 | "prior_layer_dims": [ 80 | 300, 81 | 400 82 | ] 83 | }, 84 | "rnn": { 85 | "enabled": false, 86 | "horizon": 10, 87 | "hidden_dim": 400, 88 | "rnn_type": "LSTM", 89 | "num_layers": 2, 90 | "open_loop": false, 91 | "kwargs": { 92 | "bidirectional": false 93 | } 94 | }, 95 | "transformer": { 96 | "enabled": false, 97 | "context_length": 10, 98 | "embed_dim": 512, 99 | "num_layers": 6, 100 | "num_heads": 8, 101 | "emb_dropout": 0.1, 102 | "attn_dropout": 0.1, 103 | "block_output_dropout": 0.1, 104 | "sinusoidal_embedding": false, 105 | "activation": "gelu", 106 | "supervise_all_steps": false, 107 | "nn_parameter_for_timesteps": true 108 | } 109 | } 110 | } -------------------------------------------------------------------------------- /manipgen/config/dagger/robomimic/bc_rnn_gmm.json: -------------------------------------------------------------------------------- 1 | { 2 | "algo_name": "bc", 3 | "train": { 4 | "seq_length": 8, 5 | "pad_seq_length": true, 6 | "frame_stack": 1, 7 | "pad_frame_stack": true, 8 | "batch_size": 100 9 | }, 10 | "algo": { 11 | "optim_params": { 12 | "policy": { 13 | "optimizer_type": "adam", 14 | "learning_rate": { 15 | "initial": 0.0001, 16 | "decay_factor": 0.1, 17 | "epoch_schedule": [], 18 | "scheduler_type": "multistep" 19 | }, 20 | "regularization": { 21 | "L2": 0.0 22 | } 23 | } 24 | }, 25 | "loss": { 26 | "l2_weight": 0.0, 27 | "l1_weight": 0.0, 28 | "cos_weight": 0.0 29 | }, 30 | "actor_layer_dims": [ 31 | 1024, 32 | 1024 33 | ], 34 | "gaussian": { 35 | "enabled": false, 36 | "fixed_std": false, 37 | "init_std": 0.1, 38 | "min_std": 0.01, 39 | "std_activation": "softplus", 40 | "low_noise_eval": true 41 | }, 42 | "gmm": { 43 | "enabled": true, 44 | "num_modes": 5, 45 | "min_std": 0.0001, 46 | "std_activation": "softplus", 47 | "low_noise_eval": true 48 | }, 49 | "vae": { 50 | "enabled": false, 51 | "latent_dim": 14, 52 | "latent_clip": null, 53 | "kl_weight": 1.0, 54 | "decoder": { 55 | "is_conditioned": true, 56 | "reconstruction_sum_across_elements": false 57 | }, 58 | "prior": { 59 | "learn": false, 60 | "is_conditioned": false, 61 | "use_gmm": false, 62 | "gmm_num_modes": 10, 63 | "gmm_learn_weights": false, 64 | "use_categorical": false, 65 | "categorical_dim": 10, 66 | "categorical_gumbel_softmax_hard": false, 67 | "categorical_init_temp": 1.0, 68 | "categorical_temp_anneal_step": 0.001, 69 | "categorical_min_temp": 0.3 70 | }, 71 | "encoder_layer_dims": [ 72 | 300, 73 | 400 74 | ], 75 | "decoder_layer_dims": [ 76 | 300, 77 | 400 78 | ], 79 | "prior_layer_dims": [ 80 | 300, 81 | 400 82 | ] 83 | }, 84 | "rnn": { 85 | "enabled": true, 86 | "horizon": 8, 87 | "hidden_dim": 1024, 88 | "rnn_type": "LSTM", 89 | "num_layers": 2, 90 | "open_loop": false, 91 | "kwargs": { 92 | "bidirectional": false 93 | } 94 | }, 95 | "transformer": { 96 | "enabled": false, 97 | "context_length": 10, 98 | "embed_dim": 512, 99 | "num_layers": 6, 100 | "num_heads": 8, 101 | "emb_dropout": 0.1, 102 | "attn_dropout": 0.1, 103 | "block_output_dropout": 0.1, 104 | "sinusoidal_embedding": false, 105 | "activation": "gelu", 106 | "supervise_all_steps": false, 107 | "nn_parameter_for_timesteps": true 108 | } 109 | } 110 | } -------------------------------------------------------------------------------- /manipgen/config/config_train.yaml: -------------------------------------------------------------------------------- 1 | # choose the task to be executed 2 | task_name: ${task.name} 3 | 4 | # name of the experiment - if not set, inferred from task 5 | exp_name: '' 6 | 7 | # set random seed 8 | seed: 3407 9 | 10 | # number of environments 11 | num_envs: 8192 12 | 13 | # set the maximum number of learning iterations to train for. overrides default per-environment setting 14 | max_iterations: '' 15 | 16 | # device config 17 | device: 'cuda:0' 18 | 19 | # visualization: viewer 20 | render: False 21 | 22 | # visualization: camera - if set, capture `capture_length` frames every `capture_interval` steps. 23 | # `capture_envs` specifies the number of environments to capture videos from 24 | capture_video: False 25 | capture_interval: 1920 # 16 * episode_length 26 | capture_length: ${task.env.episode_length} 27 | capture_envs: 8 28 | 29 | # set the initial progress_buf to be random - reduce correlation between samples in the experience buffer 30 | random_time: False 31 | 32 | # set checkpoint path 33 | checkpoint: '' 34 | 35 | # ratio of samples to use for validation (not used here) 36 | val_ratio: 0.0 37 | 38 | # training directory - checkpoints are saved under {train_dir}/{exp_name}/nn/ 39 | train_dir: 'runs' 40 | 41 | # initial states path - if not set, inferred from task 42 | init_states: '' 43 | 44 | # test - if set, run policy in inference mode 45 | test: False 46 | 47 | # wandb 48 | wandb_activate: False 49 | wandb_project: ${task_name} 50 | 51 | # unidexgrasp / partnet 52 | object_code: '' 53 | object_scale: 0.06 # only applicable for unidexgrasp 54 | object_list: '' # format: [[object_code,object_scale],...] for unidexgrasp, [object_code,...] for partnet 55 | 56 | # (initial states sampling) number of samples to collect 57 | num_samples: 10000 58 | 59 | defaults: 60 | - task: 'pick_cube' 61 | - train: ${task} 62 | - override hydra/job_logging: disabled 63 | - _self_ 64 | 65 | # set the directory where the output files get saved 66 | hydra: 67 | output_subdir: null 68 | run: 69 | dir: . 70 | 71 | # some of the visualization scripts also load this config file 72 | # keep the following keys in the config file for convenience 73 | local_obs: False 74 | global_obs: False 75 | task: 76 | env: 77 | local_obs: 78 | width: 84 79 | height: 84 80 | render_type: 81 | depth: True 82 | rgb: False 83 | segmentation: False 84 | horizontal_fov: 42 # real world: crop 640 x 480 to 332 x 332 85 | camera_offset: [0.06, 0.0, 0.023] 86 | camera_angle: 0.0 87 | use_back_wrist_camera: False 88 | global_obs: 89 | width: 128 90 | height: 128 91 | locations: [[0.5, -0.5, 1.5], [0.5, 0.5, 1.5], [-0.5, -0.5, 1.5], [-0.5, 0.5, 1.5]] 92 | render_type: 93 | depth: True 94 | rgb: False 95 | segmentation: False 96 | 97 | # set True for samplers 98 | sample_mode: False 99 | 100 | # only True when sampling grasp poses 101 | sample_grasp_pose: False 102 | 103 | # control gripper 104 | gripper_control: True 105 | 106 | ## Device config 107 | # 'physx' or 'flex' 108 | physics_engine: 'physx' 109 | # whether to use cpu or gpu pipeline 110 | pipeline: 'gpu' 111 | # device for running physics simulation 112 | sim_device: 'cuda:0' 113 | # device to run RL 114 | rl_device: 'cuda:0' 115 | graphics_device_id: 0 116 | 117 | ## PhysX arguments 118 | num_threads: 4 # Number of worker threads per scene used by PhysX - for CPU PhysX only. 119 | solver_type: 1 # 0: pgs, 1: tgs 120 | num_subscenes: 4 # Splits the simulation into N physics scenes and runs each one in a separate thread -------------------------------------------------------------------------------- /manipgen/config/task/place.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - IndustRealBase 3 | - _self_ 4 | 5 | name: 'place' 6 | use_init_states: True 7 | headless: True # open viewer with render=True instead 8 | physics_engine: "physx" 9 | 10 | env: 11 | numEnvs: ${resolve_default:8192,${...num_envs}} 12 | numObservations: 76 13 | numActions: 6 14 | episode_length: 120 15 | 16 | val_ratio: ${resolve_default:0.0,${...val_ratio}} 17 | 18 | # camera properties 19 | camera: 20 | width: 1440 21 | height: 960 22 | 23 | object_friction: 1.0 24 | 25 | # unidexgrasp 26 | object_code: ${...object_code} 27 | object_scale: ${...object_scale} 28 | object_list: ${...object_list} 29 | 30 | # clutter and obstacles 31 | enable_clutter_and_obstacle: True 32 | num_clutter_objects: 5 33 | clutter_object_radius: 0.05 34 | clutter_dist_max: 0.40 35 | num_obstacles: 2 36 | obstacle_width: 0.40 37 | obstacle_height: 0.30 38 | obstacle_pos_noise: 0.10 39 | obstacle_rot_noise: 0.25 40 | obstacle_show_up_prob: 0.7 41 | use_unidexgrasp_clutter: False # if False, uses capsules; if True, use unidexgrasp objects; automatically set to True in multitask dagger 42 | num_unidexgrasp_clutter_rest_poses: 16 43 | unidexgrasp_file_list: assets/unidexgrasp/trainset3419.txt 44 | 45 | sampler: 46 | # number of RL steps for pick policy 47 | num_rl_steps: 90 48 | # number of steps for closing gripper 49 | num_close_gripper_steps: 40 50 | # number of steps for lifting the arm 51 | num_lift_up_steps: 30 52 | # number of steps for randomizing franka and object pose 53 | num_randomization_steps: 20 54 | 55 | # for each pick result, we sample multiple randomized franka and object poses 56 | num_randomization_per_policy: 15 57 | 58 | # configuration for pick policy 59 | # resolve the value of policy_init_states_path and policy_checkpoint_path programmatically 60 | # too complicated to be resolved in the config file 61 | policy_init_states_path: "" 62 | policy_config_path: "config/train/pick.yaml" 63 | policy_checkpoint_path: "" 64 | 65 | # restriction for object position 66 | object_pos_noise: 0.2 67 | object_height_lower: 0.05 68 | object_height_upper: 0.12 69 | 70 | # filter initial states where the object drops 71 | # we keep Franka rest for `num_filter_steps` steps for each initial state and check if the object is still grasped 72 | num_filter_steps: 20 73 | 74 | rl: 75 | # episode length 76 | max_episode_length: ${..env.episode_length} 77 | 78 | # actions 79 | pos_action_scale: [0.05, 0.05, 0.05] 80 | rot_action_scale: [0.05, 0.05, 0.05] 81 | force_action_scale: [1.0, 1.0, 1.0] 82 | torque_action_scale: [1.0, 1.0, 1.0] 83 | 84 | clamp_rot: True 85 | clamp_rot_thresh: 1.0e-6 86 | 87 | # filter initial object poses where the grasp success rate is lower than the threshold 88 | # for place env, this should only be applied in sample mode to match obs for pick policy 89 | filter_pose_threshold: 0.01 90 | 91 | # reward settings 92 | target_xy_noise: 0.0 # manually set this to 0.15 in command during training 93 | dist_object_keypoints_reward_temp: -15.0 94 | dist_object_gripper_reward_temp: -3.0 95 | eef_pose_consistency_reward_temp: -5.0 96 | success_bonus: 15.0 97 | success_threshold: 0.04 98 | 99 | finger_contact_force_threshold: 1.0 100 | finger_contact_reward_temp: 0.0 101 | 102 | num_object_keypoints: 8 103 | gripper_keypoint_scale: 0.5 104 | gripper_keypoint_dof: 5 105 | use_object_keypoint_6d: False 106 | object_keypoint_6d_scale: 0.5 107 | 108 | randomize: 109 | franka_arm_initial_dof_pos: [0, 0.1963, 0, -2.0, 0, 2.3416, 0.7854] 110 | franka_gripper_initial_state: 0.0 111 | 112 | ctrl: 113 | task_space_impedance: 114 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 115 | task_prop_gains: [300, 300, 300, 50, 50, 50] 116 | task_deriv_gains: [34, 34, 34, 1.4, 1.4, 1.4] 117 | -------------------------------------------------------------------------------- /manipgen/config/task/IndustRealBase.yaml: -------------------------------------------------------------------------------- 1 | # See schema in factory_schema_config_base.py for descriptions of parameters. 2 | 3 | defaults: 4 | - _self_ 5 | 6 | mode: 7 | export_scene: False 8 | export_states: False 9 | 10 | sim: 11 | dt: 0.016667 12 | substeps: 2 13 | up_axis: "z" 14 | use_gpu_pipeline: True 15 | gravity: [0.0, 0.0, -9.81] 16 | add_damping: True 17 | disable_franka_collisions: False 18 | 19 | physx: 20 | solver_type: ${....solver_type} 21 | num_threads: ${....num_threads} 22 | num_subscenes: ${....num_subscenes} 23 | use_gpu: ${contains:"cuda",${....sim_device}} 24 | 25 | num_position_iterations: 8 # 16 -> 8 for speed up 26 | num_velocity_iterations: 0 27 | contact_offset: 0.01 28 | rest_offset: 0.0 29 | bounce_threshold_velocity: 0.2 30 | max_depenetration_velocity: 5.0 31 | friction_offset_threshold: 0.01 32 | friction_correlation_distance: 0.00625 33 | 34 | max_gpu_contact_pairs: 8388608 # 8 * 1024 * 1024 35 | default_buffer_size_multiplier: 2.0 36 | contact_collection: 1 # 0: CC_NEVER (don't collect contact info), 1: CC_LAST_SUBSTEP (collect only contacts on last substep), 2: CC_ALL_SUBSTEPS (broken - do not use!) 37 | 38 | env: 39 | env_spacing: 1.0 40 | franka_depth: 0.50 # Franka origin 50 cm behind table midpoint - keep consistent with parameters before refactoring 41 | table_height: 1.04 42 | franka_friction: 1.0 43 | table_friction: 1.0 44 | 45 | randomize: 46 | franka_arm_initial_dof_pos: [-1.7574766278484677, 0.8403247702305783, 2.015877580177467, -2.0924931236718334, -0.7379389376686856, 1.6256438760537268, 1.2689337870766628] 47 | fingertip_centered_pos_initial: [0.0, 0.0, 0.2] # initial position of midpoint between fingertips above table 48 | fingertip_centered_pos_noise: [0.0, 0.0, 0.0] # noise on fingertip pos 49 | fingertip_centered_rot_initial: [3.141593, 0.0, 0.0] # initial rotation of fingertips (Euler) 50 | fingertip_centered_rot_noise: [0.0, 0.0, 0.0] # noise on fingertip rotation 51 | 52 | ctrl: 53 | ctrl_type: task_space_impedance # {gym_default, 54 | # joint_space_ik, joint_space_id, 55 | # task_space_impedance, operational_space_motion, 56 | # open_loop_force, closed_loop_force, 57 | # hybrid_force_motion} 58 | all: 59 | jacobian_type: geometric 60 | gripper_prop_gains: [500, 500] 61 | gripper_deriv_gains: [2, 2] 62 | # gripper_prop_gains: [100, 100] # close to speed of real gripper 63 | # gripper_deriv_gains: [2, 2] 64 | gym_default: 65 | ik_method: dls 66 | joint_prop_gains: [40, 40, 40, 40, 40, 40, 40] 67 | joint_deriv_gains: [8, 8, 8, 8, 8, 8, 8] 68 | gripper_prop_gains: [500, 500] 69 | gripper_deriv_gains: [20, 20] 70 | joint_space_ik: 71 | ik_method: dls 72 | joint_prop_gains: [1, 1, 1, 1, 1, 1, 1] 73 | joint_deriv_gains: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1] 74 | joint_space_id: 75 | ik_method: dls 76 | joint_prop_gains: [40, 40, 40, 40, 40, 40, 40] 77 | joint_deriv_gains: [8, 8, 8, 8, 8, 8, 8] 78 | task_space_impedance: 79 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 80 | task_prop_gains: [1000, 1000, 1000, 50, 50, 50] 81 | task_deriv_gains: [63.2, 63.2, 63.2, 1.41, 1.41, 1.41] 82 | operational_space_motion: 83 | motion_ctrl_axes: [1, 1, 1, 1, 1, 1] 84 | task_prop_gains: [150, 150, 150, 150, 150, 150] 85 | task_deriv_gains: [25, 25, 25, 25, 25, 25] 86 | open_loop_force: 87 | force_ctrl_axes: [0, 0, 1, 0, 0, 0] 88 | closed_loop_force: 89 | force_ctrl_axes: [0, 0, 1, 0, 0, 0] 90 | wrench_prop_gains: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1] 91 | hybrid_force_motion: 92 | motion_ctrl_axes: [1, 1, 0, 1, 1, 1] 93 | task_prop_gains: [40, 40, 40, 40, 40, 40] 94 | task_deriv_gains: [8, 8, 8, 8, 8, 8] 95 | force_ctrl_axes: [0, 0, 1, 0, 0, 0] 96 | wrench_prop_gains: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1] 97 | -------------------------------------------------------------------------------- /manipgen/config/config_sample.yaml: -------------------------------------------------------------------------------- 1 | # choose the task to be executed 2 | task_name: ${task.name} 3 | 4 | # set random seed 5 | seed: -1 6 | 7 | # number of environments 8 | num_envs: 8192 9 | 10 | # device config 11 | device: 'cuda:0' 12 | 13 | # visualization: viewer 14 | render: False 15 | 16 | # visualization: camera - if set, capture `capture_length` frames every `capture_interval` steps. 17 | # `capture_envs` specifies the number of environments to capture videos from 18 | capture_video: False 19 | capture_interval: 3600 20 | capture_length: ${task.env.episode_length} 21 | capture_envs: 8 22 | 23 | # set the initial progress_buf to be random - reduce correlation between samples in the experience buffer 24 | random_time: False 25 | 26 | # ratio of samples to use for validation (not used here) 27 | val_ratio: 0.0 28 | 29 | # set checkpoint path 30 | checkpoint: '' 31 | 32 | # initial states path - if not set, inferred from task 33 | init_states: '' 34 | 35 | # directory to save the initial states in 36 | init_state_dir: '' 37 | 38 | # unidexgrasp / partnet 39 | object_code: '' 40 | object_scale: 0.06 # only applicable for unidexgrasp 41 | object_list: '' # format: [[object_code,object_scale],...] for unidexgrasp, [object_code,...] for partnet 42 | 43 | # number of samples to collect 44 | num_samples: 10000 45 | 46 | # maximum number of iterations: if the number of samples collected is less than `num_samples`, 47 | # the sampler will continue to collect samples until `max_iters` is reached. 48 | # This is useful when filtering is enabled. 49 | max_iters: 3 50 | 51 | # the samplers set different diversity of initial states for state-based and vision-based policies 52 | vision_based: True 53 | 54 | # filter initial states (e.g., for place cube, filter out states that will lead to cube falling off the gripper) 55 | filter: False 56 | 57 | # generate samples in easy mode (e.g., for place cube, pick up the cube and move above the target location by teleportation) 58 | easy_mode: False 59 | 60 | # filter initial states based on vision (e.g., for pick cube, filter out states where the cube is not visible. 61 | # The cube is invisible if the cube's mask takes up less than `filter_vision_threshold` of the pixels) 62 | filter_vision: False 63 | filter_vision_threshold: 0.02 64 | 65 | local_obs: False 66 | global_obs: False 67 | task: 68 | env: 69 | local_obs: 70 | width: 84 71 | height: 84 72 | render_type: 73 | depth: False 74 | rgb: False 75 | segmentation: True 76 | horizontal_fov: 42 # real world: crop 640 x 480 to 332 x 332 77 | camera_offset: [0.06, 0.0, 0.023] 78 | camera_angle: 0.0 79 | use_back_wrist_camera: False 80 | global_obs: 81 | width: 128 82 | height: 128 83 | locations: [[0.5, -0.5, 1.5], [0.5, 0.5, 1.5], [-0.5, -0.5, 1.5], [-0.5, 0.5, 1.5]] 84 | render_type: 85 | depth: True 86 | rgb: False 87 | segmentation: False 88 | 89 | sim: 90 | physx: 91 | contact_collection: 2 92 | 93 | defaults: 94 | - task: 'pick_cube' 95 | - train: ${task} 96 | - override hydra/job_logging: disabled 97 | - _self_ 98 | 99 | # set the directory where the output files get saved 100 | hydra: 101 | output_subdir: null 102 | run: 103 | dir: . 104 | 105 | # path to the sif file 106 | sif_path: '/projects/rsalakhugroup/containers/skills_planning.sif' 107 | 108 | # path to asset files (objects and codes in txt file) 109 | asset_file_path: "assets/unidexgrasp/trainset3363.txt" 110 | 111 | # set True for samplers 112 | sample_mode: True 113 | 114 | # control gripper 115 | gripper_control: True 116 | 117 | ## Device config 118 | # 'physx' or 'flex' 119 | physics_engine: 'physx' 120 | # whether to use cpu or gpu pipeline 121 | pipeline: 'gpu' 122 | # device for running physics simulation 123 | sim_device: 'cuda:0' 124 | # device to run RL 125 | rl_device: 'cuda:0' 126 | graphics_device_id: 0 127 | 128 | ## PhysX arguments 129 | num_threads: 4 # Number of worker threads per scene used by PhysX - for CPU PhysX only. 130 | solver_type: 1 # 0: pgs, 1: tgs 131 | num_subscenes: 4 # Splits the simulation into N physics scenes and runs each one in a separate thread 132 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/visual/link6.mtl: -------------------------------------------------------------------------------- 1 | # Blender MTL File: 'None' 2 | # Material Count: 17 3 | 4 | newmtl Face064_002_001_002_001 5 | Ns -1.960784 6 | Ka 1.000000 1.000000 1.000000 7 | Kd 1.000000 0.000000 0.000000 8 | Ks 0.003906 0.003906 0.003906 9 | Ke 0.000000 0.000000 0.000000 10 | Ni 1.000000 11 | d 1.000000 12 | illum 2 13 | 14 | newmtl Face065_002_001_002_001 15 | Ns -1.960784 16 | Ka 1.000000 1.000000 1.000000 17 | Kd 0.000000 1.000000 0.000000 18 | Ks 0.003906 0.003906 0.003906 19 | Ke 0.000000 0.000000 0.000000 20 | Ni 1.000000 21 | d 1.000000 22 | illum 2 23 | 24 | newmtl Face374_002_001_002_001 25 | Ns -1.960784 26 | Ka 1.000000 1.000000 1.000000 27 | Kd 1.000000 1.000000 1.000000 28 | Ks 0.003906 0.003906 0.003906 29 | Ke 0.000000 0.000000 0.000000 30 | Ni 1.000000 31 | d 1.000000 32 | illum 2 33 | 34 | newmtl Face539_002_001_002_001 35 | Ns -1.960784 36 | Ka 1.000000 1.000000 1.000000 37 | Kd 0.250980 0.250980 0.250980 38 | Ks 0.003906 0.003906 0.003906 39 | Ke 0.000000 0.000000 0.000000 40 | Ni 1.000000 41 | d 1.000000 42 | illum 2 43 | 44 | newmtl Part__Feature001_009_001_002_001 45 | Ns -1.960784 46 | Ka 1.000000 1.000000 1.000000 47 | Kd 0.250980 0.250980 0.250980 48 | Ks 0.003906 0.003906 0.003906 49 | Ke 0.000000 0.000000 0.000000 50 | Ni 1.000000 51 | d 1.000000 52 | illum 2 53 | 54 | newmtl Part__Feature002_006_001_002_001 55 | Ns -1.960784 56 | Ka 1.000000 1.000000 1.000000 57 | Kd 0.250980 0.250980 0.250980 58 | Ks 0.003906 0.003906 0.003906 59 | Ke 0.000000 0.000000 0.000000 60 | Ni 1.000000 61 | d 1.000000 62 | illum 2 63 | 64 | newmtl Shell002_002_001_002_001 65 | Ns -1.960784 66 | Ka 1.000000 1.000000 1.000000 67 | Kd 1.000000 1.000000 1.000000 68 | Ks 0.003906 0.003906 0.003906 69 | Ke 0.000000 0.000000 0.000000 70 | Ni 1.000000 71 | d 1.000000 72 | illum 2 73 | 74 | newmtl Shell003_002_001_002_001 75 | Ns -1.960784 76 | Ka 1.000000 1.000000 1.000000 77 | Kd 1.000000 1.000000 1.000000 78 | Ks 0.003906 0.003906 0.003906 79 | Ke 0.000000 0.000000 0.000000 80 | Ni 1.000000 81 | d 1.000000 82 | illum 2 83 | 84 | newmtl Shell004_001_001_002_001 85 | Ns -1.960784 86 | Ka 1.000000 1.000000 1.000000 87 | Kd 1.000000 1.000000 1.000000 88 | Ks 0.003906 0.003906 0.003906 89 | Ke 0.000000 0.000000 0.000000 90 | Ni 1.000000 91 | d 1.000000 92 | illum 2 93 | 94 | newmtl Shell005_001_001_002_001 95 | Ns -1.960784 96 | Ka 1.000000 1.000000 1.000000 97 | Kd 1.000000 1.000000 1.000000 98 | Ks 0.003906 0.003906 0.003906 99 | Ke 0.000000 0.000000 0.000000 100 | Ni 1.000000 101 | d 1.000000 102 | illum 2 103 | 104 | newmtl Shell006_003_002_001 105 | Ns -1.960784 106 | Ka 1.000000 1.000000 1.000000 107 | Kd 0.901961 0.921569 0.929412 108 | Ks 0.015625 0.015625 0.015625 109 | Ke 0.000000 0.000000 0.000000 110 | Ni 1.000000 111 | d 1.000000 112 | illum 2 113 | 114 | newmtl Shell007_002_002_001 115 | Ns -1.960784 116 | Ka 1.000000 1.000000 1.000000 117 | Kd 0.250000 0.250000 0.250000 118 | Ks 0.003906 0.003906 0.003906 119 | Ke 0.000000 0.000000 0.000000 120 | Ni 1.000000 121 | d 1.000000 122 | illum 2 123 | 124 | newmtl Shell011_002_002_001 125 | Ns -1.960784 126 | Ka 1.000000 1.000000 1.000000 127 | Kd 1.000000 1.000000 1.000000 128 | Ks 0.003906 0.003906 0.003906 129 | Ke 0.000000 0.000000 0.000000 130 | Ni 1.000000 131 | d 1.000000 132 | illum 2 133 | 134 | newmtl Shell012_002_002_001 135 | Ns -1.960784 136 | Ka 1.000000 1.000000 1.000000 137 | Kd 1.000000 1.000000 1.000000 138 | Ks 0.003906 0.003906 0.003906 139 | Ke 0.000000 0.000000 0.000000 140 | Ni 1.000000 141 | d 1.000000 142 | illum 2 143 | 144 | newmtl Shell_003_001_002_001 145 | Ns -1.960784 146 | Ka 1.000000 1.000000 1.000000 147 | Kd 0.250980 0.250980 0.250980 148 | Ks 0.003906 0.003906 0.003906 149 | Ke 0.000000 0.000000 0.000000 150 | Ni 1.000000 151 | d 1.000000 152 | illum 2 153 | 154 | newmtl Union001_001_001_002_001 155 | Ns -1.960784 156 | Ka 1.000000 1.000000 1.000000 157 | Kd 0.039216 0.541176 0.780392 158 | Ks 0.003906 0.003906 0.003906 159 | Ke 0.000000 0.000000 0.000000 160 | Ni 1.000000 161 | d 1.000000 162 | illum 2 163 | 164 | newmtl Union_001_001_002_001 165 | Ns -1.960784 166 | Ka 1.000000 1.000000 1.000000 167 | Kd 0.039216 0.541176 0.780392 168 | Ks 0.003906 0.003906 0.003906 169 | Ke 0.000000 0.000000 0.000000 170 | Ni 1.000000 171 | d 1.000000 172 | illum 2 173 | -------------------------------------------------------------------------------- /manipgen/utils/robomimic_utils.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | import json 3 | 4 | from manipgen_robomimic.algo import algo_factory 5 | from manipgen_robomimic.config import config_factory 6 | import manipgen_robomimic.utils.file_utils as FileUtils 7 | import manipgen_robomimic.utils.obs_utils as ObsUtils 8 | import manipgen_robomimic.utils.train_utils as TrainUtils 9 | 10 | 11 | def load_config(algo_cfg_path, override_cfg): 12 | base_cfg = json.load(open(f"config/dagger/robomimic/base_{override_cfg.dagger.visual_obs_type}.json", 'r')) 13 | algo_cfg = json.load(open(algo_cfg_path, 'r')) 14 | config = config_factory(algo_cfg["algo_name"]) 15 | # update config with external json - this will throw errors if 16 | # the external config has keys not present in the base algo config 17 | with config.values_unlocked(): 18 | # load json cfg values 19 | config.update(base_cfg) 20 | config.update(algo_cfg) 21 | # override other params from dagger cfg 22 | config.experiment.name = override_cfg.exp_name 23 | config.train.seed = override_cfg.seed 24 | config.train.cuda = "cuda" in override_cfg.rl_device 25 | if override_cfg.dagger.batch_size is not None: 26 | config.train.batch_size = override_cfg.dagger.batch_size 27 | if override_cfg.dagger.lr is not None: 28 | config.algo.optim_params.policy.learning_rate.initial = override_cfg.dagger.lr 29 | 30 | return config 31 | 32 | 33 | def initialize(config): 34 | ObsUtils.initialize_obs_utils_with_config(config) 35 | 36 | 37 | def get_obs_shape_meta(config, dataset_spec, in_memory=True): 38 | dataset_path, in_memory_dataset = (dataset_spec, None) if not in_memory else (None, dataset_spec) 39 | return FileUtils.get_shape_metadata_from_dataset( 40 | dataset_path=dataset_path, 41 | in_memory_dataset=in_memory_dataset, 42 | all_obs_keys=config.all_obs_keys, 43 | verbose=True, 44 | ) 45 | 46 | 47 | def get_dataset_loader(config, shape_meta, in_memory=True): 48 | if in_memory: 49 | return lambda in_mem_data: TrainUtils.dataset_factory( 50 | config=config, 51 | obs_keys=shape_meta["all_obs_keys"], 52 | filter_by_attribute=None, 53 | in_memory_data=in_mem_data, 54 | ) 55 | else: 56 | return lambda path: TrainUtils.dataset_factory( 57 | config=config, 58 | obs_keys=shape_meta["all_obs_keys"], 59 | filter_by_attribute=None, 60 | dataset_path=path, 61 | ) 62 | 63 | 64 | def build_model(config, shape_meta, device): 65 | model = algo_factory( 66 | algo_name=config.algo_name, 67 | config=config, 68 | obs_key_shapes=shape_meta["all_shapes"], 69 | ac_dim=shape_meta["ac_dim"], 70 | device=device, 71 | ) 72 | return model 73 | 74 | 75 | def run_epoch(model, data_loader, epoch, mode, n_iter=1): 76 | assert mode in ('train', 'eval') 77 | train = mode == 'train' 78 | 79 | info = TrainUtils.run_epoch( 80 | model=model, 81 | data_loader=data_loader, 82 | epoch=epoch, 83 | validate=not train, 84 | num_steps=len(data_loader) * n_iter, # loop through dataset n_iter times 85 | obs_normalization_stats=None, 86 | ) 87 | if train: 88 | model.on_epoch_end(epoch) 89 | 90 | return info 91 | 92 | 93 | _hidden_state_attributes = [ 94 | "_rnn_hidden_state" 95 | ] 96 | 97 | def get_hidden_state(model): 98 | return deepcopy(tuple(getattr(model, name, None) for name in _hidden_state_attributes)) 99 | 100 | 101 | def set_hidden_state(model, state): 102 | for name, value in zip(_hidden_state_attributes, state): 103 | if hasattr(model, name): 104 | setattr(model, name, value) 105 | 106 | 107 | def get_rollout_action(model, state_obs, visual_obs, state_frame0_obs, visual_frame0_obs): 108 | return model.get_action( 109 | obs_dict={ 110 | "state": state_obs[:, :7], # only keep first 7 elements (proprio obs) 111 | "state_frame0": state_frame0_obs[:, :7], # only keep first 7 elements (proprio obs) 112 | "visual": visual_obs, 113 | "visual_frame0": visual_frame0_obs, 114 | }, 115 | ) 116 | -------------------------------------------------------------------------------- /manipgen/assets/industreal/yaml/industreal_asset_info_pegs.yaml: -------------------------------------------------------------------------------- 1 | round_peg_hole_4mm: 2 | round_peg_4mm: 3 | urdf_path: 'industreal_round_peg_4mm' 4 | diameter: 0.003988 5 | length: 0.050 6 | density: 8000.0 7 | friction: 1.0 8 | grasp_offset: 0.04 9 | plug_width: 0.004 10 | round_hole_4mm: 11 | urdf_path: 'industreal_round_hole_4mm' 12 | diameter: 0.0041 13 | height: 0.028 14 | depth: 0.023 15 | density: 8000.0 16 | friction: 0.5 17 | 18 | round_peg_hole_8mm: 19 | round_peg_8mm: 20 | urdf_path: 'industreal_round_peg_8mm' 21 | diameter: 0.007986 22 | length: 0.050 23 | density: 8000.0 24 | friction: 1.0 25 | grasp_offset: 0.04 26 | plug_width: 0.008 27 | round_hole_8mm: 28 | urdf_path: 'industreal_round_hole_8mm' 29 | diameter: 0.0081 30 | height: 0.028 31 | depth: 0.023 32 | density: 8000.0 33 | friction: 0.5 34 | 35 | round_peg_hole_12mm: 36 | round_peg_12mm: 37 | urdf_path: 'industreal_round_peg_12mm' 38 | diameter: 0.011983 39 | length: 0.050 40 | density: 8000.0 41 | friction: 1.0 42 | grasp_offset: 0.04 43 | plug_width: 0.012 44 | round_hole_12mm: 45 | urdf_path: 'industreal_round_hole_12mm' 46 | diameter: 0.0122 47 | height: 0.028 48 | depth: 0.023 49 | density: 8000.0 50 | friction: 0.5 51 | 52 | round_peg_hole_16mm: 53 | round_peg_16mm: 54 | urdf_path: 'industreal_round_peg_16mm' 55 | diameter: 0.015983 56 | length: 0.050 57 | density: 8000.0 58 | friction: 1.0 59 | grasp_offset: 0.04 60 | plug_width: 0.016 61 | round_hole_16mm: 62 | urdf_path: 'industreal_round_hole_16mm' 63 | diameter: 0.0165 64 | height: 0.028 65 | depth: 0.023 66 | density: 8000.0 67 | friction: 0.5 68 | 69 | rectangular_peg_hole_4mm: 70 | rectangular_peg_4mm: 71 | urdf_path: 'industreal_rectangular_peg_4mm' 72 | width: 0.00397 73 | depth: 0.00397 74 | length: 0.050 75 | density: 8000.0 76 | friction: 1.0 77 | grasp_offset: 0.04 78 | plug_width: 0.004 79 | rectangular_hole_4mm: 80 | urdf_path: 'industreal_rectangular_hole_4mm' 81 | width: 0.00411 82 | height: 0.028 83 | depth: 0.023 84 | density: 8000.0 85 | friction: 0.5 86 | 87 | rectangular_peg_hole_8mm: 88 | rectangular_peg_8mm: 89 | urdf_path: 'industreal_rectangular_peg_8mm' 90 | width: 0.007964 91 | depth: 0.006910 92 | length: 0.050 93 | density: 8000.0 94 | friction: 1.0 95 | grasp_offset: 0.04 96 | plug_width: 0.008 97 | rectangular_hole_8mm: 98 | urdf_path: 'industreal_rectangular_hole_8mm' 99 | width: 0.0081444 100 | height: 0.028 101 | depth: 0.023 102 | density: 8000.0 103 | friction: 0.5 104 | 105 | rectangular_peg_hole_12mm: 106 | rectangular_peg_12mm: 107 | urdf_path: 'industreal_rectangular_peg_12mm' 108 | width: 0.011957 109 | depth: 0.007910 110 | length: 0.050 111 | density: 8000.0 112 | friction: 1.0 113 | grasp_offset: 0.04 114 | plug_width: 0.012 115 | rectangular_hole_12mm: 116 | urdf_path: 'industreal_rectangular_hole_12mm' 117 | width: 0.0121778 118 | height: 0.028 119 | depth: 0.023 120 | density: 8000.0 121 | friction: 0.5 122 | 123 | rectangular_peg_hole_16mm: 124 | rectangular_peg_16mm: 125 | urdf_path: 'industreal_rectangular_peg_16mm' 126 | width: 0.015957 127 | depth: 0.009910 128 | length: 0.050 129 | density: 8000.0 130 | friction: 1.0 131 | grasp_offset: 0.04 132 | plug_width: 0.016 133 | rectangular_hole_16mm: 134 | urdf_path: 'industreal_rectangular_hole_16mm' 135 | width: 0.0162182 136 | height: 0.028 137 | depth: 0.023 138 | density: 8000.0 139 | friction: 0.5 -------------------------------------------------------------------------------- /manipgen/train.py: -------------------------------------------------------------------------------- 1 | from isaacgym import gymapi 2 | from isaacgym import gymutil 3 | from isaacgym import gymtorch 4 | 5 | import torch 6 | from rl_games.common import env_configurations, vecenv 7 | from rl_games.torch_runner import Runner, _restore, _override_sigma 8 | from rl_games.algos_torch import players 9 | from isaacgymenvs.utils.rlgames_utils import RLGPUEnv, MultiObserver 10 | from isaacgymenvs.utils.utils import set_seed 11 | from isaacgymenvs.utils.reformat import omegaconf_to_dict 12 | 13 | import hydra 14 | from omegaconf import DictConfig, OmegaConf 15 | import time 16 | import signal 17 | import os 18 | 19 | from manipgen.envs import environments 20 | from manipgen.utils.rlgames_utils import ( 21 | PSLAlgoObserver, 22 | WandbAlgoObserver, 23 | A2CAgent_ManipGen, 24 | ) 25 | 26 | 27 | @hydra.main(version_base="1.1", config_path="./config", config_name="config_train") 28 | def train(cfg: DictConfig): 29 | 30 | # ===================== Env ===================== 31 | def create_isaacgym_env(**kwargs): 32 | if cfg.task.use_init_states: 33 | if cfg.init_states != "": 34 | init_states = torch.load(cfg.init_states) 35 | else: 36 | task_name = cfg.task_name 37 | if task_name in ("pick", "place"): 38 | code = cfg.task.env.object_code.replace("/", "-") 39 | scale = int(100 * cfg.task.env.object_scale) 40 | task_name = task_name + "_" + code + "_" + f"{scale:03d}" 41 | elif task_name in ("grasp_handle", "open", "close", "open_nograsp", "close_nograsp"): 42 | task_name = task_name + "_" + cfg.task.env.object_code 43 | init_states = torch.load(f"init_states/franka_{task_name}_init_states.pt") 44 | 45 | idx = torch.randperm(len(list(init_states.values())[0])) 46 | for key in init_states.keys(): 47 | init_states[key] = init_states[key][idx] 48 | else: 49 | init_states = None 50 | envs = environments[cfg.task_name]( 51 | cfg, 52 | init_states 53 | ) 54 | return envs 55 | 56 | set_seed(cfg.seed) 57 | env_configurations.register( 58 | "rlgpu", 59 | { 60 | "vecenv_type": "RLGPU", 61 | "env_creator": lambda **kwargs: create_isaacgym_env(**kwargs), 62 | }, 63 | ) 64 | 65 | vecenv.register( 66 | "RLGPU", 67 | lambda config_name, num_actors, **kwargs: RLGPUEnv( 68 | config_name, num_actors, **kwargs 69 | ), 70 | ) 71 | 72 | # ===================== RL ===================== 73 | rlgames_config = omegaconf_to_dict(cfg["train"]) 74 | if cfg.exp_name != "": 75 | rlgames_config["params"]["config"]["full_experiment_name"] = cfg.exp_name 76 | rlgames_config["params"]["seed"] = cfg.seed 77 | rlgames_config["params"]["config"]["device"] = cfg.device 78 | rlgames_config["params"]["config"]["train_dir"] = cfg.train_dir 79 | 80 | observers = [ 81 | PSLAlgoObserver(), 82 | ] 83 | if cfg.wandb_activate: 84 | observers.append(WandbAlgoObserver(cfg)) 85 | runner = Runner(MultiObserver(observers)) 86 | runner.algo_factory.register_builder( 87 | "a2c_continuous_sp", lambda **kwargs: A2CAgent_ManipGen(**kwargs) 88 | ) 89 | runner.player_factory.register_builder( 90 | "a2c_continuous_sp", lambda **kwargs: players.PpoPlayerContinuous(**kwargs) 91 | ) 92 | runner.load(rlgames_config) 93 | runner.reset() 94 | 95 | # set up rl agent 96 | agent = runner.algo_factory.create(runner.algo_name, base_name='run', params=runner.params) 97 | _restore(agent, {"checkpoint": cfg.checkpoint}) 98 | _override_sigma(agent, {}) 99 | def handle(signum, frame): 100 | print("Signal handler called with signal", signum) 101 | print("Saving checkpoint before exiting...") 102 | agent.save(os.path.join(agent.nn_dir, "checkpoint_latest")) 103 | exit() 104 | signal.signal(signal.SIGUSR1, handle) 105 | 106 | tik = time.time() 107 | agent.train() 108 | tok = time.time() 109 | print( 110 | "{} finished in {:.2f} seconds".format( 111 | "Training" if not cfg.test else "Testing", tok - tik 112 | ) 113 | ) 114 | 115 | 116 | if __name__ == "__main__": 117 | train() 118 | -------------------------------------------------------------------------------- /manipgen/assets/urdf/franka_description/meshes/collision/finger.obj: -------------------------------------------------------------------------------- 1 | #### 2 | # 3 | # OBJ File Generated by Meshlab 4 | # 5 | #### 6 | # Object finger.obj 7 | # 8 | # Vertices: 52 9 | # Faces: 32 10 | # 11 | #### 12 | vn 0.999991 0.003723 -0.001919 13 | v 0.010360 0.026403 0.000155 14 | vn 0.019341 -0.997893 -0.061925 15 | v 0.010449 0.002583 0.000147 16 | vn -0.999568 -0.025962 0.013789 17 | v -0.010387 0.002534 0.000132 18 | vn -0.999606 -0.009503 0.026403 19 | v -0.010479 0.016102 0.018988 20 | vn -0.000579 0.001464 -0.999999 21 | v -0.010401 0.026309 0.000167 22 | vn -0.044737 0.976483 0.210900 23 | v -0.010389 0.025220 0.019188 24 | vn -0.871286 -0.490748 0.005227 25 | v -0.008730 -0.000024 0.036165 26 | vn 0.999861 0.006488 0.015354 27 | v 0.010400 0.025253 0.019037 28 | vn 0.377718 0.867563 0.323518 29 | v 0.005840 0.014274 0.053803 30 | vn 0.736099 -0.021564 0.676530 31 | v 0.008616 0.013989 0.051328 32 | vn 0.999373 -0.008600 0.034345 33 | v 0.010495 0.015103 0.018436 34 | vn 0.013041 -0.999896 -0.006124 35 | v 0.008693 -0.000133 0.050166 36 | vn -0.998603 -0.032800 0.041418 37 | v -0.008623 -0.000057 0.050953 38 | vn -0.588468 -0.017705 0.808327 39 | v -0.005481 -0.000091 0.053725 40 | vn 0.004085 -0.008700 0.999954 41 | v -0.005278 0.014293 0.053849 42 | vn -0.691057 -0.012018 0.722700 43 | v -0.007778 0.014218 0.052366 44 | vn -0.665951 0.690851 0.281486 45 | v -0.008841 0.013918 0.050589 46 | vn 0.736099 -0.021564 0.676530 47 | v 0.006138 -0.000021 0.053578 48 | vn -0.002818 0.998255 0.058981 49 | v 0.010360 0.026403 0.000155 50 | vn 0.000073 0.000898 -1.000000 51 | v 0.010360 0.026403 0.000155 52 | vn 0.999898 -0.012431 0.007036 53 | v 0.010449 0.002583 0.000147 54 | vn 0.000724 0.000331 -1.000000 55 | v 0.010449 0.002583 0.000147 56 | vn -0.871286 -0.490748 0.005227 57 | v -0.010387 0.002534 0.000132 58 | vn 0.002403 -0.997480 -0.070914 59 | v -0.010387 0.002534 0.000132 60 | vn 0.000073 0.000898 -1.000000 61 | v -0.010387 0.002534 0.000132 62 | vn -0.004486 0.998354 0.057168 63 | v -0.010401 0.026309 0.000167 64 | vn -0.999988 0.004662 -0.001626 65 | v -0.010401 0.026309 0.000167 66 | vn -0.665951 0.690851 0.281486 67 | v -0.010389 0.025220 0.019188 68 | vn -0.999597 0.009346 0.026807 69 | v -0.010389 0.025220 0.019188 70 | vn 0.006493 -0.999457 -0.032313 71 | v -0.008730 -0.000024 0.036165 72 | vn 0.377718 0.867563 0.323518 73 | v 0.010400 0.025253 0.019037 74 | vn -0.000242 0.983230 0.182372 75 | v 0.010400 0.025253 0.019037 76 | vn 0.665647 0.002096 0.746264 77 | v 0.005840 0.014274 0.053803 78 | vn 0.008418 -0.012115 0.999891 79 | v 0.005840 0.014274 0.053803 80 | vn 0.001757 0.953702 0.300749 81 | v 0.005840 0.014274 0.053803 82 | vn 0.377718 0.867563 0.323518 83 | v 0.008616 0.013989 0.051328 84 | vn 0.998361 0.003310 0.057136 85 | v 0.008616 0.013989 0.051328 86 | vn 0.798906 -0.045001 0.599770 87 | v 0.008693 -0.000133 0.050166 88 | vn 0.998687 -0.025065 0.044683 89 | v 0.008693 -0.000133 0.050166 90 | vn -0.769031 -0.017753 0.638965 91 | v -0.008623 -0.000057 0.050953 92 | vn -0.008996 -0.999957 -0.002185 93 | v -0.008623 -0.000057 0.050953 94 | vn -0.871286 -0.490748 0.005227 95 | v -0.008623 -0.000057 0.050953 96 | vn 0.008418 -0.012115 0.999891 97 | v -0.005481 -0.000091 0.053725 98 | vn -0.002059 -0.999940 0.010793 99 | v -0.005481 -0.000091 0.053725 100 | vn -0.510143 -0.000217 0.860089 101 | v -0.005278 0.014293 0.053849 102 | vn -0.108731 0.943365 0.313433 103 | v -0.005278 0.014293 0.053849 104 | vn -0.665951 0.690851 0.281486 105 | v -0.007778 0.014218 0.052366 106 | vn -0.218924 0.920873 0.322590 107 | v -0.007778 0.014218 0.052366 108 | vn -0.858159 -0.000049 0.513385 109 | v -0.008841 0.013918 0.050589 110 | vn -0.998665 -0.002749 0.051583 111 | v -0.008841 0.013918 0.050589 112 | vn 0.006542 -0.999267 0.037718 113 | v 0.006138 -0.000021 0.053578 114 | vn 0.012751 -0.015529 0.999798 115 | v 0.006138 -0.000021 0.053578 116 | # 52 vertices, 0 vertices normals 117 | 118 | f 20//20 22//22 25//25 119 | f 3//3 4//4 27//27 120 | f 27//27 4//4 29//29 121 | f 2//2 30//30 24//24 122 | f 32//32 6//6 35//35 123 | f 25//25 5//5 20//20 124 | f 37//37 11//11 8//8 125 | f 11//11 39//39 21//21 126 | f 37//37 39//39 11//11 127 | f 42//42 23//23 7//7 128 | f 2//2 12//12 30//30 129 | f 12//12 44//44 30//30 130 | f 8//8 11//11 21//21 131 | f 8//8 21//21 1//1 132 | f 32//32 19//19 6//6 133 | f 6//6 46//46 35//35 134 | f 48//48 46//46 6//6 135 | f 40//40 14//14 16//16 136 | f 3//3 13//13 4//4 137 | f 31//31 9//9 36//36 138 | f 19//19 26//26 6//6 139 | f 4//4 50//50 29//29 140 | f 17//17 47//47 28//28 141 | f 34//34 43//43 52//52 142 | f 15//15 43//43 34//34 143 | f 12//12 51//51 44//44 144 | f 18//18 38//38 10//10 145 | f 44//44 41//41 30//30 146 | f 16//16 14//14 45//45 147 | f 13//13 50//50 4//4 148 | f 18//18 10//10 33//33 149 | f 16//16 49//49 40//40 150 | # 32 faces, 0 coords texture 151 | 152 | # End of File 153 | -------------------------------------------------------------------------------- /manipgen/real_world/utils/rl_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import cv2 4 | 5 | from manipgen.utils.dagger_utils import quat_mul, quat_conjugate, axis_angle_from_quat 6 | 7 | 8 | def transform_quat(quat): 9 | """ Transform quaternion in proprioception to match simulation. """ 10 | quat = torch.from_numpy(quat).float().reshape(-1, 4) 11 | 12 | # step 1: rotate around x-axis by 180 degrees 13 | down_q = torch.tensor([1.0, 0.0, 0.0, 0.0]).expand(quat.shape[0], -1) 14 | quat = quat_mul(quat, down_q) 15 | 16 | # step 2: follow convention w >= 0 17 | flip = quat[:, 3] < 0 18 | quat[flip] = -quat[flip] 19 | 20 | return quat[0].numpy() 21 | 22 | def get_delta_proprioception(cur_eef_pos, cur_eef_quat, prev_eef_pos, prev_eef_quat, scale=1.0): 23 | """ 24 | Compute delta proprioception to approximate velocity information. 25 | 26 | Args: 27 | cur_eef_pos (np.ndarray): Current end-effector position. 28 | cur_eef_quat (np.ndarray): Current end-effector quaternion. 29 | prev_eef_pos (np.ndarray): Previous end-effector position. 30 | prev_eef_quat (np.ndarray): Previous end-effector quaternion. 31 | scale (float): Scale factor for the delta proprioception. 32 | 33 | Returns: 34 | np.ndarray: Delta proprioception. 35 | """ 36 | cur_eef_pos = torch.from_numpy(cur_eef_pos).float().reshape(-1, 3) 37 | cur_eef_quat = torch.from_numpy(cur_eef_quat).float().reshape(-1, 4) 38 | prev_eef_pos = torch.from_numpy(prev_eef_pos).float().reshape(-1, 3) 39 | prev_eef_quat = torch.from_numpy(prev_eef_quat).float().reshape(-1, 4) 40 | 41 | delta_pos = (cur_eef_pos - prev_eef_pos) * scale 42 | delta_quat = quat_mul(cur_eef_quat, quat_conjugate(prev_eef_quat)) 43 | axis_angle = axis_angle_from_quat(delta_quat) 44 | delta_rot = axis_angle * scale 45 | 46 | delta_proprio = torch.cat([delta_pos, delta_rot], dim=-1)[0].numpy() 47 | return delta_proprio 48 | 49 | def process_depth(depth, cfg): 50 | """ Process depth image. """ 51 | depth = depth.astype(np.float32) / 1e4 # Convert to meters 52 | 53 | # clamp and scale depth 54 | depth = depth.clip(0, cfg.depth_clamp_val) 55 | depth = depth / cfg.depth_clamp_val 56 | 57 | # rotate depth to fix orientation 58 | h, w = depth.shape 59 | center = (w // 2, h // 2) 60 | M = cv2.getRotationMatrix2D(center, cfg.process_depth_rotation, 1.0) 61 | depth = cv2.warpAffine(depth, M, (w, h)) 62 | 63 | # crop to policy view: 480 x 640 -> 368 x 368 (before center cropping) 64 | h_offset = cfg.process_depth_h_offset 65 | h_boarder = (cfg.raw_depth_h - cfg.obs_depth_w) // 2 66 | w_offset = cfg.process_depth_w_offset 67 | w_boarder = (cfg.raw_depth_w - cfg.obs_depth_w) // 2 68 | depth = depth[h_boarder+h_offset:-h_boarder+h_offset, w_boarder+w_offset:-w_boarder+w_offset] 69 | 70 | # resize to 84 x 84 71 | depth = cv2.resize(depth, (cfg.obs_depth_w_down, cfg.obs_depth_w_down), interpolation=cv2.INTER_NEAREST) 72 | 73 | return depth 74 | 75 | def compute_policy_observations( 76 | obs, prev_obs=None, frame0_obs=None, seg_mask=None, 77 | cfg=None, device="cuda" 78 | ): 79 | """ 80 | Compute local policy observations. 81 | 82 | Args: 83 | obs (dict): Current observation. 84 | prev_obs (dict): Previous observation. 85 | frame0_obs (dict): First frame observation. 86 | seg_mask (np.ndarray): Local segmentation mask (for pick policy). 87 | cfg (dict): Configuration dictionary. 88 | device (str): Device to use. 89 | 90 | Returns: 91 | Tuple[torch.Tensor, torch.Tensor]: State observation and visual observation. 92 | """ 93 | # Compute proprioception 94 | cur_eef_pos = obs["eef_pos"] 95 | cur_eef_quat = obs["eef_quat"] 96 | frame0_eef_pos = frame0_obs["eef_pos"] 97 | frame0_eef_quat = frame0_obs["eef_quat"] 98 | prev_eef_pos = prev_obs["eef_pos"] 99 | prev_eef_quat = prev_obs["eef_quat"] 100 | delta_proprio = get_delta_proprioception(cur_eef_pos, cur_eef_quat, prev_eef_pos, prev_eef_quat, cfg.delta_proprio_scale) 101 | 102 | state_obs = np.concatenate([cur_eef_pos, cur_eef_quat, frame0_eef_pos, frame0_eef_quat, delta_proprio]) 103 | 104 | if cfg.offset_eef_pos_by_frame0: 105 | state_obs[:3] -= state_obs[7:10] 106 | state_obs[7:10] = 0.0 107 | 108 | # Compute visual observation 109 | cur_depth = process_depth(obs["cam1_depth"][0], cfg) 110 | frame0_depth = process_depth(frame0_obs["cam1_depth"][0], cfg) 111 | obs_list = [cur_depth, frame0_depth] 112 | if seg_mask is not None: 113 | obs_list.append(seg_mask) 114 | visual_obs = np.stack(obs_list) 115 | 116 | state_obs = torch.from_numpy(state_obs).float().to(device) 117 | visual_obs = torch.from_numpy(visual_obs).float().to(device) 118 | 119 | # create batch and time dimensions 120 | state_obs = state_obs.unsqueeze(0).unsqueeze(0) 121 | visual_obs = visual_obs.unsqueeze(0) 122 | 123 | return state_obs, visual_obs 124 | 125 | --------------------------------------------------------------------------------