├── .dockerignore ├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── 1-bug.yml │ ├── 2-enhancement.yml │ └── 3-question.yml └── workflows │ └── pre-commit.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── docs ├── Makefile ├── README.md ├── en │ ├── _static │ │ ├── css │ │ │ └── readthedocs.css │ │ ├── image │ │ │ ├── agent_baseline.png │ │ │ ├── agent_definition.png │ │ │ ├── before_rename_urdf_img.png │ │ │ ├── benchmark.png │ │ │ ├── cfg_instance.webp │ │ │ ├── follow_target_mycobot_demo.png │ │ │ ├── logo.png │ │ │ ├── logo192.png │ │ │ ├── lula_test_extension.png │ │ │ ├── main_process.webp │ │ │ ├── overview.webp │ │ │ ├── task_illustration.png │ │ │ ├── teaser.png │ │ │ └── teleop_with_vision_pro.jpg │ │ └── video │ │ │ ├── g1_locomotion.webm │ │ │ ├── gr1_teleop.webm │ │ │ ├── h1_locomotion.webm │ │ │ ├── mocap_camera_control.webm │ │ │ ├── mocap_franka_control.webm │ │ │ ├── mocap_layout_edit.webm │ │ │ ├── mocap_rgb_camera_server.webm │ │ │ ├── mocap_start.webm │ │ │ ├── tutorial_minimum.webm │ │ │ ├── tutorial_use_controller.webm │ │ │ ├── tutorial_use_robot.webm │ │ │ ├── tutorial_use_sensor.webm │ │ │ └── tutorial_use_task.webm │ ├── _templates │ │ └── classtemplate.rst │ ├── api │ │ ├── datahub.rst │ │ ├── env.rst │ │ ├── index.rst │ │ ├── register.rst │ │ ├── robot.rst │ │ ├── scene.rst │ │ ├── task.rst │ │ └── util.rst │ ├── conf.py │ ├── index.md │ └── usage │ │ ├── advanced_tutorials │ │ ├── how-to-add-controller.md │ │ ├── how-to-add-robot.md │ │ ├── how-to-add-sensor.md │ │ ├── how-to-add-task.md │ │ └── index.md │ │ ├── get_started │ │ ├── explore-grscenes.md │ │ ├── index.md │ │ ├── installation.md │ │ ├── layout_edit_with_mocap.md │ │ ├── run-benchmark-baseline.md │ │ ├── teleoperating-with-mocap.md │ │ ├── teleoperating-with-visionpro.md │ │ └── wander-with-keyboard.md │ │ ├── index.rst │ │ └── tutorials │ │ ├── core-concepts.md │ │ ├── how-to-run-benchmark-with-custom-agent.md │ │ ├── how-to-use-controller.md │ │ ├── how-to-use-robot.md │ │ ├── how-to-use-sensor.md │ │ ├── how-to-use-task.md │ │ └── index.md ├── make.bat └── requirements.txt ├── grutopia ├── __init__.py ├── core │ ├── __init__.py │ ├── config │ │ ├── __init__.py │ │ ├── metric │ │ │ └── __init__.py │ │ ├── robot │ │ │ └── __init__.py │ │ ├── scene │ │ │ ├── __init__.py │ │ │ └── object_params │ │ │ │ ├── __init__.py │ │ │ │ ├── dynamic_cube.py │ │ │ │ └── usd_obj.py │ │ └── task │ │ │ ├── __init__.py │ │ │ ├── episode.py │ │ │ └── reward.py │ ├── datahub │ │ ├── README.md │ │ ├── __init__.py │ │ ├── datahub.py │ │ ├── isaac_data.py │ │ └── model_data.py │ ├── gym_env.py │ ├── register │ │ ├── __init__.py │ │ └── register.py │ ├── robot │ │ ├── __init__.py │ │ ├── controller.py │ │ ├── robot.py │ │ └── sensor.py │ ├── runner.py │ ├── runtime │ │ ├── __init__.py │ │ ├── distributed_task_runtime_manager.py │ │ ├── local_task_runtime_manager.py │ │ └── task_runtime.py │ ├── scene │ │ ├── __init__.py │ │ ├── object.py │ │ └── scene │ │ │ ├── __init__.py │ │ │ └── util │ │ │ ├── __init__.py │ │ │ ├── type │ │ │ ├── __init__.py │ │ │ └── type_map.py │ │ │ └── usd_op.py │ ├── task │ │ ├── __init__.py │ │ ├── metric.py │ │ ├── reward.py │ │ └── task.py │ └── util │ │ ├── __init__.py │ │ ├── async_req.py │ │ ├── chat │ │ ├── __init__.py │ │ └── agent_chat.py │ │ ├── clear_task.py │ │ ├── gym.py │ │ ├── interaction.py │ │ ├── joint.py │ │ ├── log │ │ ├── __init__.py │ │ ├── config.ini │ │ └── logger.py │ │ ├── math.py │ │ ├── omni_usd_util.py │ │ ├── physics.py │ │ ├── python.py │ │ ├── rsl_rl │ │ ├── __init__.py │ │ ├── normalizer.py │ │ └── pickle.py │ │ └── space.py ├── demo │ ├── __init__.py │ ├── aliengo_locomotion.py │ ├── franka_manipulation.py │ ├── franka_manipulation_mocap_teleop.py │ ├── g1_locomotion.py │ ├── gr1_locomotion.py │ ├── gr1_teleop.py │ ├── h1_locomotion.py │ ├── h1_social_navigation.py │ ├── h1_traveled_distance.py │ ├── jetbot_locomotion.py │ └── mobile_manipulation.py ├── download_assets.py ├── login_openxlab.py ├── macros.py ├── set_assets_path.py └── setup_conda_pypi.py ├── grutopia_extension ├── __init__.py ├── agents │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── common │ │ ├── README.md │ │ ├── agent_utils │ │ │ ├── __init__.py │ │ │ ├── acyclic_enforcer.py │ │ │ ├── geometry_utils.py │ │ │ ├── img_utils.py │ │ │ ├── path_finder.py │ │ │ └── traj_visualizer.py │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── actuator_module.py │ │ │ ├── mapping │ │ │ │ ├── __init__.py │ │ │ │ ├── base_map.py │ │ │ │ ├── object_point_cloud_map.py │ │ │ │ ├── obstacle_map.py │ │ │ │ └── value_map.py │ │ │ ├── memory_module.py │ │ │ ├── planning_module.py │ │ │ └── vlm │ │ │ │ ├── __init__.py │ │ │ │ ├── blip2.py │ │ │ │ ├── blip2itm.py │ │ │ │ ├── classes.txt │ │ │ │ ├── coco_classes.py │ │ │ │ ├── detections.py │ │ │ │ ├── grounding_dino.py │ │ │ │ ├── large_model.py │ │ │ │ ├── prompt.py │ │ │ │ ├── sam.py │ │ │ │ ├── server_wrapper.py │ │ │ │ └── yolov7.py │ │ └── scripts │ │ │ └── launch_vlm_servers.sh │ ├── config │ │ └── __init__.py │ ├── core │ │ ├── __init__.py │ │ └── agent │ │ │ ├── __init__.py │ │ │ └── agent.py │ ├── dummy_agent.py │ ├── mobile_manipulation_agent │ │ ├── .gitignore │ │ ├── README.md │ │ ├── __init__.py │ │ ├── agent_config.yaml │ │ ├── agent_utils │ │ ├── generate_mm_episodes.py │ │ ├── mobile_manipulation_agent.py │ │ ├── modules │ │ ├── requirements.txt │ │ └── scripts │ ├── mobile_manipulation_agent_client.py │ ├── npc_agent │ │ ├── __init__.py │ │ ├── base.py │ │ ├── config.py │ │ ├── llm_caller.py │ │ ├── prompt.py │ │ ├── repuirements.txt │ │ └── utils.py │ ├── npc_agent_client.py │ ├── social_navigation_agent │ │ ├── .gitignore │ │ ├── README.md │ │ ├── __init__.py │ │ ├── agent_config.yaml │ │ ├── agent_utils │ │ ├── generate_sn_episodes.py │ │ ├── modules │ │ ├── requirements.txt │ │ ├── scripts │ │ └── social_navigation_agent.py │ ├── social_navigation_agent_client.py │ └── util │ │ ├── __init__.py │ │ └── agent.py ├── configs │ ├── __init__.py │ ├── controllers │ │ ├── __init__.py │ │ ├── aliengo_move_by_speed_controller.py │ │ ├── dd_controller.py │ │ ├── franka_mocap_teleop_controller.py │ │ ├── g1_move_by_speed_controller.py │ │ ├── gr1_move_by_speed_controller.py │ │ ├── gr1_teleop_controller.py │ │ ├── gripper_controller.py │ │ ├── h1_move_by_speed_controller.py │ │ ├── ik_controller.py │ │ ├── joint_controller.py │ │ ├── layout_edit_mocap_controller.py │ │ ├── move_along_path_points_controller.py │ │ ├── move_to_point_by_speed_controller.py │ │ ├── pid_controller.py │ │ ├── recover_controller.py │ │ ├── rmpflow_controller.py │ │ └── rotate_controller.py │ ├── metrics │ │ ├── __init__.py │ │ ├── candidates_reduce_metric.py │ │ ├── debug_metric.py │ │ ├── mobile_manipulation_success_metric.py │ │ ├── recording_metric.py │ │ ├── reset_time_metric.py │ │ ├── simple_metric.py │ │ └── social_navigation_success_metric.py │ ├── objects │ │ └── __init__.py │ ├── robots │ │ ├── __init__.py │ │ ├── aliengo.py │ │ ├── franka.py │ │ ├── g1.py │ │ ├── gr1.py │ │ ├── h1.py │ │ ├── h1_with_hand.py │ │ ├── jetbot.py │ │ └── mocap_controlled_franka.py │ ├── sensors │ │ └── __init__.py │ └── tasks │ │ ├── __init__.py │ │ ├── finite_step_task.py │ │ ├── manipulation_task.py │ │ ├── mobile_manipulation_task.py │ │ ├── single_inference_task.py │ │ └── social_navigation_task.py ├── controllers │ ├── __init__.py │ ├── aliengo_move_by_speed_controller.py │ ├── dd_controller.py │ ├── franka_mocap_teleop_controller.py │ ├── g1_move_by_speed_controller.py │ ├── gr1_move_by_speed_controller.py │ ├── gr1_teleop.py │ ├── gr1_teleop_controller.py │ ├── gripper_controller.py │ ├── h1_move_by_speed_controller.py │ ├── ik_controller.py │ ├── joint_controller.py │ ├── layout_edit_controller │ │ ├── __init__.py │ │ ├── fixjoint_hand.py │ │ ├── hand_control.py │ │ ├── hand_position_control.py │ │ └── save_asset.py │ ├── layout_edit_mocap_controller.py │ ├── lcmtypes │ │ ├── __init__.py │ │ ├── teleop │ │ │ ├── __init__.py │ │ │ ├── action.py │ │ │ └── joints.py │ │ └── teleop_t.lcm │ ├── models │ │ ├── __init__.py │ │ └── aliengo │ │ │ ├── __init__.py │ │ │ ├── actor_critic.py │ │ │ └── estimator_cl.py │ ├── move_along_path_points_controller.py │ ├── move_to_point_by_speed_controller.py │ ├── move_to_point_oracle_controller.py │ ├── pin_ik_solver.py │ ├── recover_controller.py │ ├── rmpflow_controller.py │ ├── rotate_controller.py │ └── rotate_oracle.py ├── interactions │ ├── __init__.py │ ├── keyboard.py │ ├── motion_capture.py │ └── visionpro │ │ ├── Preprocessor.py │ │ ├── TeleVision.py │ │ ├── __init__.py │ │ ├── constants_vuer.py │ │ ├── motion_utils.py │ │ └── visionpro.py ├── metrics │ ├── __init__.py │ ├── candidates_reduce_metric.py │ ├── debug_metric.py │ ├── mobile_manipulation_success_metric.py │ ├── prompt.py │ ├── recording_metric.py │ ├── reset_time_metric.py │ ├── simple_metric.py │ └── social_navigation_success_metric.py ├── objects │ ├── __init__.py │ ├── dynamic_cube.py │ ├── usd_object.py │ └── visual_cube.py ├── robots │ ├── __init__.py │ ├── aliengo.py │ ├── camera_oracle.py │ ├── franka.py │ ├── g1.py │ ├── gr1.py │ ├── h1.py │ ├── h1_with_hand.py │ ├── jetbot.py │ ├── mocap_controlled_franka.py │ └── npc.py ├── sensors │ ├── __init__.py │ ├── camera.py │ ├── layout_edit_mocap_controlled_camera.py │ ├── mocap_controlled_camera.py │ └── rep_camera.py └── tasks │ ├── README.md │ ├── __init__.py │ ├── finite_step_task.py │ ├── manipulation_task.py │ ├── mobile_manipulation_task.py │ ├── single_inference_task.py │ └── social_navigation_task.py ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── requirements ├── benchmark.txt ├── contribute.txt ├── docker_install_req.sh ├── runtime.txt ├── teleop.txt ├── test.txt └── webui.txt ├── setup.cfg ├── setup.py ├── setup_conda.sh ├── tests ├── conftest.py ├── e2e_test.py ├── e2e_test.yaml ├── h1_locomotion.py ├── load_scene_without_robot.py ├── rep_camera_pointcloud.py └── robots │ ├── aliengo.py │ ├── franka.py │ ├── g1.py │ ├── gr1.py │ ├── h1.py │ ├── jetbot.py │ └── test_move_to_point.py └── toolkits ├── grscenes_scripts ├── README.md ├── export_scenes.py ├── extract_objaverse.py ├── get_metadata.py ├── play_scene.py ├── preprocess.py └── warmup.py ├── indoor_scenes_generation ├── infinigen │ ├── infinigen │ │ ├── assets │ │ │ └── static_assets │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ └── static_category.py │ │ └── core │ │ │ └── placement │ │ │ └── factory.py │ └── infinigen_examples │ │ └── constraints │ │ ├── static_assets_home.py │ │ └── static_assets_semantics.py ├── readme.md └── util │ ├── data_post_processing.py │ └── preparing_static_assets.py ├── layout_edit └── layout_edit.py └── mocap ├── hamer_real_time.py ├── hamer_real_time_server.py └── rgb_camera_server.py /.dockerignore: -------------------------------------------------------------------------------- 1 | /assets 2 | /asset 3 | /grutopia/assets 4 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | show-source=True 3 | statistics=True 4 | per-file-ignores=*/__init__.py:F401 5 | # E402: Module level import not at top of file 6 | # E501: Line too long 7 | # W503: Line break before binary operator 8 | # E203: Whitespace before ':' -> conflicts with black 9 | # D401: First line should be in imperative mood 10 | # R504: Unnecessary variable assignment before return statement. 11 | # R505: Unnecessary elif after return statement 12 | # SIM102: Use a single if-statement instead of nested if-statements 13 | # SIM117: Merge with statements for context managers that have same scope. 14 | ignore=E402,E501,W503,E203,D401,R504,R505,SIM102,SIM117 15 | max-line-length = 120 16 | max-complexity = 30 17 | exclude=_*,.vscode,.git,docs/**,**/test/**,**/lcmtypes/** 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-enhancement.yml: -------------------------------------------------------------------------------- 1 | name: "🚀 Enhancement" 2 | description: Suggest a new feature request or improvement on the project 3 | title: '[Enhancement]: ' 4 | labels: 5 | - type/enhancement 6 | - triage-needed 7 | 8 | body: 9 | - type: markdown 10 | attributes: 11 | value: | 12 | A clear and concise description of what new feature or behavior you would like to see. If applicable, please describe the current behavior as well. 13 | 14 | - type: textarea 15 | id: suggestion 16 | attributes: 17 | label: What feature or enhancement are you proposing? 18 | validations: 19 | required: true 20 | 21 | - type: textarea 22 | id: motivation 23 | attributes: 24 | label: Motivation 25 | description: What is your motivation for adding / enhancing this feature, optimally described in the form of a concrete user story or use case. 26 | value: | 27 | 32 | validations: 33 | required: false 34 | 35 | - type: textarea 36 | id: additionalinfo 37 | attributes: 38 | label: Additional information 39 | description: If you think that any additional information would be useful please provide them here. 40 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/3-question.yml: -------------------------------------------------------------------------------- 1 | name: "🙏 Question" 2 | description: Ask a question 3 | title: "[Question]: " 4 | labels: ["type/question", "triage-needed"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Please make sure to [search for existing issues](https://github.com/OpenRobotLab/GRUtopia/issues) before filing a new one! 10 | 11 | - type: textarea 12 | attributes: 13 | label: Question 14 | description: Describe your question in detail. 15 | validations: 16 | required: true 17 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yaml: -------------------------------------------------------------------------------- 1 | name: Run linters using pre-commit 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [main] 7 | 8 | jobs: 9 | pre-commit: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v3 14 | with: 15 | python-version: '3.10' 16 | - uses: pre-commit/action@v3.0.0 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/en/_build/ 68 | docs/zh_cn/_build/ 69 | docs/_build 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | .vscode 108 | .idea 109 | .DS_Store 110 | 111 | # custom 112 | *.pkl 113 | *.pkl.json 114 | *.log.json 115 | docs/modelzoo_statistics.md 116 | work_dirs/ 117 | 118 | # Pytorch 119 | *.pth 120 | *.py~ 121 | *.sh~ 122 | 123 | # grutopia assets 124 | /asset 125 | assets 126 | 127 | # user scripts 128 | /scripts 129 | 130 | # sphinx build dir 131 | docs/en/_build 132 | 133 | # default value for marcos.py 134 | grutopia/default_config.py 135 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - local: tests/e2e_test.yaml 3 | 4 | 5 | stages: 6 | - code_check 7 | - e2e_test 8 | - build 9 | 10 | 11 | code_check: 12 | stage: code_check 13 | image: 14 | name: registry.cn-hangzhou.aliyuncs.com/grutopia/pre-commit:20250303 15 | script: 16 | - pre-commit run --all-files 17 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/PyCQA/flake8 3 | rev: 4.0.1 4 | hooks: 5 | - id: flake8 6 | - repo: https://github.com/PyCQA/isort 7 | rev: 5.11.5 8 | hooks: 9 | - id: isort 10 | - repo: https://github.com/psf/black 11 | rev: 22.10.0 12 | hooks: 13 | - id: black 14 | - repo: https://github.com/codespell-project/codespell 15 | rev: v2.2.1 16 | hooks: 17 | - id: codespell 18 | exclude: | 19 | (?x)( 20 | ^toolkits/grscenes_scripts/README.md| 21 | ^toolkits/indoor_scenes_generation/infinigen/infinigen_examples/constraints 22 | ) 23 | - repo: https://github.com/gitleaks/gitleaks 24 | rev: v8.24.0 25 | hooks: 26 | - id: gitleaks 27 | - repo: https://github.com/pre-commit/pre-commit-hooks 28 | rev: v3.1.0 29 | hooks: 30 | - id: trailing-whitespace 31 | - id: check-yaml 32 | - id: end-of-file-fixer 33 | exclude: '^(.*/lcmtypes/.*)' 34 | - id: requirements-txt-fixer 35 | - id: double-quote-string-fixer 36 | exclude: '^(.*/lcmtypes/.*)' 37 | - id: check-merge-conflict 38 | - id: fix-encoding-pragma 39 | args: ["--remove"] 40 | - id: mixed-line-ending 41 | args: ["--fix=lf"] 42 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to GRUtopia 2 | 3 | Thank you for your interest in contributing to GRUtopia! We welcome contributions from everyone. Please take a moment to review this guide to ensure a smooth collaboration. 4 | 5 | - [Reporting Bugs](https://github.com/OpenRobotLab/GRUtopia/issues/new/choose) 6 | - [Suggesting Enhancement](https://github.com/OpenRobotLab/GRUtopia/issues/new/choose) 7 | - [Questions and Discussions](https://github.com/OpenRobotLab/GRUtopia/issues/new/choose) 8 | - [Submitting Code Changes](#submitting-code-changes) 9 | - [Reviewing and Merging](#reviewing-and-merging) 10 | 11 | ## Submitting Code Changes 12 | 13 | - We use the `pre-commit` configuration to automatically clean up code before committing. Install and run `pre-commit` as follows: 14 | 1. Install `pre-commit`: 15 | 16 | ```bash 17 | pip install pre-commit 18 | ``` 19 | 20 | 2. Install hooks from the configuration file at the root of the repository: 21 | 22 | ```bash 23 | pre-commit install 24 | ``` 25 | 26 | After this, `pre-commit` will automatically check and clean up code whenever you make a commit. 27 | 28 | 3. Before opening a pull request, run the following command to check all files are cleaned up: 29 | 30 | ```bash 31 | pre-commit run --all-files 32 | ``` 33 | 34 | - In the title of your Pull Request, please include [BUG FIX], [FEATURE] or [MISC] to indicate the purpose. 35 | - In the description, please provide example code or commands for testing. 36 | - For commit messages, please follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) specification. 37 | 38 | ## Reviewing and Merging 39 | 40 | - PRs require at least one approval before merging. 41 | - Automated checks (e.g., CI tests) must pass. 42 | - Use `Squash and Merge` for a clean commit history. 43 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvcr.io/nvidia/isaac-sim:4.2.0 2 | 3 | RUN apt-get update 4 | RUN apt-get install -y --allow-downgrades perl-base=5.34.0-3ubuntu1 netbase 5 | RUN apt-get install -y vim git 6 | 7 | COPY . /isaac-sim/GRUtopia 8 | WORKDIR /isaac-sim/GRUtopia 9 | 10 | 11 | RUN bash -c "cd ../ && \ 12 | ./python.sh -m venv .venv && source .venv/bin/activate && \ 13 | pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple && \ 14 | export PYTHONPATH=/isaac-sim/.venv/lib/python3.10/site-packages && \ 15 | chmod +x ./GRUtopia/requirements/docker_install_req.sh && \ 16 | cp ./GRUtopia/requirements/docker_install_req.sh . && \ 17 | bash ./docker_install_req.sh && \ 18 | sed 's/^\$python_exe/#\$python_exe/g' ./python.sh > python.env.init && \ 19 | echo 'source /isaac-sim/.venv/bin/activate' >> /root/.bashrc && \ 20 | echo ' . /isaac-sim/python.env.init' >> /root/.bashrc && \ 21 | echo 'set +e' >> /root/.bashrc && \ 22 | echo 'export MDL_SYSTEM_PATH=/isaac-sim/materials/' >> /root/.bashrc" 23 | WORKDIR /isaac-sim 24 | ENTRYPOINT ["/bin/bash"] 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) OpenRobotLab, Shanghai AI Laboratory. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = ./en 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # GRUtopia Documentation (English Version) 2 | 3 | 1. Install [GRUtopia](https://grutopia.github.io/get_started/installation.html). 4 | 5 | 1. Install Sphinx and other dependencies. 6 | 7 | ```bash 8 | pip install -r requirements.txt 9 | ``` 10 | 11 | 1. Build the documentation and watch the change lively 12 | 13 | ```bash 14 | rm -rf _build/; make html; 15 | ``` 16 | -------------------------------------------------------------------------------- /docs/en/_static/css/readthedocs.css: -------------------------------------------------------------------------------- 1 | .header-logo { 2 | background-image: url("../image/logo.png"); 3 | background-size: 180px 40px; 4 | height: 40px; 5 | width: 180px; 6 | } 7 | -------------------------------------------------------------------------------- /docs/en/_static/image/agent_baseline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/agent_baseline.png -------------------------------------------------------------------------------- /docs/en/_static/image/agent_definition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/agent_definition.png -------------------------------------------------------------------------------- /docs/en/_static/image/before_rename_urdf_img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/before_rename_urdf_img.png -------------------------------------------------------------------------------- /docs/en/_static/image/benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/benchmark.png -------------------------------------------------------------------------------- /docs/en/_static/image/cfg_instance.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/cfg_instance.webp -------------------------------------------------------------------------------- /docs/en/_static/image/follow_target_mycobot_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/follow_target_mycobot_demo.png -------------------------------------------------------------------------------- /docs/en/_static/image/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/logo.png -------------------------------------------------------------------------------- /docs/en/_static/image/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/logo192.png -------------------------------------------------------------------------------- /docs/en/_static/image/lula_test_extension.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/lula_test_extension.png -------------------------------------------------------------------------------- /docs/en/_static/image/main_process.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/main_process.webp -------------------------------------------------------------------------------- /docs/en/_static/image/overview.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/overview.webp -------------------------------------------------------------------------------- /docs/en/_static/image/task_illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/task_illustration.png -------------------------------------------------------------------------------- /docs/en/_static/image/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/teaser.png -------------------------------------------------------------------------------- /docs/en/_static/image/teleop_with_vision_pro.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/image/teleop_with_vision_pro.jpg -------------------------------------------------------------------------------- /docs/en/_static/video/g1_locomotion.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/g1_locomotion.webm -------------------------------------------------------------------------------- /docs/en/_static/video/gr1_teleop.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/gr1_teleop.webm -------------------------------------------------------------------------------- /docs/en/_static/video/h1_locomotion.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/h1_locomotion.webm -------------------------------------------------------------------------------- /docs/en/_static/video/mocap_camera_control.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/mocap_camera_control.webm -------------------------------------------------------------------------------- /docs/en/_static/video/mocap_franka_control.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/mocap_franka_control.webm -------------------------------------------------------------------------------- /docs/en/_static/video/mocap_layout_edit.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/mocap_layout_edit.webm -------------------------------------------------------------------------------- /docs/en/_static/video/mocap_rgb_camera_server.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/mocap_rgb_camera_server.webm -------------------------------------------------------------------------------- /docs/en/_static/video/mocap_start.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/mocap_start.webm -------------------------------------------------------------------------------- /docs/en/_static/video/tutorial_minimum.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/tutorial_minimum.webm -------------------------------------------------------------------------------- /docs/en/_static/video/tutorial_use_controller.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/tutorial_use_controller.webm -------------------------------------------------------------------------------- /docs/en/_static/video/tutorial_use_robot.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/tutorial_use_robot.webm -------------------------------------------------------------------------------- /docs/en/_static/video/tutorial_use_sensor.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/tutorial_use_sensor.webm -------------------------------------------------------------------------------- /docs/en/_static/video/tutorial_use_task.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/docs/en/_static/video/tutorial_use_task.webm -------------------------------------------------------------------------------- /docs/en/_templates/classtemplate.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | .. currentmodule:: {{ module }} 4 | 5 | 6 | {{ name | underline}} 7 | 8 | .. autoclass:: {{ name }} 9 | :members: 10 | 11 | 12 | .. 13 | autogenerated from source/_templates/classtemplate.rst 14 | note it does not have :inherited-members: 15 | -------------------------------------------------------------------------------- /docs/en/api/datahub.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.datahub 2 | =================================== 3 | 4 | datahub 5 | ------- 6 | 7 | .. autoclass:: grutopia.core.datahub.IsaacData 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/en/api/env.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.env 2 | ================= 3 | 4 | env 5 | --- 6 | 7 | .. autoclass:: grutopia.core.gym_env.Env 8 | :members: 9 | 10 | runner 11 | ------ 12 | 13 | .. autoclass:: grutopia.core.runner.SimulatorRunner 14 | :members: 15 | -------------------------------------------------------------------------------- /docs/en/api/index.rst: -------------------------------------------------------------------------------- 1 | API 2 | ===== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: API Reference 7 | 8 | grutopia.core.env 9 | grutopia.core.register 10 | grutopia.core.robot 11 | grutopia.core.scene 12 | grutopia.core.task 13 | grutopia.core.util 14 | -------------------------------------------------------------------------------- /docs/en/api/register.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.register 2 | =================================== 3 | 4 | register 5 | -------- 6 | 7 | .. automodule:: grutopia.core.register.register 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/en/api/robot.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.robot 2 | =================================== 3 | 4 | robot 5 | ----- 6 | 7 | .. automodule:: grutopia.core.robot.robot 8 | :members: 9 | 10 | 11 | controller 12 | ---------- 13 | 14 | .. automodule:: grutopia.core.robot.controller 15 | :members: 16 | 17 | 18 | sensor 19 | ------ 20 | 21 | .. automodule:: grutopia.core.robot.sensor 22 | :members: 23 | -------------------------------------------------------------------------------- /docs/en/api/scene.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.scene 2 | =================================== 3 | 4 | 5 | object 6 | ------ 7 | 8 | .. automodule:: grutopia.core.scene.object 9 | :members: 10 | 11 | 12 | usd_op 13 | ------ 14 | 15 | 16 | .. automodule:: grutopia.core.scene.scene.util.usd_op 17 | :members: 18 | -------------------------------------------------------------------------------- /docs/en/api/task.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.task 2 | ================== 3 | 4 | task 5 | ---- 6 | 7 | .. automodule:: grutopia.core.task.task 8 | :members: 9 | 10 | metric 11 | ---- 12 | 13 | .. automodule:: grutopia.core.task.metric 14 | :members: 15 | 16 | reward 17 | ---- 18 | 19 | .. automodule:: grutopia.core.task.reward 20 | :members: 21 | -------------------------------------------------------------------------------- /docs/en/api/util.rst: -------------------------------------------------------------------------------- 1 | grutopia.core.util 2 | ================== 3 | 4 | joint 5 | ----------- 6 | 7 | .. automodule:: grutopia.core.util.joint 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/en/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar: false 3 | myst: 4 | html_meta: 5 | "description lang=en": | 6 | Documentation for users who wish to build sphinx sites with 7 | pydata-sphinx-theme. 8 | --- 9 | 10 | # Welcome to GRUtopia's documentation! 11 | 12 | This document introduces installation steps, tutorials, and reference APIs of [GRUtopia](https://github.com/OpenRobotLab/GRUtopia). 13 | 14 | GRUtopia, built on NVIDIA Isaac Sim, is an embodied AI research platform. It aims to reduce the burden of engineering implementation, tackle the data scarcity in the field of embodied intelligence, and offer simulations that are more reflective of real-world scenarios. 15 | 16 | Here is a brief overview of GRUtopia: 17 | 18 | ![overview](../_static/image/overview.webp) 19 | 20 | We provide some demostrations to show what GRUtopia can do after [installation](./usage/get_started/installation.md): 21 | 22 | | Name | Description | Requirement | Reference | 23 | | --- | --- | --- | --- | 24 | | Robot simulation | Control a humanoid robot to explore a simulation world | A host with NVIDIA RTX GPU | [Wander with Keyboard](../usage/get_started/wander-with-keyboard.html) | 25 | | Teleoperating with VisionPro | Teleoperate a humanoid robot with a VR headset (Apple VisionPro) | * A host with NVIDIA RTX GPU
* [Apple VisionPro](https://www.apple.com/apple-vision-pro/) | [Teleoperate with VisionPro](../usage/get_started/teleoperating-with-visionpro.html) | 26 | | Teleoperating with Mocap | Teleoperate a robot arm with camera-based motion capture system | * A host with NVIDIA RTX GPU
* A camera | [Teleoperate with Mocap](../usage/get_started/teleoperating-with-mocap.html) | 27 | | Layout Edit with Mocap | Interactively edit scene with camera-based motion capture system | * A host with NVIDIA RTX GPU
* A camera | [Layout Edit with Mocap](../usage/get_started/layout_edit_with_mocap.html) | 28 | | Benchmark | Run a social-navigation/mobile-manipulation benchmark with agent | A host with NVIDIA RTX GPU
(32GB+ VRAM and 64GB+ RAM) | [Run Benchmark Baseline](../usage/get_started/run-benchmark-baseline.html) | 29 | 30 | 31 | 32 | ```{toctree} 33 | :maxdepth: 1 34 | 35 | usage/index 36 | api/index 37 | 38 | ``` 39 | -------------------------------------------------------------------------------- /docs/en/usage/advanced_tutorials/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | myst: 3 | html_meta: 4 | "description lang=en": | 5 | Documentation for users who wish to build sphinx sites with 6 | pydata-sphinx-theme. 7 | --- 8 | 9 | # Advanced Tutorials 10 | 11 | ```{toctree} 12 | :caption: Advanced Tutorials 13 | :maxdepth: 2 14 | 15 | how-to-add-robot 16 | how-to-add-controller 17 | how-to-add-sensor 18 | how-to-add-task 19 | ``` 20 | -------------------------------------------------------------------------------- /docs/en/usage/get_started/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | myst: 3 | html_meta: 4 | "description lang=en": | 5 | Documentation for users who wish to build sphinx sites with 6 | pydata-sphinx-theme. 7 | --- 8 | 9 | # Get Started 10 | 11 | ```{toctree} 12 | :caption: Get Started 13 | :maxdepth: 2 14 | 15 | installation 16 | wander-with-keyboard 17 | explore-grscenes 18 | teleoperating-with-mocap 19 | teleoperating-with-visionpro 20 | layout_edit_with_mocap.md 21 | run-benchmark-baseline 22 | ``` 23 | -------------------------------------------------------------------------------- /docs/en/usage/get_started/wander-with-keyboard.md: -------------------------------------------------------------------------------- 1 | # Wander with keyboard 2 | 3 | > This tutorial guides you to wander with keyboard as g1 robot. 4 | 5 | ```bash 6 | $ python -m grutopia.demo.g1_locomotion 7 | ``` 8 | 9 | You can control the g1 robot with keyboard command: 10 | 11 | - I: Move Forward 12 | - K: Move Backward 13 | - J: Move Left 14 | - L: Move Right 15 | - U: Turn Left 16 | - O: Turn Right 17 | 18 | 21 | 22 | ## Brief Explanation 23 | 24 | The keyboard is abstracted as an interaction device. A vector is used to denote which key is being pressed, and this vector is then translated into the robot's actions at each step. 25 | 26 | ```python 27 | from grutopia_extension.interactions.keyboard import KeyboardInteraction 28 | 29 | keyboard = KeyboardInteraction() 30 | 31 | while env.simulation_app.is_running(): 32 | i += 1 33 | command = keyboard.get_input() 34 | x_speed = command[0] - command[1] 35 | y_speed = command[2] - command[3] 36 | z_speed = command[4] - command[5] 37 | env_action = { 38 | move_by_speed_cfg.name: (x_speed, y_speed, z_speed), 39 | } 40 | obs, _, terminated, _, _ = env.step(action=env_action) 41 | ... 42 | ``` 43 | 44 | You can refer to [`GRUtopia/grutopia/demo/g1_locomotion.py`](https://github.com/OpenRobotLab/GRUtopia/blob/main/grutopia/demo/g1_locomotion.py) for a complete example. 45 | -------------------------------------------------------------------------------- /docs/en/usage/index.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | get_started/index 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | tutorials/index 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | 17 | advanced_tutorials/index 18 | -------------------------------------------------------------------------------- /docs/en/usage/tutorials/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | myst: 3 | html_meta: 4 | "description lang=en": | 5 | Documentation for users who wish to build sphinx sites with 6 | pydata-sphinx-theme. 7 | --- 8 | 9 | # Tutorials 10 | 11 | ```{toctree} 12 | :caption: Tutorials 13 | :maxdepth: 2 14 | 15 | core-concepts 16 | how-to-use-task 17 | how-to-use-robot 18 | how-to-use-controller 19 | how-to-use-sensor 20 | how-to-use-objects 21 | 22 | [//]: # (how-to-run-benchmark-with-custom-agent) 23 | ``` 24 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | myst_parser 2 | pydata-sphinx-theme 3 | sphinx==6.2.1 4 | sphinx_copybutton==0.5.2 5 | sphinx_markdown_tables==0.0.17 6 | -------------------------------------------------------------------------------- /grutopia/__init__.py: -------------------------------------------------------------------------------- 1 | # from grutopia.core import runtime, datahub, launch, register, robot, scene, task, util, runner, env 2 | # 3 | # __all__ = ['runtime', 'datahub', 'launch', 'register', 'robot', 'scene', 'task', 'util', 'runner', 'env'] 4 | -------------------------------------------------------------------------------- /grutopia/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia/core/__init__.py -------------------------------------------------------------------------------- /grutopia/core/config/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config.robot import RobotCfg 6 | from grutopia.core.config.scene import ObjectCfg, Scene 7 | from grutopia.core.config.task import TaskCfg 8 | from grutopia.core.config.task.episode import EpisodeCfg, EpisodeConfigFile 9 | 10 | 11 | class SimConfig(BaseModel): 12 | """ 13 | Config of isaac simulator 14 | """ 15 | 16 | physics_dt: Optional[float | str] = 1 / 60 17 | rendering_dt: Optional[float | str] = 1 / 60 18 | rendering_interval: Optional[int] = None 19 | use_fabric: Optional[bool] = False 20 | 21 | 22 | class DistributionConfig(BaseModel): 23 | """ 24 | Config of distribution, only for distributed operation mode 25 | """ 26 | 27 | worker_num: Optional[int] = 1 28 | 29 | 30 | class Config(BaseModel): 31 | """ 32 | Config validator for input file (yaml -> dict). 33 | """ 34 | 35 | simulator: Optional[SimConfig] = SimConfig() 36 | task_config: TaskCfg 37 | -------------------------------------------------------------------------------- /grutopia/core/config/metric/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional, Union 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class MetricCfg(BaseModel): 7 | """ 8 | A configuration model for users to define metrics. 9 | 10 | This class represents a user-defined configuration for metrics, which includes 11 | the type of metric, an optional name for the metric, and an optional dictionary 12 | of additional metric configurations. It is designed to be used in conjunction 13 | with systems that evaluate or monitor performance based on specified metrics. 14 | 15 | Attributes: 16 | type (str): The type of the metric, specifying how the metric is calculated or what it measures. 17 | name (Optional[str]): An optional name for the metric, allowing users to provide a descriptive label. 18 | metric_config (Optional[Dict]): An optional dictionary containing specific configuration parameters 19 | for the metric, which may vary depending on the metric type. 20 | 21 | Example Usage: 22 | ```python 23 | config = MetricCfg( 24 | type="accuracy", 25 | name="Training Accuracy", 26 | metric_config={"threshold": 0.9} 27 | ) 28 | ``` 29 | """ 30 | 31 | type: str 32 | name: Optional[str] = None 33 | metric_config: Optional[Union[Dict, BaseModel]] = {} 34 | -------------------------------------------------------------------------------- /grutopia/core/config/scene/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config.scene.object_params import DynamicCube, UsdObj 6 | 7 | 8 | class Scene(BaseModel): 9 | """ 10 | A class representing a scene, inheriting from BaseModel. 11 | 12 | This class provides a structure for storing information related to a scene in a project. 13 | It includes basic attributes like type, name, and path, which can be used to categorize and locate scenes within a larger context. 14 | 15 | Attributes: 16 | type (str): The type of the scene, indicating its category or nature. 17 | name (Optional[str]): The name of the scene, which can be used for identification. Defaults to None. 18 | path (Optional[str]): The file system path to the scene, enabling direct access. Defaults to None. 19 | """ 20 | 21 | type: str 22 | name: Optional[str] = None 23 | path: Optional[str] = None 24 | 25 | 26 | class ObjectCfg(BaseModel): 27 | """ 28 | Represents a customizable object within a scene, with support for physics and different object types. 29 | 30 | This class defines an object that can be placed within a virtual environment, providing essential properties for both graphical representation and physical simulation. It supports varying object types with specific parameters, ensuring flexibility and customizability for different use cases. 31 | 32 | Attributes: 33 | name (str): The unique identifier for the object. 34 | prim_path (str): The path indicating the object's position within the scene hierarchy. 35 | position (Tuple[float, float, float], optional): The 3D coordinates of the object's position. Defaults to (0.0, 0.0, 0.0). 36 | orientation (Tuple[float, float, float, float], optional): The quaternion representation of the object's orientation. Defaults to (1.0, 0.0, 0.0, 0.0). 37 | scale (Tuple[float, float, float], optional): Scaling factors along each axis. Defaults to (1.0, 1.0, 1.0). 38 | type (str): Specifies the type of the object. 39 | """ 40 | 41 | # common 42 | name: str 43 | prim_path: str 44 | position: Optional[Tuple[float, float, float]] = (0.0, 0.0, 0.0) 45 | orientation: Optional[Tuple[float, float, float, float]] = (1.0, 0.0, 0.0, 0.0) 46 | scale: Optional[Tuple[float, float, float]] = (1.0, 1.0, 1.0) 47 | 48 | type: str 49 | -------------------------------------------------------------------------------- /grutopia/core/config/scene/object_params/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config.scene.object_params.dynamic_cube import DynamicCube 2 | from grutopia.core.config.scene.object_params.usd_obj import UsdObj 3 | -------------------------------------------------------------------------------- /grutopia/core/config/scene/object_params/dynamic_cube.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class DynamicCube(BaseModel): 7 | color: Optional[List[float]] = None 8 | -------------------------------------------------------------------------------- /grutopia/core/config/scene/object_params/usd_obj.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class UsdObj(BaseModel): 5 | """ 6 | Represents a USD object with a specific file path. 7 | 8 | This class extends the BaseModel to include a 'usd_path' attribute, which stores the file path to a USD (Universal Scene Description) file. The class provides a foundation for working with USD data within the application. 9 | 10 | TODO: Add more param here. 11 | 12 | Attributes: 13 | usd_path (str): The file system path to the USD file associated with this object. 14 | """ 15 | 16 | usd_path: str 17 | -------------------------------------------------------------------------------- /grutopia/core/config/task/episode.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Optional, Tuple 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config.robot import RobotCfg 6 | from grutopia.core.config.scene import ObjectCfg as Object 7 | 8 | 9 | class EpisodeCfg(BaseModel, extra='allow'): 10 | """ 11 | Represents the configuration details for an episode in a simulation or robotics application. 12 | 13 | This class defines the structure of an episode's settings, including the scene's assets, scaling, positioning, orientation, 14 | as well as the robots and objects involved. It is designed to be flexible and extensible, allowing for customization of various aspects of the simulation environment. 15 | 16 | Attributes: 17 | scene_asset_path (Optional[str]): The file path to the scene asset. Defaults to None. 18 | scene_scale (Optional[Tuple[float, float, float]]): Scaling factors applied to the scene along the x, y, and z axes. Defaults to (1.0, 1.0, 1.0). 19 | scene_position (Optional[Tuple[float, float, float]]): The position of the scene's origin in world coordinates. Defaults to (0, 0, 0). 20 | scene_orientation (Optional[Tuple[float, float, float, float]]): The quaternion representing the scene's orientation. Defaults to (1.0, 0, 0, 0). 21 | robots (Optional[List[RobotModel]]): A list of configurations for robots participating in the episode. Defaults to an empty list. 22 | objects (Optional[List[Object]]): A list of objects present in the scene. Defaults to an empty list. 23 | extra (Optional[Dict[str, Any]]): Additional configuration options not covered by the predefined attributes. Defaults to an empty dictionary. 24 | 25 | Note: 26 | The class inherits from `BaseModel` and specifies `extra='allow'` to permit additional keys in the configuration without raising an error. 27 | """ 28 | 29 | scene_asset_path: Optional[str] = None 30 | scene_scale: Optional[Tuple[float, float, float]] = (1.0, 1.0, 1.0) 31 | scene_position: Optional[Tuple[float, float, float]] = (0, 0, 0) 32 | scene_orientation: Optional[Tuple[float, float, float, float]] = (1.0, 0, 0, 0) 33 | robots: Optional[List[RobotCfg]] = [] 34 | objects: Optional[List[Object]] = [] 35 | extra: Optional[Any] = None 36 | 37 | 38 | class EpisodeConfigFile(BaseModel, extra='allow'): 39 | """ 40 | Episode config file model. 41 | """ 42 | 43 | episodes: List[EpisodeCfg] 44 | -------------------------------------------------------------------------------- /grutopia/core/config/task/reward.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class RewardCfg(BaseModel, extra='allow'): 7 | """ 8 | Configures reward settings for a system. 9 | 10 | This class defines the structure of reward configurations, which includes specifying the type of reward and optional settings associated with that reward. It is designed to be used as a base model for creating reward configurations in various applications like machine learning algorithms, game development, or any system where rewards are instrumental in guiding behavior. 11 | 12 | Attributes: 13 | reward_type (str): Specifies the type of reward, determining the nature of the feedback provided by the system. 14 | reward_settings (Optional[Dict[str, Any]], optional): Additional settings tailored to the specified reward type. Defaults to None. This dictionary can hold a variety of parameters depending on the complexity required for configuring the reward mechanism. 15 | """ 16 | 17 | reward_type: str 18 | reward_settings: Optional[Dict[str, Any]] = None 19 | -------------------------------------------------------------------------------- /grutopia/core/datahub/README.md: -------------------------------------------------------------------------------- 1 | # DataHub 2 | 3 | Share date between isaac sim backend and WebUI backend with **Python class objects**. 4 | -------------------------------------------------------------------------------- /grutopia/core/datahub/__init__.py: -------------------------------------------------------------------------------- 1 | import grutopia.core.datahub.model_data 2 | from grutopia.core.datahub.datahub import DataHub 3 | from grutopia.core.datahub.isaac_data import ActionData, IsaacData 4 | -------------------------------------------------------------------------------- /grutopia/core/register/__init__.py: -------------------------------------------------------------------------------- 1 | import grutopia.core.register.register 2 | from grutopia.core.register.register import import_all_modules_for_register 3 | -------------------------------------------------------------------------------- /grutopia/core/register/register.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | 4 | from grutopia.core.util import log 5 | 6 | ALL_MODULES = [] 7 | MODEL_MODULES = [ 8 | 'controllers', 9 | 'objects', 10 | 'metrics', 11 | 'robots', 12 | 'sensors', 13 | 'tasks', 14 | 'interactions', 15 | 'agents', 16 | ] 17 | 18 | DEFAULT_EXTENSION_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], '../../../grutopia_extension') 19 | 20 | 21 | def _handle_errors(errors): 22 | """ 23 | Log out and possibly reraise errors during import. 24 | 25 | Args: 26 | errors: errors dict to be logged 27 | """ 28 | if not errors: 29 | return 30 | for name, err in errors: 31 | log.warning('Module {} import failed: {}'.format(name, err)) 32 | 33 | 34 | def import_all_modules_for_register(custom_module_paths=None, extension_path=None): 35 | """ 36 | Import all modules for register. 37 | 38 | Args: 39 | custom_module_paths: custom module paths, e.g. ['xxx.lib1', 'xxx.lib2', 'xxx.lib3'] 40 | extension_path: Extension path(integrated in grutopia_extension as default) 41 | """ 42 | if extension_path is None: 43 | extension_path = DEFAULT_EXTENSION_PATH 44 | for _mod in MODEL_MODULES: 45 | # grutopia_extension's default path 46 | path = os.path.join(extension_path, _mod) 47 | m = [m.split('.py')[0] for m in os.listdir(path) if m.endswith('.py') and m != '__init__.py'] 48 | ALL_MODULES.append((_mod, m)) 49 | modules = [] 50 | for base_dir, mods in ALL_MODULES: 51 | for name in mods: 52 | full_name = 'grutopia_extension.' + base_dir + '.' + name 53 | modules.append(full_name) 54 | if isinstance(custom_module_paths, list): 55 | modules += custom_module_paths 56 | errors = [] 57 | for module in modules: 58 | try: 59 | importlib.import_module(module) 60 | except ImportError as error: 61 | errors.append((module, error)) 62 | _handle_errors(errors) 63 | -------------------------------------------------------------------------------- /grutopia/core/robot/__init__.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from omni.isaac.core.scenes import Scene 4 | 5 | from grutopia.core.robot.robot import BaseRobot, create_robots 6 | from grutopia.core.runtime.task_runtime import TaskRuntime 7 | 8 | 9 | def init_robots(runtime: TaskRuntime, scene: Scene) -> typing.Dict[str, BaseRobot]: 10 | return create_robots(runtime, scene) 11 | -------------------------------------------------------------------------------- /grutopia/core/runtime/distributed_task_runtime_manager.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | 3 | from grutopia.core.config import TaskCfg 4 | from grutopia.core.runtime.task_runtime import BaseTaskRuntimeManager, Env, TaskRuntime 5 | 6 | 7 | # TODO: use ray to implement task runtime manager for the distributed version 8 | @BaseTaskRuntimeManager.register('DistributedTaskRuntimeManager') 9 | class DistributedTaskRuntimeManager(BaseTaskRuntimeManager): 10 | def __init__(self, task_user_config: TaskCfg = None): 11 | pass 12 | 13 | def get_next_task_runtime(self, last_env: Optional[Env] = None) -> Union[TaskRuntime, None]: 14 | pass 15 | -------------------------------------------------------------------------------- /grutopia/core/scene/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.scene.object import create_object 2 | from grutopia.core.scene.scene import create_scene 3 | from grutopia.core.scene.scene.util import usd_op 4 | from grutopia.core.scene.scene.util.usd_op import delete_prim_in_stage 5 | -------------------------------------------------------------------------------- /grutopia/core/scene/object.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | from omni.isaac.core.scenes import Scene 4 | 5 | from grutopia.core.config import ObjectCfg as ObjectConfig 6 | 7 | 8 | class ObjectCommon: 9 | """ 10 | Object common class. 11 | """ 12 | 13 | objs = {} 14 | 15 | def __init__(self, config: ObjectConfig): 16 | self._config = config 17 | 18 | def set_up_scene(self, scene: Scene): 19 | raise NotImplementedError 20 | 21 | @classmethod 22 | def register(cls, name: str): 23 | """ 24 | Register an object class with the given name(decorator). 25 | 26 | Args: 27 | name(str): name of the object 28 | """ 29 | 30 | def decorator(object_class): 31 | cls.objs[name] = object_class 32 | 33 | @wraps(object_class) 34 | def wrapped_function(*args, **kwargs): 35 | return object_class(*args, **kwargs) 36 | 37 | return wrapped_function 38 | 39 | return decorator 40 | 41 | 42 | def create_object(config: ObjectConfig): 43 | """ 44 | Create an object. 45 | Args: 46 | config (ObjectConfig): configuration of the objects 47 | """ 48 | assert config.type in ObjectCommon.objs, 'unknown objects type {}'.format(config.type) 49 | obj: ObjectCommon = ObjectCommon.objs[config.type](config) 50 | return obj 51 | -------------------------------------------------------------------------------- /grutopia/core/scene/scene/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.scene.scene.util import usd_op 2 | 3 | 4 | def create_scene(config_json_path: str, prim_path_root: str = 'background'): 5 | """ 6 | TODO: rename. 7 | Create a scene from config.(But just input usd file yet.) 8 | Args: 9 | config_json_path (str): path to scene config file(use to be a .usd file) 10 | prim_path_root (str): path to root prim 11 | 12 | Returns: 13 | config_json_path (str): path to config file 14 | world_prim_path (str): path to world prim 15 | """ 16 | world_prim_path = '/' + prim_path_root 17 | if config_json_path.endswith('usd') or config_json_path.endswith('usda') or config_json_path.endswith('usdc'): 18 | # Add usd directly 19 | return config_json_path, world_prim_path 20 | raise RuntimeError('Env file path needs to end with .usd, .usda or .usdc .') 21 | -------------------------------------------------------------------------------- /grutopia/core/scene/scene/util/__init__.py: -------------------------------------------------------------------------------- 1 | import grutopia.core.scene.scene.util.usd_op as usd_op 2 | -------------------------------------------------------------------------------- /grutopia/core/scene/scene/util/type/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.scene.scene.util.type.type_map import ( 2 | dtype_map, 3 | get_xformop_precision, 4 | get_xformop_type, 5 | ) 6 | 7 | 8 | def get_usd_data_type(dtype: str): 9 | return dtype_map[dtype] 10 | -------------------------------------------------------------------------------- /grutopia/core/scene/scene/util/type/type_map.py: -------------------------------------------------------------------------------- 1 | from pxr import Gf, UsdGeom 2 | 3 | dtype_map = { 4 | 'double2': Gf.Vec2d, 5 | 'double3': Gf.Vec3d, 6 | 'double4': Gf.Vec4d, 7 | 'float2': Gf.Vec2f, 8 | 'float3': Gf.Vec3f, 9 | 'float4': Gf.Vec4f, 10 | 'half2': Gf.Vec2h, 11 | 'half3': Gf.Vec3h, 12 | 'half4': Gf.Vec4h, 13 | 'int2': Gf.Vec2i, 14 | 'int3': Gf.Vec3i, 15 | 'int4': Gf.Vec4i, 16 | 'matrix2d': Gf.Matrix2d, 17 | 'matrix3d': Gf.Matrix3d, 18 | 'matrix4d': Gf.Matrix4d, 19 | 'quatd': Gf.Quatd, 20 | 'quatf': Gf.Quatf, 21 | 'quath': Gf.Quath, 22 | } 23 | 24 | 25 | def get_xformop_precision(precision: str) -> UsdGeom.XformOp.Precision: 26 | if 'double' in precision: 27 | return UsdGeom.XformOp.PrecisionDouble 28 | elif 'float' in precision: 29 | return UsdGeom.XformOp.PrecisionFloat 30 | else: 31 | return UsdGeom.XformOp.PrecisionHalf 32 | 33 | 34 | def get_xformop_type(x_type: str) -> UsdGeom.XformOp.Type: 35 | if ':' in x_type: 36 | x_type = x_type.split(':')[1] 37 | return UsdGeom.XformOp.Type.GetValueFromName('Type' + x_type[0].capitalize() + x_type[1:]) 38 | -------------------------------------------------------------------------------- /grutopia/core/task/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.task.metric import BaseMetric 2 | from grutopia.core.task.task import BaseTask 3 | -------------------------------------------------------------------------------- /grutopia/core/task/metric.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from functools import wraps 3 | 4 | from grutopia.core.config.metric import MetricCfg 5 | from grutopia.core.runtime.task_runtime import TaskRuntime 6 | 7 | 8 | class BaseMetric(ABC): 9 | metrics = {} 10 | 11 | def __init__(self, config: MetricCfg, task_runtime: TaskRuntime): 12 | self.config = config 13 | self.name = config.name 14 | self.task_runtime = task_runtime 15 | self.metric_config = config.metric_config 16 | 17 | @abstractmethod 18 | def reset(self): 19 | raise NotImplementedError(f'`reset` function of {self.name} is not implemented') 20 | 21 | @abstractmethod 22 | def update(self, *args): 23 | """ 24 | This function is called at each world step. 25 | """ 26 | raise NotImplementedError(f'`update` function of {self.name} is not implemented') 27 | 28 | @abstractmethod 29 | def calc(self): 30 | """ 31 | This function is called to calculate the metrics when the episode is terminated. 32 | """ 33 | raise NotImplementedError(f'`calc` function of {self.name} is not implemented') 34 | 35 | @classmethod 36 | def register(cls, name: str): 37 | """ 38 | This function is used to register a metric class.(decorator) 39 | Args: 40 | name(str): name of the metric 41 | """ 42 | 43 | def decorator(metric_class): 44 | cls.metrics[name] = metric_class 45 | 46 | @wraps(metric_class) 47 | def wrapped_function(*args, **kwargs): 48 | return metric_class(*args, **kwargs) 49 | 50 | return wrapped_function 51 | 52 | return decorator 53 | 54 | 55 | def create_metric(config: MetricCfg, task_runtime: TaskRuntime): 56 | if config.type not in BaseMetric.metrics: 57 | raise KeyError( 58 | f"""The metric {config.type} is not registered, please register it using `@BaseMetric.register`""" 59 | ) 60 | metric_cls = BaseMetric.metrics[config.type] 61 | return metric_cls(config, task_runtime) 62 | -------------------------------------------------------------------------------- /grutopia/core/task/reward.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from functools import wraps 3 | from typing import Any, Dict 4 | 5 | from grutopia.core.config.task import RewardCfg 6 | from grutopia.core.task import BaseTask 7 | 8 | 9 | class BaseReward(ABC): 10 | rewards = {} 11 | 12 | def __init__(self, task: BaseTask, settings: Dict[str, Any]): 13 | self.state = None 14 | self.task = task 15 | self.settings = settings 16 | self.init_setting() 17 | 18 | def init_setting(self): 19 | pass 20 | 21 | @abstractmethod 22 | def reset(self): 23 | self.state = None 24 | 25 | @abstractmethod 26 | def calc(self) -> float: 27 | raise NotImplementedError(f'`calc` function of {self.name} is not implemented') 28 | 29 | @abstractmethod 30 | def _calc_next_state(self): 31 | raise NotImplementedError(f'`_calc_next_state` function of {self.name} is not implemented') 32 | 33 | @classmethod 34 | def register(cls, name: str): 35 | """ 36 | This function is used to register a reward class.(decorator) 37 | Args: 38 | name(str): name of the reward 39 | """ 40 | 41 | def decorator(reward_class): 42 | cls.rewards[name] = reward_class 43 | 44 | @wraps(reward_class) 45 | def wrapped_function(*args, **kwargs): 46 | return reward_class(*args, **kwargs) 47 | 48 | return wrapped_function 49 | 50 | return decorator 51 | 52 | 53 | def create_reward(reward_config: RewardCfg, task: BaseTask): 54 | if reward_config.reward_type not in BaseReward.rewards: 55 | raise KeyError( 56 | f"""The reward {reward_config.reward_type} is not registered, please register it using `@BaseReward.register`""" 57 | ) 58 | reward_cls = BaseReward.rewards[reward_config.reward_type] 59 | return reward_cls(task, reward_config.reward_settings) 60 | -------------------------------------------------------------------------------- /grutopia/core/util/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from grutopia.core.util.async_req import AsyncRequest 4 | from grutopia.core.util.log import log 5 | 6 | if not AsyncRequest.loop: 7 | AsyncRequest.start_loop() 8 | 9 | 10 | def is_in_container() -> bool: 11 | try: 12 | os.stat('/.dockerenv') 13 | return True 14 | except FileNotFoundError: 15 | return False 16 | except Exception as e: 17 | log.error(f'Error while checking if in container, assume False: {e}') 18 | return False 19 | 20 | 21 | def has_display() -> bool: 22 | try: 23 | display = os.environ['DISPLAY'] 24 | if display is not None and display != '': 25 | return True 26 | return False 27 | except KeyError: 28 | return False 29 | except Exception as e: 30 | log.error(f'Error while checking if has display, assume False: {e}') 31 | return False 32 | -------------------------------------------------------------------------------- /grutopia/core/util/chat/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.util.chat.agent_chat import AgentChat 2 | -------------------------------------------------------------------------------- /grutopia/core/util/chat/agent_chat.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List 2 | 3 | from grutopia.core.datahub import DataHub 4 | 5 | 6 | class AgentChat: 7 | """ 8 | Utility for agents communicating with each other in Datahub. 9 | """ 10 | 11 | def __init__(self, task_name: str, robot_name: str): 12 | self.read_message_index = 0 13 | self.task_name = task_name 14 | self.robot_name = robot_name 15 | 16 | def send_message(self, message: str, at: List[str], parent_idx: int, role: str = 'agent'): 17 | """ 18 | Send a message to Chatbox. 19 | 20 | Args: 21 | message (str): message to send 22 | at (List[str]): note which agent need process this messages. None as default(means everyone). 23 | parent_idx (int): index of parent message(this message is a reply to parent message). 24 | role (str): role of agent (agent as default, don't change this pls). 25 | """ 26 | DataHub.send_chat_control( 27 | nickname=self.robot_name, task_name=self.task_name, role=role, text=message, at=at, parent_idx=parent_idx 28 | ) 29 | 30 | def get_message(self) -> List[Dict]: 31 | """ 32 | Get unread messages from Chatbox. 33 | 34 | Returns: 35 | List: list of unread messages. 36 | """ 37 | new_chat_data = DataHub.get_chat_control(self.task_name, self.read_message_index) 38 | self.read_message_index += len(new_chat_data) 39 | return new_chat_data 40 | -------------------------------------------------------------------------------- /grutopia/core/util/clear_task.py: -------------------------------------------------------------------------------- 1 | from omni.isaac.core.utils.prims import get_prim_type_name 2 | from omni.usd.commands import DeletePrimsCommand 3 | 4 | 5 | def clear_stage_by_prim_path(prim_path: str = None) -> None: 6 | """Deletes all prims in the stage without populating the undo command buffer 7 | 8 | Args: 9 | prim_path (str, optional): path of the stage. Defaults to None. 10 | """ 11 | # Note: Need to import this here to prevent circular dependencies. 12 | from omni.isaac.core.utils.prims import ( 13 | get_all_matching_child_prims, 14 | get_prim_path, 15 | is_prim_ancestral, 16 | is_prim_hidden_in_stage, 17 | is_prim_no_delete, 18 | ) 19 | 20 | def default_predicate(path: str): 21 | # prim = get_prim_at_path(prim_path) 22 | # skip prims that we cannot delete 23 | if is_prim_no_delete(path): 24 | return False 25 | if is_prim_hidden_in_stage(path): 26 | return False 27 | if is_prim_ancestral(path): 28 | return False 29 | if path == '/': 30 | return False 31 | if get_prim_type_name(prim_path=path) == 'PhysicsScene': 32 | return False 33 | if path == '/World': 34 | return False 35 | # Don't remove any /Render prims as that can cause crashes 36 | if path.startswith('/Render'): 37 | return False 38 | return True 39 | 40 | prims = get_all_matching_child_prims(prim_path, default_predicate) 41 | prim_paths_to_delete = [get_prim_path(prim) for prim in prims] 42 | DeletePrimsCommand(prim_paths_to_delete).do() 43 | -------------------------------------------------------------------------------- /grutopia/core/util/gym.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | 5 | 6 | class gym_adapter: 7 | def __init__(self, joint_names_gym: List[str], joint_names_sim: List[str]): 8 | self.joint_names_gym = joint_names_gym 9 | self.joint_names_sim = joint_names_sim 10 | 11 | @staticmethod 12 | def rearange_order(src: list | np.ndarray, from_joint_names: List[str], to_joint_names: List[str]): 13 | src = src.tolist() 14 | sim_actions = [None] * len(from_joint_names) 15 | for i, gym_joint in enumerate(from_joint_names): 16 | sim_joint_index = to_joint_names.index(gym_joint) 17 | sim_actions[sim_joint_index] = src[i] 18 | 19 | return np.array(sim_actions) 20 | 21 | def gym2sim(self, action): 22 | return self.rearange_order(action, self.joint_names_gym, self.joint_names_sim) 23 | 24 | def sim2gym(self, obs): 25 | return self.rearange_order(obs, self.joint_names_sim, self.joint_names_gym) 26 | -------------------------------------------------------------------------------- /grutopia/core/util/interaction.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from functools import wraps 3 | 4 | import numpy as np 5 | 6 | 7 | class BaseInteraction(ABC): 8 | interactions = {} 9 | 10 | def __init__(self): 11 | self._type = None 12 | 13 | @property 14 | def type(self): 15 | return self._type 16 | 17 | @abstractmethod 18 | def get_input(self) -> np.ndarray: 19 | """ 20 | Get input from UID. 21 | Returns: 22 | nd.array: Input data. 23 | """ 24 | raise NotImplementedError 25 | 26 | @classmethod 27 | def register(cls, name): 28 | """ 29 | Register an interaction (like keyboard,gamepad or vr controllers) class with its name(decorator). 30 | Args: 31 | name(str): name of the robot class. 32 | """ 33 | 34 | def decorator(interaction_class): 35 | cls.interactions[name] = interaction_class 36 | 37 | @wraps(interaction_class) 38 | def wrapped_function(*args, **kwargs): 39 | return interaction_class(*args, **kwargs) 40 | 41 | return wrapped_function 42 | 43 | return decorator 44 | -------------------------------------------------------------------------------- /grutopia/core/util/log/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Optional 3 | 4 | import toml 5 | from pydantic import BaseModel 6 | 7 | from grutopia.core.util.log.logger import Logger 8 | 9 | 10 | class LogConfig(BaseModel): 11 | filename: Optional[str] = None 12 | level: Optional[str] = 'info' 13 | fmt: Optional[str] = '[%(asctime)s][%(levelname)s] %(pathname)s[line:%(lineno)d] -: %(message)s' 14 | 15 | 16 | # with open(os.path.join(os.path.split(os.path.realpath(__file__))[0], 'config.ini'), 'r') as f: 17 | # config = LogConfig(**(toml.loads(f.read())['log'])) 18 | 19 | log_dict = {'level': 'error', 'fmt': '[%(asctime)s][%(levelname)s] %(pathname)s[line:%(lineno)d] -: %(message)s'} 20 | 21 | config = LogConfig(**log_dict) 22 | 23 | # Use this rather than `Logger` 24 | log = Logger( 25 | filename=config.filename, 26 | level=config.level, 27 | fmt=config.fmt, 28 | ).log 29 | -------------------------------------------------------------------------------- /grutopia/core/util/log/config.ini: -------------------------------------------------------------------------------- 1 | [log] 2 | # filename = "debug.log" 3 | level = "error" 4 | #fmt = '[%(asctime)s][%(levelname)s] %(message)s' # https://docs.python.org/zh-cn/3/library/logging.html FYI. 5 | fmt = '[%(asctime)s][%(levelname)s] %(pathname)s[line:%(lineno)d] -: %(message)s' 6 | -------------------------------------------------------------------------------- /grutopia/core/util/log/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class Logger(object): 5 | """global logger 6 | 7 | Args: 8 | filename (str, optional): log file name. Defaults to None. 9 | level (str, optional): log level( debug info warning error critical ). Defaults to 'info'. 10 | fmt (str, optional): log format. Defaults to '[%(asctime)s][%(levelname)s] %(message)s'. 11 | PS: 12 | more format details at : https://docs.python.org/zh-cn/3/library/logging.html 13 | """ 14 | 15 | level_relations = { 16 | 'debug': logging.DEBUG, 17 | 'info': logging.INFO, 18 | 'warning': logging.WARNING, 19 | 'error': logging.ERROR, 20 | 'critical': logging.CRITICAL, 21 | } 22 | 23 | # '[%(asctime)s][%(levelname)s] %(pathname)s[line:%(lineno)d] -: %(message)s' 24 | def __init__( 25 | self, filename: str = None, level: str = 'info', fmt: str = '[%(asctime)s][%(levelname)s] %(message)s' 26 | ): 27 | if filename == 'None': 28 | filename = None 29 | self.log = logging.getLogger(filename) 30 | format_str = logging.Formatter(fmt) 31 | self.log.setLevel(self.level_relations.get(level)) 32 | sh = logging.StreamHandler() 33 | sh.setFormatter(format_str) 34 | self.log.addHandler(sh) 35 | # Logging file 36 | if filename is not None: 37 | th = logging.FileHandler(filename=filename, encoding='utf-8') 38 | th.setFormatter(format_str) 39 | self.log.addHandler(th) 40 | -------------------------------------------------------------------------------- /grutopia/core/util/math.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | @torch.jit.script 6 | def quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor: 7 | """Rotate a vector by the inverse of a quaternion. 8 | 9 | Args: 10 | q: The quaternion in (w, x, y, z). Shape is (N, 4). 11 | v: The vector in (x, y, z). Shape is (N, 3). 12 | 13 | Returns: 14 | The rotated vector in (x, y, z). Shape is (N, 3). 15 | """ 16 | shape = q.shape 17 | q_w = q[:, 0] 18 | q_vec = q[:, 1:] 19 | a = v * (2.0 * q_w**2 - 1.0).unsqueeze(-1) 20 | b = torch.cross(q_vec, v, dim=-1) * q_w.unsqueeze(-1) * 2.0 21 | c = q_vec * torch.bmm(q_vec.view(shape[0], 1, 3), v.view(shape[0], 3, 1)).squeeze(-1) * 2.0 22 | return a - b + c 23 | 24 | 25 | def quaternion_multiply(q1, q2): 26 | w1, x1, y1, z1 = q1 27 | w2, x2, y2, z2 = q2 28 | 29 | w = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2 30 | x = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2 31 | y = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2 32 | z = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2 33 | 34 | return np.array([w, x, y, z]) 35 | 36 | 37 | def quaternion_conjugate(q): 38 | w, x, y, z = q 39 | return np.array([w, -x, -y, -z]) 40 | 41 | 42 | def quaternion_to_gravity_component(quaternion: np.ndarray): 43 | # Assuming quaternion as (w, x, y, z) 44 | gravity_vector_global = np.array([0, 0, -1]) 45 | 46 | # Convert gravity vector to a quaternion 47 | gravity_quaternion_global = np.array([0, *gravity_vector_global]) 48 | 49 | # Rotate the gravity quaternion by the object's quaternion 50 | rotated_gravity_quaternion = quaternion_multiply( 51 | quaternion, quaternion_multiply(gravity_quaternion_global, quaternion_conjugate(quaternion)) 52 | ) 53 | 54 | # Extract the result as a numpy array 55 | gravity_component_local = np.array( 56 | [rotated_gravity_quaternion[1], rotated_gravity_quaternion[2], rotated_gravity_quaternion[3]] 57 | ) 58 | 59 | return gravity_component_local 60 | -------------------------------------------------------------------------------- /grutopia/core/util/rsl_rl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia/core/util/rsl_rl/__init__.py -------------------------------------------------------------------------------- /grutopia/core/util/rsl_rl/pickle.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | 4 | class Unpickler(pickle.Unpickler): 5 | def find_class(self, module, name): 6 | if name == 'Normalizer' and module == 'rsl_rl.utils.utils': 7 | from grutopia.core.util.rsl_rl.normalizer import Normalizer 8 | 9 | return Normalizer 10 | return super().find_class(module, name) 11 | -------------------------------------------------------------------------------- /grutopia/core/util/space.py: -------------------------------------------------------------------------------- 1 | import gymnasium as gym 2 | import numpy as np 3 | 4 | from grutopia.core.config import Config 5 | 6 | # from grutopia.core.robot import BaseRobot 7 | # from grutopia.core.robot.controller import BaseController 8 | # from grutopia.core.robot.sensor import BaseSensor 9 | 10 | 11 | # TODO get action space based on the specific task, currently the hardcoded value will be returned. 12 | def get_action_space_by_task(config: Config) -> gym.Space: 13 | return gym.spaces.Dict( 14 | { 15 | 'move_along_path': gym.spaces.Sequence( 16 | gym.spaces.Tuple( 17 | ( 18 | gym.spaces.Box(low=-np.inf, high=np.inf, shape=(), dtype=np.float32), 19 | gym.spaces.Box(low=-np.inf, high=np.inf, shape=(), dtype=np.float32), 20 | gym.spaces.Box(low=-np.inf, high=np.inf, shape=(), dtype=np.float32), 21 | ) 22 | ) 23 | ) 24 | } 25 | ) 26 | 27 | 28 | # TODO get observation space based on the specific task, currently the hardcoded value will be returned. 29 | def get_observation_space_by_task(config: Config) -> gym.Space: 30 | return gym.spaces.Dict( 31 | { 32 | 'position': gym.spaces.Box(low=-np.inf, high=np.inf, shape=(3,), dtype=np.float32), 33 | 'orientation': gym.spaces.Box(low=-np.inf, high=np.inf, shape=(4,), dtype=np.float32), 34 | } 35 | ) 36 | -------------------------------------------------------------------------------- /grutopia/demo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia/demo/__init__.py -------------------------------------------------------------------------------- /grutopia/demo/aliengo_locomotion.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.core.util import has_display 5 | from grutopia.macros import gm 6 | from grutopia_extension import import_extensions 7 | from grutopia_extension.configs.robots.aliengo import AliengoRobotCfg, move_to_point_cfg 8 | from grutopia_extension.configs.tasks import ( 9 | SingleInferenceEpisodeCfg, 10 | SingleInferenceTaskCfg, 11 | ) 12 | 13 | headless = False 14 | webrtc = False 15 | 16 | if not has_display(): 17 | headless = True 18 | webrtc = True 19 | 20 | 21 | config = Config( 22 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 23 | task_config=SingleInferenceTaskCfg( 24 | episodes=[ 25 | SingleInferenceEpisodeCfg( 26 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 27 | robots=[ 28 | AliengoRobotCfg( 29 | position=[0.0, 0.0, 1.05], 30 | controllers=[move_to_point_cfg], 31 | ) 32 | ], 33 | ), 34 | ], 35 | ), 36 | ) 37 | 38 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 39 | 40 | import_extensions() 41 | # import custom extensions here. 42 | 43 | env = Env(sim_runtime) 44 | obs, _ = env.reset() 45 | 46 | i = 0 47 | 48 | env_action = { 49 | move_to_point_cfg.name: [(3.0, 3.0, 0.0)], 50 | } 51 | 52 | print(f'actions: {env_action}') 53 | 54 | while env.simulation_app.is_running(): 55 | i += 1 56 | obs, _, terminated, _, _ = env.step(action=env_action) 57 | 58 | if i % 1000 == 0: 59 | print(i) 60 | 61 | env.close() 62 | -------------------------------------------------------------------------------- /grutopia/demo/franka_manipulation_mocap_teleop.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.macros import gm 5 | from grutopia_extension import import_extensions 6 | from grutopia_extension.configs.metrics import RecordingMetricCfg 7 | from grutopia_extension.configs.robots.mocap_controlled_franka import ( 8 | MocapControlledFrankaRobotCfg, 9 | lh_controlled_camera_cfg, 10 | teleop_cfg, 11 | ) 12 | from grutopia_extension.configs.tasks import ( 13 | ManipulationEpisodeCfg, 14 | ManipulationExtra, 15 | ManipulationTaskCfg, 16 | ManipulationTaskSetting, 17 | ) 18 | 19 | franka = MocapControlledFrankaRobotCfg( 20 | position=[-0.35, 0.0, 1.05], 21 | controllers=[ 22 | teleop_cfg, 23 | ], 24 | sensors=[lh_controlled_camera_cfg], 25 | ) 26 | 27 | config = Config( 28 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 29 | task_config=ManipulationTaskCfg( 30 | metrics=[ 31 | RecordingMetricCfg( 32 | robot_name='franka', 33 | fields=['joint_action'], 34 | ) 35 | ], 36 | episodes=[ 37 | ManipulationEpisodeCfg( 38 | scene_asset_path=gm.ASSET_PATH + '/scenes/demo_scenes/franka_mocap_teleop/table_scene.usd', 39 | robots=[franka], 40 | extra=ManipulationExtra( 41 | prompt='Prompt test 1', 42 | target='franka_manipulation_mocap_teleop', 43 | episode_idx=0, 44 | ), 45 | ), 46 | ], 47 | task_settings=ManipulationTaskSetting(max_step=10000), 48 | ), 49 | ) 50 | 51 | sim_runtime = SimulatorRuntime(config_class=config, headless=False, webrtc=False, native=True) 52 | 53 | import_extensions() 54 | from grutopia_extension.interactions.motion_capture import MocapInteraction 55 | 56 | env = Env(sim_runtime) 57 | obs, _ = env.reset() 58 | 59 | mocap_url = 'http://127.0.0.1:5001' 60 | mocap_interaction = MocapInteraction(mocap_url) 61 | 62 | while env.simulation_app.is_running(): 63 | cur_mocap_info = mocap_interaction.step() 64 | arm_action = {teleop_cfg.name: [cur_mocap_info]} 65 | 66 | obs, _, _, _, _ = env.step(action=arm_action) 67 | 68 | mocap_interaction.server_stop() 69 | env.close() 70 | -------------------------------------------------------------------------------- /grutopia/demo/g1_locomotion.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.core.util import has_display 5 | from grutopia.macros import gm 6 | from grutopia_extension import import_extensions 7 | from grutopia_extension.configs.robots.g1 import G1RobotCfg, move_by_speed_cfg 8 | from grutopia_extension.configs.tasks import ( 9 | SingleInferenceEpisodeCfg, 10 | SingleInferenceTaskCfg, 11 | ) 12 | 13 | headless = False 14 | 15 | if not has_display(): 16 | headless = True 17 | 18 | 19 | config = Config( 20 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 21 | task_config=SingleInferenceTaskCfg( 22 | episodes=[ 23 | SingleInferenceEpisodeCfg( 24 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 25 | robots=[ 26 | G1RobotCfg( 27 | position=[0.0, 0.0, 0.8], 28 | controllers=[move_by_speed_cfg], 29 | ) 30 | ], 31 | ), 32 | ], 33 | ), 34 | ) 35 | 36 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 37 | 38 | import_extensions() 39 | # import custom extensions here. 40 | 41 | from grutopia_extension.interactions.keyboard import KeyboardInteraction 42 | 43 | env = Env(sim_runtime) 44 | obs, _ = env.reset() 45 | 46 | i = 0 47 | 48 | env_action = { 49 | move_by_speed_cfg.name: (0.0, 0.0, 0.0), 50 | } 51 | 52 | print(f'actions: {env_action}') 53 | 54 | keyboard = KeyboardInteraction() 55 | while env.simulation_app.is_running(): 56 | i += 1 57 | command = keyboard.get_input() 58 | x_speed = command[0] - command[1] 59 | y_speed = command[2] - command[3] 60 | z_speed = command[4] - command[5] 61 | env_action = { 62 | move_by_speed_cfg.name: (x_speed, y_speed, z_speed), 63 | } 64 | obs, _, terminated, _, _ = env.step(action=env_action) 65 | 66 | if i % 1000 == 0: 67 | print(i) 68 | 69 | env.close() 70 | -------------------------------------------------------------------------------- /grutopia/demo/gr1_locomotion.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.core.util import has_display 5 | from grutopia.macros import gm 6 | from grutopia_extension import import_extensions 7 | from grutopia_extension.configs.robots.gr1 import GR1RobotCfg, move_to_point_cfg 8 | from grutopia_extension.configs.tasks import ( 9 | SingleInferenceEpisodeCfg, 10 | SingleInferenceTaskCfg, 11 | ) 12 | 13 | headless = False 14 | 15 | if not has_display(): 16 | headless = True 17 | 18 | 19 | config = Config( 20 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 21 | task_config=SingleInferenceTaskCfg( 22 | episodes=[ 23 | SingleInferenceEpisodeCfg( 24 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 25 | robots=[ 26 | GR1RobotCfg( 27 | position=[0.0, 0.0, 0.95], 28 | controllers=[move_to_point_cfg], 29 | ) 30 | ], 31 | ), 32 | ], 33 | ), 34 | ) 35 | 36 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 37 | 38 | import_extensions() 39 | # import custom extensions here. 40 | 41 | env = Env(sim_runtime) 42 | obs, _ = env.reset() 43 | 44 | i = 0 45 | 46 | env_action = { 47 | move_to_point_cfg.name: [(3.0, 3.0, 0.0)], 48 | } 49 | 50 | print(f'actions: {env_action}') 51 | 52 | while env.simulation_app.is_running(): 53 | i += 1 54 | obs, _, terminated, _, _ = env.step(action=env_action) 55 | 56 | if i % 1000 == 0: 57 | print(i) 58 | 59 | env.close() 60 | -------------------------------------------------------------------------------- /grutopia/demo/h1_locomotion.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.core.util import has_display 5 | from grutopia.macros import gm 6 | from grutopia_extension import import_extensions 7 | from grutopia_extension.configs.robots.h1 import ( 8 | H1RobotCfg, 9 | h1_camera_cfg, 10 | h1_tp_camera_cfg, 11 | move_along_path_cfg, 12 | move_by_speed_cfg, 13 | rotate_cfg, 14 | ) 15 | from grutopia_extension.configs.tasks import ( 16 | SingleInferenceEpisodeCfg, 17 | SingleInferenceTaskCfg, 18 | ) 19 | 20 | headless = False 21 | if not has_display(): 22 | headless = True 23 | 24 | h1_1 = H1RobotCfg( 25 | position=(0.0, 0.0, 1.05), 26 | controllers=[ 27 | move_by_speed_cfg, 28 | move_along_path_cfg, 29 | rotate_cfg, 30 | ], 31 | sensors=[ 32 | h1_camera_cfg.update(name='camera', resolution=(320, 240), enable=False), 33 | h1_tp_camera_cfg.update(enable=False), 34 | ], 35 | ) 36 | 37 | config = Config( 38 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 39 | task_config=SingleInferenceTaskCfg( 40 | episodes=[ 41 | SingleInferenceEpisodeCfg( 42 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 43 | scene_scale=(0.01, 0.01, 0.01), 44 | robots=[h1_1], 45 | ), 46 | ], 47 | ), 48 | ) 49 | 50 | print(config.model_dump_json(indent=4)) 51 | 52 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 53 | 54 | import_extensions() 55 | # import custom extensions here. 56 | 57 | env = Env(sim_runtime) 58 | obs, _ = env.reset() 59 | print(f'========INIT OBS{obs}=============') 60 | 61 | path = [(1.0, 0.0, 0.0), (1.0, 1.0, 0.0), (3.0, 4.0, 0.0)] 62 | i = 0 63 | 64 | move_action = {move_along_path_cfg.name: [path]} 65 | 66 | while env.simulation_app.is_running(): 67 | i += 1 68 | action = move_action 69 | obs, _, terminated, _, _ = env.step(action=action) 70 | if i % 100 == 0: 71 | print(i) 72 | 73 | env.close() 74 | -------------------------------------------------------------------------------- /grutopia/demo/h1_traveled_distance.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.core.util import has_display 5 | from grutopia.macros import gm 6 | from grutopia_extension import import_extensions 7 | from grutopia_extension.configs.metrics.simple_metric import SimpleMetricCfg 8 | from grutopia_extension.configs.robots.h1 import ( 9 | H1RobotCfg, 10 | h1_camera_cfg, 11 | h1_tp_camera_cfg, 12 | move_along_path_cfg, 13 | move_by_speed_cfg, 14 | rotate_cfg, 15 | ) 16 | from grutopia_extension.configs.tasks import FiniteStepTaskCfg, FiniteStepTaskEpisodeCfg 17 | 18 | headless = False 19 | if not has_display(): 20 | headless = True 21 | 22 | h1_1 = H1RobotCfg( 23 | position=(0.0, 0.0, 1.05), 24 | controllers=[ 25 | move_by_speed_cfg, 26 | move_along_path_cfg, 27 | rotate_cfg, 28 | ], 29 | sensors=[ 30 | h1_camera_cfg.update(name='camera', resolution=(320, 240), enable=True), 31 | h1_tp_camera_cfg.update(enable=False), 32 | ], 33 | ) 34 | 35 | config = Config( 36 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 37 | task_config=FiniteStepTaskCfg( 38 | task_settings={'max_step': 300}, 39 | metrics=[SimpleMetricCfg(metric_config={'robot_name': 'h1'})], 40 | metrics_save_path='./h1_simple_metric.jsonl', 41 | episodes=[ 42 | FiniteStepTaskEpisodeCfg( 43 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 44 | scene_scale=(0.01, 0.01, 0.01), 45 | robots=[h1_1], 46 | ), 47 | ], 48 | ), 49 | ) 50 | 51 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 52 | 53 | import_extensions() 54 | # import custom extensions here. 55 | 56 | env = Env(sim_runtime) 57 | obs, _ = env.reset() 58 | print(f'========INIT OBS{obs}=============') 59 | 60 | path = [(1.0, 0.0, 0.0), (1.0, 1.0, 0.0), (3.0, 4.0, 0.0)] 61 | i = 0 62 | 63 | move_action = {move_along_path_cfg.name: [path]} 64 | 65 | while env.simulation_app.is_running(): 66 | i += 1 67 | action = move_action 68 | obs, _, terminated, _, _ = env.step(action=action) 69 | 70 | if terminated: 71 | obs, info = env.reset() 72 | if env.RESET_INFO_TASK_RUNTIME not in info: # No more episode 73 | break 74 | 75 | if i % 100 == 0: 76 | print(i) 77 | 78 | env.close() 79 | -------------------------------------------------------------------------------- /grutopia/demo/jetbot_locomotion.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.core.util import has_display 5 | from grutopia.macros import gm 6 | from grutopia_extension import import_extensions 7 | from grutopia_extension.configs.robots.jetbot import ( 8 | JetbotRobotCfg, 9 | move_along_path_cfg, 10 | move_to_point_cfg, 11 | ) 12 | from grutopia_extension.configs.tasks import ( 13 | SingleInferenceEpisodeCfg, 14 | SingleInferenceTaskCfg, 15 | ) 16 | 17 | headless = False 18 | 19 | if not has_display(): 20 | headless = True 21 | 22 | 23 | config = Config( 24 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 25 | task_config=SingleInferenceTaskCfg( 26 | episodes=[ 27 | SingleInferenceEpisodeCfg( 28 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 29 | robots=[ 30 | JetbotRobotCfg( 31 | position=(0.0, 0.0, 0.0), 32 | controllers=[move_to_point_cfg, move_along_path_cfg], 33 | scale=(5.0, 5.0, 5.0), 34 | ) 35 | ], 36 | ), 37 | ], 38 | ), 39 | ) 40 | 41 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 42 | 43 | path = [(3.0, 3.0, 0.0), (1.0, 0.0, 0.0), (0.0, -1.0, 0.0), (-1.0, 0.5, 0.0)] 44 | 45 | import_extensions() 46 | # import custom extensions here. 47 | import numpy as np 48 | 49 | env = Env(sim_runtime) 50 | obs, _ = env.reset() 51 | 52 | i = 0 53 | 54 | env_action = { 55 | move_along_path_cfg.name: [path], 56 | } 57 | 58 | print(f'actions: {env_action}') 59 | 60 | while env.simulation_app.is_running(): 61 | i += 1 62 | obs, _, terminated, _, _ = env.step(action=env_action) 63 | 64 | if i % 1000 == 0: 65 | print(i) 66 | current_point = obs['controllers'][move_along_path_cfg.name]['current_point'] 67 | error = np.linalg.norm(obs['position'][:2] - current_point[:2]) 68 | print(f'position: {obs["position"]}, error: {error}') 69 | 70 | env.simulation_app.close() 71 | -------------------------------------------------------------------------------- /grutopia/login_openxlab.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openxlab 4 | 5 | 6 | def login_to_openxlab(ak, sk): 7 | """Logs into OpenXLab with the provided Access Key and Secret Key.""" 8 | try: 9 | openxlab.login(ak=ak, sk=sk) 10 | print('Login successful!') 11 | except Exception as e: 12 | print(f'Login failed: {e}') 13 | raise 14 | 15 | 16 | if __name__ == '__main__': 17 | # Read AK and SK from environment variables 18 | ak = os.environ.get('OPENXLAB_AK') 19 | sk = os.environ.get('OPENXLAB_SK') 20 | 21 | if ak and sk: 22 | login_to_openxlab(ak, sk) 23 | -------------------------------------------------------------------------------- /grutopia/macros.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from addict import Dict 4 | 5 | 6 | class MacroDict(Dict): 7 | def __init__(self, *args, **kwargs): 8 | super().__init__(*args, **kwargs) 9 | self['_read'] = set() 10 | 11 | def __setattr__(self, name, value): 12 | if name in self.get('_read', set()): 13 | raise AttributeError(f'Cannot set attribute {name} in MacroDict, it has already been used.') 14 | # Use the super's setattr for setting attributes, but handle _read directly to avoid recursion. 15 | if name == '_read': 16 | self[name] = value 17 | else: 18 | super().__setattr__(name, value) 19 | 20 | def __getattr__(self, item): 21 | # Directly check and modify '_read' to avoid going through __getattr__ or __setattr__. 22 | if item != '_read': 23 | self['_read'].add(item) 24 | # Use direct dictionary access to avoid infinite recursion. 25 | try: 26 | return self[item] 27 | except KeyError: 28 | raise AttributeError(f"'MacroDict' object has no attribute '{item}'") 29 | 30 | 31 | # Initialize settings 32 | macros = MacroDict() 33 | gm = macros.globals 34 | 35 | 36 | def determine_gm_path(default_path, env_var_name): 37 | # Start with the default path 38 | path = default_path 39 | # Override with the environment variable, if set 40 | if env_var_name in os.environ: 41 | path = os.environ[env_var_name] 42 | # Expand the user directory (~) 43 | path = os.path.expanduser(path) 44 | # Make the path absolute if it's not already 45 | if not os.path.isabs(path): 46 | path = os.path.join(os.path.dirname(os.path.realpath(__file__)), path) 47 | return path 48 | 49 | 50 | # Users can override the path if needed 51 | 52 | default_assets_path = 'assets' 53 | try: 54 | import grutopia.default_config as dc 55 | 56 | default_assets_path = dc.DEFAULT_ASSETS_PATH 57 | except Exception: 58 | pass 59 | 60 | gm.ASSET_PATH = determine_gm_path(default_assets_path, 'GRUTOPIA_ASSETS_PATH') 61 | -------------------------------------------------------------------------------- /grutopia/set_assets_path.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from grutopia.download_assets import unzip_all 4 | from grutopia.macros import gm 5 | 6 | RED = '\033[31m' 7 | GREEN = '\033[32m' 8 | YELLOW = '\033[33m' 9 | BLUE = '\033[34m' 10 | END = '\033[0m' 11 | 12 | 13 | def main(): 14 | print(f'Current assets path: {gm.ASSET_PATH}') 15 | target_path = '' 16 | while True: 17 | target_path = input('Please enter the new assets path (must be absolute path): ').strip() 18 | if target_path.startswith('/'): 19 | break 20 | print('target path must be absolute path') 21 | if not os.path.isdir(target_path): 22 | print( 23 | f'{RED}ERROR{END}: {target_path} is not an existing directory.\nPlease ensure the assets have been placed at {target_path}.' 24 | ) 25 | return 26 | 27 | config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'default_config.py') 28 | with open(config_file, 'w') as f: 29 | f.write(f'DEFAULT_ASSETS_PATH = "{target_path}"') 30 | print(f'Assets path has been set to: {target_path}') 31 | 32 | unzip = input('Need to unzip all the assets? (assets should only be unzipped once) (y/N) ').strip().lower() 33 | if unzip == 'y': 34 | unzip_all(target_path) 35 | 36 | 37 | if __name__ == '__main__': 38 | main() 39 | -------------------------------------------------------------------------------- /grutopia_extension/__init__.py: -------------------------------------------------------------------------------- 1 | def import_extensions(): 2 | import grutopia_extension.agents 3 | import grutopia_extension.controllers 4 | import grutopia_extension.interactions 5 | import grutopia_extension.metrics 6 | import grutopia_extension.objects 7 | import grutopia_extension.robots 8 | import grutopia_extension.sensors 9 | import grutopia_extension.tasks 10 | -------------------------------------------------------------------------------- /grutopia_extension/agents/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/.gitignore -------------------------------------------------------------------------------- /grutopia_extension/agents/README.md: -------------------------------------------------------------------------------- 1 | # Agents 2 | 3 | > This is a directory for Agents. 4 | > 5 | > Here are some of our agent implementations, and other agents are not obligated to be implemented in the same manner. Please note that breaking changes may occur in the future in our agent implementations. 6 | 7 | # Import New Agent 8 | 9 | When adding an Agent, you need to follow these steps: 10 | 11 | 1. Create a subclass of `grutopia_extension.agents.core.agent.BaseAgent` like `grutopia_extension.agents.dummy_agent`. 12 | 2. Implement `decision_making` method. 13 | 3. If necessary, you can create directories to implement the complex logic you need. 14 | 15 | > We will officially integrate some Agents into this project as submodules (like `npc_agent`) 16 | -------------------------------------------------------------------------------- /grutopia_extension/agents/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | from grutopia.core.util import log 4 | 5 | for module in [ 6 | 'grutopia_extension.agents.dummy_agent', 7 | 'grutopia_extension.agents.npc_agent_client', 8 | 'grutopia_extension.agents.social_navigation_agent_client', 9 | 'grutopia_extension.agents.mobile_manipulation_agent_client', 10 | ]: 11 | try: 12 | importlib.import_module(module) 13 | except ImportError as e: 14 | log.error(e) 15 | continue 16 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/README.md: -------------------------------------------------------------------------------- 1 | # Readme 2 | 3 | Common folder for social_navigation_agent and mobile_manipulation_agent. 4 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/agent_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/common/agent_utils/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/common/agent_utils/acyclic_enforcer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved. 2 | 3 | from typing import Any, Set 4 | 5 | import numpy as np 6 | 7 | 8 | class StateAction: 9 | def __init__(self, position: np.ndarray, action: Any, other: Any = None): 10 | self.position = position 11 | self.action = action 12 | self.other = other 13 | 14 | def __hash__(self) -> int: 15 | string_repr = f'{self.position}_{self.action}_{self.other}' 16 | return hash(string_repr) 17 | 18 | 19 | class AcyclicEnforcer: 20 | history: Set[StateAction] = set() 21 | 22 | def check_cyclic(self, position: np.ndarray, action: Any, other: Any = None) -> bool: 23 | state_action = StateAction(position, action, other) 24 | cyclic = state_action in self.history 25 | return cyclic 26 | 27 | def add_state_action(self, position: np.ndarray, action: Any, other: Any = None) -> None: 28 | state_action = StateAction(position, action, other) 29 | self.history.add(state_action) 30 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/common/modules/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/common/modules/mapping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/common/modules/mapping/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/common/modules/mapping/base_map.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved. 2 | 3 | from typing import Any, List 4 | 5 | import numpy as np 6 | from agent_utils.traj_visualizer import TrajectoryVisualizer 7 | 8 | 9 | class BaseMap: 10 | _camera_positions: List[np.ndarray] = [] 11 | _last_camera_yaw: float = 0.0 12 | _map_dtype: np.dtype = np.dtype(np.float32) 13 | 14 | def __init__(self, size, pixels_per_meter, *args: Any, **kwargs: Any): 15 | """ 16 | Args: 17 | size: The size of the map in pixels. 18 | """ 19 | self.pixels_per_meter = pixels_per_meter 20 | self.size = size 21 | self._map = np.zeros((size, size), dtype=self._map_dtype) 22 | self._episode_pixel_origin = np.array([size // 2, size // 2]) 23 | self._traj_vis = TrajectoryVisualizer(self._episode_pixel_origin, self.pixels_per_meter) 24 | 25 | def reset(self) -> None: 26 | self._map.fill(0) 27 | self._camera_positions = [] 28 | self._traj_vis = TrajectoryVisualizer(self._episode_pixel_origin, self.pixels_per_meter) 29 | 30 | def update_agent_traj(self, robot_xy: np.ndarray, robot_heading: float) -> None: 31 | self._camera_positions.append(robot_xy) 32 | self._last_camera_yaw = robot_heading 33 | 34 | def _xy_to_px(self, points: np.ndarray) -> np.ndarray: 35 | """Converts an array of (x, y) coordinates to pixel coordinates. 36 | 37 | Args: 38 | points: The array of (x, y) coordinates to convert. 39 | 40 | Returns: 41 | The array of (x, y) pixel coordinates. 42 | """ 43 | px = np.rint(points[:, ::-1] * self.pixels_per_meter) + self._episode_pixel_origin 44 | px[:, 0] = self._map.shape[0] - px[:, 0] 45 | return px.astype(int) 46 | 47 | def _px_to_xy(self, px: np.ndarray) -> np.ndarray: 48 | """Converts an array of pixel coordinates to (x, y) coordinates. 49 | 50 | Args: 51 | px: The array of pixel coordinates to convert. 52 | 53 | Returns: 54 | The array of (x, y) coordinates. 55 | """ 56 | px_copy = px.copy() 57 | px_copy[:, 0] = self._map.shape[0] - px_copy[:, 0] 58 | points = (px_copy - self._episode_pixel_origin) / self.pixels_per_meter 59 | return points[:, ::-1] 60 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/modules/vlm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved. 2 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/modules/vlm/classes.txt: -------------------------------------------------------------------------------- 1 | framed photograph 2 | cabinet 3 | pillow 4 | nightstand 5 | sink 6 | stool 7 | towel 8 | shower 9 | bathtub 10 | counter 11 | fireplace 12 | gym equipment 13 | seating 14 | clothes 15 | cupboard 16 | table 17 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/modules/vlm/coco_classes.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved. 2 | 3 | COCO_CLASSES = [ 4 | 'person', 5 | 'bicycle', 6 | 'car', 7 | 'motorcycle', 8 | 'airplane', 9 | 'bus', 10 | 'train', 11 | 'truck', 12 | 'boat', 13 | 'traffic light', 14 | 'fire hydrant', 15 | 'stop sign', 16 | 'parking meter', 17 | 'bench', 18 | 'bird', 19 | 'cat', 20 | 'dog', 21 | 'horse', 22 | 'sheep', 23 | 'cow', 24 | 'elephant', 25 | 'bear', 26 | 'zebra', 27 | 'giraffe', 28 | 'backpack', 29 | 'umbrella', 30 | 'handbag', 31 | 'tie', 32 | 'suitcase', 33 | 'frisbee', 34 | 'skis', 35 | 'snowboard', 36 | 'sports ball', 37 | 'kite', 38 | 'baseball bat', 39 | 'baseball glove', 40 | 'skateboard', 41 | 'surfboard', 42 | 'tennis racket', 43 | 'bottle', 44 | 'wine glass', 45 | 'cup', 46 | 'fork', 47 | 'knife', 48 | 'spoon', 49 | 'bowl', 50 | 'banana', 51 | 'apple', 52 | 'sandwich', 53 | 'orange', 54 | 'broccoli', 55 | 'carrot', 56 | 'hot dog', 57 | 'pizza', 58 | 'donut', 59 | 'cake', 60 | 'chair', 61 | 'couch', 62 | 'potted plant', 63 | # "bed", 64 | 'dining table', 65 | 'toilet', 66 | 'tv', 67 | 'laptop', 68 | 'mouse', 69 | 'remote', 70 | 'keyboard', 71 | 'cell phone', 72 | 'microwave', 73 | 'oven', 74 | 'toaster', 75 | 'sink', 76 | 'refrigerator', 77 | 'book', 78 | 'clock', 79 | 'vase', 80 | 'scissors', 81 | 'teddy bear', 82 | 'hair drier', 83 | 'toothbrush', 84 | ] 85 | -------------------------------------------------------------------------------- /grutopia_extension/agents/common/scripts/launch_vlm_servers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright [2023] Boston Dynamics AI Institute, Inc. 3 | 4 | # Ensure you have 'export VLFM_PYTHON=' in your .bashrc, where 5 | # is the path to the python executable for your conda env 6 | # (e.g., PATH_TO_PYTHON=`conda activate && which python`) 7 | 8 | export VLFM_PYTHON=${VLFM_PYTHON:-`which python`} 9 | export VIRTUAL_ENV=$(basename $(dirname $(dirname ${VLFM_PYTHON}))) 10 | export MOBILE_SAM_CHECKPOINT=${MOBILE_SAM_CHECKPOINT:-data/mobile_sam.pt} 11 | export GROUNDING_DINO_CONFIG=${GROUNDING_DINO_CONFIG:-GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py} 12 | export GROUNDING_DINO_WEIGHTS=${GROUNDING_DINO_WEIGHTS:-data/groundingdino_swint_ogc.pth} 13 | export CLASSES_PATH=${CLASSES_PATH:-vlfm/vlm/classes.txt} 14 | export GROUNDING_DINO_PORT=${GROUNDING_DINO_PORT:-12181} 15 | export BLIP2ITM_PORT=${BLIP2ITM_PORT:-12182} 16 | export SAM_PORT=${SAM_PORT:-12183} 17 | export YOLOV7_PORT=${YOLOV7_PORT:-12184} 18 | 19 | session_name=vlm_servers_${RANDOM} 20 | 21 | # Create a detached tmux session 22 | tmux new-session -d -s ${session_name} 23 | 24 | # Split the window vertically 25 | tmux split-window -v -t ${session_name}:0 26 | 27 | # Split both panes horizontally 28 | tmux split-window -h -t ${session_name}:0.0 29 | tmux split-window -h -t ${session_name}:0.2 30 | 31 | # Run commands in each pane 32 | tmux send-keys -t ${session_name}:0.0 "conda activate ${VIRTUAL_ENV}" C-m 33 | tmux send-keys -t ${session_name}:0.0 "${VLFM_PYTHON} -m modules.vlm.grounding_dino --port ${GROUNDING_DINO_PORT}" C-m 34 | tmux send-keys -t ${session_name}:0.1 "conda activate ${VIRTUAL_ENV}" C-m 35 | tmux send-keys -t ${session_name}:0.1 "${VLFM_PYTHON} -m modules.vlm.blip2itm --port ${BLIP2ITM_PORT}" C-m 36 | tmux send-keys -t ${session_name}:0.2 "conda activate ${VIRTUAL_ENV}" C-m 37 | tmux send-keys -t ${session_name}:0.2 "${VLFM_PYTHON} -m modules.vlm.sam --port ${SAM_PORT}" C-m 38 | tmux send-keys -t ${session_name}:0.3 "conda activate ${VIRTUAL_ENV}" C-m 39 | tmux send-keys -t ${session_name}:0.3 "${VLFM_PYTHON} -m modules.vlm.yolov7 --port ${YOLOV7_PORT}" C-m 40 | 41 | # Attach to the tmux session to view the windows 42 | echo "Created tmux session '${session_name}'. You must wait up to 90 seconds for the model weights to finish being loaded." 43 | echo "Run the following to monitor all the server commands:" 44 | echo "tmux attach-session -t ${session_name}" 45 | -------------------------------------------------------------------------------- /grutopia_extension/agents/config/__init__.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Any, Dict, Optional, Union 3 | 4 | from pydantic import BaseModel 5 | 6 | 7 | class AgentModeEnum(str, Enum): 8 | sync_mode = 'sync' 9 | async_mode = 'async' 10 | 11 | 12 | class AgentCfg(BaseModel): 13 | """ 14 | Class representing the configuration for an agent, including its type, robot association, synchronization mode, and additional settings. 15 | 16 | This class is designed to encapsulate the necessary parameters to configure an agent within a system. It inherits from `BaseModel`, indicating it's a data model class likely used within a framework that supports data validation and parsing, such as Pydantic. 17 | 18 | Attributes: 19 | type (str): Specifies the type of the agent. This could refer to the functional category or a specific implementation detail of the agent. 20 | 21 | robot_name (Optional[Union[str, None]], optional): The name of the robot associated with the agent. Currently marked as optional but is planned to become a required field in the future, potentially also facilitating non-physical agents (e.g., NPCs). 22 | 23 | sync_mode (Optional[AgentModeEnum], optional): Defines the synchronization mode for the agent, defaulting to `AgentModeEnum.sync_mode`. This enum determines how the agent operates concerning real-time updates or processing. 24 | 25 | agent_config (Dict[str, Any]): Additional configuration settings for the agent, provided as a dictionary where keys are strings and values can be of any type. This allows for flexible extension of agent-specific configurations without altering the base class structure. 26 | 27 | Note: Type hints are provided for parameters, hence types are not redundantly described in the docstring. 28 | """ 29 | 30 | type: str 31 | # TODO: `robot_name` will be changed to a required parameter later. 32 | # And consider adding a disembodied robot for NPCs. 33 | robot_name: Optional[Union[str, None]] = None 34 | sync_mode: Optional[AgentModeEnum] = AgentModeEnum.sync_mode 35 | agent_config: Optional[Dict[str, Any]] = {} 36 | -------------------------------------------------------------------------------- /grutopia_extension/agents/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/core/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/core/agent/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.agents.core.agent.agent import BaseAgent, create_agent 2 | 3 | if BaseAgent.loop is None: 4 | BaseAgent.agent_loop_start() 5 | -------------------------------------------------------------------------------- /grutopia_extension/agents/dummy_agent.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | from typing import Any, Dict 4 | 5 | from grutopia.core.runtime.task_runtime import TaskRuntime 6 | from grutopia.core.util import log 7 | from grutopia_extension.agents.core.agent import BaseAgent 8 | 9 | 10 | @BaseAgent.register('DummyAgent') 11 | class DummyAgentClient(BaseAgent): 12 | """ 13 | Dummy Agent that does nothing. And set is_done at the 2nd call. 14 | """ 15 | 16 | def __init__( 17 | self, task_name: str, robot_name: str | None, agent_config: Dict, sync_mode: str, task_runtime: TaskRuntime 18 | ): 19 | super().__init__(task_name, robot_name, agent_config, sync_mode, task_runtime) 20 | self.counter = 0 21 | log.debug(f'=============== agent_config: {agent_config} ===============') 22 | 23 | def decision_making(self, obs: Any) -> Any: 24 | time.sleep(float(random.randint(3, 5))) 25 | self.counter += 1 26 | log.info(f'Task_id: {self.task_name}, DummyAgent.counter: {self.counter}') 27 | if self.counter % 2 == 0: 28 | log.info(self.counter) 29 | return self.terminate() 30 | 31 | def terminate(self) -> Any: 32 | pass 33 | -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/.gitignore: -------------------------------------------------------------------------------- 1 | # Python cache files 2 | __pycache__/ 3 | *.pyc 4 | 5 | # Editor and environment configurations 6 | .vscode/ 7 | 8 | # Temporary and backup files 9 | lockfiles/ 10 | 11 | # Output and generated files 12 | result/ 13 | images/ 14 | 15 | # Project-specific files 16 | yolov7 17 | GroundingDINO 18 | api_key/ 19 | GroundingDINO/ 20 | yolov7/ 21 | 22 | # Local models or large files 23 | data/ 24 | data 25 | local_model 26 | traced_model.pt 27 | -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/mobile_manipulation_agent/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/agent_config.yaml: -------------------------------------------------------------------------------- 1 | vlm: gpt 2 | task_config: 3 | dialogue_turn: 0 4 | no_explore_threshold: 2 5 | fall_threshold: 0.5 6 | robot_type: 'oracle' 7 | verbose: True 8 | agent_path: 'GRUtopia/grutopia_extension/agents/mobile_manipulation_agent' 9 | # agent_path: './' 10 | 11 | map_config: 12 | size: 1000 13 | min_depth: 0.7 14 | max_depth: 5.0 15 | agent_radius: 0.25 16 | pixels_per_meter: 20 17 | 18 | obstacle_map: 19 | min_height: 0.3 20 | max_height: 1.7 21 | area_thresh: 1.5 22 | hole_area_thresh: 100000 23 | 24 | value_map: 25 | text_prompt: 'Seems like there is a target_object ahead.' 26 | value_channels: 1 27 | 28 | object_map: 29 | use_vqa: False 30 | vqa_prompt: 'Is this ' 31 | coco_threshold: 0.7 32 | non_coco_threshold: 0.3 33 | erosion_size: 5 34 | -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/agent_utils: -------------------------------------------------------------------------------- 1 | ../common/agent_utils -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/modules: -------------------------------------------------------------------------------- 1 | ../common/modules -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/requirements.txt: -------------------------------------------------------------------------------- 1 | dashscope 2 | deepdiff 3 | einops 4 | flask==2.3.2 5 | git+https://github.com/brean/python-pathfinding.git 6 | git+https://github.com/ChaoningZhang/MobileSAM.git 7 | git+https://github.com/IDEA-Research/GroundingDINO.git 8 | git+https://github.com/naokiyokoyama/depth_camera_filtering 9 | git+https://github.com/naokiyokoyama/frontier_exploration.git 10 | gymnasium==0.28.1 11 | httpcore==1.0.5 12 | httpx==0.25.2 13 | matplotlib==3.8.1 14 | open3d==0.18.0 15 | openai==1.29.0 16 | opencv-python==4.5.5.64 17 | openpyxl==3.1.2 18 | pandas==2.2.2 19 | peft 20 | psutil==5.7.2 21 | salesforce-lavis==1.0.2 22 | scikit-image 23 | scikit-learn==1.4.2 24 | scipy==1.10.1 25 | seaborn==0.12.2 26 | shapely 27 | sniffio==1.3.0 28 | spacy==3.7.4 29 | tiktoken 30 | timm 31 | torch 32 | tqdm==4.66.4 33 | transformers 34 | usd-core==23.11 35 | -------------------------------------------------------------------------------- /grutopia_extension/agents/mobile_manipulation_agent/scripts: -------------------------------------------------------------------------------- 1 | ../common/scripts -------------------------------------------------------------------------------- /grutopia_extension/agents/npc_agent/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.agents.npc_agent.base import NPC 2 | -------------------------------------------------------------------------------- /grutopia_extension/agents/npc_agent/config.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class NPCUserConfig(BaseModel): 7 | model_name: str 8 | openai_api_key: str 9 | max_interaction_turn: Optional[int] = 5 10 | api_base_url: Optional[str] = 'https://api.openai.com/v1/chat/completions' 11 | -------------------------------------------------------------------------------- /grutopia_extension/agents/npc_agent/repuirements.txt: -------------------------------------------------------------------------------- 1 | ipython 2 | -------------------------------------------------------------------------------- /grutopia_extension/agents/npc_agent_client.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from grutopia.core.datahub import DataHub 4 | from grutopia.core.runtime.task_runtime import TaskRuntime 5 | from grutopia.core.util import log 6 | from grutopia_extension.agents.core.agent import BaseAgent 7 | 8 | # Load NPC from NPC repo 9 | from grutopia_extension.agents.npc_agent import NPC 10 | from grutopia_extension.agents.npc_agent.config import NPCUserConfig 11 | 12 | 13 | @BaseAgent.register('NPCAgent') 14 | class NPCAgentClient(BaseAgent): 15 | """ 16 | NPC Agent. 17 | 18 | This agent won't be terminated. 19 | """ 20 | 21 | def __init__( 22 | self, task_name: str, robot_name: str | None, agent_config: Dict, sync_mode: str, task_runtime: TaskRuntime 23 | ): 24 | super().__init__(task_name, robot_name, agent_config, sync_mode, task_runtime) 25 | log.debug(f'=============== agent_config: {agent_config} ===============') 26 | try: 27 | cfg = NPCUserConfig(**agent_config) 28 | except Exception as e: 29 | log.error('agent_config of this agent(NPC) is not valid (By grutopia.core.runtime.npc.NPCUserConfig)') 30 | raise e 31 | self.npc = NPC(cfg, task_runtime.extra) 32 | 33 | # TODO: Figure out a better way to implement this method. 34 | def decision_making(self, obs: Any) -> Any: 35 | """ 36 | This agent won't be terminated. 37 | """ 38 | obs = DataHub.get_obs_by_task_name(self.task_name) 39 | response_list = self.npc.feed(self.robot_name, obs, self.chat.get_message()) 40 | for response in response_list: 41 | self.chat.send_message(**response.model_dump()) 42 | return {} 43 | -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/.gitignore: -------------------------------------------------------------------------------- 1 | # Python cache files 2 | __pycache__/ 3 | *.pyc 4 | 5 | # Editor and environment configurations 6 | .vscode/ 7 | 8 | # Temporary and backup files 9 | lockfiles/ 10 | 11 | # Output and generated files 12 | result/ 13 | images/ 14 | 15 | # Project-specific files 16 | yolov7 17 | GroundingDINO 18 | api_key/ 19 | GroundingDINO/ 20 | yolov7/ 21 | 22 | # Local models or large files 23 | data/ 24 | data 25 | local_model 26 | traced_model.pt 27 | -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/social_navigation_agent/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/agent_config.yaml: -------------------------------------------------------------------------------- 1 | vlm: gpt 2 | task_config: 3 | dialogue_turn: 3 4 | no_explore_threshold: 2 5 | fall_threshold: 0.5 6 | robot_type: 'oracle' 7 | verbose: True 8 | agent_path: 'GRUtopia/grutopia_extension/agents/social_navigation_agent' 9 | # agent_path: './' 10 | 11 | map_config: 12 | size: 1000 13 | min_depth: 0.5 14 | max_depth: 5.0 15 | agent_radius: 0.25 16 | pixels_per_meter: 20 17 | 18 | obstacle_map: 19 | min_height: 0.3 20 | max_height: 1.7 21 | area_thresh: 1.5 22 | hole_area_thresh: 100000 23 | 24 | value_map: 25 | text_prompt: 'Seems like there is a target_object ahead.' 26 | value_channels: 1 27 | 28 | object_map: 29 | use_vqa: False 30 | vqa_prompt: 'Is this ' 31 | coco_threshold: 0.7 32 | non_coco_threshold: 0.3 33 | erosion_size: 5 34 | -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/agent_utils: -------------------------------------------------------------------------------- 1 | ../common/agent_utils -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/modules: -------------------------------------------------------------------------------- 1 | ../common/modules -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/requirements.txt: -------------------------------------------------------------------------------- 1 | dashscope 2 | deepdiff 3 | einops 4 | flask==2.3.2 5 | git+https://github.com/brean/python-pathfinding.git 6 | git+https://github.com/ChaoningZhang/MobileSAM.git 7 | git+https://github.com/IDEA-Research/GroundingDINO.git 8 | git+https://github.com/naokiyokoyama/depth_camera_filtering 9 | git+https://github.com/naokiyokoyama/frontier_exploration.git 10 | gymnasium==0.28.1 11 | httpcore==1.0.5 12 | httpx==0.25.2 13 | matplotlib==3.8.1 14 | open3d==0.18.0 15 | openai==1.29.0 16 | opencv-python==4.5.5.64 17 | openpyxl==3.1.2 18 | pandas==2.2.2 19 | peft 20 | psutil==5.7.2 21 | salesforce-lavis==1.0.2 22 | scikit-image 23 | scikit-learn==1.4.2 24 | scipy==1.10.1 25 | seaborn==0.12.2 26 | shapely 27 | sniffio==1.3.0 28 | spacy==3.7.4 29 | tiktoken 30 | timm 31 | torch 32 | tqdm==4.66.4 33 | transformers 34 | usd-core==23.11 35 | -------------------------------------------------------------------------------- /grutopia_extension/agents/social_navigation_agent/scripts: -------------------------------------------------------------------------------- 1 | ../common/scripts -------------------------------------------------------------------------------- /grutopia_extension/agents/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/agents/util/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/agents/util/agent.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from grutopia.core.gym_env import Env 4 | from grutopia.core.runtime.task_runtime import TaskRuntime 5 | from grutopia_extension.agents.config import AgentCfg 6 | from grutopia_extension.agents.core.agent import BaseAgent 7 | from grutopia_extension.agents.core.agent import create_agent as _create_agent 8 | 9 | 10 | def create_agents(agent_cfgs: List[AgentCfg], reset_info: dict) -> list[BaseAgent]: 11 | """ 12 | Creates a list of agent clients based on the provided configurations and reset information. 13 | 14 | Args: 15 | agent_cfgs: A list of configurations for each agent. 16 | reset_info: A dictionary containing reset information, which may include task runtime details. 17 | 18 | Returns: 19 | A list of BaseAgent instances created from the provided configurations and reset information. 20 | """ 21 | agents: List[BaseAgent] = [] 22 | if Env.RESET_INFO_TASK_RUNTIME in reset_info: 23 | current_task: TaskRuntime = reset_info[Env.RESET_INFO_TASK_RUNTIME] 24 | for agent_cfg in agent_cfgs: 25 | agents.append(_create_agent(config=agent_cfg, task_name=current_task.name, task_runtime=current_task)) 26 | 27 | return agents 28 | 29 | 30 | def create_agent(agent_cfg: AgentCfg, reset_info: dict) -> BaseAgent: 31 | """ 32 | Creates an agent based on the provided configuration and reset information. 33 | 34 | Args: 35 | agent_cfg (AgentCfg): Configuration for the agent. 36 | reset_info (dict): Information used to reset the environment, which may 37 | include task runtime details. 38 | 39 | Returns: 40 | BaseAgent: The created agent, or None if the necessary reset information is not present. 41 | """ 42 | agent = None 43 | if Env.RESET_INFO_TASK_RUNTIME in reset_info: 44 | current_task: TaskRuntime = reset_info[Env.RESET_INFO_TASK_RUNTIME] 45 | agent = _create_agent(config=agent_cfg, task_name=current_task.name, task_runtime=current_task) 46 | return agent 47 | -------------------------------------------------------------------------------- /grutopia_extension/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/configs/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.configs.controllers.aliengo_move_by_speed_controller import ( 2 | AliengoMoveBySpeedControllerCfg, 3 | ) 4 | from grutopia_extension.configs.controllers.dd_controller import ( 5 | DifferentialDriveControllerCfg, 6 | ) 7 | from grutopia_extension.configs.controllers.franka_mocap_teleop_controller import ( 8 | FrankaMocapTeleopControllerCfg, 9 | ) 10 | from grutopia_extension.configs.controllers.g1_move_by_speed_controller import ( 11 | G1MoveBySpeedControllerCfg, 12 | ) 13 | from grutopia_extension.configs.controllers.gr1_move_by_speed_controller import ( 14 | GR1MoveBySpeedControllerCfg, 15 | ) 16 | from grutopia_extension.configs.controllers.gr1_teleop_controller import ( 17 | GR1TeleOpControllerCfg, 18 | ) 19 | from grutopia_extension.configs.controllers.gripper_controller import ( 20 | GripperControllerCfg, 21 | ) 22 | from grutopia_extension.configs.controllers.h1_move_by_speed_controller import ( 23 | H1MoveBySpeedControllerCfg, 24 | ) 25 | from grutopia_extension.configs.controllers.ik_controller import ( 26 | InverseKinematicsControllerCfg, 27 | ) 28 | from grutopia_extension.configs.controllers.joint_controller import JointControllerCfg 29 | from grutopia_extension.configs.controllers.layout_edit_mocap_controller import ( 30 | LayoutEditMocapControllerCfg, 31 | ) 32 | from grutopia_extension.configs.controllers.move_along_path_points_controller import ( 33 | MoveAlongPathPointsControllerCfg, 34 | ) 35 | from grutopia_extension.configs.controllers.move_to_point_by_speed_controller import ( 36 | MoveToPointBySpeedControllerCfg, 37 | ) 38 | from grutopia_extension.configs.controllers.recover_controller import ( 39 | RecoverControllerCfg, 40 | ) 41 | from grutopia_extension.configs.controllers.rmpflow_controller import ( 42 | RMPFlowControllerCfg, 43 | ) 44 | from grutopia_extension.configs.controllers.rotate_controller import RotateControllerCfg 45 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/aliengo_move_by_speed_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class AliengoMoveBySpeedControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'AliengoMoveBySpeedController' 9 | joint_names: List[str] 10 | policy_weights_path: str 11 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/dd_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class DifferentialDriveControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'DifferentialDriveController' 9 | wheel_radius: float 10 | wheel_base: float 11 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/franka_mocap_teleop_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class FrankaMocapTeleopControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'FrankaMocapTeleopController' 8 | scale: Tuple[float, float, float] 9 | target_position: Tuple[float, float, float] 10 | origin_xyz: Optional[Tuple[float, float, float]] = None 11 | origin_xyz_angle: Optional[Tuple[float, float, float]] = None 12 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/g1_move_by_speed_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class G1MoveBySpeedControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'G1MoveBySpeedController' 8 | joint_names: List[str] 9 | policy_weights_path: str 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/gr1_move_by_speed_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class GR1MoveBySpeedControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'GR1MoveBySpeedController' 8 | joint_names: List[str] 9 | policy_weights_path: str 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/gr1_teleop_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class GR1TeleOpControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'GR1TeleOpController' 9 | joint_names: List[str] 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/gripper_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class GripperControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'GripperController' 8 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/h1_move_by_speed_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class H1MoveBySpeedControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'H1MoveBySpeedController' 9 | joint_names: List[str] 10 | policy_weights_path: str 11 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/ik_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class InverseKinematicsControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'InverseKinematicsController' 8 | robot_description_path: str 9 | robot_urdf_path: str 10 | end_effector_frame_name: str 11 | threshold: float 12 | reference: Optional[str] = None 13 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/joint_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class JointControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'JointController' 9 | joint_names: List[str] 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/layout_edit_mocap_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class LayoutEditMocapControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'LayoutEditMocapController' 9 | scale: Tuple[float, float, float] 10 | target_position: Tuple[float, float, float] 11 | rh_origin_xyz: Optional[Tuple[float, float, float]] = None 12 | lh_origin_xyz: Optional[Tuple[float, float, float]] = None 13 | origin_xyz_angle: Optional[Tuple[float, float, float]] = None 14 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/move_along_path_points_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class MoveAlongPathPointsControllerCfg(ControllerCfg): 7 | name: Optional[str] = 'move_along_path' 8 | type: Optional[str] = 'MoveAlongPathPointsController' 9 | forward_speed: Optional[float] = None 10 | rotation_speed: Optional[float] = None 11 | threshold: Optional[float] = None 12 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/move_to_point_by_speed_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class MoveToPointBySpeedControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'MoveToPointBySpeedController' 9 | forward_speed: Optional[float] = None 10 | rotation_speed: Optional[float] = None 11 | threshold: Optional[float] = None 12 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/recover_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class RecoverControllerCfg(ControllerCfg): 7 | 8 | type: Optional[str] = 'RecoverController' 9 | recover_height: float 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/rmpflow_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class RMPFlowControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'RMPFlowController' 8 | robot_description_path: str 9 | robot_urdf_path: str 10 | rmpflow_config_path: str 11 | end_effector_frame_name: str 12 | -------------------------------------------------------------------------------- /grutopia_extension/configs/controllers/rotate_controller.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.robot import ControllerCfg 4 | 5 | 6 | class RotateControllerCfg(ControllerCfg): 7 | type: Optional[str] = 'RotateController' 8 | rotation_speed: Optional[float] = None 9 | threshold: Optional[float] = None 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.configs.metrics.candidates_reduce_metric import ECRMetricCfg 2 | from grutopia_extension.configs.metrics.debug_metric import DebugMetricCfg 3 | from grutopia_extension.configs.metrics.mobile_manipulation_success_metric import ( 4 | MobileManipulationSuccessMetricCfg, 5 | ) 6 | from grutopia_extension.configs.metrics.recording_metric import RecordingMetricCfg 7 | from grutopia_extension.configs.metrics.reset_time_metric import ResetTimeMetricCfg 8 | from grutopia_extension.configs.metrics.simple_metric import SimpleMetricCfg 9 | from grutopia_extension.configs.metrics.social_navigation_success_metric import ( 10 | SocialNavigationSuccessMetricCfg, 11 | SocialNavigationSuccessMetricConfig, 12 | ) 13 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/candidates_reduce_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.metric import MetricCfg 4 | 5 | 6 | class ECRMetricCfg(MetricCfg): 7 | type: Optional[str] = 'ECRMetric' 8 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/debug_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.metric import MetricCfg 4 | 5 | 6 | class DebugMetricCfg(MetricCfg): 7 | type: Optional[str] = 'DebugMetric' 8 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/mobile_manipulation_success_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.metric import MetricCfg 4 | 5 | 6 | class MobileManipulationSuccessMetricCfg(MetricCfg): 7 | type: Optional[str] = 'MobileManipulationSuccessMetric' 8 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/recording_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.metric import MetricCfg 4 | 5 | 6 | class RecordingMetricCfg(MetricCfg): 7 | type: Optional[str] = 'RecordingMetric' 8 | robot_name: str 9 | fields: list = None # fields that need to be recorded. 10 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/reset_time_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.metric import MetricCfg 4 | 5 | 6 | class ResetTimeMetricCfg(MetricCfg): 7 | type: Optional[str] = 'ResetTimeMetric' 8 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/simple_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config.metric import MetricCfg 4 | 5 | 6 | class SimpleMetricCfg(MetricCfg): 7 | type: Optional[str] = 'SimpleMetric' 8 | -------------------------------------------------------------------------------- /grutopia_extension/configs/metrics/social_navigation_success_metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config.metric import MetricCfg 6 | 7 | 8 | class SocialNavigationSuccessMetricConfig(BaseModel): 9 | navigation_error_threshold: int 10 | 11 | 12 | class SocialNavigationSuccessMetricCfg(MetricCfg): 13 | type: Optional[str] = 'SocialNavigationSuccessMetric' 14 | metric_config: SocialNavigationSuccessMetricConfig 15 | -------------------------------------------------------------------------------- /grutopia_extension/configs/objects/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from grutopia.core.config.scene import ObjectCfg 4 | 5 | 6 | class DynamicCubeCfg(ObjectCfg): 7 | type: Optional[str] = 'DynamicCube' 8 | color: Optional[Tuple[float, float, float]] = None 9 | mass: Optional[float] = None 10 | density: Optional[float] = None 11 | collider: Optional[bool] = True 12 | 13 | 14 | class VisualCubeCfg(ObjectCfg): 15 | type: Optional[str] = 'VisualCube' 16 | color: Optional[Tuple[float, float, float]] = None 17 | 18 | 19 | class UsdObjCfg(ObjectCfg): 20 | type: Optional[str] = 'UsdObject' 21 | usd_path: str 22 | collider: Optional[bool] = True 23 | -------------------------------------------------------------------------------- /grutopia_extension/configs/robots/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/configs/robots/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/configs/robots/aliengo.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config import RobotCfg 4 | from grutopia.macros import gm 5 | from grutopia_extension.configs.controllers import ( 6 | AliengoMoveBySpeedControllerCfg, 7 | MoveAlongPathPointsControllerCfg, 8 | MoveToPointBySpeedControllerCfg, 9 | RotateControllerCfg, 10 | ) 11 | from grutopia_extension.configs.sensors import CameraCfg 12 | 13 | move_by_speed_cfg = AliengoMoveBySpeedControllerCfg( 14 | name='move_by_speed', 15 | policy_weights_path=gm.ASSET_PATH + '/robots/aliengo/policy/move_by_speed/aliengo_loco_model_4000.pt', 16 | joint_names=[ 17 | 'FL_hip_joint', 18 | 'FR_hip_joint', 19 | 'RL_hip_joint', 20 | 'RR_hip_joint', 21 | 'FL_thigh_joint', 22 | 'FR_thigh_joint', 23 | 'RL_thigh_joint', 24 | 'RR_thigh_joint', 25 | 'FL_calf_joint', 26 | 'FR_calf_joint', 27 | 'RL_calf_joint', 28 | 'RR_calf_joint', 29 | ], 30 | ) 31 | 32 | move_to_point_cfg = MoveToPointBySpeedControllerCfg( 33 | name='move_to_point', 34 | forward_speed=1.0, 35 | rotation_speed=4.0, 36 | threshold=0.05, 37 | sub_controllers=[move_by_speed_cfg], 38 | ) 39 | 40 | move_along_path_cfg = MoveAlongPathPointsControllerCfg( 41 | name='move_along_path', 42 | forward_speed=1.0, 43 | rotation_speed=4.0, 44 | threshold=0.1, 45 | sub_controllers=[move_to_point_cfg], 46 | ) 47 | 48 | rotate_cfg = RotateControllerCfg( 49 | name='rotate', 50 | rotation_speed=2.0, 51 | threshold=0.02, 52 | sub_controllers=[move_by_speed_cfg], 53 | ) 54 | 55 | camera_cfg = CameraCfg( 56 | name='camera', 57 | prim_path='trunk/Camera', 58 | ) 59 | 60 | 61 | class AliengoRobotCfg(RobotCfg): 62 | # meta info 63 | name: Optional[str] = 'aliengo' 64 | type: Optional[str] = 'AliengoRobot' 65 | prim_path: Optional[str] = '/aliengo' 66 | create_robot: Optional[bool] = True 67 | usd_path: Optional[str] = gm.ASSET_PATH + '/robots/aliengo/aliengo_camera.usdz' 68 | -------------------------------------------------------------------------------- /grutopia_extension/configs/robots/franka.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config import RobotCfg 4 | from grutopia.macros import gm 5 | from grutopia_extension.configs.controllers import ( 6 | GripperControllerCfg, 7 | InverseKinematicsControllerCfg, 8 | ) 9 | 10 | arm_ik_cfg = InverseKinematicsControllerCfg( 11 | name='arm_ik_controller', 12 | robot_description_path=gm.ASSET_PATH + '/robots/franka/rmpflow/robot_descriptor.yaml', 13 | robot_urdf_path=gm.ASSET_PATH + '/robots/franka/lula_franka_gen.urdf', 14 | end_effector_frame_name='right_gripper', 15 | threshold=0.01, 16 | ) 17 | 18 | gripper_cfg = GripperControllerCfg( 19 | name='gripper_controller', 20 | ) 21 | 22 | 23 | class FrankaRobotCfg(RobotCfg): 24 | # meta info 25 | name: Optional[str] = 'franka' 26 | type: Optional[str] = 'FrankaRobot' 27 | prim_path: Optional[str] = '/franka' 28 | create_robot: Optional[bool] = True 29 | usd_path: Optional[str] = gm.ASSET_PATH + '/robots/franka/franka.usd' 30 | -------------------------------------------------------------------------------- /grutopia_extension/configs/robots/g1.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config import RobotCfg 4 | from grutopia.macros import gm 5 | from grutopia_extension.configs.controllers import ( 6 | G1MoveBySpeedControllerCfg, 7 | MoveAlongPathPointsControllerCfg, 8 | MoveToPointBySpeedControllerCfg, 9 | RotateControllerCfg, 10 | ) 11 | 12 | move_by_speed_cfg = G1MoveBySpeedControllerCfg( 13 | name='move_by_speed', 14 | policy_weights_path=gm.ASSET_PATH + '/robots/g1/policy/move_by_speed/g1_15000.onnx', 15 | joint_names=[ 16 | 'left_hip_pitch_joint', 17 | 'right_hip_pitch_joint', 18 | 'waist_yaw_joint', 19 | 'left_hip_roll_joint', 20 | 'right_hip_roll_joint', 21 | 'left_hip_yaw_joint', 22 | 'right_hip_yaw_joint', 23 | 'left_knee_joint', 24 | 'right_knee_joint', 25 | 'left_shoulder_pitch_joint', 26 | 'right_shoulder_pitch_joint', 27 | 'left_ankle_pitch_joint', 28 | 'right_ankle_pitch_joint', 29 | 'left_shoulder_roll_joint', 30 | 'right_shoulder_roll_joint', 31 | 'left_ankle_roll_joint', 32 | 'right_ankle_roll_joint', 33 | 'left_shoulder_yaw_joint', 34 | 'right_shoulder_yaw_joint', 35 | 'left_elbow_joint', 36 | 'right_elbow_joint', 37 | 'left_wrist_roll_joint', 38 | 'right_wrist_roll_joint', 39 | 'left_wrist_pitch_joint', 40 | 'right_wrist_pitch_joint', 41 | 'left_wrist_yaw_joint', 42 | 'right_wrist_yaw_joint', 43 | ], 44 | ) 45 | 46 | move_to_point_cfg = MoveToPointBySpeedControllerCfg( 47 | name='move_to_point', 48 | forward_speed=1.0, 49 | rotation_speed=2.0, 50 | threshold=0.05, 51 | sub_controllers=[move_by_speed_cfg], 52 | ) 53 | 54 | 55 | move_along_path_cfg = MoveAlongPathPointsControllerCfg( 56 | name='move_along_path', 57 | forward_speed=1.0, 58 | rotation_speed=4.0, 59 | threshold=0.1, 60 | sub_controllers=[move_to_point_cfg], 61 | ) 62 | 63 | rotate_cfg = RotateControllerCfg( 64 | name='rotate', 65 | rotation_speed=2.0, 66 | threshold=0.02, 67 | sub_controllers=[move_by_speed_cfg], 68 | ) 69 | 70 | 71 | class G1RobotCfg(RobotCfg): 72 | # meta info 73 | name: Optional[str] = 'g1' 74 | type: Optional[str] = 'G1Robot' 75 | prim_path: Optional[str] = '/g1' 76 | create_robot: Optional[bool] = True 77 | usd_path: Optional[str] = gm.ASSET_PATH + '/robots/g1/g1_29dof_color.usd' 78 | -------------------------------------------------------------------------------- /grutopia_extension/configs/robots/jetbot.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from grutopia.core.config import RobotCfg 4 | from grutopia.macros import gm 5 | from grutopia_extension.configs.controllers import ( 6 | DifferentialDriveControllerCfg, 7 | MoveAlongPathPointsControllerCfg, 8 | MoveToPointBySpeedControllerCfg, 9 | ) 10 | from grutopia_extension.configs.sensors import CameraCfg 11 | 12 | move_by_speed_cfg = DifferentialDriveControllerCfg(name='move_by_speed', wheel_base=0.1125, wheel_radius=0.03) 13 | 14 | move_to_point_cfg = MoveToPointBySpeedControllerCfg( 15 | name='move_to_point', 16 | forward_speed=1.0, 17 | rotation_speed=1.0, 18 | threshold=0.1, 19 | sub_controllers=[move_by_speed_cfg], 20 | ) 21 | 22 | move_along_path_cfg = MoveAlongPathPointsControllerCfg( 23 | name='move_along_path', 24 | forward_speed=1.0, 25 | rotation_speed=1.0, 26 | threshold=0.1, 27 | sub_controllers=[move_to_point_cfg], 28 | ) 29 | 30 | 31 | camera_cfg = CameraCfg( 32 | name='camera', 33 | prim_path='chassis/rgb_camera/jetbot_camera', 34 | resolution=(640, 360), 35 | ) 36 | 37 | 38 | class JetbotRobotCfg(RobotCfg): 39 | # meta info 40 | name: Optional[str] = 'jetbot' 41 | type: Optional[str] = 'JetbotRobot' 42 | prim_path: Optional[str] = '/World/jetbot' 43 | create_robot: Optional[bool] = True 44 | usd_path: Optional[str] = gm.ASSET_PATH + '/robots/jetbot/jetbot.usd' 45 | -------------------------------------------------------------------------------- /grutopia_extension/configs/sensors/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from grutopia.core.config.robot import SensorCfg 4 | 5 | 6 | class CameraCfg(SensorCfg): 7 | # Fields from params. 8 | type: Optional[str] = 'Camera' 9 | enable: Optional[bool] = True 10 | resolution: Optional[Tuple[int, int]] = None 11 | 12 | 13 | class RepCameraCfg(SensorCfg): 14 | # Fields from params. 15 | type: Optional[str] = 'RepCamera' 16 | enable: Optional[bool] = True 17 | resolution: Optional[Tuple[int, int]] = None # Camera only 18 | depth: Optional[bool] = False 19 | 20 | 21 | class MocapControlledCameraCfg(SensorCfg): 22 | # Fields from params. 23 | type: Optional[str] = 'MocapControlledCamera' 24 | enable: Optional[bool] = True 25 | resolution: Optional[Tuple[int, int]] = None # Camera only 26 | translation: Optional[Tuple[float, float, float]] = None 27 | orientation: Optional[Tuple[float, float, float, float]] = None # Quaternion in local frame 28 | 29 | 30 | class LayoutEditMocapControlledCameraCfg(SensorCfg): 31 | type: Optional[str] = 'LayoutEditMocapControlledCamera' 32 | enable: Optional[bool] = True 33 | resolution: Optional[Tuple[int, int]] = None # Camera only 34 | translation: Optional[Tuple[float, float, float]] = None 35 | orientation: Optional[Tuple[float, float, float, float]] = None # Quaternion in local frame 36 | -------------------------------------------------------------------------------- /grutopia_extension/configs/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.configs.tasks.finite_step_task import ( 2 | FiniteStepTaskCfg, 3 | FiniteStepTaskEpisodeCfg, 4 | ) 5 | from grutopia_extension.configs.tasks.manipulation_task import ( 6 | ManipulationEpisodeCfg, 7 | ManipulationExtra, 8 | ManipulationTaskCfg, 9 | ) 10 | from grutopia_extension.configs.tasks.manipulation_task import ( 11 | TaskSettingCfg as ManipulationTaskSetting, 12 | ) 13 | from grutopia_extension.configs.tasks.mobile_manipulation_task import ( 14 | MobileManipulationEpisodeCfg, 15 | MobileManipulationTaskCfg, 16 | ) 17 | from grutopia_extension.configs.tasks.mobile_manipulation_task import ( 18 | TaskSettingCfg as MobileManipulationTaskSetting, 19 | ) 20 | from grutopia_extension.configs.tasks.single_inference_task import ( 21 | SingleInferenceEpisodeCfg, 22 | SingleInferenceTaskCfg, 23 | ) 24 | from grutopia_extension.configs.tasks.social_navigation_task import ( 25 | SocialNavigationEpisodeCfg, 26 | SocialNavigationTaskCfg, 27 | ) 28 | from grutopia_extension.configs.tasks.social_navigation_task import ( 29 | TaskSettingCfg as SocialNavigationTaskSetting, 30 | ) 31 | -------------------------------------------------------------------------------- /grutopia_extension/configs/tasks/finite_step_task.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from grutopia.core.config import EpisodeCfg 4 | from grutopia.core.config.task import TaskCfg 5 | 6 | 7 | class FiniteStepTaskEpisodeCfg(EpisodeCfg): 8 | pass 9 | 10 | 11 | class FiniteStepTaskCfg(TaskCfg): 12 | type: Optional[str] = 'FiniteStepTask' 13 | episodes: List[FiniteStepTaskEpisodeCfg] 14 | -------------------------------------------------------------------------------- /grutopia_extension/configs/tasks/manipulation_task.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config import EpisodeCfg 6 | from grutopia.core.config.task import TaskCfg 7 | 8 | 9 | class TaskSettingCfg(BaseModel): 10 | max_step: int 11 | 12 | 13 | class ManipulationExtra(BaseModel): 14 | prompt: Optional[str] = '' 15 | target: str 16 | episode_idx: int 17 | 18 | 19 | class ManipulationEpisodeCfg(EpisodeCfg): 20 | extra: ManipulationExtra 21 | 22 | 23 | class ManipulationTaskCfg(TaskCfg): 24 | type: Optional[str] = 'ManipulationTask' 25 | task_settings: TaskSettingCfg 26 | episodes: List[ManipulationEpisodeCfg] 27 | -------------------------------------------------------------------------------- /grutopia_extension/configs/tasks/mobile_manipulation_task.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config import EpisodeCfg 6 | from grutopia.core.config.task import TaskCfg 7 | 8 | 9 | class TaskSettingCfg(BaseModel): 10 | max_step: int 11 | verbose: Optional[bool] = False 12 | 13 | 14 | class MobileManipulationEpisodeCfg(EpisodeCfg): 15 | extra: Optional[Dict[str, Any]] = {} 16 | 17 | 18 | class MobileManipulationTaskCfg(TaskCfg): 19 | type: Optional[str] = 'MobileManipulationTask' 20 | task_settings: TaskSettingCfg 21 | episodes: List[MobileManipulationEpisodeCfg] 22 | -------------------------------------------------------------------------------- /grutopia_extension/configs/tasks/single_inference_task.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Optional 2 | 3 | from grutopia.core.config import EpisodeCfg 4 | from grutopia.core.config.task import TaskCfg 5 | 6 | 7 | class SingleInferenceEpisodeCfg(EpisodeCfg): 8 | extra: Optional[Any] = None 9 | 10 | 11 | class SingleInferenceTaskCfg(TaskCfg): 12 | type: Optional[str] = 'SingleInferenceTask' 13 | episodes: List[SingleInferenceEpisodeCfg] 14 | -------------------------------------------------------------------------------- /grutopia_extension/configs/tasks/social_navigation_task.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from grutopia.core.config import EpisodeCfg 6 | from grutopia.core.config.task import TaskCfg 7 | 8 | 9 | class TaskSettingCfg(BaseModel): 10 | max_step: int 11 | verbose: Optional[bool] = False 12 | 13 | 14 | class SocialNavigationEpisodeCfg(EpisodeCfg): 15 | extra: Optional[Dict[str, Any]] = {} 16 | 17 | 18 | class SocialNavigationTaskCfg(TaskCfg): 19 | type: Optional[str] = 'SocialNavigationTask' 20 | task_settings: TaskSettingCfg 21 | episodes: List[SocialNavigationEpisodeCfg] 22 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.controllers import ( 2 | aliengo_move_by_speed_controller, 3 | dd_controller, 4 | franka_mocap_teleop_controller, 5 | g1_move_by_speed_controller, 6 | gr1_move_by_speed_controller, 7 | gr1_teleop_controller, 8 | gripper_controller, 9 | h1_move_by_speed_controller, 10 | ik_controller, 11 | joint_controller, 12 | layout_edit_mocap_controller, 13 | move_along_path_points_controller, 14 | move_to_point_by_speed_controller, 15 | move_to_point_oracle_controller, 16 | recover_controller, 17 | rmpflow_controller, 18 | rotate_controller, 19 | rotate_oracle, 20 | ) 21 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/dd_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | from omni.isaac.core.scenes import Scene 5 | from omni.isaac.core.utils.types import ArticulationAction 6 | 7 | from grutopia.core.robot.controller import BaseController 8 | from grutopia.core.robot.robot import BaseRobot 9 | from grutopia_extension.configs.controllers import DifferentialDriveControllerCfg 10 | 11 | 12 | @BaseController.register('DifferentialDriveController') 13 | class DifferentialDriveController(BaseController): 14 | def __init__(self, config: DifferentialDriveControllerCfg, robot: BaseRobot, scene: Scene) -> None: 15 | super().__init__(config=config, robot=robot, scene=scene) 16 | self._robot_scale = self.robot.get_robot_scale()[0] 17 | self._wheel_base = config.wheel_base * self._robot_scale 18 | self._wheel_radius = config.wheel_radius * self._robot_scale 19 | 20 | def forward( 21 | self, 22 | forward_speed: float = 0, 23 | rotation_speed: float = 0, 24 | ) -> ArticulationAction: 25 | left_wheel_vel = ((2 * forward_speed) - (rotation_speed * self._wheel_base)) / (2 * self._wheel_radius) 26 | right_wheel_vel = ((2 * forward_speed) + (rotation_speed * self._wheel_base)) / (2 * self._wheel_radius) 27 | # A controller has to return an ArticulationAction 28 | return ArticulationAction(joint_velocities=[left_wheel_vel, right_wheel_vel]) 29 | 30 | def action_to_control(self, action: List | np.ndarray) -> ArticulationAction: 31 | """ 32 | Args: 33 | action (List | np.ndarray): n-element 1d array containing: 34 | 0. forward_speed (float) 35 | 1. rotation_speed (float) 36 | """ 37 | assert len(action) == 2, 'action must contain 2 elements' 38 | return self.forward( 39 | forward_speed=action[0], 40 | rotation_speed=action[1], 41 | ) 42 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/gripper_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | from omni.isaac.core.scenes import Scene 5 | from omni.isaac.core.utils.types import ArticulationAction 6 | 7 | from grutopia.core.robot.controller import BaseController 8 | from grutopia.core.robot.robot import BaseRobot 9 | from grutopia_extension.configs.controllers import GripperControllerCfg 10 | 11 | 12 | @BaseController.register('GripperController') 13 | class GripperController(BaseController): 14 | def __init__(self, config: GripperControllerCfg, robot: BaseRobot, scene: Scene): 15 | self._gripper = robot.isaac_robot.gripper # for franka is OK 16 | 17 | super().__init__(config, robot, scene) 18 | 19 | def forward(self, action: str) -> ArticulationAction: 20 | return self._gripper.forward(action) 21 | 22 | def action_to_control(self, action: List | np.ndarray) -> ArticulationAction: 23 | """ 24 | Args: 25 | action (List | np.ndarray): 1-element 1d array. 26 | """ 27 | assert len(action) == 1 and action[0] in [ 28 | 'open', 29 | 'close', 30 | ], f'action must be a list with one str elem, which is one of "open" / "close", but got {action}' 31 | return self.forward(action[0]) 32 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/layout_edit_controller/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/controllers/layout_edit_controller/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/controllers/layout_edit_controller/hand_control.py: -------------------------------------------------------------------------------- 1 | from pxr import Gf, UsdGeom 2 | 3 | 4 | class HandControl: 5 | def __init__(self, stage): 6 | self.stage = stage 7 | 8 | def add_hand_to_stage(self, hand_path, hand_prim_path, hand_scale): 9 | try: 10 | # Create a Prim and add a USD reference. 11 | asset_prim = self.stage.DefinePrim(hand_prim_path) 12 | if not asset_prim.GetReferences().AddReference(hand_path): 13 | raise RuntimeError(f'Failed to add reference: {hand_path}') 14 | 15 | # Set the scale. 16 | xform = UsdGeom.Xformable(asset_prim) 17 | scale_op = xform.AddXformOp(UsdGeom.XformOp.TypeScale) 18 | scale_op.Set(Gf.Vec3f(hand_scale)) 19 | 20 | print('Successfully added asset with scale') 21 | return True 22 | except Exception as e: 23 | print(f'Error: {str(e)}') 24 | return False 25 | 26 | def delete_hand(self, left_prim_path, right_prim_path): 27 | self.delete_asset(left_prim_path) 28 | self.delete_asset(right_prim_path) 29 | 30 | def delete_asset(self, prim_path): 31 | print(prim_path) 32 | try: 33 | if self.stage.GetPrimAtPath(prim_path).IsValid(): 34 | # Delete the operation of the hand in the virtual scene. 35 | success = self.stage.RemovePrim(prim_path) 36 | print(f'Deleted {prim_path} : {success}') 37 | return success 38 | else: 39 | print(f'Prim {prim_path} does not exist') 40 | return False 41 | except Exception as e: 42 | print(f'Delete error: {str(e)}') 43 | return False 44 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/layout_edit_controller/save_asset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import omni 4 | import omni.usd 5 | from omni.isaac.core import SimulationContext 6 | from pxr import Sdf, Usd 7 | 8 | 9 | class SaveAsset: 10 | def __init__(self, stage): 11 | self.stage = stage 12 | 13 | def save_entire_scene(self, target_folder: str, robot_save_bool: bool, filename: str = 'saved_scene.usd'): 14 | try: 15 | sim = SimulationContext.instance() 16 | sim.pause() 17 | if not robot_save_bool: 18 | self.deleted_robots = self.delete_robots(self.stage.GetPseudoRoot()) 19 | os.makedirs(target_folder, exist_ok=True) 20 | save_path = os.path.join(target_folder, filename) 21 | self.stage.GetRootLayer().Export(save_path) 22 | print(f'asset save as {save_path}') 23 | sim.stop() 24 | omni.kit.app.get_app().shutdown() 25 | except Exception as e: 26 | print(f'Delete error: {str(e)}') 27 | 28 | def delete_robots(self, root_prim): 29 | deleted_robots = [] 30 | for prim in Usd.PrimRange(root_prim): 31 | prim_path = prim.GetPath().pathString 32 | if 'robots' in prim_path: 33 | deleted_robots.append(prim_path) 34 | prim.SetActive(False) 35 | print(f'Deleted robot: {prim_path}') 36 | 37 | for prim_path in deleted_robots: 38 | self.stage.RemovePrim(Sdf.Path(prim_path)) 39 | print(f'Deleted robot: {prim_path}') 40 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/lcmtypes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/controllers/lcmtypes/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/controllers/lcmtypes/teleop/__init__.py: -------------------------------------------------------------------------------- 1 | """LCM package __init__.py file 2 | This file automatically generated by lcm-gen. 3 | DO NOT MODIFY BY HAND!!!! 4 | lcm-gen 1.5.0 5 | """ 6 | 7 | from .action import action 8 | from .joints import joints 9 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/lcmtypes/teleop/joints.py: -------------------------------------------------------------------------------- 1 | """LCM type definitions 2 | This file automatically generated by lcm. 3 | DO NOT MODIFY BY HAND!!!! 4 | """ 5 | 6 | from io import BytesIO 7 | import struct 8 | 9 | class joints(object): 10 | 11 | __slots__ = ["joint_num", "joint_positions"] 12 | 13 | __typenames__ = ["int32_t", "float"] 14 | 15 | __dimensions__ = [None, ["joint_num"]] 16 | 17 | def __init__(self): 18 | self.joint_num = 0 19 | """ LCM Type: int32_t """ 20 | self.joint_positions = [] 21 | """ LCM Type: float[joint_num] """ 22 | 23 | def encode(self): 24 | buf = BytesIO() 25 | buf.write(joints._get_packed_fingerprint()) 26 | self._encode_one(buf) 27 | return buf.getvalue() 28 | 29 | def _encode_one(self, buf): 30 | buf.write(struct.pack(">i", self.joint_num)) 31 | buf.write(struct.pack('>%df' % self.joint_num, *self.joint_positions[:self.joint_num])) 32 | 33 | @staticmethod 34 | def decode(data): 35 | if hasattr(data, 'read'): 36 | buf = data 37 | else: 38 | buf = BytesIO(data) 39 | if buf.read(8) != joints._get_packed_fingerprint(): 40 | raise ValueError("Decode error") 41 | return joints._decode_one(buf) 42 | 43 | @staticmethod 44 | def _decode_one(buf): 45 | self = joints() 46 | self.joint_num = struct.unpack(">i", buf.read(4))[0] 47 | self.joint_positions = struct.unpack('>%df' % self.joint_num, buf.read(self.joint_num * 4)) 48 | return self 49 | 50 | @staticmethod 51 | def _get_hash_recursive(parents): 52 | if joints in parents: return 0 53 | tmphash = (0x242b6dadb9e53b97) & 0xffffffffffffffff 54 | tmphash = (((tmphash<<1)&0xffffffffffffffff) + (tmphash>>63)) & 0xffffffffffffffff 55 | return tmphash 56 | _packed_fingerprint = None 57 | 58 | @staticmethod 59 | def _get_packed_fingerprint(): 60 | if joints._packed_fingerprint is None: 61 | joints._packed_fingerprint = struct.pack(">Q", joints._get_hash_recursive([])) 62 | return joints._packed_fingerprint 63 | 64 | def get_hash(self): 65 | """Get the LCM hash of the struct""" 66 | return struct.unpack(">Q", joints._get_packed_fingerprint())[0] 67 | 68 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/lcmtypes/teleop_t.lcm: -------------------------------------------------------------------------------- 1 | package teleop; 2 | 3 | struct action 4 | { 5 | float head_mat[4][4]; 6 | float left_wrist_mat[4][4]; 7 | float right_wrist_mat[4][4]; 8 | float left_hand_mat[25][3]; 9 | float right_hand_mat[25][3]; 10 | } 11 | 12 | struct joints 13 | { 14 | int32_t joint_num; 15 | float joint_positions[joint_num]; 16 | } 17 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/controllers/models/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/controllers/models/aliengo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/controllers/models/aliengo/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/controllers/recover_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | from omni.isaac.core.scenes import Scene 5 | from omni.isaac.core.utils.types import ArticulationAction 6 | 7 | from grutopia.core.robot.controller import BaseController 8 | from grutopia.core.robot.robot import BaseRobot 9 | from grutopia.core.util import log 10 | from grutopia_extension.configs.controllers import RecoverControllerCfg 11 | 12 | 13 | @BaseController.register('RecoverController') 14 | class RecoverController(BaseController): 15 | """Controller for recovering from pose failure.""" 16 | 17 | def __init__(self, config: RecoverControllerCfg, robot: BaseRobot, scene: Scene) -> None: 18 | super().__init__(config=config, robot=robot, scene=scene) 19 | 20 | self.recover_height = config.recover_height 21 | self.target_position = None 22 | self.num_joints = None 23 | 24 | def forward( 25 | self, target_position: List | np.ndarray, target_orientation: List | np.ndarray = np.array([1.0, 0.0, 0.0, 0.0]) 26 | ) -> ArticulationAction: 27 | if self.num_joints is None: 28 | self.num_joints = self.get_joint_subset().num_joints 29 | current_position = self.robot.get_world_pose()[0] 30 | log.info(f'current pos is {current_position}, recovering to {target_position}') 31 | self.robot.isaac_robot.set_world_pose(target_position, target_orientation) 32 | self.robot.isaac_robot.set_world_velocity(np.zeros(6)) 33 | 34 | self.robot.isaac_robot.set_joint_velocities(np.zeros(len(self.robot.isaac_robot.dof_names))) 35 | self.robot.isaac_robot.set_joint_positions(np.zeros(len(self.robot.isaac_robot.dof_names))) 36 | self.robot.isaac_robot.set_linear_velocity(np.zeros(3)) 37 | 38 | return self.sub_controllers[0].forward( 39 | joint_positions=np.zeros(self.num_joints), 40 | joint_velocities=np.zeros(self.num_joints), 41 | ) 42 | 43 | def action_to_control(self, action: List | np.ndarray) -> ArticulationAction: 44 | """Convert input action (in 1d array format) to joint signals to apply. 45 | 46 | Args: 47 | action (List | np.ndarray): 0-element 1d array. 48 | 49 | Returns: 50 | ArticulationAction: joint signals to apply. 51 | """ 52 | # assert len(action) == 0, 'action must be empty' 53 | if len(action) > 1: 54 | return self.forward(action[0], action[1]) 55 | 56 | return self.forward(action[0]) 57 | -------------------------------------------------------------------------------- /grutopia_extension/controllers/rmpflow_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | from omni.isaac.core.scenes import Scene 5 | from omni.isaac.motion_generation import ArticulationMotionPolicy, RmpFlow 6 | 7 | from grutopia.core.robot.controller import BaseController 8 | from grutopia.core.robot.robot import BaseRobot 9 | from grutopia_extension.configs.controllers import RMPFlowControllerCfg 10 | 11 | 12 | @BaseController.register('RMPFlowController') 13 | class RMPFlowController(BaseController): 14 | def __init__(self, config: RMPFlowControllerCfg, robot: BaseRobot, scene: Scene): 15 | super().__init__(config=config, robot=robot, scene=scene) 16 | self.rmpflow = RmpFlow( 17 | robot_description_path=config.robot_description_path, 18 | urdf_path=config.robot_urdf_path, 19 | rmpflow_config_path=config.rmpflow_config_path, 20 | end_effector_frame_name=config.end_effector_frame_name, 21 | maximum_substep_size=1 / 120, 22 | ) 23 | self._articulation_rmpflow = ArticulationMotionPolicy( 24 | robot.isaac_robot, self.rmpflow, default_physics_dt=1 / 120 25 | ) 26 | 27 | self.success = False 28 | 29 | def forward(self, eef_target_position: np.ndarray, eef_target_orientation: np.ndarray): 30 | self.rmpflow.set_end_effector_target(np.asarray(eef_target_position), eef_target_orientation) 31 | 32 | return self._articulation_rmpflow.get_next_articulation_action(), True 33 | 34 | def action_to_control(self, action: List | np.ndarray): 35 | """ 36 | Args: 37 | action (np.ndarray): n-element 1d array containing: 38 | 0. eef_target_position 39 | 1. eef_target_orientation 40 | """ 41 | assert len(action) == 2, 'action must contain 2 elements' 42 | 43 | eef_target_position = None if action[0] is None else np.array(action[0]) 44 | eef_target_orientation = None if action[1] is None else np.array(action[1]) 45 | 46 | result, self.success = self.forward( 47 | eef_target_position=eef_target_position, 48 | eef_target_orientation=eef_target_orientation, 49 | ) 50 | return result 51 | -------------------------------------------------------------------------------- /grutopia_extension/interactions/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.interactions import keyboard 2 | -------------------------------------------------------------------------------- /grutopia_extension/interactions/keyboard.py: -------------------------------------------------------------------------------- 1 | import carb 2 | import numpy as np 3 | import omni 4 | 5 | from grutopia.core.util.interaction import BaseInteraction 6 | 7 | 8 | class KeyboardController: 9 | def __init__(self): 10 | self.command = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) 11 | self.subscribe() 12 | 13 | def subscribe(self): 14 | """ 15 | subscribe to keyboard events 16 | """ 17 | # subscribe to keyboard events 18 | app_window = omni.appwindow.get_default_app_window() # noqa 19 | key_input = carb.input.acquire_input_interface() # noqa 20 | key_input.subscribe_to_keyboard_events(app_window.get_keyboard(), self._sub_keyboard_event) 21 | 22 | def _sub_keyboard_event(self, event, *args, **kwargs): 23 | """subscribe to keyboard events, map to str""" 24 | if ( 25 | event.type == carb.input.KeyboardEventType.KEY_PRESS 26 | or event.type == carb.input.KeyboardEventType.KEY_REPEAT 27 | ): 28 | if event.input == carb.input.KeyboardInput.I: 29 | self.command = np.array([1.0, 0.0, 0.0, 0.0, 0.0, 0.0]) 30 | if event.input == carb.input.KeyboardInput.K: 31 | self.command = np.array([0.0, 1.0, 0.0, 0.0, 0.0, 0.0]) 32 | if event.input == carb.input.KeyboardInput.J: 33 | self.command = np.array([0.0, 0.0, 1.0, 0.0, 0.0, 0.0]) 34 | if event.input == carb.input.KeyboardInput.L: 35 | self.command = np.array([0.0, 0.0, 0.0, 1.0, 0.0, 0.0]) 36 | if event.input == carb.input.KeyboardInput.U: 37 | self.command = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 0.0]) 38 | if event.input == carb.input.KeyboardInput.O: 39 | self.command = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 1.0]) 40 | if event.type == carb.input.KeyboardEventType.KEY_RELEASE: 41 | self.command = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) 42 | return True 43 | 44 | 45 | @BaseInteraction.register('Keyboard') 46 | class KeyboardInteraction(BaseInteraction): 47 | """Get keyboard input event(i, k, j, l, u, o)""" 48 | 49 | def __init__(self): 50 | super().__init__() 51 | self._type = 'Keyboard' 52 | self.controller = KeyboardController() 53 | 54 | def get_input(self) -> np.ndarray: 55 | """ 56 | Read input of Keyboard. 57 | Returns: 58 | np.ndarray, len == 6, representing (i, k, j, l, u, o) key pressed or not. 59 | """ 60 | return self.controller.command 61 | -------------------------------------------------------------------------------- /grutopia_extension/interactions/motion_capture.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import requests 4 | 5 | 6 | class MocapInteraction: 7 | def __init__(self, mocap_url): 8 | self.base_url = mocap_url 9 | self.is_runing = False 10 | self.server_start() 11 | time.sleep(1) # asynchronous calculation of buffer time 12 | 13 | def server_start(self): 14 | if self.is_runing: 15 | return 16 | 17 | response = requests.get(f'{self.base_url}/start') 18 | if response.status_code == 400: 19 | self.is_runing = True 20 | return 21 | 22 | if response.status_code != 200: 23 | raise RuntimeError(f'server start failed: {response.json()}') 24 | 25 | self.is_runing = True 26 | 27 | def step(self): 28 | response = requests.get(f'{self.base_url}/results') 29 | if response.status_code != 200: 30 | raise RuntimeError(f'mocap server get result failed: {response.json()}') 31 | cur_mocap_info = response.json() 32 | 33 | return cur_mocap_info 34 | 35 | def server_stop(self): 36 | if not self.is_runing: 37 | return 38 | 39 | response = requests.get(f'{self.base_url}/stop') 40 | if response.status_code == 400: 41 | self.is_runing = False 42 | return 43 | 44 | if response.status_code != 200: 45 | raise RuntimeError(f'server stop failed: {response.json()}') 46 | 47 | self.is_runing = False 48 | -------------------------------------------------------------------------------- /grutopia_extension/interactions/visionpro/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenRobotLab/GRUtopia/38e94d0465f26ab0db217edcaf74bc1fa46da0b7/grutopia_extension/interactions/visionpro/__init__.py -------------------------------------------------------------------------------- /grutopia_extension/interactions/visionpro/constants_vuer.py: -------------------------------------------------------------------------------- 1 | # yapf: disable 2 | import numpy as np 3 | 4 | tip_indices = [4, 9, 14, 19, 24] 5 | 6 | hand2inspire_l_arm = np.array([[1, 0, 0, 0], 7 | [0, 0, -1, 0], 8 | [1, 1, 0, 0], 9 | [0, 0, 0, 1]]) 10 | 11 | hand2inspire_r_arm = np.array([[1, 0, 0, 0], 12 | [0, 0, 1, 0], 13 | [1, -1, 0, 0], 14 | [0, 0, 0, 1]]) 15 | 16 | hand2inspire_l_finger = np.array([[0, -1, 0, 0], 17 | [0, 0, -1, 0], 18 | [1, 0, 0, 0], 19 | [0, 0, 0, 1]]) 20 | 21 | hand2inspire_r_finger = np.array([[0, -1, 0, 0], 22 | [0, 0, -1, 0], 23 | [1, 0, 0, 0], 24 | [0, 0, 0, 1]]) 25 | 26 | 27 | grd_yup2grd_zup = np.array([[0, 0, -1, 0], 28 | [-1, 0, 0, 0], 29 | [0, 1, 0, 0], 30 | [0, 0, 0, 1]]) 31 | 32 | 33 | align_transform_l = np.array([ 34 | [0, 0, 1], 35 | [0, 1, 0], 36 | [-1, 0, 0], 37 | ]) 38 | 39 | align_transform_r = np.array([ 40 | [1, 0, 0], 41 | [0, -1, 0], 42 | [0, 0, -1] 43 | ]) @ np.array([ 44 | [0, 0, 1], 45 | [0, -1, 0], 46 | [1, 0, 0], 47 | ]) 48 | 49 | align_transform_head = np.array([ 50 | [0, -1, 0], 51 | [1, 0, 0], 52 | [0, 0, 1] 53 | ]) 54 | -------------------------------------------------------------------------------- /grutopia_extension/interactions/visionpro/motion_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def mat_update(prev_mat, mat): 5 | if np.linalg.det(mat) == 0: 6 | return prev_mat 7 | else: 8 | return mat 9 | 10 | 11 | def fast_mat_inv(mat): 12 | ret = np.eye(4) 13 | ret[:3, :3] = mat[:3, :3].T 14 | ret[:3, 3] = -mat[:3, :3].T @ mat[:3, 3] 15 | return ret 16 | -------------------------------------------------------------------------------- /grutopia_extension/interactions/visionpro/visionpro.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Event, Queue, shared_memory 2 | 3 | import numpy as np 4 | 5 | from .Preprocessor import VuerPreprocessor 6 | from .TeleVision import OpenTeleVision 7 | 8 | 9 | class VuerTeleop: 10 | def __init__( 11 | self, cert_file='./GRUtopia/mkcert/cert.pem', key_file='./GRUtopia/mkcert/key.pem', resolution=(720, 1280) 12 | ): 13 | self.resolution = resolution 14 | self.crop_size_w = 0 15 | self.crop_size_h = 0 16 | self.resolution_cropped = (self.resolution[0] - self.crop_size_h, self.resolution[1] - 2 * self.crop_size_w) 17 | 18 | self.img_shape = (self.resolution_cropped[0], 2 * self.resolution_cropped[1], 3) 19 | self.img_height, self.img_width = self.resolution_cropped[:2] 20 | 21 | self.shm = shared_memory.SharedMemory(create=True, size=np.prod(self.img_shape) * np.uint8().itemsize) 22 | self.img_array = np.ndarray((self.img_shape[0], self.img_shape[1], 3), dtype=np.uint8, buffer=self.shm.buf) 23 | image_queue = Queue() 24 | toggle_streaming = Event() 25 | self.tv = OpenTeleVision( 26 | self.resolution_cropped, self.shm.name, image_queue, toggle_streaming, cert_file, key_file 27 | ) 28 | self.processor = VuerPreprocessor() 29 | 30 | def step(self): 31 | head_mat, left_wrist_mat, right_wrist_mat, left_hand_mat, right_hand_mat = self.processor.process(self.tv) 32 | begin_move = self.tv.begin_move.value 33 | 34 | return head_mat, left_wrist_mat, right_wrist_mat, left_hand_mat, right_hand_mat, begin_move 35 | 36 | def cleanup(self): 37 | self.tv.cleanup() 38 | -------------------------------------------------------------------------------- /grutopia_extension/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | from grutopia.core.util import log 4 | 5 | for module in [ 6 | 'grutopia_extension.metrics.candidates_reduce_metric', 7 | 'grutopia_extension.metrics.debug_metric', 8 | 'grutopia_extension.metrics.mobile_manipulation_success_metric', 9 | 'grutopia_extension.metrics.reset_time_metric', 10 | 'grutopia_extension.metrics.simple_metric', 11 | 'grutopia_extension.metrics.social_navigation_success_metric', 12 | 'grutopia_extension.metrics.recording_metric', 13 | ]: 14 | 15 | try: 16 | importlib.import_module(module) 17 | except ImportError as e: 18 | log.error(e) 19 | -------------------------------------------------------------------------------- /grutopia_extension/metrics/recording_metric.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.runtime.task_runtime import TaskRuntime 2 | from grutopia.core.task.metric import BaseMetric 3 | from grutopia_extension.configs.metrics.recording_metric import RecordingMetricCfg 4 | 5 | 6 | @BaseMetric.register('RecordingMetric') 7 | class RecordingMetric(BaseMetric): 8 | """ 9 | Record any controllers output or joint actions during playing or teleoperating 10 | """ 11 | 12 | def __init__(self, config: RecordingMetricCfg, task_runtime: TaskRuntime): 13 | super().__init__(config, task_runtime) 14 | self.param = config 15 | _robot_name = self.param.robot_name 16 | self.robot_name = ( 17 | _robot_name + '_' + str(self.task_runtime.env.env_id) 18 | ) # real robot name in isaac sim: {robot_name}_{env_id} 19 | self.fields = self.param.fields 20 | self.step = 0 21 | self.obs = {} 22 | 23 | def reset(self): 24 | self.step = 0 25 | 26 | def update(self, task_obs: dict): 27 | """ 28 | This function is called at each world step. 29 | """ 30 | if self.fields is None: 31 | self.obs[self.step] = task_obs 32 | else: 33 | assert len(task_obs) == 1, 'only support one task currently.' 34 | robot_name = list(task_obs.keys())[0] 35 | self.obs[self.step] = {robot_name: {}} 36 | for record_field in self.fields: 37 | self.obs[self.step][robot_name][record_field] = task_obs[robot_name][record_field] 38 | 39 | self.step += 1 40 | 41 | def calc(self): 42 | """ 43 | This function is called to calculate the metrics when the episode is terminated. 44 | """ 45 | return self.obs 46 | -------------------------------------------------------------------------------- /grutopia_extension/metrics/reset_time_metric.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config.metric import MetricCfg 2 | from grutopia.core.datahub.datahub import DataHub 3 | from grutopia.core.runtime.task_runtime import TaskRuntime 4 | from grutopia.core.task.metric import BaseMetric 5 | from grutopia.core.util import log 6 | 7 | 8 | @BaseMetric.register('ResetTimeMetric') 9 | class ResetTimeMetric(BaseMetric): 10 | """ 11 | Calculate the fall times of the robot 12 | """ 13 | 14 | def __init__(self, config: MetricCfg, task_runtime: TaskRuntime): 15 | super().__init__(config, task_runtime) 16 | self.reset() 17 | 18 | def reset(self): 19 | self.fall_times = 0 20 | 21 | def update(self, task_obs: dict): 22 | """ 23 | This function is called at each world step. 24 | """ 25 | action = DataHub.get_actions_by_task_name(self.task_runtime.name) 26 | # log.debug(f"======== get action: {action} ========") 27 | self.fall_times = ( 28 | self.fall_times + 1 29 | if action and len(action['controllers']) > 0 and 'recover' == action['controllers'][0]['controller'] 30 | else self.fall_times 31 | ) 32 | 33 | def calc(self, task_info: dict): 34 | """ 35 | This function is called to calculate the metrics when the episode is terminated. 36 | """ 37 | log.info('ResetTimeMetric calc() called.') 38 | 39 | return self.fall_times 40 | -------------------------------------------------------------------------------- /grutopia_extension/metrics/simple_metric.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pydantic import BaseModel 3 | 4 | from grutopia.core.config.metric import MetricCfg 5 | from grutopia.core.runtime.task_runtime import TaskRuntime 6 | from grutopia.core.task.metric import BaseMetric 7 | from grutopia.core.util import log 8 | 9 | 10 | class SimpleMetricParam(BaseModel): 11 | robot_name: str 12 | 13 | 14 | @BaseMetric.register('SimpleMetric') 15 | class SimpleMetric(BaseMetric): 16 | """ 17 | Calculate the total distance a robot moves 18 | """ 19 | 20 | def __init__(self, config: MetricCfg, task_runtime: TaskRuntime): 21 | super().__init__(config, task_runtime) 22 | self.distance: float = 0.0 23 | self.position = None 24 | self.param = SimpleMetricParam(**config.metric_config) 25 | _robot_name = self.param.robot_name 26 | self.robot_name = ( 27 | _robot_name + '_' + str(self.task_runtime.env.env_id) 28 | ) # real robot name in isaac sim: {robot_name}_{env_id} 29 | 30 | def reset(self): 31 | self.distance = 0.0 32 | 33 | def update(self, task_obs: dict): 34 | """ 35 | This function is called at each world step. 36 | """ 37 | if self.position is None: 38 | self.position = task_obs[self.robot_name]['position'] 39 | return 40 | self.distance += np.linalg.norm(self.position - task_obs[self.robot_name]['position']) 41 | self.position = task_obs[self.robot_name]['position'] 42 | # log.info(f'distance: {self.distance}') 43 | return 44 | 45 | def calc(self): 46 | """ 47 | This function is called to calculate the metrics when the episode is terminated. 48 | """ 49 | log.info('SimpleMetric calc() called.') 50 | return self.distance 51 | -------------------------------------------------------------------------------- /grutopia_extension/objects/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.objects import dynamic_cube, usd_object, visual_cube 2 | -------------------------------------------------------------------------------- /grutopia_extension/objects/dynamic_cube.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from omni.isaac.core.objects import DynamicCuboid 3 | 4 | from grutopia.core.scene.object import ObjectCommon, Scene 5 | from grutopia_extension.configs.objects import DynamicCubeCfg 6 | 7 | 8 | @ObjectCommon.register('DynamicCube') 9 | class DynamicCube(ObjectCommon): 10 | def __init__(self, config: DynamicCubeCfg): 11 | super().__init__(config=config) 12 | self._config = config 13 | 14 | def set_up_scene(self, scene: Scene): 15 | scene.add( 16 | DynamicCuboid( 17 | prim_path=self._config.prim_path, 18 | name=self._config.name, 19 | position=np.array(self._config.position), 20 | orientation=np.array(self._config.orientation), 21 | scale=np.array(self._config.scale), 22 | color=np.array(self._config.color), 23 | ) 24 | ) 25 | -------------------------------------------------------------------------------- /grutopia_extension/objects/visual_cube.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from omni.isaac.core.objects.cuboid import VisualCuboid 3 | 4 | from grutopia.core.scene.object import ObjectCommon, Scene 5 | from grutopia_extension.configs.objects import VisualCubeCfg 6 | 7 | 8 | @ObjectCommon.register('VisualCube') 9 | class VisualCube(ObjectCommon): 10 | def __init__(self, config: VisualCubeCfg): 11 | super().__init__(config=config) 12 | self._config = config 13 | 14 | def set_up_scene(self, scene: Scene): 15 | scene.add( 16 | VisualCuboid( 17 | prim_path=self._config.prim_path, 18 | name=self._config.name, 19 | position=np.array(self._config.position), 20 | scale=np.array([self._config.scale]), 21 | color=np.array([self._config.color]), 22 | ) 23 | ) # noqa: F401,F403 24 | -------------------------------------------------------------------------------- /grutopia_extension/robots/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.robots import ( 2 | aliengo, 3 | camera_oracle, 4 | franka, 5 | g1, 6 | gr1, 7 | h1, 8 | h1_with_hand, 9 | jetbot, 10 | mocap_controlled_franka, 11 | npc, 12 | ) 13 | -------------------------------------------------------------------------------- /grutopia_extension/robots/npc.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from omni.isaac.core.scenes import Scene 4 | 5 | from grutopia.core.config.robot import RobotCfg 6 | from grutopia.core.robot.robot import BaseRobot 7 | from grutopia.core.util import log 8 | 9 | 10 | @BaseRobot.register('NPC') 11 | class NPC(BaseRobot): 12 | def __init__(self, robot_model: RobotCfg, scene: Scene): 13 | super().__init__(robot_model, scene) 14 | self.name_of_robots_in_scene = [] 15 | if robot_model.prim_path is not None: 16 | # TODO implement when NPCs need a body. 17 | log.info('NPC has a body, but this is not implemented yet.') 18 | log.debug(f'NPC {self.name} initialized') 19 | 20 | def set_up_to_scene(self, scene: Scene): 21 | # No need to add to scene as this is a non-prim robot 22 | pass 23 | 24 | def post_reset(self): 25 | log.debug(f'Post-reset for NPC {self.name}') 26 | pass 27 | 28 | def apply_action(self, action: Dict): 29 | log.debug(f'Applying action for NPC {self.name}: {action}') 30 | pass 31 | 32 | def get_obs(self) -> Dict: 33 | log.debug(f'Getting observation for NPC {self.name}') 34 | # npc can get all the robot obs in the scene, but this method return nothing. 35 | return {} 36 | -------------------------------------------------------------------------------- /grutopia_extension/sensors/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.sensors import ( 2 | camera, 3 | layout_edit_mocap_controlled_camera, 4 | mocap_controlled_camera, 5 | rep_camera, 6 | ) 7 | -------------------------------------------------------------------------------- /grutopia_extension/sensors/camera.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from omni.isaac.sensor import Camera as i_Camera 4 | 5 | from grutopia.core.robot.robot import BaseRobot, Scene 6 | from grutopia.core.robot.sensor import BaseSensor 7 | from grutopia.core.util import log 8 | from grutopia_extension.configs.sensors import CameraCfg 9 | 10 | 11 | @BaseSensor.register('Camera') 12 | class Camera(BaseSensor): 13 | """ 14 | wrap of isaac sim's Camera class 15 | """ 16 | 17 | def __init__(self, config: CameraCfg, robot: BaseRobot, name: str = None, scene: Scene = None): 18 | super().__init__(config, robot, scene) 19 | self.name = name 20 | self.config = config 21 | 22 | def post_reset(self): 23 | if self.config.enable: 24 | self._camera = self.create_camera() 25 | self._camera.initialize() 26 | # self._camera.add_pointcloud_to_frame() 27 | # self._camera.add_distance_to_image_plane_to_frame() 28 | # self._camera.add_semantic_segmentation_to_frame() 29 | # self._camera.add_instance_segmentation_to_frame() 30 | # self._camera.add_instance_id_segmentation_to_frame() 31 | self._camera.add_bounding_box_2d_tight_to_frame() 32 | 33 | def create_camera(self) -> i_Camera: 34 | """Create an isaac-sim camera object. 35 | 36 | Initializes the camera's resolution and prim path based on configuration. 37 | 38 | Returns: 39 | i_Camera: The initialized camera object. 40 | """ 41 | # Initialize the default resolution for the camera 42 | resolution = (320, 240) 43 | # Use the configured camera resolution if provided. 44 | if self.config.resolution is not None: 45 | resolution = self.config.resolution 46 | 47 | prim_path = self._robot.config.prim_path + '/' + self.config.prim_path 48 | log.debug('camera_prim_path: ' + prim_path) 49 | log.debug('name : ' + self.config.name) 50 | log.debug(f'resolution : {resolution}') 51 | return i_Camera(prim_path=prim_path, resolution=resolution) 52 | 53 | def get_data(self) -> Dict: 54 | if self.config.enable: 55 | rgba = self._camera.get_rgba() 56 | frame = self._camera.get_current_frame() 57 | return {'rgba': rgba, 'frame': frame} 58 | return {} 59 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/README.md: -------------------------------------------------------------------------------- 1 | # Tasks 2 | 3 | > Task Definition 4 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from grutopia_extension.tasks import ( 2 | finite_step_task, 3 | manipulation_task, 4 | mobile_manipulation_task, 5 | single_inference_task, 6 | social_navigation_task, 7 | ) 8 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/finite_step_task.py: -------------------------------------------------------------------------------- 1 | from omni.isaac.core.scenes import Scene 2 | 3 | from grutopia.core.runtime.task_runtime import TaskRuntime 4 | from grutopia.core.task import BaseTask 5 | 6 | 7 | @BaseTask.register('FiniteStepTask') 8 | class FiniteStepTask(BaseTask): 9 | def __init__(self, runtime: TaskRuntime, scene: Scene): 10 | super().__init__(runtime, scene) 11 | self.stop_count = 0 12 | self.max_step = 500 13 | # Validate task setting 14 | if not runtime.task_settings: 15 | pass 16 | elif not isinstance(runtime.task_settings, dict): 17 | raise ValueError('task_settings of FiniteStepTask must be a dict') 18 | if 'max_step' in runtime.task_settings: 19 | self.max_step = runtime.task_settings['max_step'] 20 | 21 | def is_done(self) -> bool: 22 | self.stop_count += 1 23 | return self.stop_count > self.max_step 24 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/manipulation_task.py: -------------------------------------------------------------------------------- 1 | from omni.isaac.core.scenes import Scene 2 | 3 | from grutopia.core.datahub import DataHub 4 | from grutopia.core.runtime.task_runtime import TaskRuntime 5 | from grutopia.core.task import BaseTask 6 | from grutopia.core.util import log 7 | from grutopia_extension.configs.tasks.manipulation_task import ( 8 | ManipulationExtra, 9 | TaskSettingCfg, 10 | ) 11 | 12 | 13 | @BaseTask.register('ManipulationTask') 14 | class ManipulationTask(BaseTask): 15 | def __init__(self, runtime: TaskRuntime, scene: Scene): 16 | super().__init__(runtime, scene) 17 | self.step_counter = 0 18 | if isinstance(runtime.task_settings, TaskSettingCfg): 19 | self.settings = runtime.task_settings 20 | else: 21 | raise ValueError('task_settings must be a TaskSettingCfg') 22 | if isinstance(runtime.extra, ManipulationExtra): 23 | self.extra = runtime.extra 24 | else: 25 | raise ValueError('extra must be a ManipulationExtra') 26 | log.info(f'task_settings: max_step : {self.settings.max_step}.)') 27 | # Episode 28 | log.info(f'Episode meta : prompt : {self.extra.prompt}.)') 29 | 30 | def is_done(self) -> bool: 31 | self.step_counter = self.step_counter + 1 32 | return DataHub.get_episode_finished(self.runtime.name) or self.step_counter > self.settings.max_step 33 | 34 | def individual_reset(self): 35 | for name, metric in self.metrics.items(): 36 | metric.reset() 37 | 38 | def update_metrics(self): 39 | return super().update_metrics() 40 | 41 | def calculate_metrics(self) -> dict: 42 | metrics_res = {} 43 | for name, metric in self.metrics.items(): 44 | metrics_res[name] = metric.calc() 45 | 46 | return metrics_res 47 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/mobile_manipulation_task.py: -------------------------------------------------------------------------------- 1 | # import time 2 | import traceback 3 | from typing import Any, Dict 4 | 5 | from omni.isaac.core.scenes import Scene 6 | 7 | from grutopia.core.datahub import DataHub 8 | from grutopia.core.runtime.task_runtime import TaskRuntime 9 | from grutopia.core.task import BaseTask 10 | from grutopia.core.util import log 11 | from grutopia_extension.configs.tasks.mobile_manipulation_task import TaskSettingCfg 12 | 13 | 14 | @BaseTask.register('MobileManipulationTask') 15 | class MobileManipulationTask(BaseTask): 16 | def __init__(self, runtime: TaskRuntime, scene: Scene): 17 | super().__init__(runtime, scene) 18 | self.step_counter = 0 19 | if isinstance(runtime.task_settings, TaskSettingCfg): 20 | self.settings = runtime.task_settings 21 | else: 22 | raise ValueError('task_settings must be a TaskSettingCfg') 23 | self.episode_meta: Dict[str, Any] = runtime.extra 24 | self.instruction = self.episode_meta['instruction'] 25 | 26 | log.info(f'task_settings: max_step : {self.settings.max_step}.)') 27 | # Episode 28 | log.info(f'Episode meta : instruction : {self.instruction}.)') 29 | 30 | def get_observations(self) -> Dict[str, Any]: 31 | """ 32 | Returns current observations from the objects needed for the behavioral layer. 33 | 34 | Return: 35 | Dict[str, Any]: observation of robots in this task 36 | """ 37 | if not self.work: 38 | return {} 39 | obs = {} 40 | for robot_name, robot in self.robots.items(): 41 | try: 42 | obs[robot_name] = robot.get_obs() 43 | obs[robot_name]['sim_step'] = self.steps 44 | except Exception as e: 45 | log.error(self.name) 46 | log.error(e) 47 | traceback.print_exc() 48 | return {} 49 | return obs 50 | 51 | def calculate_metrics(self) -> Dict: 52 | metrics_res = {} 53 | for name, metric in self.metrics.items(): 54 | metrics_res[name] = metric.calc(dict(self.episode_meta)) 55 | 56 | return metrics_res 57 | 58 | def is_done(self) -> bool: 59 | self.step_counter = self.step_counter + 1 60 | return DataHub.get_episode_finished(self.runtime.name) or self.step_counter > self.settings.max_step 61 | 62 | def individual_reset(self): 63 | for name, metric in self.metrics.items(): 64 | metric.reset() 65 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/single_inference_task.py: -------------------------------------------------------------------------------- 1 | from omni.isaac.core.scenes import Scene 2 | 3 | from grutopia.core.runtime.task_runtime import TaskRuntime 4 | from grutopia.core.task import BaseTask 5 | 6 | 7 | @BaseTask.register('SingleInferenceTask') 8 | class SimpleInferenceTask(BaseTask): 9 | def __init__(self, runtime: TaskRuntime, scene: Scene): 10 | super().__init__(runtime, scene) 11 | 12 | def calculate_metrics(self) -> dict: 13 | pass 14 | 15 | def is_done(self) -> bool: 16 | return False 17 | 18 | def individual_reset(self): 19 | for name, metric in self.metrics.items(): 20 | metric.reset() 21 | -------------------------------------------------------------------------------- /grutopia_extension/tasks/social_navigation_task.py: -------------------------------------------------------------------------------- 1 | # import time 2 | import traceback 3 | from typing import Any, Dict 4 | 5 | from omni.isaac.core.scenes import Scene 6 | 7 | from grutopia.core.datahub import DataHub 8 | from grutopia.core.runtime.task_runtime import TaskRuntime 9 | from grutopia.core.task import BaseTask 10 | from grutopia.core.util import log 11 | from grutopia_extension.configs.tasks.social_navigation_task import TaskSettingCfg 12 | 13 | 14 | @BaseTask.register('SocialNavigationTask') 15 | class SocialNavigationTask(BaseTask): 16 | def __init__(self, runtime: TaskRuntime, scene: Scene): 17 | super().__init__(runtime, scene) 18 | self.step_counter = 0 19 | if isinstance(runtime.task_settings, TaskSettingCfg): 20 | self.settings = runtime.task_settings 21 | else: 22 | raise ValueError('task_settings must be a TaskSettingCfg') 23 | self.episode_meta = runtime.extra 24 | self.question = self.episode_meta['question'] 25 | 26 | log.info(f'task_settings: max_step : {self.settings.max_step}.)') 27 | # Episode 28 | log.info(f'Episode meta : question : {self.question}.)') 29 | 30 | def get_observations(self) -> Dict[str, Any]: 31 | """ 32 | Returns current observations from the objects needed for the behavioral layer. 33 | 34 | Return: 35 | Dict[str, Any]: observation of robots in this task 36 | """ 37 | if not self.work: 38 | return {} 39 | obs = {} 40 | for robot_name, robot in self.robots.items(): 41 | try: 42 | obs[robot_name] = robot.get_obs() 43 | obs[robot_name]['sim_step'] = self.steps 44 | except Exception as e: 45 | log.error(self.name) 46 | log.error(e) 47 | traceback.print_exc() 48 | return {} 49 | return obs 50 | 51 | def calculate_metrics(self) -> Dict: 52 | metrics_res = {} 53 | for name, metric in self.metrics.items(): 54 | metrics_res[name] = metric.calc(dict(self.episode_meta)) 55 | 56 | return metrics_res 57 | 58 | def is_done(self) -> bool: 59 | self.step_counter = self.step_counter + 1 60 | return DataHub.get_episode_finished(self.runtime.name) or self.step_counter > self.settings.max_step 61 | 62 | def individual_reset(self): 63 | for name, metric in self.metrics.items(): 64 | metric.reset() 65 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 120 3 | skip-string-normalization = true 4 | force-exclude = ''' 5 | lcmtypes 6 | ''' 7 | 8 | [tool.isort] 9 | profile = "black" 10 | skip_glob = '**/lcmtypes/**' 11 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --strict-markers 3 | markers = 4 | P0: the first priority in test cases 5 | P1: the second priority in test cases 6 | P2: the third priority in test cases 7 | P3: the fourth priority in test cases 8 | smoking: smoking test case 9 | performance: performance test case 10 | stress: stress test case 11 | function: function test case 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements/runtime.txt 2 | -r requirements/contribute.txt 3 | -r requirements/test.txt 4 | -r requirements/webui.txt 5 | -------------------------------------------------------------------------------- /requirements/benchmark.txt: -------------------------------------------------------------------------------- 1 | dashscope 2 | deepdiff 3 | git+https://github.com/naokiyokoyama/depth_camera_filtering 4 | git+https://github.com/naokiyokoyama/frontier_exploration.git 5 | open3d==0.18.0 6 | pathfinding 7 | shapely 8 | tiktoken 9 | -------------------------------------------------------------------------------- /requirements/contribute.txt: -------------------------------------------------------------------------------- 1 | codespell~=2.2.1 2 | flake8==4.0.1 3 | isort==5.11.5 4 | pre-commit==3.6.2 5 | -------------------------------------------------------------------------------- /requirements/docker_install_req.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | error_exit() 5 | { 6 | echo "There was an error running python" 7 | exit 1 8 | } 9 | 10 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 11 | # MY_DIR="$(realpath -s "$SCRIPT_DIR")" 12 | 13 | # Setup python env from generated file (generated by tools/repoman/build.py) 14 | export CARB_APP_PATH=$SCRIPT_DIR/kit 15 | export ISAAC_PATH=$SCRIPT_DIR 16 | export EXP_PATH=$SCRIPT_DIR/apps 17 | source "${SCRIPT_DIR}/setup_python_env.sh" 18 | 19 | # shellcheck disable=SC2237 20 | if ! [[ -z "${CONDA_PREFIX}" ]]; then 21 | echo "Warning: running in conda env, please deactivate before executing this script" 22 | echo "If conda is desired please source setup_conda_env.sh in your python 3.10 conda env and run python normally" 23 | fi 24 | 25 | # Check if we are running in a docker container 26 | if [ -f /.dockerenv ]; then 27 | # Check for vulkan in docker container 28 | if [[ -f "${SCRIPT_DIR}/vulkan_check.sh" ]]; then 29 | "${SCRIPT_DIR}/vulkan_check.sh" 30 | fi 31 | fi 32 | 33 | # Show icon if not running headless 34 | export RESOURCE_NAME="IsaacSim" 35 | # WAR for missing libcarb.so 36 | export LD_PRELOAD=$SCRIPT_DIR/kit/libcarb.so 37 | 38 | source /isaac-sim/.venv/bin/activate 39 | 40 | python -m pip install -e GRUtopia -i https://pypi.tuna.tsinghua.edu.cn/simple 41 | python -m pip install -r ./GRUtopia/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple 42 | python -m pip install openxlab==0.1.2 huggingface-hub==0.29.1 modelscope==1.23.0 -i https://pypi.tuna.tsinghua.edu.cn/simple 43 | -------------------------------------------------------------------------------- /requirements/runtime.txt: -------------------------------------------------------------------------------- 1 | addict==2.4.0 2 | aiohttp_cors==0.7.0 3 | httpcore==1.0.8 4 | httpx==0.25.2 5 | inputs==0.5 6 | ipython==8.20.0 7 | lcm==1.5.0 8 | onnxruntime==1.19.2 9 | openai==1.29.0 10 | pyyaml==6.0.1 11 | rsl-rl-lib==2.2.1 12 | transformers==4.26.1 13 | vuer==0.0.32rc7 14 | -------------------------------------------------------------------------------- /requirements/teleop.txt: -------------------------------------------------------------------------------- 1 | dex_retargeting==0.1.1 2 | lcm==1.5.0 3 | meshcat==0.3.2 4 | nlopt==2.7.1 5 | pin-pink==3.0.0 6 | trimesh==4.5.1 7 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | pytest==7.3.1 2 | pytest-cov==4.0.0 3 | -------------------------------------------------------------------------------- /requirements/webui.txt: -------------------------------------------------------------------------------- 1 | fastapi>=0.104.1 2 | psutil==5.9.6 3 | toml>=0.10.2 4 | uvicorn>=0.23.2 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [yapf] 2 | based_on_style = pep8 3 | blank_line_before_nested_class_or_def = true 4 | split_before_expression_after_opening_paren = true 5 | 6 | [isort] 7 | line_length = 79 8 | multi_line_output = 0 9 | extra_standard_library = pkg_resources,setuptools 10 | known_first_party = grutopia 11 | no_lines_before = STDLIB,LOCALFOLDER 12 | default_section = THIRDPARTY 13 | 14 | # ignore-words-list needs to be lowercase format. For example, if we want to 15 | # ignore word "BA", then we need to append "ba" to ignore-words-list rather 16 | # than "BA" 17 | [codespell] 18 | quiet-level = 3 19 | ignore-words-list = patten,nd,ty,mot,hist,formating,jetbot,wth 20 | skip = *.js 21 | 22 | [flake8] 23 | # The E251 check is conflict with yapf in some situation. 24 | # See https://github.com/google/yapf/issues/393 25 | extend-ignore = E251 26 | # The F401 check is wrong if the `__all__` variable is modified 27 | # in `objects.py` 28 | per-file-ignores = 29 | */__init__.py: F401 30 | max-line-length = 120 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open('README.md', 'r') as fh: 4 | long_description = fh.read() 5 | 6 | with open('requirements/runtime.txt', 'r') as fr: 7 | requirements = fr.readlines() 8 | 9 | setuptools.setup( 10 | name='grutopia', 11 | version='2.1.1', 12 | author='APX103', 13 | author_email='lijialun@pjlab.org.cn', 14 | description='Easy to use omniverse isaac sim standalone package', 15 | long_description=long_description, 16 | long_description_content_type='text/markdown', 17 | packages=setuptools.find_packages(), 18 | classifiers=[ 19 | 'Programming Language :: Python :: 3', 20 | 'Operating System :: OS Independent', 21 | ], 22 | install_requires=[i.strip() for i in requirements], 23 | python_requires='>=3.10', 24 | ) 25 | -------------------------------------------------------------------------------- /tests/e2e_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | import pytest 7 | 8 | 9 | def common_body(cmd_line): 10 | with subprocess.Popen( 11 | cmd_line, 12 | stdin=subprocess.PIPE, 13 | stderr=sys.stderr, 14 | close_fds=True, 15 | stdout=sys.stdout, 16 | universal_newlines=True, 17 | shell=True, 18 | bufsize=1, 19 | ) as cmd: 20 | cmd.communicate() 21 | assert cmd.returncode == 0, f'real exit code is {cmd.returncode}' 22 | 23 | 24 | def update_jsonl_from_json(json_file_path, jsonl_file_path, update_item): 25 | with open(json_file_path, 'r', encoding='utf-8') as json_file: 26 | data = json.load(json_file) 27 | data = {**update_item, **data} 28 | if not isinstance(data, list): 29 | data = [data] 30 | with open(jsonl_file_path, 'a', encoding='utf-8') as jsonl_file: 31 | for item in data: 32 | json_line = json.dumps(item, ensure_ascii=False) 33 | jsonl_file.write(json_line + '\n') 34 | 35 | 36 | def teardown_function(function): 37 | if os.path.exists('./test_result.json'): 38 | case_info = {} 39 | test_name = function.__name__ 40 | case_info['case_info'] = test_name + '_' + os.environ.get('JOB_ID') 41 | update_jsonl_from_json('./test_result.json', '../total_result.jsonl', case_info) 42 | else: 43 | print('Warning! There is no test_result.json') 44 | 45 | 46 | @pytest.mark.P0 47 | def test_h1_locomotion(): 48 | start_command = 'python ./tests/h1_locomotion.py' 49 | common_body(start_command) 50 | 51 | 52 | @pytest.mark.P0 53 | def test_load_scene_without_robot(): 54 | start_command = 'python ./tests/load_scene_without_robot.py' 55 | common_body(start_command) 56 | 57 | 58 | @pytest.mark.P0 59 | def test_rep_camera_pointcloud(): 60 | start_command = 'python ./tests/rep_camera_pointcloud.py' 61 | common_body(start_command) 62 | 63 | 64 | @pytest.mark.P0 65 | def test_robots(): 66 | start_command = 'ls ./tests/robots/*.py | grep -v test_ | while read f; do echo "run $f" && python $f; done' 67 | common_body(start_command) 68 | -------------------------------------------------------------------------------- /tests/e2e_test.yaml: -------------------------------------------------------------------------------- 1 | variables: 2 | PYTHONUNBUFFERED: 1 3 | 4 | e2e-demo-tests: 5 | stage: e2e_test 6 | tags: 7 | - grutopia_0510 8 | script: 9 | - export PATH=/isaac-sim/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH 10 | - export JOB_ID=${CI_JOB_ID} 11 | - ln -s /isaac-sim/GRUtopia/testk ./grutopia/assets 12 | - pytest -s -v -m "P0" ./tests/e2e_test.py 13 | rules: 14 | - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^release/ || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "proto") 15 | -------------------------------------------------------------------------------- /tests/load_scene_without_robot.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | from grutopia.core.config import Config, SimConfig 3 | from grutopia.core.gym_env import Env 4 | from grutopia.core.runtime import SimulatorRuntime 5 | from grutopia.core.util import has_display 6 | from grutopia.macros import gm 7 | from grutopia_extension import import_extensions 8 | from grutopia_extension.configs.tasks import ( 9 | SingleInferenceEpisodeCfg, 10 | SingleInferenceTaskCfg, 11 | ) 12 | 13 | headless = False 14 | if not has_display(): 15 | headless = True 16 | 17 | config = Config( 18 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 19 | task_config=SingleInferenceTaskCfg( 20 | episodes=[ 21 | SingleInferenceEpisodeCfg( 22 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 23 | scene_scale=(0.01, 0.01, 0.01), 24 | robots=[], 25 | ), 26 | ], 27 | ), 28 | ) 29 | 30 | print(config.model_dump_json(indent=4)) 31 | 32 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 33 | 34 | import_extensions() 35 | 36 | env = Env(sim_runtime) 37 | obs, _ = env.reset() 38 | print(f'========INIT OBS{obs}=============') 39 | 40 | i = 0 41 | 42 | while env.simulation_app.is_running(): 43 | i += 1 44 | obs, _, terminated, _, _ = env.step(action={}) 45 | 46 | if i == 2000: 47 | break 48 | 49 | env.close() 50 | 51 | 52 | if __name__ == '__main__': 53 | try: 54 | main() 55 | except Exception as e: 56 | print(f'exception is {e}') 57 | import sys 58 | import traceback 59 | 60 | traceback.print_exc() 61 | sys.exit(1) 62 | -------------------------------------------------------------------------------- /tests/robots/aliengo.py: -------------------------------------------------------------------------------- 1 | from test_move_to_point import run 2 | 3 | from grutopia_extension.configs.robots.aliengo import AliengoRobotCfg, move_to_point_cfg 4 | 5 | if __name__ == '__main__': 6 | try: 7 | target = (3.0, 2.0, 0.0) 8 | case = { 9 | 'robot': AliengoRobotCfg( 10 | position=[0.0, 0.0, 1.05], 11 | controllers=[move_to_point_cfg], 12 | ), 13 | 'action': {move_to_point_cfg.name: [target]}, 14 | 'target': target, 15 | } 16 | run(**case) 17 | except Exception as e: 18 | print(f'exception is {e}') 19 | import sys 20 | import traceback 21 | 22 | traceback.print_exc() 23 | sys.exit(1) 24 | -------------------------------------------------------------------------------- /tests/robots/franka.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.macros import gm 5 | from grutopia_extension import import_extensions 6 | from grutopia_extension.configs.robots.franka import ( 7 | FrankaRobotCfg, 8 | arm_ik_cfg, 9 | gripper_cfg, 10 | ) 11 | from grutopia_extension.configs.tasks import ( 12 | SingleInferenceEpisodeCfg, 13 | SingleInferenceTaskCfg, 14 | ) 15 | 16 | franka = FrankaRobotCfg( 17 | position=[0, 0, 0], 18 | controllers=[ 19 | arm_ik_cfg, 20 | gripper_cfg, 21 | ], 22 | ) 23 | 24 | config = Config( 25 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=True), 26 | task_config=SingleInferenceTaskCfg( 27 | episodes=[ 28 | SingleInferenceEpisodeCfg( 29 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 30 | robots=[franka], 31 | ), 32 | ], 33 | ), 34 | ) 35 | 36 | sim_runtime = SimulatorRuntime(config_class=config, headless=False, webrtc=False, native=True) 37 | 38 | import_extensions() 39 | import numpy as np 40 | from omni.isaac.core.utils.rotations import euler_angles_to_quat 41 | 42 | env = Env(sim_runtime) 43 | obs, _ = env.reset() 44 | print(f'========INIT OBS{obs}=============') 45 | 46 | actions = [ 47 | {arm_ik_cfg.name: [np.array([0.4, 0, 0.45]), euler_angles_to_quat((0.0, 0.0, 0.0))]}, 48 | {gripper_cfg.name: ['open']}, 49 | {arm_ik_cfg.name: [np.array([0.4, 0.4, 0.1]), euler_angles_to_quat((np.pi / 2, np.pi / 2, np.pi / 2))]}, 50 | {gripper_cfg.name: ['close']}, 51 | ] 52 | 53 | i = 0 54 | action_idx = 0 55 | max_steps = 2000 56 | while env.simulation_app.is_running(): 57 | i += 1 58 | env_action = actions[action_idx] 59 | 60 | obs, _, terminated, _, _ = env.step(action=env_action) 61 | finished = obs['controllers'][arm_ik_cfg.name]['finished'] 62 | if finished: 63 | print(f'action finished: env_action={env_action}, obs={obs}') 64 | action_idx += 1 65 | if action_idx >= len(actions): 66 | print('all actions finished') 67 | break 68 | 69 | assert i < max_steps, f'max steps reached, env_action={env_action}, obs={obs}' 70 | if i % 1000 == 0: 71 | print(i) 72 | print(obs) 73 | 74 | 75 | env.close() 76 | -------------------------------------------------------------------------------- /tests/robots/g1.py: -------------------------------------------------------------------------------- 1 | from test_move_to_point import run 2 | 3 | from grutopia_extension.configs.robots.g1 import G1RobotCfg, move_to_point_cfg 4 | 5 | if __name__ == '__main__': 6 | try: 7 | target = (3.0, 2.0, 0.0) 8 | case = { 9 | 'robot': G1RobotCfg( 10 | position=[0.0, 0.0, 0.8], 11 | controllers=[move_to_point_cfg], 12 | ), 13 | 'action': {move_to_point_cfg.name: [target]}, 14 | 'target': target, 15 | } 16 | run(**case) 17 | except Exception as e: 18 | print(f'exception is {e}') 19 | import sys 20 | import traceback 21 | 22 | traceback.print_exc() 23 | sys.exit(1) 24 | -------------------------------------------------------------------------------- /tests/robots/gr1.py: -------------------------------------------------------------------------------- 1 | from test_move_to_point import run 2 | 3 | from grutopia_extension.configs.robots.gr1 import GR1RobotCfg, move_to_point_cfg 4 | 5 | if __name__ == '__main__': 6 | try: 7 | target = (3.0, 2.0, 0.0) 8 | case = { 9 | 'robot': GR1RobotCfg( 10 | position=[0.0, 0.0, 0.95], 11 | controllers=[move_to_point_cfg], 12 | ), 13 | 'action': {move_to_point_cfg.name: [target]}, 14 | 'target': target, 15 | } 16 | run(**case) 17 | except Exception as e: 18 | print(f'exception is {e}') 19 | import sys 20 | import traceback 21 | 22 | traceback.print_exc() 23 | sys.exit(1) 24 | -------------------------------------------------------------------------------- /tests/robots/h1.py: -------------------------------------------------------------------------------- 1 | from test_move_to_point import run 2 | 3 | from grutopia_extension.configs.robots.h1 import H1RobotCfg, move_to_point_cfg 4 | 5 | if __name__ == '__main__': 6 | try: 7 | target = (3.0, 2.0, 0.0) 8 | case = { 9 | 'robot': H1RobotCfg( 10 | position=[0.0, 0.0, 1.05], 11 | controllers=[move_to_point_cfg], 12 | ), 13 | 'action': {move_to_point_cfg.name: [target]}, 14 | 'target': target, 15 | } 16 | run(**case) 17 | except Exception as e: 18 | print(f'exception is {e}') 19 | import sys 20 | import traceback 21 | 22 | traceback.print_exc() 23 | sys.exit(1) 24 | -------------------------------------------------------------------------------- /tests/robots/jetbot.py: -------------------------------------------------------------------------------- 1 | from test_move_to_point import run 2 | 3 | from grutopia_extension.configs.robots.jetbot import JetbotRobotCfg, move_to_point_cfg 4 | 5 | if __name__ == '__main__': 6 | try: 7 | target = (3.0, 2.0, 0.0) 8 | case = { 9 | 'robot': JetbotRobotCfg( 10 | position=[0.0, 0.0, 0.0], 11 | controllers=[move_to_point_cfg], 12 | ), 13 | 'action': {move_to_point_cfg.name: [target]}, 14 | 'target': target, 15 | } 16 | run(**case) 17 | except Exception as e: 18 | print(f'exception is {e}') 19 | import sys 20 | import traceback 21 | 22 | traceback.print_exc() 23 | sys.exit(1) 24 | -------------------------------------------------------------------------------- /tests/robots/test_move_to_point.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from typing import Any, Dict, Tuple 3 | 4 | from grutopia.core.config import Config, SimConfig 5 | from grutopia.core.gym_env import Env 6 | from grutopia.core.runtime import SimulatorRuntime 7 | from grutopia.core.util import has_display 8 | from grutopia.macros import gm 9 | from grutopia_extension import import_extensions 10 | from grutopia_extension.configs.tasks import ( 11 | SingleInferenceEpisodeCfg, 12 | SingleInferenceTaskCfg, 13 | ) 14 | 15 | headless = not has_display() 16 | 17 | 18 | def run(robot, action: Dict[str, Any], target: Tuple[float, float, float], max_steps=5000): 19 | config = Config( 20 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=True), 21 | task_config=SingleInferenceTaskCfg( 22 | episodes=[ 23 | SingleInferenceEpisodeCfg( 24 | scene_asset_path=gm.ASSET_PATH + '/scenes/empty.usd', 25 | robots=[robot], 26 | ), 27 | ], 28 | ), 29 | ) 30 | 31 | sim_runtime = SimulatorRuntime(config_class=config, headless=headless, native=headless) 32 | 33 | import_extensions() 34 | # import custom extensions here. 35 | import numpy as np 36 | 37 | env = Env(sim_runtime) 38 | obs, _ = env.reset() 39 | 40 | i = 0 41 | 42 | env_action = action 43 | 44 | print(f'actions: {env_action}') 45 | confirm = 0 46 | 47 | while env.simulation_app.is_running(): 48 | i += 1 49 | obs, _, _, _, _ = env.step(action=env_action) 50 | 51 | position = deepcopy(obs['position']) 52 | position[2] = target[2] # Make sure we ignore z axis 53 | error = np.linalg.norm(position - target) 54 | 55 | if error < 0.1: 56 | confirm += 1 57 | if confirm == 10: 58 | print(f'step {i}: target={target} reached with position={position}') 59 | break 60 | assert i < max_steps, f'max steps reached, position={position}, target={target}, error={error}' 61 | 62 | env.close() 63 | 64 | 65 | if __name__ == '__main__': 66 | pass 67 | -------------------------------------------------------------------------------- /toolkits/grscenes_scripts/play_scene.py: -------------------------------------------------------------------------------- 1 | """This script is used to load and play a scene in the Isaac Sim GUI. 2 | 3 | Usage: 4 | python play_scene.py -f {scene_usd_file} 5 | """ 6 | import argparse 7 | import os 8 | 9 | parser = argparse.ArgumentParser(description='Load and play the scene in the Isaac Sim GUI.') 10 | parser.add_argument('-f', '--file', required=True, type=str, help='the usd file to play') 11 | 12 | args = parser.parse_args() 13 | 14 | f_abs_path = os.path.abspath(args.file) 15 | if not os.path.exists(f_abs_path): 16 | print(f'Error! {args.file} not found!') 17 | else: 18 | from isaacsim import SimulationApp 19 | 20 | CONFIG = {'headless': False} 21 | kit = SimulationApp(launch_config=CONFIG) 22 | 23 | import omni.physx.bindings._physx as physx_bindings 24 | 25 | kit.set_setting(physx_bindings.SETTING_UJITSO_COLLISION_COOKING, False) 26 | 27 | kit.context.open_stage(f_abs_path) 28 | stage = kit.context.get_stage() 29 | kit._wait_for_viewport() 30 | 31 | import omni.timeline 32 | 33 | omni.timeline.get_timeline_interface().play() 34 | while kit.is_running(): 35 | kit.update() 36 | -------------------------------------------------------------------------------- /toolkits/indoor_scenes_generation/infinigen/infinigen/assets/static_assets/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import StaticAssetFactory 2 | from .static_category import ( 3 | StaticBackpackFactory, 4 | StaticBasketFactory, 5 | StaticBathtubFactory, 6 | StaticBedFactory, 7 | StaticBicycleFactory, 8 | StaticBlanketFactory, 9 | StaticBookFactory, 10 | StaticBookshelfFactory, 11 | StaticBottleFactory, 12 | StaticBowlFactory, 13 | StaticBoxFactory, 14 | StaticCabinetFactory, 15 | StaticCartFactory, 16 | StaticCeilinglightFactory, 17 | StaticChairFactory, 18 | StaticChestofdrawersFactory, 19 | StaticClockFactory, 20 | StaticClothesFactory, 21 | StaticCoffeemakerFactory, 22 | StaticCouchFactory, 23 | StaticCounterFactory, 24 | StaticCupFactory, 25 | StaticCurtainFactory, 26 | StaticDecorationFactory, 27 | StaticDeskFactory, 28 | StaticDishwasherFactory, 29 | StaticDoorFactory, 30 | StaticElectriccookerFactory, 31 | StaticFanFactory, 32 | StaticFaucetFactory, 33 | StaticHearthFactory, 34 | StaticKeyboardFactory, 35 | StaticLampFactory, 36 | StaticLaptopFactory, 37 | StaticLightFactory, 38 | StaticMicrowaveFactory, 39 | StaticMirrorFactory, 40 | StaticMonitorFactory, 41 | StaticMouseFactory, 42 | StaticMusicalinstrumentFactory, 43 | StaticNightstandFactory, 44 | StaticOtherFactory, 45 | StaticOvenFactory, 46 | StaticPanFactory, 47 | StaticPenFactory, 48 | StaticPersonFactory, 49 | StaticPianoFactory, 50 | StaticPictureFactory, 51 | StaticPillowFactory, 52 | StaticPlantFactory, 53 | StaticPlateFactory, 54 | StaticPotFactory, 55 | StaticRefrigeratorFactory, 56 | StaticShelfFactory, 57 | StaticShoecabinetFactory, 58 | StaticShoeFactory, 59 | StaticSideboardcabinetFactory, 60 | StaticSinkFactory, 61 | StaticSofachairFactory, 62 | StaticSpeakerFactory, 63 | StaticStoolFactory, 64 | StaticTableFactory, 65 | StaticTeatableFactory, 66 | StaticTelephoneFactory, 67 | StaticToiletFactory, 68 | StaticTowelFactory, 69 | StaticToyFactory, 70 | StaticTrashcanFactory, 71 | StaticTrayFactory, 72 | StaticTvFactory, 73 | StaticTvstandFactory, 74 | StaticWallFactory, 75 | StaticWashingmachineFactory, 76 | static_category_factory, 77 | ) 78 | -------------------------------------------------------------------------------- /toolkits/layout_edit/layout_edit.py: -------------------------------------------------------------------------------- 1 | from grutopia.core.config import Config, SimConfig 2 | from grutopia.core.gym_env import Env 3 | from grutopia.core.runtime import SimulatorRuntime 4 | from grutopia.macros import gm 5 | from grutopia_extension import import_extensions 6 | from grutopia_extension.configs.robots.mocap_controlled_franka import ( 7 | MocapControlledFrankaRobotCfg, 8 | layout_cfg, 9 | layout_controlled_camera_cfg, 10 | ) 11 | from grutopia_extension.configs.tasks import ( 12 | ManipulationEpisodeCfg, 13 | ManipulationExtra, 14 | ManipulationTaskCfg, 15 | ManipulationTaskSetting, 16 | ) 17 | 18 | franka = MocapControlledFrankaRobotCfg( 19 | position=[-0.35, 100.0, 1.05], 20 | controllers=[ 21 | layout_cfg, 22 | ], 23 | sensors=[layout_controlled_camera_cfg], 24 | ) 25 | 26 | config = Config( 27 | simulator=SimConfig(physics_dt=1 / 240, rendering_dt=1 / 240, use_fabric=False), 28 | task_config=ManipulationTaskCfg( 29 | episodes=[ 30 | ManipulationEpisodeCfg( 31 | scene_asset_path=gm.ASSET_PATH + '/scenes/demo_scenes/franka_mocap_teleop/table_scene.usd', 32 | robots=[franka], 33 | extra=ManipulationExtra( 34 | prompt='Prompt test 1', 35 | target='layout_edit', 36 | episode_idx=0, 37 | ), 38 | ), 39 | ], 40 | task_settings=ManipulationTaskSetting(max_step=10000), 41 | ), 42 | ) 43 | 44 | sim_runtime = SimulatorRuntime(config_class=config, headless=False, webrtc=False, native=True) 45 | 46 | import_extensions() 47 | from grutopia_extension.interactions.motion_capture import MocapInteraction 48 | 49 | env = Env(sim_runtime) 50 | obs, _ = env.reset() 51 | 52 | mocap_url = 'http://127.0.0.1:5001' 53 | mocap_interaction = MocapInteraction(mocap_url) 54 | 55 | while env.simulation_app.is_running(): 56 | cur_mocap_info = mocap_interaction.step() 57 | arm_action = {layout_cfg.name: [cur_mocap_info]} 58 | 59 | obs, _, _, _, _ = env.step(action=arm_action) 60 | 61 | mocap_interaction.server_stop() 62 | env.close() 63 | -------------------------------------------------------------------------------- /toolkits/mocap/hamer_real_time_server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from flask import Flask, jsonify 4 | from hamer_real_time import CameraProcessor 5 | 6 | 7 | def create_app(video_url): 8 | app = Flask(__name__) 9 | 10 | processor = CameraProcessor(video_url=video_url) 11 | 12 | @app.route('/start', methods=['GET']) 13 | def start_service(): 14 | if not processor.running: 15 | processor.start() 16 | print('Video streaming started!') 17 | return jsonify({'status': 'started', 'message': 'Video processing started.'}), 200 18 | else: 19 | return jsonify({'status': 'error', 'message': 'Already running.'}), 400 20 | 21 | @app.route('/stop', methods=['GET']) 22 | def stop_service(): 23 | if processor.running: 24 | processor.stop() 25 | print('Video streaming stopped!') 26 | return jsonify({'status': 'stopped', 'message': 'Video processing stopped.'}), 200 27 | else: 28 | return jsonify({'status': 'error', 'message': 'Not running.'}), 400 29 | 30 | @app.route('/results', methods=['GET']) 31 | def get_results(): 32 | results = processor.get_results() 33 | if results: 34 | return jsonify(results), 200 35 | else: 36 | return jsonify({'status': 'error', 'message': 'No results available.'}), 200 37 | 38 | return app 39 | 40 | 41 | if __name__ == '__main__': 42 | parser = argparse.ArgumentParser(description='Run Flask service with video processing.') 43 | parser.add_argument('--video_url', type=str, required=True, help='URL of the video stream to process.') 44 | args = parser.parse_args() 45 | 46 | app = create_app(video_url=args.video_url) 47 | app.run(host='0.0.0.0', port=5001) 48 | -------------------------------------------------------------------------------- /toolkits/mocap/rgb_camera_server.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from flask import Flask, Response 3 | 4 | app = Flask(__name__) 5 | cap = cv2.VideoCapture(0) 6 | 7 | 8 | def generate_frames(): 9 | while True: 10 | success, frame = cap.read() 11 | if not success: 12 | break 13 | else: 14 | ret, buffer = cv2.imencode('.jpg', frame) 15 | frame = buffer.tobytes() 16 | yield (b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n') 17 | 18 | 19 | @app.route('/video') 20 | def video(): 21 | return Response(generate_frames(), mimetype='multipart/x-mixed-replace; boundary=frame') 22 | 23 | 24 | if __name__ == '__main__': 25 | app.run(host='0.0.0.0', port=5000) 26 | --------------------------------------------------------------------------------