├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── LICENSE_HEADER.py ├── LICENSE_README.md ├── README.md ├── add_license_headers.py ├── isaac.sim.mcp_extension ├── .cursorrules ├── config │ └── extension.toml ├── examples │ ├── franka.py │ ├── franka_grid.py │ ├── g1.py │ └── go1.py └── isaac_sim_mcp_extension │ ├── __init__.py │ ├── extension.py │ ├── gen3d.py │ └── usd.py ├── isaac_mcp ├── __init__.py └── server.py └── media ├── add_more_robot_into_party.gif └── add_more_robot_into_party.mp4 /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | MANIFEST 23 | 24 | # Virtual environments 25 | .env 26 | .venv 27 | env/ 28 | venv/ 29 | ENV/ 30 | env.bak/ 31 | venv.bak/ 32 | 33 | # IDE - VSCode 34 | .vscode/* 35 | !.vscode/settings.json 36 | !.vscode/tasks.json 37 | !.vscode/launch.json 38 | !.vscode/extensions.json 39 | 40 | # IDE - PyCharm 41 | .idea/ 42 | *.iml 43 | *.iws 44 | *.ipr 45 | .idea_modules/ 46 | 47 | # IDE - Cursor 48 | .cursor/ 49 | 50 | # Logs 51 | logs/ 52 | *.log 53 | npm-debug.log* 54 | yarn-debug.log* 55 | yarn-error.log* 56 | 57 | # OS specific 58 | .DS_Store 59 | .DS_Store? 60 | ._* 61 | .Spotlight-V100 62 | .Trashes 63 | ehthumbs.db 64 | Thumbs.db 65 | 66 | # Isaac Sim specific 67 | *.usd 68 | *.usda 69 | *.usdc 70 | *.usdz 71 | local_cache/ 72 | *checkpoint.pth 73 | /results/ 74 | 75 | # MCP specific 76 | mcp_cache/ 77 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to the isaac-sim-mcp project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | 9 | ## [0.3.0] - 2024-04-22 10 | 11 | ### Added 12 | - USD asset search integration with `search_3d_usd_by_text` tool 13 | - Ability to search and load pre-existing 3D models from USD libraries 14 | - Support for custom positioning and scaling of USD models 15 | - Direct model transformation capabilities with the improved `transform` tool 16 | - Enhanced scene management with multi-object placement 17 | 18 | ### Improved 19 | - Scene object manipulation with precise positioning controls 20 | - Asset loading performance and reliability 21 | - Error handling for model search and placement 22 | - Integration with existing physics scene management 23 | 24 | ### Technical Details 25 | - Advanced USD model retrieval system 26 | - Optimized asset loading pipeline 27 | - Position and scale customization for USD models 28 | - Better compatibility with Isaac Sim's native USD handling 29 | 30 | ## [0.2.1] - 2024-04-15 31 | 32 | ### Added 33 | - Beaver3D integration for 3D model generation from text prompts and images 34 | - Asynchronous model loading with asyncio support 35 | - Task caching system to prevent duplicate model generation 36 | - New MCP tools: 37 | - `generate_3d_from_text_or_image` for AI-powered 3D asset creation 38 | - `transform` for manipulating generated 3D models in the scene 39 | - Texture and material binding for generated 3D models 40 | 41 | ### Improved 42 | - Asynchronous command execution with `run_coroutine` 43 | - Error handling and reporting for 3D generation tasks 44 | - Performance optimizations for model loading 45 | 46 | ### Technical Details 47 | - Integration with Beaver3D API for 3D generation 48 | - Task monitoring with callback support 49 | - Position and scale customization for generated models 50 | 51 | ## [0.1.0] - 2025-04-02 52 | 53 | ### Added 54 | - Initial implementation of Isaac Sim MCP Extension 55 | - Natural language control interface for Isaac Sim through MCP framework 56 | - Core robot manipulation capabilities: 57 | - Dynamic placement and positioning of robots (Franka, G1, Go1, Jetbot) 58 | - Robot movement controls with position updates 59 | - Multi-robot grid creation (3x3 arrangement support) 60 | - Advanced simulation features: 61 | - Quadruped robot walking simulation with waypoint navigation 62 | - Physics-based interactions between robots and environment 63 | - Custom lighting controls for better scene visualization 64 | - Environment enrichment: 65 | - Various obstacle types: boxes, spheres, cylinders, cones 66 | - Wall creation for maze-like environments 67 | - Dynamic obstacle placement with customizable properties 68 | - Development tools: 69 | - MCP server integration with Cursor AI 70 | - Debug interface accessible via local web server 71 | - Connection status verification with `get_scene_info` 72 | - Documentation: 73 | - Installation instructions 74 | - Example prompts for common simulation scenarios 75 | - Configuration guidelines 76 | 77 | ### Technical Details 78 | - Extension server running on localhost:8766 79 | - Compatible with NVIDIA Isaac Sim 4.2.0 80 | - Support for Python 3.9+ 81 | - MIT License for open development -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 omni-mcp 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE_HEADER.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | # How to use this header: 26 | # 1. Copy the license text above (excluding this instruction block) 27 | # 2. Paste at the beginning of each Python source file 28 | # 3. Optionally, add a brief description of the file below the license 29 | # 30 | # Example: 31 | # """ 32 | # MIT License 33 | # 34 | # Copyright (c) 2023-2025 omni-mcp 35 | # 36 | # [License text...] 37 | # """ 38 | # 39 | # Description: This module implements the core functionality for the Isaac Sim MCP extension. -------------------------------------------------------------------------------- /LICENSE_README.md: -------------------------------------------------------------------------------- 1 | # Source Code Licensing 2 | 3 | This project is licensed under the MIT License. To properly maintain licensing information across the codebase, we provide utilities for applying license headers to all Python source files. 4 | 5 | ## License Files 6 | 7 | - `LICENSE` - The main MIT license file for the project 8 | - `LICENSE_HEADER.py` - Example license header for Python files 9 | - `add_license_headers.py` - Utility script to automatically add license headers to Python files 10 | 11 | ## Adding License Headers to Source Files 12 | 13 | You can add the MIT license header to all Python files in your project by running: 14 | 15 | ```bash 16 | python add_license_headers.py 17 | ``` 18 | 19 | This will recursively search through the project directory and add the license header to any Python file that doesn't already have one. 20 | 21 | You can also specify a specific directory to process: 22 | 23 | ```bash 24 | python add_license_headers.py path/to/directory 25 | ``` 26 | 27 | ## License Header Format 28 | 29 | Each Python source file should have the following license header at the top: 30 | 31 | ```python 32 | """ 33 | MIT License 34 | 35 | Copyright (c) 2023-2025 omni-mcp 36 | 37 | Permission is hereby granted, free of charge, to any person obtaining a copy 38 | of this software and associated documentation files (the "Software"), to deal 39 | in the Software without restriction, including without limitation the rights 40 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 41 | copies of the Software, and to permit persons to whom the Software is 42 | furnished to do so, subject to the following conditions: 43 | 44 | The above copyright notice and this permission notice shall be included in all 45 | copies or substantial portions of the Software. 46 | 47 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 48 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 49 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 50 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 51 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 52 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 53 | SOFTWARE. 54 | """ 55 | ``` 56 | 57 | Followed by a brief description of the file's purpose: 58 | 59 | ```python 60 | # Description: This file implements... 61 | ``` 62 | 63 | ## Adding License Headers Manually 64 | 65 | If you prefer to add headers manually, or are creating a new file, simply copy the header from `LICENSE_HEADER.py` and paste it at the beginning of your Python file. 66 | 67 | ## Excluded Directories 68 | 69 | The automated script skips the following directories: 70 | - `.git` 71 | - `.vscode` 72 | - `__pycache__` 73 | - `venv`, `env`, `.env` 74 | - `build` 75 | - `dist` 76 | 77 | If you need to modify this list, edit the `SKIP_DIRS` variable in `add_license_headers.py`. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Isaac Sim MCP Extension and MCP Server 2 | 3 | The MCP Server and its extension leverage the Model Context Protocol (MCP) framework to enable natural language control of NVIDIA Isaac Sim, transforming conversational AI inputs into precise simulation manipulation. This expansion bridges the MCP ecosystem with embodied intelligence applications. 4 | 5 | ## Features 6 | 7 | - Natural language control of Isaac Sim 8 | - Dynamic robot positioning and movement 9 | - Custom lighting and scene creation 10 | - Advanced robot simulations with obstacle navigation 11 | - Interactive code preview before execution 12 | 13 | ## Requirements 14 | 15 | - NVIDIA Isaac Sim 4.2.0 or higher 16 | - Python 3.9+ 17 | - Cursor AI editor for MCP integration 18 | 19 | ## **Mandatory** Pre-requisite 20 | 21 | - Install uv/uvx: [https://github.com/astral-sh/uv](https://github.com/astral-sh/uv) 22 | - Install mcp[cli] to base env: [uv pip install "mcp[cli]"](https://pypi.org/project/mcp/) 23 | 24 | ## Installation 25 | 26 | ```bash 27 | cd ~/Documents 28 | git clone https://github.com/omni-mcp/isaac-sim-mcp 29 | ``` 30 | 31 | ### Install and Enable Extension 32 | 33 | Isaac Sim extension folder should point to your project folder: 34 | - Extension location: `~/Documents/isaac-sim-mcp` 35 | - Extension ID: `isaac.sim.mcp_extension` 36 | 37 | ```bash 38 | # Enable extension in Isaac Simulation 39 | # cd to your Isaac Sim installation directory 40 | # You can change assets root to local with --/persistent/isaac/asset_root/default="" 41 | # By default it is an AWS bucket, e.g. --/persistent/isaac/asset_root/default="/share/Assets/Isaac/4.2" 42 | # Setup API KEY for Beaver3d and NVIDIA 43 | export BEAVER3D_MODEL= 44 | export export ARK_API_KEY= 45 | export NVIDIA_API_KEY="" 46 | 47 | cd ~/.local/share/ov/pkg/isaac-sim-4.2.0 48 | ./isaac-sim.sh --ext-folder /home/ubuntu/Documents/isaac-sim-mcp/ --enable isaac.sim.mcp_extension 49 | ``` 50 | 51 | Verify the extension starts successfully. The output should look like: 52 | 53 | ``` 54 | [7.160s] [ext: isaac.sim.mcp_extension-0.1.0] startup 55 | trigger on_startup for: isaac.sim.mcp_extension-0.1.0 56 | settings: {'envPath': '/home/ubuntu/.local/share/ov/data/Kit/Isaac-Sim/4.2/pip3-envs/default', 'archiveDirs': {}, 'installCheckIgnoreVersion': False, 'allowOnlineIndex': True, 'tryUpgradePipOnFirstUse': False} 57 | Server thread startedIsaac Sim MCP server started on localhost:8766 58 | ``` 59 | 60 | The extension should be listening at **localhost:8766** by default. 61 | 62 | 63 | 64 | ### Install MCP Server 65 | 66 | 1. Go to terminal and run, make sure mcp server could start sucessfully at terminal with base venv. 67 | ``` 68 | uv pip install "mcp[cli]" 69 | uv run /home/ubuntu/Documents/isaac-sim-mcp/isaac_mcp/server.py 70 | ``` 71 | 2. Start Cursor and open the folder `~/Documents/isaac-sim-mcp` 72 | 3. Go to Cursor preferences, choose MCP and add a global MCP server: 73 | 74 | ```json 75 | { 76 | "mcpServers": { 77 | "isaac-sim": { 78 | "command": "uv run /home/ubuntu/Documents/isaac-sim-mcp/isaac_mcp/server.py" 79 | } 80 | } 81 | } 82 | ``` 83 | 84 | ### Development Mode 85 | 86 | To develop the MCP Server, start the MCP inspector: 87 | 88 | ```bash 89 | uv run mcp dev ~/Documents/isaac-sim-mcp/isaac_mcp/server.py 90 | ``` 91 | 92 | You can visit the debug page through http://localhost:5173 93 | 94 | ## Example Prompts for Simulation 95 | Notice: Switch to Agent mode in top left of Chat dialog before you type prompt and choose sonnet 3.7 for better coding. 96 | 97 | ### Robot Party 98 | ``` 99 | # Create robots and improve lighting 100 | create 3x3 frankas robots in these current stage across location [3, 0, 0] and [6, 3, 0] 101 | always check connection with get_scene_info before execute code. 102 | add more light in the stage 103 | 104 | 105 | # Add specific robots at positions 106 | create a g1 robot at [3, 9, 0] 107 | add Go1 robot at location [2, 1, 0] 108 | move go1 robot to [1, 1, 0] 109 | ``` 110 | 111 | ### Factory Setup 112 | ``` 113 | # Create multiple robots in a row 114 | acreate 3x3 frankas robots in these current stage across location [3, 0, 0] and [6, 3, 0] 115 | always check connection with get_scene_info before execute code. 116 | add more light in the stage 117 | 118 | 119 | ``` 120 | ### Vibe Coding from scratch 121 | ``` 122 | reference to g1.py to create an new g1 robot simulation and allow robot g1 walk straight from [0, 0, 0] to [3, 0, 0] and [3, 3, 0] 123 | create more obstacles in the stage 124 | 125 | ``` 126 | ### Gen3D with beaver3d model support 127 | 128 | ``` 129 | Use following images to generate beaver 3d objects and place them into a grid area across [0, 0, 0] to [40, 40, 0] with scale [3, 3, 3] 130 | 131 | 132 | ``` 133 | 134 | ### USD search 135 | ``` 136 | search a rusty desk and place it at [0, 5, 0] with scale [3, 3, 3] 137 | ``` 138 | 139 | ## MCP Tools 140 | 141 | The Isaac Sim MCP Extension provides several specialized tools that can be accessed through natural language in Cursor AI. These tools enable you to control and manipulate NVIDIA Isaac Sim with simple commands: 142 | 143 | ### Connection and Scene Management 144 | 145 | - **get_scene_info** - Pings the Isaac Sim Extension Server to verify connection status and retrieve basic scene information. Always use this first to ensure the connection is active. 146 | 147 | ### Physics and Environment Creation 148 | 149 | - **create_physics_scene** - Creates a physics scene with configurable parameters: 150 | - `objects`: List of objects to create (each with type and position) 151 | - `floor`: Whether to create a ground plane (default: true) 152 | - `gravity`: Vector defining gravity direction and magnitude (default: [0, -0.981, 0]) 153 | - `scene_name`: Name for the scene (default: "physics_scene") 154 | 155 | ### Robot Creation and Control 156 | 157 | - **create_robot** - Creates a robot in the scene at a specified position: 158 | - `robot_type`: Type of robot to create (options: "franka", "jetbot", "carter", "g1", "go1") 159 | - `position`: [x, y, z] position coordinates 160 | 161 | ### Omniverse Kit and Scripting 162 | 163 | - **omni_kit_command** - Executes an Omni Kit command: 164 | - `command`: The Omni Kit command to execute (e.g., "CreatePrim") 165 | - `prim_type`: The primitive type for the command (e.g., "Sphere") 166 | 167 | - **execute_script** - Executes arbitrary Python code in Isaac Sim: 168 | - `code`: Python code to execute 169 | 170 | ### Usage Best Practices 171 | 172 | 1. Always check connection with `get_scene_info` before executing any commands 173 | 2. Initialize a physics scene with `create_physics_scene` before adding robots 174 | 3. Use `create_robot` for standard robot placement before trying custom scripts 175 | 4. For complex simulations, use `execute_script` with proper async patterns 176 | 5. Preview code in chat before execution for verification 177 | 178 | ## Contributing 179 | 180 | Contributions are welcome! Please feel free to submit a Pull Request. 181 | 182 | ## License 183 | 184 | This project is licensed under the MIT License - see the LICENSE file for details. 185 | 186 | ## Video Demonstrations 187 | 188 | Below are demonstrations of the Isaac Sim MCP Extension in action: 189 | 190 | ### Robot Party Demo 191 | 192 | ![Robot Party Demo](media/add_more_robot_into_party.gif) 193 | 194 | *GIF: Adding more robots to the simulation using natural language commands* 195 | 196 | 197 | ### Video Format (MP4) 198 | 199 | For higher quality video, you can access the MP4 version directly: 200 | 201 | - [Robot Party Demo (MP4)](media/add_more_robot_into_party.mp4) 202 | 203 | When viewing on GitHub, you can click the link above to view or download the MP4 file. 204 | -------------------------------------------------------------------------------- /add_license_headers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | MIT License 4 | 5 | Copyright (c) 2023-2025 omni-mcp 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | """ 25 | 26 | # Description: Script to add license headers to all Python files in the project. 27 | 28 | import os 29 | import sys 30 | import re 31 | 32 | # The license header text 33 | LICENSE_HEADER = '''""" 34 | MIT License 35 | 36 | Copyright (c) 2023-2025 omni-mcp 37 | 38 | Permission is hereby granted, free of charge, to any person obtaining a copy 39 | of this software and associated documentation files (the "Software"), to deal 40 | in the Software without restriction, including without limitation the rights 41 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 42 | copies of the Software, and to permit persons to whom the Software is 43 | furnished to do so, subject to the following conditions: 44 | 45 | The above copyright notice and this permission notice shall be included in all 46 | copies or substantial portions of the Software. 47 | 48 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 49 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 50 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 51 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 52 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 53 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54 | SOFTWARE. 55 | """ 56 | 57 | ''' 58 | 59 | # Directories to skip 60 | SKIP_DIRS = ['.git', '.vscode', '__pycache__', 'venv', 'env', '.env', 'build', 'dist'] 61 | 62 | # Check if the file already has a license header 63 | def has_license(content): 64 | return 'MIT License' in content[:500] and 'Copyright' in content[:500] 65 | 66 | def process_file(file_path): 67 | with open(file_path, 'r', encoding='utf-8') as f: 68 | content = f.read() 69 | 70 | # Skip if file already has a license header 71 | if has_license(content): 72 | print(f"Skipping {file_path} - already has license header") 73 | return False 74 | 75 | # Handle shebang line if present 76 | if content.startswith('#!'): 77 | shebang_end = content.find('\n') + 1 78 | new_content = content[:shebang_end] + '\n' + LICENSE_HEADER + content[shebang_end:] 79 | else: 80 | new_content = LICENSE_HEADER + content 81 | 82 | with open(file_path, 'w', encoding='utf-8') as f: 83 | f.write(new_content) 84 | 85 | print(f"Added license header to {file_path}") 86 | return True 87 | 88 | def process_directory(directory): 89 | files_processed = 0 90 | 91 | for root, dirs, files in os.walk(directory): 92 | # Skip directories in SKIP_DIRS 93 | dirs[:] = [d for d in dirs if d not in SKIP_DIRS] 94 | 95 | for file in files: 96 | if file.endswith('.py') and file != 'add_license_headers.py': 97 | file_path = os.path.join(root, file) 98 | if process_file(file_path): 99 | files_processed += 1 100 | 101 | return files_processed 102 | 103 | if __name__ == '__main__': 104 | # Get the directory to process, default to current directory 105 | directory = sys.argv[1] if len(sys.argv) > 1 else '.' 106 | 107 | if not os.path.isdir(directory): 108 | print(f"Error: {directory} is not a valid directory") 109 | sys.exit(1) 110 | 111 | print(f"Adding license headers to Python files in {directory}") 112 | count = process_directory(directory) 113 | print(f"Added license headers to {count} files") -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/.cursorrules: -------------------------------------------------------------------------------- 1 | # Isaac Sim MCP Rules 2 | 3 | # General Rules 4 | - Before executing any code, always check if the scene is properly initialized by calling get_scene_info() 5 | - When working with robots, try using create_robot() first before using execute_script() 6 | - If execute_script() fails due to communication error, retry up to 3 times at most 7 | - For any creation of robot, call create_physics_scene() first 8 | - Always print the formatted code into chat to confirm before execution 9 | 10 | # Physics Rules 11 | - If the scene is empty, create a physics scene with create_physics_scene() 12 | - For physics simulation, avoid using simulation_context to run simulations in the main thread 13 | - Use the World class with async methods for initializing physics and running simulations 14 | - When needed, use my_world.play() followed by multiple step_async() calls to wait for physics to stabilize 15 | 16 | # Robot Creation Rules 17 | - Before creating a robot, verify availability of connection with get_scene_info() 18 | - Available robot types: "franka", "jetbot", "carter", "g1", "go1" 19 | - Position robots using their appropriate parameters 20 | - For custom robot configurations, use execute_script() only when create_robot() is insufficient 21 | 22 | # Physics Scene Rules 23 | - Objects should include 'type' and 'position' at minimum 24 | - Object format example: {"path": "/World/Cube", "type": "Cube", "size": 20, "position": [0, 100, 0]} 25 | - Default gravity is [0, 0, -981.0] (cm/s^2) 26 | - Set floor=True to create a default ground plane 27 | 28 | # Script Execution Rules 29 | - Use World class instead of SimulationContext when possible 30 | - Initialize physics before trying to control any articulations 31 | - When controlling robots, make sure to step the physics at least once before interaction 32 | - For robot joint control, first initialize the articulation, then get the controller -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/config/extension.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "isaac_sim_mcp_extension" 3 | version = "0.3.0" 4 | title = "Isaac Sim MCP Server" 5 | description = "MCP server extension for Isaac Sim to enable communication with Claude and other AI assistants" 6 | category = "Robotics" 7 | keywords = ["mcp", "server", "socket", "simulation", "robotics"] 8 | 9 | [dependencies] 10 | "omni.isaac.core" = {} 11 | "omni.kit.uiapp" = {} 12 | "omni.isaac.ui" = {} 13 | 14 | 15 | 16 | # Main python module this extension provides, it will be publicly available as "import isaac_sim_mcp_extension". 17 | [[python.module]] 18 | name = "isaac_sim_mcp_extension" 19 | 20 | 21 | 22 | [python.pipapi] 23 | 24 | 25 | [settings] 26 | server.socket = 8766 27 | server.host = "localhost" 28 | -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/examples/franka.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import asyncio 26 | from time import sleep 27 | from omni.isaac.core import SimulationContext 28 | from omni.isaac.core.utils.prims import create_prim 29 | from omni.isaac.core.utils.stage import add_reference_to_stage, is_stage_loading 30 | from omni.isaac.nucleus import get_assets_root_path 31 | from omni.isaac.core.prims import XFormPrim 32 | from omni.isaac.core.articulations import Articulation 33 | import numpy as np 34 | from omni.isaac.core import World 35 | 36 | 37 | 38 | assets_root_path = get_assets_root_path() 39 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 40 | franka_robot = add_reference_to_stage(asset_path, "/World/Franka") 41 | #cre_prim("/DistantLight", "DistantLight") 42 | 43 | # set the position of the robot 44 | # robot_prim = XFormPrim(prim_path="/World/Franka") 45 | # robot_prim.set_world_pose(position=np.array([2.0, 1.5, 0.0])) 46 | for row in range(5): 47 | for col in range(5): 48 | object_path = f"/World/Franka_{row}_{col}" 49 | add_reference_to_stage(asset_path, object_path) 50 | robot_prim = XFormPrim(prim_path=object_path) 51 | robot_prim.set_world_pose(position=np.array([2.0 + row * 1 - 4, 1.5 + col * 1 - 4, 0.0])) 52 | 53 | 54 | physics_initialized = False 55 | simulation_context = None 56 | 57 | # async def initialize_physics(physics_initialized, simulation_context): 58 | # """Initializes the physics simulation, handling potential errors.""" 59 | 60 | # if physics_initialized: 61 | # print("Physics already initialized, skipping.") 62 | # return 63 | 64 | # try: 65 | # print("Initializing physics...") 66 | # # Configure simulation parameters 67 | # sim_config = { 68 | # "physics_engine": "PhysX", 69 | # "physx": { 70 | # "use_gpu": True, # Enable GPU acceleration 71 | # "solver_type": 1, 72 | # "gpu_found_lost_pairs_capacity": 2048, 73 | # "gpu_found_lost_aggregate_pairs_capacity": 2048 74 | # }, 75 | # "stage_units_in_meters": 1.0, 76 | # } 77 | 78 | # simulation_context = SimulationContext() 79 | # simulation_context.initialize_physics(**sim_config) # Pass sim_config 80 | 81 | # if not simulation_context.is_physics_running(): 82 | # print("Physics is not running! Playing the simulation.") 83 | # simulation_context.play() 84 | 85 | 86 | # physics_initialized = True 87 | # print("Physics initialization complete.") 88 | 89 | # except Exception as e: 90 | # print(f"Error during physics initialization: {e}") 91 | # raise # Re-raise the exception to halt the simulation setup 92 | # Create and run the simulation 93 | async def main(): 94 | 95 | 96 | 97 | await asyncio.sleep(3) 98 | # need to initialize physics getting any articulation..etc 99 | # simulation_context = SimulationContext() 100 | # simulation_context.initialize_physics() 101 | # await initialize_physics(physics_initialized, simulation_context) 102 | # Create world 103 | my_world = World(physics_dt=1.0 / 60.0, rendering_dt=1.0 / 60.0, stage_units_in_meters=1.0) 104 | 105 | # Get the stage 106 | # stage = my_world.get_stage() 107 | simulation_context = SimulationContext(physics_dt=1.0 / 60.0, rendering_dt=1.0 / 60.0, stage_units_in_meters=1.0) 108 | my_world.initialize_physics() 109 | 110 | # Make sure the world is playing before initializing the robot 111 | if not my_world.is_playing(): 112 | my_world.play() 113 | # Wait a few frames for physics to stabilize 114 | for _ in range(1000): 115 | my_world.step_async() 116 | 117 | 118 | 119 | # Now initialize the robot 120 | franka_robot_art = Articulation(prim_path="/World/Franka", name="Franka") 121 | franka_robot_art.initialize(simulation_context.physics_sim_view) 122 | # Now get the controller after initialization 123 | franka_controller = franka_robot_art.get_articulation_controller() 124 | 125 | # art = Articulation(prim_path="/World/G1") 126 | 127 | # simulation_context.stop() 128 | 129 | # Launch the async function 130 | # sleep(1) 131 | asyncio.ensure_future(main()) 132 | print("Franka simulation launched in background") -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/examples/franka_grid.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import asyncio 26 | import numpy as np 27 | from omni.isaac.core import World, SimulationContext 28 | from omni.isaac.core.utils.stage import add_reference_to_stage 29 | from omni.isaac.core.prims import XFormPrim 30 | from omni.isaac.core.articulations import Articulation 31 | from omni.isaac.nucleus import get_assets_root_path 32 | 33 | # Get the path to the Franka robot asset 34 | assets_root_path = get_assets_root_path() 35 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 36 | 37 | # Create a 3x3 grid of Franka robots 38 | # The grid will span from [-3, -3, 0] to [3, 3, 0] 39 | def create_franka_grid(): 40 | # Calculate spacing between robots 41 | num_robots = 3 # 3x3 grid 42 | x_min, y_min = -3, -3 43 | x_max, y_max = 3, 3 44 | x_spacing = (x_max - x_min) / (num_robots - 1) if num_robots > 1 else 0 45 | y_spacing = (y_max - y_min) / (num_robots - 1) if num_robots > 1 else 0 46 | 47 | # Create each robot in the grid 48 | for row in range(num_robots): 49 | for col in range(num_robots): 50 | # Calculate position for this robot 51 | x_pos = x_min + row * x_spacing 52 | y_pos = y_min + col * y_spacing 53 | 54 | # Create a unique path for each robot 55 | object_path = f"/World/Franka_{row}_{col}" 56 | 57 | # Add robot to the stage 58 | add_reference_to_stage(asset_path, object_path) 59 | 60 | # Set position 61 | robot_prim = XFormPrim(prim_path=object_path) 62 | robot_prim.set_world_pose(position=np.array([x_pos, y_pos, 0.0])) 63 | 64 | print(f"Created Franka robot at position [{x_pos}, {y_pos}, 0.0]") 65 | 66 | async def main(): 67 | # Create the robots 68 | create_franka_grid() 69 | 70 | # Wait for robots to be fully loaded 71 | await asyncio.sleep(2) 72 | 73 | # Create world and initialize physics 74 | my_world = World(physics_dt=1.0/60.0, rendering_dt=1.0/60.0, stage_units_in_meters=1.0) 75 | simulation_context = SimulationContext(physics_dt=1.0/60.0, rendering_dt=1.0/60.0, stage_units_in_meters=1.0) 76 | 77 | # Initialize physics 78 | my_world.initialize_physics() 79 | 80 | # Make sure the world is playing 81 | if not my_world.is_playing(): 82 | my_world.play() 83 | 84 | # Run simulation for some time 85 | for _ in range(500): 86 | my_world.step_async() 87 | await asyncio.sleep(0.01) # Small delay to prevent blocking 88 | 89 | # Initialize all robots 90 | for row in range(3): 91 | for col in range(3): 92 | robot_path = f"/World/Franka_{row}_{col}" 93 | try: 94 | robot = Articulation(prim_path=robot_path, name=f"Franka_{row}_{col}") 95 | robot.initialize(simulation_context.physics_sim_view) 96 | print(f"Initialized robot at {robot_path}") 97 | except Exception as e: 98 | print(f"Error initializing robot at {robot_path}: {e}") 99 | 100 | # Launch the async function 101 | asyncio.ensure_future(main()) 102 | print("Franka robot grid simulation launched") -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/examples/g1.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import asyncio 27 | from time import sleep 28 | import omni 29 | from omni.isaac.core import World 30 | from omni.isaac.core.robots import Robot 31 | from omni.isaac.core.utils.types import ArticulationAction 32 | from omni.isaac.core.utils.stage import add_reference_to_stage 33 | from omni.isaac.core.objects import DynamicCuboid 34 | from omni.isaac.nucleus import get_assets_root_path 35 | import numpy as np 36 | from omni.isaac.core import SimulationContext 37 | from omni.isaac.core import PhysicsContext 38 | from omni.isaac.core.prims import XFormPrim 39 | 40 | class G1Simulation: 41 | """Simulation of G1 robot in a factory environment using the BaseSample pattern.""" 42 | 43 | def __init__(self): 44 | self.my_world = None 45 | self.g1_robot = None 46 | self.simulation_context = None 47 | self.g1_controller = None 48 | self.num_joints = 0 49 | self.assets_root_path = None 50 | self.factory_elements = [] 51 | self.trot_sequence = [] 52 | 53 | async def setup_scene(self): 54 | """Set up the initial scene with ground plane and environment.""" 55 | print("Setting up scene...") 56 | 57 | # Create world 58 | self.my_world = World(stage_units_in_meters=1.0) 59 | 60 | # Get the stage 61 | stage = self.my_world.stage 62 | 63 | # Check if ground plane exists before adding it 64 | if not stage.GetPrimAtPath("/World/groundPlane"): 65 | print("Adding default ground plane") 66 | self.my_world.scene.add_default_ground_plane() 67 | else: 68 | print("Ground plane already exists, skipping creation") 69 | 70 | # Get assets root path 71 | self.assets_root_path = get_assets_root_path() 72 | print(f"Assets root path: {self.assets_root_path}") 73 | 74 | # Add factory environment objects 75 | await self._create_factory_environment() 76 | 77 | # Mn 78 | physics_context = PhysicsContext() 79 | physics_context.set_physics_dt(0.0167) 80 | physics_context.enable_gpu_dynamics(True) # 81 | 82 | # 83 | self.simulation_context = SimulationContext() 84 | # self.simulation_context.set_physics_context(physics_context) 85 | # Initialize physics directly from simulation_context 86 | # This method handles the physics initialization internally 87 | # without needing PhysicsContext 88 | # await self.simulation_context.initialize_physics_async() 89 | # await self.simulation_context.start_async() # 90 | 91 | async def _create_factory_environment(self): 92 | """Create the factory environment objects.""" 93 | # Define factory obstacles 94 | factory_obstacles = [ 95 | # Format: name, position, scale, color 96 | ("machine_1", [2.0, 1.0, 0.5], [1.0, 0.5, 1.0], [0.7, 0.7, 0.8]), 97 | ("machine_2", [-2.0, 1.5, 0.5], [0.8, 0.8, 1.0], [0.8, 0.7, 0.7]), 98 | ("workbench_1", [0.0, 2.0, 0.3], [2.0, 0.8, 0.6], [0.6, 0.5, 0.4]), 99 | ("storage_shelf", [-1.5, -1.5, 0.75], [0.5, 1.5, 1.5], [0.5, 0.5, 0.5]), 100 | ("conveyor_belt", [2.5, -1.0, 0.2], [3.0, 0.7, 0.4], [0.3, 0.3, 0.3]) 101 | ] 102 | 103 | # Add factory objects (checking if they already exist) 104 | stage = self.my_world.stage 105 | for name, position, scale, color in factory_obstacles: 106 | object_path = f"/World/{name}" 107 | 108 | # Check if object already exists 109 | if not stage.GetPrimAtPath(object_path): 110 | print(f"Adding factory object: {name}") 111 | obj =DynamicCuboid( 112 | prim_path=object_path, 113 | name=name, 114 | position=np.array(position), 115 | scale=np.array(scale), 116 | color=np.array(color) 117 | ) 118 | # obj = self.my_world.scene.add( 119 | 120 | # ) 121 | self.factory_elements.append(obj) 122 | else: 123 | print(f"Factory object {name} already exists, skipping creation") 124 | 125 | async def load_robot(self): 126 | """Load the G1 robot into the scene.""" 127 | print("Loading G1 robot...") 128 | 129 | # Define potential G1 paths 130 | g1_paths = [] 131 | if self.assets_root_path: 132 | g1_paths.extend([ 133 | f"{self.assets_root_path}/Isaac/Robots/Unitree/G1/g1.usd" 134 | #f"{self.assets_root_path}/Isaac/Robots/Unitree/Go1/go1.usd" 135 | #f"{self.assets_root_path}/Robots/Unitree/G1/g1.usd" 136 | ]) 137 | g1_paths.extend([ 138 | "/Isaac/Robots/Unitree/G1/g1.usd" 139 | #"/Isaac/Robots/Unitree/Go1/go1.usd" 140 | #"/Robots/Unitree/G1/g1.usd" 141 | ]) 142 | 143 | # Check if G1 robot already exists 144 | stage = self.my_world.stage 145 | g1_exists = stage.GetPrimAtPath("/World/G1").IsValid() 146 | robot_prim = XFormPrim(prim_path="/World/G1") 147 | robot_prim.set_world_pose(position=np.array([4.0 , 6, 0.0])) 148 | 149 | 150 | if not g1_exists: 151 | print("G1 robot not found, adding it to stage...") 152 | # Add G1 robot reference to stage 153 | g1_added = False 154 | for path in g1_paths: 155 | try: 156 | print(f"Trying to add G1 from: {path}") 157 | add_reference_to_stage(usd_path=path, prim_path="/World/G1") 158 | g1_added = True 159 | print(f"Successfully added G1 from: {path}") 160 | break 161 | except Exception as e: 162 | print(f"Could not add G1 from {path}: {e}") 163 | 164 | if not g1_added: 165 | raise Exception("Could not add G1 from any known path") 166 | else: 167 | print("G1 robot already exists in stage, skipping creation") 168 | g1_added = True 169 | 170 | return g1_added 171 | 172 | async def post_load(self): 173 | """Setup after loading - initialize the robot and controllers.""" 174 | print("Post-load setup...") 175 | 176 | # Add the G1 robot to the scene 177 | # self.g1_robot = self.my_world.stage.GetPrimAtPath("/World/G1") 178 | # from omni.isaac.articulations import Articulation 179 | # self.g1_robot_articulation = Articulation(prim_path=robot_prim_path) 180 | from omni.isaac.core.articulations import Articulation 181 | # art = Articulation(prim_path="/World/G1") 182 | 183 | self.g1_robot = Articulation(prim_path="/World/G1", name="G1") 184 | 185 | # self.simulation_context.add_articulation(art) 186 | 187 | # await art.initialize() 188 | # self.g1_robot = self.my_world.stage.add(Robot(prim_path="/World/G2", name="g1_robot")) 189 | 190 | # Get joint names to understand the robot's structure 191 | # print(art.dof_properties["name"]) 192 | # joint_names = art.dof_properties["name"] 193 | 194 | 195 | # Create articulation controller 196 | # Initialize the G1 robot as an articulation 197 | 198 | # self.g1_robot = self.my_world.scene.add(Robot(prim_path="/World/G1", name="g1_robot")) 199 | 200 | # Wait for the robot to be fully initialized 201 | # self.my_world.scene.initialize_physics() 202 | 203 | # Get joint information 204 | # joint_names = self.g1_robot_articulation.dof_properties["name"] 205 | # self.num_joints = len(joint_names) 206 | # print(f"G1 has {self.num_joints} joints: {joint_names}") 207 | 208 | # Create the articulation controller for the robot 209 | # Initialize the robot first to ensure it's properly set up 210 | # Initialize physics scene first to avoid "no active physics scene" error 211 | # from omni.isaac.core import get_physics_context 212 | physics_context = self.my_world.get_physics_context() 213 | if physics_context is None: 214 | print("Warning: Physics context is None") 215 | # Don't attempt to reset the world here as it causes the error 216 | # Instead, ensure physics is initialized before this point 217 | print("Make sure physics scene is created before initializing the robot") 218 | elif not physics_context.is_initialized(): 219 | print("Physics context exists but not initialized") 220 | # Don't call reset() directly as it's causing the error 221 | 222 | # Check if physics scene exists 223 | if not self.my_world.physics_sim_view: 224 | print("No physics simulation view found - need to initialize physics first") 225 | # Try playing the simulation first 226 | if not self.my_world.is_playing(): 227 | self.my_world.play() 228 | print("Started simulation to initialize physics") 229 | # Make sure the world is playing before initializing the robot 230 | if not self.my_world.is_playing(): 231 | self.my_world.play() 232 | # Wait a few frames for physics to stabilize 233 | for _ in range(10): 234 | self.my_world.step_async() 235 | 236 | # Now initialize the robot 237 | self.g1_robot.initialize(self.my_world.physics_sim_view) 238 | # Now get the controller after initialization 239 | self.g1_controller = self.g1_robot.get_articulation_controller() 240 | 241 | print("joint_names", self.g1_robot.dof_names) 242 | joint_names = self.g1_robot.dof_names 243 | if joint_names is not None: 244 | self.num_joints = len(joint_names) 245 | print(f"G1 has {self.num_joints} joints: {joint_names}") 246 | 247 | # Set control parameters for better stability 248 | self.g1_controller.set_gains(kps=[100.0] * self.num_joints, 249 | kds=[10.0] * self.num_joints) 250 | 251 | # print("G1 articulation controller created successfully") 252 | # from omni.isaac.core_nodes.IsaacArticulationController import IsaacArticulationController 253 | 254 | 255 | # self.g1_controller = IsaacArticulationController(self.g1_robot) 256 | 257 | # Define trot sequence for quadruped walking 258 | if self.num_joints >= 12: 259 | # Create full joint position arrays with zeros for all joints 260 | # The error shows we need arrays of shape (1,37) instead of (1,12) 261 | base_positions = np.zeros(self.num_joints) 262 | 263 | # Create two different walking poses 264 | trot_pose_1 = base_positions.copy() 265 | trot_pose_2 = base_positions.copy() 266 | 267 | # Only modify the leg joint positions (assuming first 12 are leg joints) 268 | # Front right leg 269 | trot_pose_1[0:3] = [0.1, 0.4, -0.8] # up and forward 270 | trot_pose_2[0:3] = [-0.1, 0.4, -0.6] # down and back 271 | 272 | # Front left leg 273 | trot_pose_1[3:6] = [-0.1, 0.4, -0.6] # down and back 274 | trot_pose_2[3:6] = [0.1, 0.4, -0.8] # up and forward 275 | 276 | # Rear right leg 277 | trot_pose_1[6:9] = [-0.1, 0.4, -0.6] # down and back 278 | trot_pose_2[6:9] = [0.1, 0.4, -0.8] # up and forward 279 | 280 | # Rear left leg 281 | trot_pose_1[9:12] = [0.1, 0.4, -0.8] # up and forward 282 | trot_pose_2[9:12] = [-0.1, 0.4, -0.6] # down and back 283 | 284 | self.trot_sequence = [trot_pose_1, trot_pose_2] 285 | 286 | async def initialize_simulation(self): 287 | """Initialize the simulation with proper robot pose.""" 288 | # Define standing pose 289 | standing_pose = np.zeros(self.num_joints) 290 | if self.num_joints >= 12: # Safety check in case joint structure is different 291 | # Set knee joints to a slight bend for stability 292 | standing_pose[2] = -0.5 # front right knee 293 | standing_pose[5] = -0.5 # front left knee 294 | standing_pose[8] = -0.5 # rear right knee 295 | standing_pose[11] = -0.5 # rear left knee 296 | 297 | # Move to standing pose 298 | print("Moving to standing pose...") 299 | self.g1_controller.apply_action(ArticulationAction(joint_positions=standing_pose)) 300 | # self.g1_robot._articulation_view.set_joint_positions(standing_pose) 301 | # Allow time to stabilize 302 | for _ in range(60): 303 | self.my_world.step_async() 304 | 305 | async def run_simulation(self): 306 | """Run the main simulation sequence.""" 307 | if self.num_joints < 12: 308 | print("Not enough joints for walking animation, simulation will be static") 309 | return 310 | 311 | # Perform a walking movement in a circle 312 | print("Starting walking pattern...") 313 | steps = 120 314 | radius = 1.0 315 | 316 | # Initial position 317 | initial_position = self.g1_robot.get_world_pose()[0] 318 | center = initial_position + np.array([radius, 0, 0]) 319 | 320 | for i in range(steps): 321 | # Calculate angle around circle 322 | angle = i * 2 * np.pi / steps 323 | 324 | # Calculate new position on circle 325 | new_x = center[0] - radius * np.cos(angle) 326 | new_y = center[1] + radius * np.sin(angle) 327 | new_position = np.array([new_x, new_y, initial_position[2]]) 328 | 329 | # Calculate direction (tangent to circle) 330 | direction_angle = angle + np.pi/2 331 | 332 | # Set rotation to face direction of travel 333 | # Convert angle to quaternion (w,x,y,z format) 334 | orientation = np.array([np.cos(direction_angle/2), 0, 0, np.sin(direction_angle/2)]) 335 | 336 | # Alternate between trot poses 337 | pose_idx = i % 2 338 | self.g1_controller.apply_action(ArticulationAction(joint_positions=self.trot_sequence[pose_idx])) 339 | 340 | # Update position 341 | self.g1_robot.set_world_pose(position=new_position, orientation=orientation) 342 | 343 | # Step the simulation 344 | for _ in range(3): 345 | self.my_world.step_async() 346 | 347 | # Return to standing pose 348 | print("Returning to standing pose...") 349 | standing_pose = np.zeros(self.num_joints) 350 | if self.num_joints >= 12: 351 | standing_pose[2] = -0.5 352 | standing_pose[5] = -0.5 353 | standing_pose[8] = -0.5 354 | standing_pose[11] = -0.5 355 | 356 | self.g1_controller.apply_action(ArticulationAction(joint_positions=standing_pose)) 357 | 358 | # Allow time to stabilize 359 | for _ in range(60): 360 | self.my_world.step_async() 361 | 362 | async def cleanup(self): 363 | """Clean up resources after simulation.""" 364 | print("Cleaning up simulation resources...") 365 | # Any specific cleanup if needed 366 | 367 | async def clear_async(self): 368 | """Clear the simulation completely.""" 369 | print("Clearing simulation...") 370 | # Stop physics simulation if running 371 | if self.my_world and self.my_world.is_playing(): 372 | await self.my_world.stop_async() 373 | 374 | # Additional cleanup as needed 375 | self.g1_robot = None 376 | self.g1_controller = None 377 | self.factory_elements = [] 378 | 379 | async def run(self): 380 | """Run the complete simulation sequence.""" 381 | try: 382 | print("Starting G1 robot simulation using asyncio...") 383 | 384 | # Setup scene 385 | await self.setup_scene() 386 | 387 | # Load robot 388 | robot_loaded = await self.load_robot() 389 | if not robot_loaded: 390 | print("Failed to load G1 robot, aborting simulation") 391 | return 392 | 393 | # Reset physics with async method 394 | print("Resetting world with async method...") 395 | # Simply use reset_async without checking for physics_context 396 | try: 397 | self.my_world.reset() 398 | 399 | except Exception as e: 400 | print(f"Error during reset: {str(e)}") 401 | # Initialize physics if needed 402 | try: 403 | await self.my_world.initialize_physics_async() 404 | except Exception as e: 405 | print(f"Error initializing physics: {str(e)}") 406 | 407 | physics_context = self.my_world.get_physics_context() 408 | print("physics_context", physics_context) 409 | # Make sure physics is properly set up 410 | if physics_context is not None and not physics_context.is_initialized(): 411 | print("Warning: Unable to initialize physics context") 412 | 413 | # Post-load setup 414 | await self.post_load() 415 | 416 | # Start simulation using async method 417 | print("Starting simulation with async method...") 418 | # Simply use play_async without checking for physics_context 419 | try: 420 | self.my_world.play() 421 | except Exception as e: 422 | print(f"Error during play: {str(e)}") 423 | try: 424 | await self.my_world.initialize_physics_async() 425 | await self.my_world.play_async() 426 | except Exception as e: 427 | print(f"Error initializing and playing physics: {str(e)}") 428 | 429 | # Initialize the simulation 430 | await self.initialize_simulation() 431 | 432 | # Run simulation sequence 433 | await self.run_simulation() 434 | 435 | print("G1 robot simulation completed successfully!") 436 | 437 | except Exception as e: 438 | print(f"Error in G1 simulation: {str(e)}") 439 | import traceback 440 | print(traceback.format_exc()) 441 | finally: 442 | # Cleanup 443 | await self.cleanup() 444 | 445 | # Create and run the simulation 446 | async def main(): 447 | sim = G1Simulation() 448 | await sim.run() 449 | 450 | # Launch the async function 451 | asyncio.ensure_future(main()) 452 | print("G1 simulation launched in background") -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/examples/go1.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import asyncio 27 | from time import sleep 28 | import omni 29 | from omni.isaac.core import World 30 | from omni.isaac.core.robots import Robot 31 | from omni.isaac.core.utils.types import ArticulationAction 32 | from omni.isaac.core.utils.stage import add_reference_to_stage 33 | from omni.isaac.core.objects import DynamicCuboid 34 | from omni.isaac.nucleus import get_assets_root_path 35 | import numpy as np 36 | from omni.isaac.core import SimulationContext 37 | from omni.isaac.core import PhysicsContext 38 | 39 | class Go1Simulation: 40 | """Simulation of Go1 robot in a factory environment using the BaseSample pattern.""" 41 | 42 | def __init__(self): 43 | self.my_world = None 44 | self.go1_robot = None 45 | self.simulation_context = None 46 | self.go1_controller = None 47 | self.num_joints = 0 48 | self.assets_root_path = None 49 | self.factory_elements = [] 50 | self.trot_sequence = [] 51 | 52 | async def setup_scene(self): 53 | """Set up the initial scene with ground plane and environment.""" 54 | print("Setting up scene...") 55 | 56 | # Create world 57 | self.my_world = World(stage_units_in_meters=1.0) 58 | 59 | # Get the stage 60 | stage = self.my_world.stage 61 | 62 | # Check if ground plane exists before adding it 63 | if not stage.GetPrimAtPath("/World/groundPlane"): 64 | print("Adding default ground plane") 65 | self.my_world.scene.add_default_ground_plane() 66 | else: 67 | print("Ground plane already exists, skipping creation") 68 | 69 | # Get assets root path 70 | self.assets_root_path = get_assets_root_path() 71 | print(f"Assets root path: {self.assets_root_path}") 72 | 73 | # Add factory environment objects 74 | await self._create_factory_environment() 75 | 76 | # Mn 77 | physics_context = PhysicsContext() 78 | physics_context.set_physics_dt(0.0167) 79 | physics_context.enable_gpu_dynamics(True) # 80 | 81 | # 82 | self.simulation_context = SimulationContext() 83 | # self.simulation_context.set_physics_context(physics_context) 84 | # Initialize physics directly from simulation_context 85 | # This method handles the physics initialization internally 86 | # without needing PhysicsContext 87 | # await self.simulation_context.initialize_physics_async() 88 | # await self.simulation_context.start_async() # 89 | 90 | async def _create_factory_environment(self): 91 | """Create the factory environment objects.""" 92 | # Define factory obstacles 93 | factory_obstacles = [ 94 | # Format: name, position, scale, color 95 | ("machine_1", [2.0, 1.0, 0.5], [1.0, 0.5, 1.0], [0.7, 0.7, 0.8]), 96 | ("machine_2", [-2.0, 1.5, 0.5], [0.8, 0.8, 1.0], [0.8, 0.7, 0.7]), 97 | ("workbench_1", [0.0, 2.0, 0.3], [2.0, 0.8, 0.6], [0.6, 0.5, 0.4]), 98 | ("storage_shelf", [-1.5, -1.5, 0.75], [0.5, 1.5, 1.5], [0.5, 0.5, 0.5]), 99 | ("conveyor_belt", [2.5, -1.0, 0.2], [3.0, 0.7, 0.4], [0.3, 0.3, 0.3]) 100 | ] 101 | 102 | # Add factory objects (checking if they already exist) 103 | stage = self.my_world.stage 104 | for name, position, scale, color in factory_obstacles: 105 | object_path = f"/World/{name}" 106 | 107 | # Check if object already exists 108 | if not stage.GetPrimAtPath(object_path): 109 | print(f"Adding factory object: {name}") 110 | obj =DynamicCuboid( 111 | prim_path=object_path, 112 | name=name, 113 | position=np.array(position), 114 | scale=np.array(scale), 115 | color=np.array(color) 116 | ) 117 | # obj = self.my_world.scene.add( 118 | 119 | # ) 120 | self.factory_elements.append(obj) 121 | else: 122 | print(f"Factory object {name} already exists, skipping creation") 123 | 124 | async def load_robot(self): 125 | """Load the Go1 robot into the scene.""" 126 | print("Loading Go1 robot...") 127 | 128 | # Define potential Go1 paths 129 | go1_paths = [] 130 | if self.assets_root_path: 131 | go1_paths.extend([ 132 | #f"{self.assets_root_path}/Isaac/Robots/Unitree/Go1/go1.usd", 133 | f"{self.assets_root_path}/Isaac/Robots/Unitree/Go1/go1.usd" 134 | #f"{self.assets_root_path}/Robots/Unitree/Go1/go1.usd" 135 | ]) 136 | go1_paths.extend([ 137 | #"/Isaac/Robots/Unitree/Go1/go1.usd", 138 | # "/Isaac/Robots/Unitree/Go1/go1.usd" 139 | "/Robots/Unitree/Go1/go1.usd" 140 | ]) 141 | 142 | # Check if Go1 robot already exists 143 | stage = self.my_world.stage 144 | go1_exists = stage.GetPrimAtPath("/World/Go1").IsValid() 145 | 146 | if not go1_exists: 147 | print("Go1 robot not found, adding it to stage...") 148 | # Add Go1 robot reference to stage 149 | go1_added = False 150 | for path in go1_paths: 151 | try: 152 | print(f"Trying to add Go1 from: {path}") 153 | add_reference_to_stage(usd_path=path, prim_path="/World/Go1") 154 | go1_added = True 155 | print(f"Successfully added Go1 from: {path}") 156 | break 157 | except Exception as e: 158 | print(f"Could not add Go1 from {path}: {e}") 159 | 160 | if not go1_added: 161 | raise Exception("Could not add Go1 from any known path") 162 | else: 163 | print("Go1 robot already exists in stage, skipping creation") 164 | go1_added = True 165 | 166 | return go1_added 167 | 168 | async def initialize_sim_view(self): 169 | """Initialize the physics simulation view if not already initialized.""" 170 | 171 | physics_context = self.my_world.get_physics_context() 172 | if physics_context is None: 173 | print("Warning: Physics context is None") 174 | # Don't attempt to reset the world here as it causes the error 175 | # Instead, ensure physics is initialized before this point 176 | print("Make sure physics scene is created before initializing the robot") 177 | elif not physics_context.is_initialized(): 178 | print("Physics context exists but not initialized") 179 | # Don't call reset() directly as it's causing the error 180 | 181 | # # Check if physics scene exists 182 | # if not self.my_world.physics_sim_view: 183 | # print("No physics simulation view found - need to initialize physics first") 184 | # # Try playing the simulation first 185 | # if not self.my_world.is_playing(): 186 | # self.my_world.play() 187 | # print("Started simulation to initialize physics") 188 | # Make sure the world is playing before initializing the robot 189 | if not self.my_world.is_playing(): 190 | self.my_world.play() 191 | # Wait a few frames for physics to stabilize 192 | for _ in range(10): 193 | self.my_world.step_async() 194 | 195 | # Now initialize the robot with the physics_sim_view 196 | # sleep(1) 197 | self.go1_robot.initialize(self.my_world.physics_sim_view) 198 | print("Initializing physics simulation view...") 199 | # sleep(3) 200 | # return self.my_world.physics_sim_view 201 | 202 | async def step_async(self): 203 | """Step the simulation asynchronously.""" 204 | self.my_world.step_async() 205 | 206 | async def post_load(self): 207 | """Setup after loading - initialize the robot and controllers.""" 208 | print("Post-load setup...") 209 | 210 | # Add the Go1 robot to the scene 211 | # self.go1_robot = self.my_world.stage.GetPrimAtPath("/World/Go1") 212 | # from omni.isaac.articulations import Articulation 213 | # self.go1_robot_articulation = Articulation(prim_path=robot_prim_path) 214 | from omni.isaac.core.articulations import Articulation 215 | # art = Articulation(prim_path="/World/Go1") 216 | 217 | self.go1_robot = Articulation(prim_path="/World/Go1", name="Go1") 218 | 219 | 220 | 221 | 222 | # Now initialize the robot 223 | await self.initialize_sim_view() 224 | 225 | 226 | # Now get the controller after initialization 227 | self.go1_controller = self.go1_robot.get_articulation_controller() 228 | 229 | print("joint_names", self.go1_robot.dof_names) 230 | joint_names = self.go1_robot.dof_names 231 | if joint_names is not None: 232 | self.num_joints = len(joint_names) 233 | print(f"Go1 has {self.num_joints} joints: {joint_names}") 234 | 235 | # Set control parameters for better stability 236 | self.go1_controller.set_gains(kps=[100.0] * self.num_joints, 237 | kds=[10.0] * self.num_joints) 238 | 239 | 240 | 241 | # Define trot sequence for quadruped walking 242 | if self.num_joints >= 12: 243 | # Create full joint position arrays with zeros for all joints 244 | # The error shows we need arrays of shape (1,37) instead of (1,12) 245 | base_positions = np.zeros(self.num_joints) 246 | 247 | # Create two different walking poses 248 | trot_pose_1 = base_positions.copy() 249 | trot_pose_2 = base_positions.copy() 250 | 251 | # Based on actual joint names: 252 | # FL_hip_joint, FR_hip_joint, RL_hip_joint, RR_hip_joint, 253 | # FL_thigh_joint, FR_thigh_joint, RL_thigh_joint, RR_thigh_joint, 254 | # FL_calf_joint, FR_calf_joint, RL_calf_joint, RR_calf_joint 255 | 256 | # Front Left leg (FL) - indices 0, 4, 8 257 | trot_pose_1[0] = 0.1 # hip joint - swing outward 258 | trot_pose_1[4] = 0.5 # thigh joint - lift up 259 | trot_pose_1[8] = -0.9 # calf joint - bend for ground clearance 260 | 261 | trot_pose_2[0] = -0.1 # hip joint - swing inward 262 | trot_pose_2[4] = 0.3 # thigh joint - lower down 263 | trot_pose_2[8] = -0.6 # calf joint - extend for stance 264 | 265 | # Front Right leg (FR) - indices 1, 5, 9 266 | trot_pose_1[1] = -0.1 # hip joint - swing inward 267 | trot_pose_1[5] = 0.3 # thigh joint - lower down 268 | trot_pose_1[9] = -0.6 # calf joint - extend for stance 269 | 270 | trot_pose_2[1] = 0.1 # hip joint - swing outward 271 | trot_pose_2[5] = 0.5 # thigh joint - lift up 272 | trot_pose_2[9] = -0.9 # calf joint - bend for ground clearance 273 | 274 | # Rear Left leg (RL) - indices 2, 6, 10 275 | trot_pose_1[2] = -0.1 # hip joint - swing inward 276 | trot_pose_1[6] = 0.3 # thigh joint - lower down 277 | trot_pose_1[10] = -0.6 # calf joint - extend for stance 278 | 279 | trot_pose_2[2] = 0.1 # hip joint - swing outward 280 | trot_pose_2[6] = 0.5 # thigh joint - lift up 281 | trot_pose_2[10] = -0.9 # calf joint - bend for ground clearance 282 | 283 | # Rear Right leg (RR) - indices 3, 7, 11 284 | trot_pose_1[3] = 0.1 # hip joint - swing outward 285 | trot_pose_1[7] = 0.5 # thigh joint - lift up 286 | trot_pose_1[11] = -0.9 # calf joint - bend for ground clearance 287 | 288 | trot_pose_2[3] = -0.1 # hip joint - swing inward 289 | trot_pose_2[7] = 0.3 # thigh joint - lower down 290 | trot_pose_2[11] = -0.6 # calf joint - extend for stance 291 | 292 | self.trot_sequence = [trot_pose_1, trot_pose_2] 293 | 294 | async def initialize_simulation(self): 295 | """Initialize the simulation with proper robot pose.""" 296 | # Define standing pose 297 | self.standing_pose = np.zeros(self.num_joints) 298 | if self.num_joints >= 12: # Safety check in case joint structure is different 299 | # Set standing pose for all joints based on joint names 300 | # Hip joints (slight outward angle) 301 | self.standing_pose[0] = 0.0 # FL_hip_joint 302 | self.standing_pose[1] = 0.0 # FR_hip_joint 303 | self.standing_pose[2] = 0.0 # RL_hip_joint 304 | self.standing_pose[3] = 0.0 # RR_hip_joint 305 | 306 | # Thigh joints (slight forward angle) 307 | self.standing_pose[4] = 0.4 # FL_thigh_joint 308 | self.standing_pose[5] = 0.4 # FR_thigh_joint 309 | self.standing_pose[6] = 0.4 # RL_thigh_joint 310 | self.standing_pose[7] = 0.4 # RR_thigh_joint 311 | 312 | # Calf joints (bend for stability) 313 | self.standing_pose[8] = -0.8 # FL_calf_joint 314 | self.standing_pose[9] = -0.8 # FR_calf_joint 315 | self.standing_pose[10] = -0.8 # RL_calf_joint 316 | self.standing_pose[11] = -0.8 # RR_calf_joint 317 | 318 | # Move to standing pose 319 | print("Moving to standing pose...") 320 | self.go1_controller.apply_action(ArticulationAction(joint_positions=self.standing_pose)) 321 | # self.go1_robot._articulation_view.set_joint_positions(standing_pose) 322 | # Allow time to stabilize 323 | for _ in range(60): 324 | self.my_world.step_async() 325 | 326 | async def run_simulation(self): 327 | """Run the main simulation sequence.""" 328 | if self.num_joints < 12: 329 | print("Not enough joints for walking animation, simulation will be static") 330 | return 331 | 332 | # Perform a walking movement in a circle 333 | print("Starting walking pattern...") 334 | steps = 120 335 | radius = 1.0 336 | 337 | # Initial position 338 | initial_position = self.go1_robot.get_world_pose()[0] 339 | center = initial_position + np.array([radius, 0, 0]) 340 | 341 | for i in range(steps): 342 | # Calculate angle around circle 343 | angle = i * 2 * np.pi / steps 344 | 345 | # Calculate new position on circle 346 | new_x = center[0] - radius * np.cos(angle) 347 | new_y = center[1] + radius * np.sin(angle) 348 | new_position = np.array([new_x, new_y, initial_position[2]]) 349 | 350 | # Calculate direction (tangent to circle) 351 | direction_angle = angle + np.pi/2 352 | 353 | # Set rotation to face direction of travel 354 | # Convert angle to quaternion (w,x,y,z format) 355 | orientation = np.array([np.cos(direction_angle/2), 0, 0, np.sin(direction_angle/2)]) 356 | 357 | # Alternate between trot poses 358 | pose_idx = i % 2 359 | self.go1_controller.apply_action(ArticulationAction(joint_positions=self.trot_sequence[pose_idx])) 360 | 361 | # Update position 362 | self.go1_robot.set_world_pose(position=new_position, orientation=orientation) 363 | 364 | # Step the simulation 365 | for _ in range(3): 366 | self.my_world.step_async() 367 | 368 | # Return to standing pose 369 | print("Returning to standing pose...") 370 | # standing_pose = np.zeros(self.num_joints) 371 | # if self.num_joints >= 12: 372 | # standing_pose[2] = -0.5 373 | # standing_pose[5] = -0.5 374 | # standing_pose[8] = -0.5 375 | # standing_pose[11] = -0.5 376 | 377 | self.go1_controller.apply_action(ArticulationAction(joint_positions=self.standing_pose)) 378 | 379 | # Allow time to stabilize 380 | for _ in range(60): 381 | self.my_world.step_async() 382 | 383 | async def cleanup(self): 384 | """Clean up resources after simulation.""" 385 | print("Cleaning up simulation resources...") 386 | # Any specific cleanup if needed 387 | 388 | async def clear_async(self): 389 | """Clear the simulation completely.""" 390 | print("Clearing simulation...") 391 | # Stop physics simulation if running 392 | if self.my_world and self.my_world.is_playing(): 393 | await self.my_world.stop_async() 394 | 395 | # Additional cleanup as needed 396 | self.go1_robot = None 397 | self.go1_controller = None 398 | self.factory_elements = [] 399 | 400 | async def run(self): 401 | """Run the complete simulation sequence.""" 402 | try: 403 | print("Starting Go1 robot simulation using asyncio...") 404 | 405 | # Setup scene 406 | await self.setup_scene() 407 | 408 | # Load robot 409 | robot_loaded = await self.load_robot() 410 | if not robot_loaded: 411 | print("Failed to load Go1 robot, aborting simulation") 412 | return 413 | 414 | # Reset physics with async method 415 | print("Resetting world with async method...") 416 | # Simply use reset_async without checking for physics_context 417 | try: 418 | self.my_world.reset() 419 | 420 | except Exception as e: 421 | print(f"Error during reset: {str(e)}") 422 | # Initialize physics if needed 423 | try: 424 | await self.my_world.initialize_physics_async() 425 | except Exception as e: 426 | print(f"Error initializing physics: {str(e)}") 427 | 428 | physics_context = self.my_world.get_physics_context() 429 | print("physics_context", physics_context) 430 | # Make sure physics is properly set up 431 | if physics_context is not None and not physics_context.is_initialized(): 432 | print("Warning: Unable to initialize physics context") 433 | 434 | # Post-load setup 435 | await self.post_load() 436 | 437 | # Start simulation using async method 438 | print("Starting simulation with async method...") 439 | # Simply use play_async without checking for physics_context 440 | try: 441 | self.my_world.play() 442 | except Exception as e: 443 | print(f"Error during play: {str(e)}") 444 | try: 445 | await self.my_world.initialize_physics_async() 446 | await self.my_world.play_async() 447 | except Exception as e: 448 | print(f"Error initializing and playing physics: {str(e)}") 449 | 450 | # Initialize the simulation 451 | await self.initialize_simulation() 452 | 453 | # Run simulation sequence 454 | await self.run_simulation() 455 | 456 | print("Go1 robot simulation completed successfully!") 457 | 458 | except Exception as e: 459 | print(f"Error in Go1 simulation: {str(e)}") 460 | import traceback 461 | print(traceback.format_exc()) 462 | finally: 463 | # Cleanup 464 | await self.cleanup() 465 | 466 | # Create and run the simulation 467 | async def main(): 468 | sim = Go1Simulation() 469 | await sim.run() 470 | 471 | # Launch the async function 472 | asyncio.ensure_future(main()) 473 | print("Go1 simulation launched in background") -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/isaac_sim_mcp_extension/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | from .extension import MCPExtension 26 | __all__ = ["MCPExtension"] -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/isaac_sim_mcp_extension/extension.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | """Extension module for Isaac Sim MCP.""" 26 | 27 | import asyncio 28 | import carb 29 | # import omni.ext 30 | # import omni.ui as ui 31 | import omni.usd 32 | import threading 33 | import time 34 | import socket 35 | import json 36 | import traceback 37 | 38 | import gc 39 | from pxr import Usd, UsdGeom, Sdf, Gf 40 | 41 | import omni 42 | import omni.kit.commands 43 | import omni.physx as _physx 44 | import omni.timeline 45 | from typing import Dict, Any, List, Optional, Union 46 | from omni.isaac.nucleus import get_assets_root_path 47 | from omni.isaac.core.prims import XFormPrim 48 | import numpy as np 49 | from omni.isaac.core import World 50 | # Import Beaver3d and USDLoader 51 | from isaac_sim_mcp_extension.gen3d import Beaver3d 52 | from isaac_sim_mcp_extension.usd import USDLoader 53 | from isaac_sim_mcp_extension.usd import USDSearch3d 54 | import requests 55 | 56 | # Extension Methods required by Omniverse Kit 57 | # Any class derived from `omni.ext.IExt` in top level module (defined in `python.modules` of `extension.toml`) will be 58 | # instantiated when extension gets enabled and `on_startup(ext_id)` will be called. Later when extension gets disabled 59 | # on_shutdown() is called. 60 | class MCPExtension(omni.ext.IExt): 61 | def __init__(self) -> None: 62 | """Initialize the extension.""" 63 | super().__init__() 64 | self.ext_id = None 65 | self.running = False 66 | self.host = None 67 | self.port = None 68 | self.socket = None 69 | self.server_thread = None 70 | self._usd_context = None 71 | self._physx_interface = None 72 | self._timeline = None 73 | self._window = None 74 | self._status_label = None 75 | self._server_thread = None 76 | self._models = None 77 | self._settings = carb.settings.get_settings() 78 | self._image_url_cache = {} # cache for image url 79 | self._text_prompt_cache = {} # cache for text prompt 80 | 81 | 82 | def on_startup(self, ext_id: str): 83 | """Initialize extension and UI elements""" 84 | print("trigger on_startup for: ", ext_id) 85 | print("settings: ", self._settings.get("/exts/omni.kit.pipapi")) 86 | self.port = self._settings.get("/exts/isaac.sim.mcp/server, port") or 8766 87 | self.host = self._settings.get("/exts/isaac.sim.mcp/server.host") or "localhost" 88 | if not hasattr(self, 'running'): 89 | self.running = False 90 | 91 | self.ext_id = ext_id 92 | self._usd_context = omni.usd.get_context() 93 | # omni.kit.commands.execute("CreatePrim", prim_type="Sphere") 94 | 95 | # print("sphere created") 96 | # result = self.execute_script('omni.kit.commands.execute("CreatePrim", prim_type="Cube")') 97 | # print("script executed", result) 98 | self._start() 99 | # result = self.execute_script('omni.kit.commands.execute("CreatePrim", prim_type="Cube")') 100 | # print("script executed", result) 101 | 102 | def on_shutdown(self): 103 | print("trigger on_shutdown for: ", self.ext_id) 104 | self._models = {} 105 | gc.collect() 106 | self._stop() 107 | 108 | def _start(self): 109 | if self.running: 110 | print("Server is already running") 111 | return 112 | 113 | self.running = True 114 | 115 | try: 116 | # Create socket 117 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 118 | self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 119 | self.socket.bind((self.host, self.port)) 120 | self.socket.listen(1) 121 | 122 | # Start server thread 123 | self.server_thread = threading.Thread(target=self._server_loop) 124 | self.server_thread.daemon = True 125 | self.server_thread.start() 126 | 127 | print(f"Isaac Sim MCP server started on {self.host}:{self.port}") 128 | except Exception as e: 129 | print(f"Failed to start server: {str(e)}") 130 | self.stop() 131 | 132 | def _stop(self): 133 | self.running = False 134 | 135 | # Close socket 136 | if self.socket: 137 | try: 138 | self.socket.close() 139 | except: 140 | pass 141 | self.socket = None 142 | 143 | # Wait for thread to finish 144 | if self.server_thread: 145 | try: 146 | if self.server_thread.is_alive(): 147 | self.server_thread.join(timeout=1.0) 148 | except: 149 | pass 150 | self.server_thread = None 151 | 152 | print("Isaac Sim MCP server stopped") 153 | 154 | def _server_loop(self): 155 | """Main server loop in a separate thread""" 156 | print("Server thread started") 157 | self.socket.settimeout(1.0) # Timeout to allow for stopping 158 | if not hasattr(self, 'running'): 159 | self.running = False 160 | 161 | while self.running: 162 | try: 163 | # Accept new connection 164 | try: 165 | client, address = self.socket.accept() 166 | print(f"Connected to client: {address}") 167 | 168 | # Handle client in a separate thread 169 | client_thread = threading.Thread( 170 | target=self._handle_client, 171 | args=(client,) 172 | ) 173 | client_thread.daemon = True 174 | client_thread.start() 175 | except socket.timeout: 176 | # Just check running condition 177 | continue 178 | except Exception as e: 179 | print(f"Error accepting connection: {str(e)}") 180 | time.sleep(0.5) 181 | except Exception as e: 182 | print(f"Error in server loop: {str(e)}") 183 | if not self.running: 184 | break 185 | time.sleep(0.5) 186 | 187 | print("Server thread stopped") 188 | 189 | def _handle_client(self, client): 190 | """Handle connected client""" 191 | print("Client handler started") 192 | client.settimeout(None) # No timeout 193 | buffer = b'' 194 | 195 | try: 196 | while self.running: 197 | # Receive data 198 | try: 199 | data = client.recv(16384) 200 | if not data: 201 | print("Client disconnected") 202 | break 203 | 204 | buffer += data 205 | try: 206 | # Try to parse command 207 | command = json.loads(buffer.decode('utf-8')) 208 | buffer = b'' 209 | 210 | # Execute command in Isaac Sim's main thread 211 | async def execute_wrapper(): 212 | try: 213 | response = self.execute_command(command) 214 | response_json = json.dumps(response) 215 | print("response_json: ", response_json) 216 | try: 217 | client.sendall(response_json.encode('utf-8')) 218 | except: 219 | print("Failed to send response - client disconnected") 220 | except Exception as e: 221 | print(f"Error executing command: {str(e)}") 222 | traceback.print_exc() 223 | try: 224 | error_response = { 225 | "status": "error", 226 | "message": str(e) 227 | } 228 | client.sendall(json.dumps(error_response).encode('utf-8')) 229 | except: 230 | pass 231 | return None 232 | # import omni.kit.commands 233 | # import omni.kit.async 234 | from omni.kit.async_engine import run_coroutine 235 | task = run_coroutine(execute_wrapper()) 236 | # import asyncio 237 | # asyncio.ensure_future(execute_wrapper()) 238 | #time.sleep(30) 239 | 240 | 241 | # 242 | # omni.kit.async.get_event_loop().create_task(create_sphere_async()) 243 | # TODO:Schedule execution in main thread 244 | # bpy.app.timers.register(execute_wrapper, first_interval=0.0) 245 | # omni.kit.app.get_app().post_to_main_thread(execute_wrapper()) 246 | # carb.apputils.get_app().get_update_event_loop().post(execute_wrapper) 247 | 248 | # from omni.kit.async_engine import run_coroutine 249 | # run_coroutine(execute_wrapper()) 250 | # omni.kit.app.get_app().get_update_event_stream().push(0, 0, {"fn": execute_wrapper}) 251 | except json.JSONDecodeError: 252 | # Incomplete data, wait for more 253 | pass 254 | except Exception as e: 255 | print(f"Error receiving data: {str(e)}") 256 | break 257 | except Exception as e: 258 | print(f"Error in client handler: {str(e)}") 259 | finally: 260 | try: 261 | client.close() 262 | except: 263 | pass 264 | print("Client handler stopped") 265 | 266 | # TODO: This is a temporary function to execute commands in the main thread 267 | def execute_command(self, command): 268 | """Execute a command in the main thread""" 269 | try: 270 | cmd_type = command.get("type") 271 | params = command.get("params", {}) 272 | 273 | # TODO: Ensure we're in the right context 274 | if cmd_type in ["create_object", "modify_object", "delete_object"]: 275 | self._usd_context = omni.usd.get_context() 276 | self._execute_command_internal(command) 277 | else: 278 | return self._execute_command_internal(command) 279 | 280 | except Exception as e: 281 | print(f"Error executing command: {str(e)}") 282 | traceback.print_exc() 283 | return {"status": "error", "message": str(e)} 284 | 285 | def _execute_command_internal(self, command): 286 | """Internal command execution with proper context""" 287 | cmd_type = command.get("type") 288 | params = command.get("params", {}) 289 | 290 | #todo: add a handler for extend simulation method if necessary 291 | handlers = { 292 | # "get_scene_info": self.get_scene_info, 293 | # "create_object": self.create_object, 294 | # "modify_object": self.modify_object, 295 | # "delete_object": self.delete_object, 296 | # "get_object_info": self.get_object_info, 297 | "execute_script": self.execute_script, 298 | "get_scene_info": self.get_scene_info, 299 | "omini_kit_command": self.omini_kit_command, 300 | "create_physics_scene": self.create_physics_scene, 301 | "create_robot": self.create_robot, 302 | "generate_3d_from_text_or_image": self.generate_3d_from_text_or_image, 303 | "transform": self.transform, 304 | "search_3d_usd_by_text": self.search_3d_usd_by_text, 305 | } 306 | 307 | handler = handlers.get(cmd_type) 308 | if handler: 309 | try: 310 | print(f"Executing handler for {cmd_type}") 311 | result = handler(**params) 312 | print(f"Handler execution complete: /n", result) 313 | # return result 314 | if result and result.get("status") == "success": 315 | return {"status": "success", "result": result} 316 | else: 317 | return {"status": "error", "message": result.get("message", "Unknown error")} 318 | except Exception as e: 319 | print(f"Error in handler: {str(e)}") 320 | traceback.print_exc() 321 | return {"status": "error", "message": str(e)} 322 | else: 323 | return {"status": "error", "message": f"Unknown command type: {cmd_type}"} 324 | 325 | 326 | 327 | 328 | def execute_script(self, code: str) : 329 | """Execute a Python script within the Isaac Sim context. 330 | 331 | Args: 332 | code: The Python script to execute. 333 | 334 | Returns: 335 | Dictionary with execution result. 336 | """ 337 | try: 338 | # Create a local namespace 339 | local_ns = {} 340 | 341 | # Add frequently used modules to the namespace 342 | local_ns["omni"] = omni 343 | local_ns["carb"] = carb 344 | local_ns["Usd"] = Usd 345 | local_ns["UsdGeom"] = UsdGeom 346 | local_ns["Sdf"] = Sdf 347 | local_ns["Gf"] = Gf 348 | # code = script["code"] 349 | 350 | # Execute the script 351 | exec(code, local_ns) 352 | 353 | # Get the result if any 354 | # result = local_ns.get("result", None) 355 | result = None 356 | 357 | 358 | return { 359 | "status": "success", 360 | "message": "Script executed successfully", 361 | "result": result 362 | } 363 | except Exception as e: 364 | carb.log_error(f"Error executing script: {e}") 365 | import traceback 366 | carb.log_error(traceback.format_exc()) 367 | return { 368 | "status": "error", 369 | "message": str(e), 370 | "traceback": traceback.format_exc() 371 | } 372 | 373 | def get_scene_info(self): 374 | self._stage = omni.usd.get_context().get_stage() 375 | assert self._stage is not None 376 | stage_path = self._stage.GetRootLayer().realPath 377 | assets_root_path = get_assets_root_path() 378 | return {"status": "success", "message": "pong", "assets_root_path": assets_root_path} 379 | 380 | def omini_kit_command(self, command: str, prim_type: str) -> Dict[str, Any]: 381 | omni.kit.commands.execute(command, prim_type=prim_type) 382 | print("command executed") 383 | return {"status": "success", "message": "command executed"} 384 | 385 | def create_robot(self, robot_type: str = "g1", position: List[float] = [0, 0, 0]): 386 | from omni.isaac.core.utils.prims import create_prim 387 | from omni.isaac.core.utils.stage import add_reference_to_stage, is_stage_loading 388 | from omni.isaac.nucleus import get_assets_root_path 389 | 390 | 391 | stage = omni.usd.get_context().get_stage() 392 | assets_root_path = get_assets_root_path() 393 | print("position: ", position) 394 | 395 | if robot_type.lower() == "franka": 396 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 397 | add_reference_to_stage(asset_path, "/Franka") 398 | robot_prim = XFormPrim(prim_path="/Franka") 399 | robot_prim.set_world_pose(position=np.array(position)) 400 | return {"status": "success", "message": f"{robot_type} robot created"} 401 | elif robot_type.lower() == "jetbot": 402 | asset_path = assets_root_path + "/Isaac/Robots/Jetbot/jetbot.usd" 403 | add_reference_to_stage(asset_path, "/Jetbot") 404 | robot_prim = XFormPrim(prim_path="/Jetbot") 405 | robot_prim.set_world_pose(position=np.array(position)) 406 | return {"status": "success", "message": f"{robot_type} robot created"} 407 | elif robot_type.lower() == "carter": 408 | asset_path = assets_root_path + "/Isaac/Robots/Carter/carter.usd" 409 | add_reference_to_stage(asset_path, "/Carter") 410 | robot_prim = XFormPrim(prim_path="/Carter") 411 | robot_prim.set_world_pose(position=np.array(position)) 412 | return {"status": "success", "message": f"{robot_type} robot created"} 413 | elif robot_type.lower() == "g1": 414 | asset_path = assets_root_path + "/Isaac/Robots/Unitree/G1/g1.usd" 415 | add_reference_to_stage(asset_path, "/G1") 416 | robot_prim = XFormPrim(prim_path="/G1") 417 | robot_prim.set_world_pose(position=np.array(position)) 418 | return {"status": "success", "message": f"{robot_type} robot created"} 419 | elif robot_type.lower() == "go1": 420 | asset_path = assets_root_path + "/Isaac/Robots/Unitree/Go1/go1.usd" 421 | add_reference_to_stage(asset_path, "/Go1") 422 | robot_prim = XFormPrim(prim_path="/Go1") 423 | robot_prim.set_world_pose(position=np.array(position)) 424 | return {"status": "success", "message": f"{robot_type} robot created"} 425 | else: 426 | # Default to Franka if unknown robot type 427 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 428 | add_reference_to_stage(asset_path, "/Franka") 429 | robot_prim = XFormPrim(prim_path="/Franka") 430 | robot_prim.set_world_pose(position=np.array(position)) 431 | return {"status": "success", "message": f"{robot_type} robot created"} 432 | 433 | def create_physics_scene( 434 | self, 435 | objects: List[Dict[str, Any]] = [], 436 | floor: bool = True, 437 | gravity: List[float] = (0.0, -9.81, 0.0), 438 | scene_name: str = "None" 439 | ) -> Dict[str, Any]: 440 | """Create a physics scene with multiple objects.""" 441 | try: 442 | # Set default values 443 | gravity = gravity or [0, -9.81, 0] 444 | scene_name = scene_name or "physics_scene" 445 | 446 | 447 | # Create a new stage 448 | #omni.kit.commands.execute("CreateNewStage") 449 | 450 | 451 | stage = omni.usd.get_context().get_stage() 452 | print("stage: ", stage) 453 | 454 | # print("start to create new sphere") 455 | # # import omni.kit.commands 456 | # omni.kit.commands.execute("CreatePrim", prim_type="Sphere") 457 | # print("create sphere successfully") 458 | 459 | # Set up the physics scene 460 | scene_path = "/World/PhysicsScene" 461 | omni.kit.commands.execute( 462 | "CreatePrim", 463 | prim_path=scene_path, 464 | prim_type="PhysicsScene", 465 | 466 | ) 467 | #attributes={"physxScene:enabled": True , "physxScene:gravity": gravity}, 468 | 469 | 470 | # Initialize simulation context with physics 471 | # simulation_context = SimulationContext() 472 | # my_world = World(physics_dt=1.0 / 60.0, rendering_dt=1.0 / 60.0, stage_units_in_meters=1.0) 473 | 474 | # # Make sure the world is playing before initializing the robot 475 | # if not my_world.is_playing(): 476 | # my_world.play() 477 | # # Wait a few frames for physics to stabilize 478 | # for _ in range(1000): 479 | # my_world.step_async() 480 | # my_world.initialize_physics() 481 | 482 | # print("created physics scene: ", scene_path) 483 | 484 | # Create the World prim as a Xform 485 | world_path = "/World" 486 | omni.kit.commands.execute( 487 | "CreatePrim", 488 | prim_path=world_path, 489 | prim_type="Xform", 490 | ) 491 | print("create world: ", world_path) 492 | # Create a ground plane if requested 493 | if floor: 494 | floor_path = "/World/ground" 495 | omni.kit.commands.execute( 496 | "CreatePrim", 497 | prim_path=floor_path, 498 | prim_type="Plane", 499 | attributes={"size": 100.0} # Large ground plane 500 | ) 501 | 502 | # Add physics properties to the ground 503 | # omni.kit.commands.execute( 504 | # "CreatePhysics", 505 | # prim_path=floor_path, 506 | # physics_type="collider", 507 | # attributes={ 508 | # "static": True, 509 | # "collision_enabled": True 510 | # } 511 | # ) 512 | # objects = [ 513 | # {"path": "/World/Cube", "type": "Cube", "size": 20, "position": (0, 100, 0), "rotation": [1, 2, 3, 0], "scale": [1, 1, 1], "color": [0.5, 0.5, 0.5, 1.0], "physics_enabled": True, "mass": 1.0, "is_kinematic": False}, 514 | # {"path": "/World/Sphere", "type": "Sphere", "radius": 5, "position": (5, 200, 0)}, 515 | # {"path": "/World/Cone", "type": "Cone", "height": 8, "radius": 3, "position": (-5, 150, 0)} 516 | # ] 517 | print("start create objects: ", objects) 518 | objects_created = 0 519 | # Create each object 520 | for i, obj in enumerate(objects): 521 | obj_name = obj.get("name", f"object_{i}") 522 | obj_type = obj.get("type", "Cube") 523 | obj_position = obj.get("position", [0, 0, 0]) 524 | obj_rotation = obj.get("rotation", [1, 0, 0, 0]) # Default is no rotation (identity quaternion) 525 | obj_scale = obj.get("scale", [1, 1, 1]) 526 | obj_color = obj.get("color", [0.5, 0.5, 0.5, 1.0]) 527 | obj_physics = obj.get("physics_enabled", True) 528 | obj_mass = obj.get("mass", 1.0) 529 | obj_kinematic = obj.get("is_kinematic", False) 530 | 531 | # Create the object 532 | obj_path = obj.get("path", f"/World/{obj_name}") 533 | print("obj_path: ", obj_path) 534 | if stage.GetPrimAtPath(obj_path): 535 | print("obj_path already exists and skip creating") 536 | continue 537 | 538 | # Create the primitive based on type 539 | if obj_type in ["Cube", "Sphere", "Cylinder", "Cone", "Plane"]: 540 | omni.kit.commands.execute( 541 | "CreatePrim", 542 | prim_path=obj_path, 543 | prim_type=obj_type, 544 | attributes={ 545 | "size": obj.get("size", 100.0), 546 | "position": obj_position, 547 | "rotation": obj_rotation, 548 | "scale": obj_scale, 549 | "color": obj_color, 550 | "physics_enabled": obj_physics, 551 | "mass": obj_mass, 552 | "is_kinematic": obj_kinematic} if obj_type in ["Cube", "Sphere","Plane"] else {}, 553 | ) 554 | print(f"Created {obj_type} at {obj_path}") 555 | else: 556 | return {"status": "error", "message": f"Invalid object type: {obj_type}"} 557 | 558 | # Set the transform 559 | omni.kit.commands.execute( 560 | "TransformPrimSRT", 561 | path=obj_path, 562 | new_translation=obj_position, 563 | new_rotation_euler=[0, 0, 0], # We'll set the quaternion separately 564 | new_scale=obj_scale, 565 | ) 566 | print(f"Created TransformPrimSRT at {obj_position}") 567 | # Set rotation as quaternion 568 | xform = UsdGeom.Xformable(stage.GetPrimAtPath(obj_path)) 569 | if xform and obj_rotation != [1, 0, 0, 0]: 570 | quat = Gf.Quatf(obj_rotation[0], obj_rotation[1], obj_rotation[2], obj_rotation[3]) 571 | xform_op = xform.AddRotateOp() 572 | xform_op.Set(quat) 573 | 574 | # Add physics properties if enabled 575 | if obj_physics: 576 | omni.kit.commands.execute( 577 | "CreatePhysics", 578 | prim_path=obj_path, 579 | physics_type="rigid_body" if not obj_kinematic else "kinematic_body", 580 | attributes={ 581 | "mass": obj_mass, 582 | "collision_enabled": True, 583 | "kinematic": obj_kinematic 584 | } 585 | ) 586 | print(f"Created Physics at {obj_path}") 587 | # Set the color 588 | if obj_color: 589 | material_path = f"{obj_path}/material" 590 | omni.kit.commands.execute( 591 | "CreatePrim", 592 | prim_path=material_path, 593 | prim_type="Material", 594 | attributes={ 595 | "diffuseColor": obj_color[:3], 596 | "opacity": obj_color[3] if len(obj_color) > 3 else 1.0 597 | } 598 | ) 599 | print(f"Created Material at {material_path}") 600 | # Bind the material to the object 601 | omni.kit.commands.execute( 602 | "BindMaterial", 603 | material_path=material_path, 604 | prim_path=obj_path 605 | ) 606 | 607 | print(f"Bound Material to {obj_path}") 608 | # increment the number of objects created 609 | objects_created += 1 610 | return { 611 | "status": "success", 612 | "message": f"Created physics scene with {objects_created} objects", 613 | "result": scene_name 614 | } 615 | 616 | except Exception as e: 617 | import traceback 618 | return { 619 | "status": "error", 620 | "message": str(e), 621 | "traceback": traceback.format_exc() 622 | } 623 | 624 | def generate_3d_from_text_or_image(self, text_prompt=None, image_url=None, position=(0, 0, 50), scale=(10, 10, 10)): 625 | """ 626 | Generate a 3D model from text or image, load it into the scene and transform it. 627 | 628 | Args: 629 | text_prompt (str, optional): Text prompt for 3D generation 630 | image_url (str, optional): URL of image for 3D generation 631 | position (tuple, optional): Position to place the model 632 | scale (tuple, optional): Scale of the model 633 | 634 | Returns: 635 | dict: Dictionary with the task_id and prim_path 636 | """ 637 | try: 638 | # Initialize Beaver3d 639 | beaver = Beaver3d() 640 | 641 | # Determine generation method based on inputs 642 | # if image_url and text_prompt: 643 | # # Generate 3D from image with text prompt as options 644 | # task_id = beaver.generate_3d_from_image(image_url, text_prompt) 645 | # print(f"3D model generation from image with text options started with task ID: {task_id}") 646 | # Check if we have cached task IDs for this input 647 | if not hasattr(self, '_image_url_cache'): 648 | self._image_url_cache = {} # Cache for image URL to task_id mapping 649 | 650 | if not hasattr(self, '_text_prompt_cache'): 651 | self._text_prompt_cache = {} # Cache for text prompt to task_id mapping 652 | 653 | # Check if we can retrieve task_id from cache 654 | task_id = None 655 | if image_url and image_url in self._image_url_cache: 656 | task_id = self._image_url_cache[image_url] 657 | print(f"Using cached task ID: {task_id} for image URL: {image_url}") 658 | elif text_prompt and text_prompt in self._text_prompt_cache: 659 | task_id = self._text_prompt_cache[text_prompt] 660 | print(f"Using cached task ID: {task_id} for text prompt: {text_prompt}") 661 | 662 | if task_id: #cache hit 663 | print(f"Using cached model ID: {task_id}") 664 | elif image_url: 665 | # Generate 3D from image only 666 | task_id = beaver.generate_3d_from_image(image_url) 667 | print(f"3D model generation from image started with task ID: {task_id}") 668 | elif text_prompt: 669 | # Generate 3D from text 670 | task_id = beaver.generate_3d_from_text(text_prompt) 671 | print(f"3D model generation from text started with task ID: {task_id}") 672 | else: 673 | return { 674 | "status": "error", 675 | "message": "Either text_prompt or image_url must be provided" 676 | } 677 | 678 | # Monitor the task and download the result 679 | # result_path = beaver.monitor_task_status(task_id) 680 | # task = asyncio.create_task( 681 | # beaver.monitor_task_status_async( 682 | # task_id, on_complete_callback=load_model_into_scene)) 683 | #await task 684 | def load_model_into_scene(task_id, status, result_path): 685 | print(f"{task_id} is {status}, 3D model downloaded to: {result_path}") 686 | # Only cache the task_id after successful download 687 | if image_url and image_url not in self._image_url_cache: 688 | self._image_url_cache[image_url] = task_id 689 | elif text_prompt and text_prompt not in self._text_prompt_cache: 690 | self._text_prompt_cache[text_prompt] = task_id 691 | # Load the model into the scene 692 | loader = USDLoader() 693 | prim_path = loader.load_usd_model(task_id=task_id) 694 | 695 | # Load texture and create material 696 | try: 697 | texture_path, material = loader.load_texture_and_create_material(task_id=task_id) 698 | 699 | # Bind texture to model 700 | loader.bind_texture_to_model() 701 | except Exception as e: 702 | print(f"Warning: Texture loading failed, continuing without texture: {str(e)}") 703 | 704 | # Transform the model 705 | loader.transform(position=position, scale=scale) 706 | 707 | return { 708 | "status": "success", 709 | "task_id": task_id, 710 | "prim_path": prim_path 711 | } 712 | 713 | from omni.kit.async_engine import run_coroutine 714 | task = run_coroutine(beaver.monitor_task_status_async( 715 | task_id, on_complete_callback=load_model_into_scene)) 716 | 717 | return { 718 | "status": "success", 719 | "task_id": task_id, 720 | "message": f"3D model generation started with task ID: {task_id}" 721 | } 722 | 723 | 724 | 725 | except Exception as e: 726 | print(f"Error generating 3D model: {str(e)}") 727 | traceback.print_exc() 728 | return { 729 | "status": "error", 730 | "message": str(e) 731 | } 732 | 733 | def search_3d_usd_by_text(self, text_prompt:str, target_path:str, position=(0, 0, 50), scale=(10, 10, 10)): 734 | """ 735 | Search a USD assets in USD Search service, load it into the scene and transform it. 736 | 737 | Args: 738 | text_prompt (str, optional): Text prompt for 3D generation 739 | target_path (str, ): target path in current scene stage 740 | position (tuple, optional): Position to place the model 741 | scale (tuple, optional): Scale of the model 742 | 743 | Returns: 744 | dict: Dictionary with prim_path 745 | """ 746 | try: 747 | if text_prompt: 748 | print(f"3D model generation from text: {text_prompt}") 749 | else: 750 | return { 751 | "status": "error", 752 | "message": "text_prompt must be provided" 753 | } 754 | 755 | searcher3d = USDSearch3d() 756 | url = searcher3d.search( text_prompt ) 757 | 758 | loader = USDLoader() 759 | prim_path = loader.load_usd_from_url( url, target_path ) 760 | print(f"loaded url {url} to scene, prim path is: {prim_path}") 761 | # TODO: transform the model, need to fix the transform function for loaded USD 762 | # loader.transform(prim=prim_path, position=position, scale=scale) 763 | 764 | return { 765 | "status": "success", 766 | "prim_path": prim_path, 767 | "message": f"3D model searching with prompt: {text_prompt}, return url: {url}, prim path in current scene: {prim_path}" 768 | } 769 | except Exception as e: 770 | print(f"Error searching 3D model: {str(e)}") 771 | traceback.print_exc() 772 | return { 773 | "status": "error", 774 | "message": str(e) 775 | } 776 | 777 | def transform(self, prim_path, position=(0, 0, 50), scale=(10, 10, 10)): 778 | """ 779 | Transform a USD model by applying position and scale. 780 | 781 | Args: 782 | prim_path (str): Path to the USD prim to transform 783 | position (tuple, optional): The position to set (x, y, z) 784 | scale (tuple, optional): The scale to set (x, y, z) 785 | 786 | Returns: 787 | dict: Result information 788 | """ 789 | try: 790 | # Get the USD context 791 | stage = omni.usd.get_context().get_stage() 792 | 793 | # Get the prim 794 | prim = stage.GetPrimAtPath(prim_path) 795 | if not prim: 796 | return { 797 | "status": "error", 798 | "message": f"Prim not found at path: {prim_path}" 799 | } 800 | 801 | # Initialize USDLoader 802 | loader = USDLoader() 803 | 804 | # Transform the model 805 | xformable = loader.transform(prim=prim, position=position, scale=scale) 806 | 807 | return { 808 | "status": "success", 809 | "message": f"Model at {prim_path} transformed successfully", 810 | "position": position, 811 | "scale": scale 812 | } 813 | except Exception as e: 814 | print(f"Error transforming model: {str(e)}") 815 | traceback.print_exc() 816 | return { 817 | "status": "error", 818 | "message": str(e) 819 | } 820 | -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/isaac_sim_mcp_extension/gen3d.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | import time 4 | import json 5 | import requests 6 | import zipfile 7 | import shutil 8 | from pathlib import Path 9 | 10 | class Beaver3d: 11 | def __init__(self): 12 | """Initialize Beaver3d with model name and API key from environment variables""" 13 | self.api_key = os.environ.get("ARK_API_KEY") 14 | self.model_name = os.environ.get("BEAVER3D_MODEL") 15 | self._working_dir = Path(os.environ.get("USD_WORKING_DIR", "/tmp/usd")) 16 | self.base_url = os.environ.get("ARK_BASE_URL", "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks") 17 | 18 | if not self.api_key: 19 | raise Exception("ARK_API_KEY environment variable not set, Beaver3D service is not available untill ARK_API_KEY is set") 20 | if not self.model_name: 21 | raise Exception("BEAVER3D_MODEL environment variable not set, Beaver3D service is not available untill BEAVER3D_MODEL is set") 22 | 23 | 24 | def _get_headers(self): 25 | """Get request headers with authorization""" 26 | return { 27 | "Content-Type": "application/json", 28 | "Authorization": f"Bearer {self.api_key}" 29 | } 30 | 31 | def _download_files_for_completed_task(self, task_id, file_url): 32 | """ 33 | Process a completed task by downloading and extracting the result file 34 | 35 | Args: 36 | task_id (str): The task ID 37 | file_url (str): URL to download the result file 38 | 39 | Returns: 40 | str: Path to the extracted task directory 41 | """ 42 | # Create directories if they don't exist 43 | tmp_dir = self._working_dir 44 | tmp_dir.mkdir(parents=True, exist_ok=True) 45 | 46 | task_dir = tmp_dir / task_id 47 | task_dir.mkdir(parents=True, exist_ok=True) 48 | 49 | # Download the file 50 | zip_path = tmp_dir / f"{task_id}.zip" 51 | response = requests.get(file_url) 52 | with open(zip_path, "wb") as f: 53 | f.write(response.content) 54 | 55 | # Extract the zip file 56 | with zipfile.ZipFile(zip_path, 'r') as zip_ref: 57 | zip_ref.extractall(task_dir) 58 | 59 | return str(task_dir) 60 | 61 | def monitor_task_status(self, task_id): 62 | """Monitor task status until succeeded, then download and extract the result file""" 63 | task_url = f"{self.base_url}/{task_id}" 64 | elapsed_time_in_seconds = 0 65 | estimated_time_in_seconds = 75 # 75 seconds is the estimated time for a high subdivision level USD model 66 | while True: 67 | response = requests.get(task_url, headers=self._get_headers()) 68 | if response.status_code != 200: 69 | raise Exception(f"Error fetching task status: {response.text}") 70 | 71 | task_data = response.json() 72 | status = task_data.get("status") 73 | 74 | if status == "succeeded": 75 | file_url = task_data.get("content", {}).get("file_url") 76 | if not file_url: 77 | raise Exception("No file URL found in the response") 78 | 79 | return self._download_files_for_completed_task(task_id, file_url) 80 | 81 | if status == "failed": 82 | raise Exception(f"Task failed: {task_data}") 83 | elif status == "running": 84 | # Assuming generation takes about 70s total, calculate an estimated completion percentage 85 | # Each iteration is 5s, so after 70s we should be at 100% 86 | 87 | completion_ratio = min(100, int((elapsed_time_in_seconds / estimated_time_in_seconds) * 100)) 88 | print(f"Task {task_id} is generating. Progress: {completion_ratio}% complete. Waiting for completion...") 89 | 90 | # Sleep for 5 seconds before checking again 91 | time.sleep(5) 92 | elapsed_time_in_seconds += 5 93 | 94 | async def monitor_task_status_async(self, task_id, on_complete_callback=None): 95 | """ 96 | Asynchronously monitor task status until succeeded, then download and extract the result file 97 | 98 | Args: 99 | task_id (str): The task ID to monitor 100 | on_complete (callable, optional): Callback function to call when task completes 101 | with the task directory as argument 102 | 103 | Returns: 104 | str: Path to the extracted task directory 105 | """ 106 | import asyncio 107 | 108 | task_url = f"{self.base_url}/{task_id}" 109 | elapsed_time_in_seconds = 0 110 | estimated_time_in_seconds = 75 # 75 seconds is the estimated time for a high subdivision level USD model 111 | 112 | while True: 113 | response = requests.get(task_url, headers=self._get_headers()) 114 | if response.status_code != 200: 115 | raise Exception(f"Error fetching task status: {response.text}") 116 | 117 | task_data = response.json() 118 | status = task_data.get("status") 119 | 120 | if status == "succeeded": 121 | file_url = task_data.get("content", {}).get("file_url") 122 | if not file_url: 123 | raise Exception("No file URL found in the response") 124 | 125 | result_path = self._download_files_for_completed_task(task_id, file_url) 126 | 127 | # Call the callback if provided 128 | if on_complete_callback and callable(on_complete_callback): 129 | on_complete_callback(task_id, status, result_path) 130 | 131 | return result_path 132 | 133 | if status == "failed": 134 | raise Exception(f"Task failed: {task_data}") 135 | elif status == "running": 136 | # Calculate an estimated completion percentage 137 | completion_ratio = min(100, int((elapsed_time_in_seconds / estimated_time_in_seconds) * 100)) 138 | print(f"Task {task_id} is generating. Progress: {completion_ratio}% complete. Waiting for completion...") 139 | 140 | # Asynchronously sleep for 5 seconds before checking again 141 | await asyncio.sleep(5) 142 | elapsed_time_in_seconds += 5 143 | 144 | def generate_3d_from_text(self, text_prompt): 145 | """Generate a 3D model from text input and return the task ID""" 146 | # Add default options for USD generation if not already present 147 | if "--subdivision_level" not in text_prompt: 148 | text_prompt += " --subdivision_level high" 149 | if "--fileformat" not in text_prompt: 150 | text_prompt += " --fileformat usd" 151 | 152 | payload = { 153 | "model": self.model_name, 154 | "content": [ 155 | { 156 | "type": "text", 157 | "text": text_prompt 158 | } 159 | ] 160 | } 161 | 162 | response = requests.post( 163 | self.base_url, 164 | headers=self._get_headers(), 165 | json=payload 166 | ) 167 | 168 | if response.status_code != 200: 169 | raise Exception(f"Error generating 3D model: {response.text}") 170 | 171 | return response.json().get("id") 172 | 173 | def generate_3d_from_image(self, image_url, text_options="--subdivision_level high --fileformat usd --watermark true"): 174 | """Generate a 3D model from an image URL and return the task ID""" 175 | """ 176 | Generate a 3D model from an image URL and return the task ID 177 | 178 | Args: 179 | image_url (str): URL of the image to generate 3D model from 180 | text_options (str): Additional options for the generation 181 | 182 | Returns: 183 | str: Task ID for the generation job 184 | """ 185 | # Add default options for USD generation if not already present 186 | if "--subdivision_level" not in text_options: 187 | text_options += " --subdivision_level high" 188 | if "--fileformat" not in text_options: 189 | text_options += " --fileformat usd" 190 | if "--watermark" not in text_options: 191 | text_options += " --watermark true" 192 | payload = { 193 | "model": self.model_name, 194 | "content": [ 195 | { 196 | "type": "image_url", 197 | "image_url": { 198 | "url": image_url 199 | } 200 | }, 201 | { 202 | "type": "text", 203 | "text": text_options 204 | } 205 | ] 206 | } 207 | 208 | response = requests.post( 209 | self.base_url, 210 | headers=self._get_headers(), 211 | json=payload 212 | ) 213 | 214 | if response.status_code != 200: 215 | raise Exception(f"Error generating 3D model from image: {response.text}") 216 | 217 | return response.json().get("id") 218 | 219 | 220 | def main(): 221 | """Main function to test the Beaver3d class""" 222 | try: 223 | # Initialize the Beaver3d class 224 | beaver = Beaver3d() 225 | 226 | # Generate a 3D model from text 227 | text_prompt = "A gothic castle with 4 towers surrounding a central tower, inspired by Notre-Dame de Paris --subdivision_level high --fileformat usd" 228 | task_id = beaver.generate_3d_from_text(text_prompt) 229 | print(f"3D model generation task started with ID: {task_id}") 230 | 231 | # Monitor the task and download the result 232 | result_path = beaver.monitor_task_status(task_id) 233 | print(f"3D model downloaded to: {result_path}") 234 | 235 | # Generate a 3D model from an image 236 | image_url = "https://lvlecheng.tos-cn-beijing.volces.com/chore/apple.jpg" 237 | task_id = beaver.generate_3d_from_image(image_url) 238 | print(f"3D model generation from image task started with ID: {task_id}") 239 | 240 | # Monitor the task and download the result 241 | result_path = beaver.monitor_task_status(task_id) 242 | print(f"3D model from image downloaded to: {result_path}") 243 | 244 | except Exception as e: 245 | print(f"Error: {str(e)}") 246 | 247 | async def test_async(): 248 | # Test initialization 249 | beaver = Beaver3d() 250 | assert beaver.api_key, "API key should be set" 251 | assert beaver.model_name, "Model name should be set" 252 | 253 | # Test async task monitoring with callback 254 | def call_back_fn(task_id, status=None, result_path=None): 255 | print(f"Callback invoked: Task {task_id} status is {status}") 256 | if result_path: 257 | print(f"Callback received result path: {result_path}") 258 | return True 259 | 260 | # Test async monitoring 261 | async_task_id = beaver.generate_3d_from_text("A simple chair") 262 | assert async_task_id, "Async task ID should be returned" 263 | print(f"Starting async monitoring for task ID: {async_task_id}") 264 | 265 | # Monitor the task asynchronously with callback 266 | result_path = await beaver.monitor_task_status_async(async_task_id, on_complete_callback=call_back_fn) 267 | print(f"Async monitoring initiated for task ID: {async_task_id}") 268 | print(f"Async monitoring completed, result path: {result_path}") 269 | print("Async test completed") 270 | return result_path 271 | 272 | def test(): 273 | """Unit test for the Beaver3d class""" 274 | try: 275 | # Test initialization 276 | beaver = Beaver3d() 277 | assert beaver.api_key, "API key should be set" 278 | assert beaver.model_name, "Model name should be set" 279 | 280 | 281 | 282 | # Test text generation 283 | text_prompt = "An fresh apple in red --subdivision_level high --fileformat usd" 284 | task_id = beaver.generate_3d_from_text(text_prompt) 285 | assert task_id, "Task ID should be returned" 286 | print(f"Text generation test passed, task ID: {task_id}") 287 | result_path = beaver.monitor_task_status(task_id) 288 | result_path_obj = Path(result_path) 289 | assert result_path_obj.exists(), f"Downloaded file does not exist at {result_path}" 290 | assert result_path_obj.is_dir(), f"Expected directory at {result_path}" 291 | 292 | # Check if there are USD files in the extracted directory 293 | usd_files = list(result_path_obj.glob("*.usd")) + list(result_path_obj.glob("*.usda")) + list(result_path_obj.glob("*.usdc")) 294 | assert len(usd_files) > 0, f"No USD files found in {result_path}" 295 | print(f"Verified: USD file successfully downloaded and extracted to {result_path}") 296 | 297 | # Test text generation with Chinese text 298 | chinese_text_prompt = "一个哥特式的城堡,参考巴黎圣母院 " 299 | task_id = beaver.generate_3d_from_text(chinese_text_prompt) 300 | assert task_id, "Task ID should be returned" 301 | print(f"Chinese text generation test passed, task ID: {task_id}") 302 | result_path = beaver.monitor_task_status(task_id) 303 | result_path_obj = Path(result_path) 304 | assert result_path_obj.exists(), f"Downloaded file does not exist at {result_path}" 305 | assert result_path_obj.is_dir(), f"Expected directory at {result_path}" 306 | 307 | # Check if there are USD files in the extracted directory 308 | usd_files = list(result_path_obj.glob("*.usd")) + list(result_path_obj.glob("*.usda")) + list(result_path_obj.glob("*.usdc")) 309 | assert len(usd_files) > 0, f"No USD files found in {result_path}" 310 | print(f"Verified: USD file from Chinese text successfully downloaded and extracted to {result_path}") 311 | 312 | # Test image generation 313 | image_url = "https://lvlecheng.tos-cn-beijing.volces.com/chore/apple.jpg" 314 | task_id = beaver.generate_3d_from_image(image_url) 315 | assert task_id, "Task ID should be returned" 316 | print(f"Image generation test passed, task ID: {task_id}") 317 | result_path = beaver.monitor_task_status(task_id) 318 | result_path_obj = Path(result_path) 319 | assert result_path_obj.exists(), f"Downloaded file does not exist at {result_path}" 320 | assert result_path_obj.is_dir(), f"Expected directory at {result_path}" 321 | 322 | # Check if there are USD files in the extracted directory 323 | usd_files = list(result_path_obj.glob("*.usd")) + list(result_path_obj.glob("*.usda")) + list(result_path_obj.glob("*.usdc")) 324 | assert len(usd_files) > 0, f"No USD files found in {result_path}" 325 | print(f"Verified: USD file successfully downloaded and extracted to {result_path}") 326 | 327 | 328 | 329 | print("All tests passed!") 330 | 331 | except Exception as e: 332 | print(f"Test failed: {str(e)}") 333 | 334 | 335 | if __name__ == "__main__": 336 | 337 | 338 | asyncio.run(test_async()) 339 | # task = asyncio.create_task(test_async()) 340 | test() 341 | 342 | 343 | # Schedule the test to run in the background 344 | # task = asyncio.create_task(run_test_in_background()) 345 | # task.add_done_callback(lambda _: print(f"Test completed: {result_path}")) 346 | 347 | # print("Test scheduled to run in background") 348 | # while result_path is None: 349 | # sleep(1) 350 | # print(f"Async test completed, result path: {result_path}") 351 | 352 | 353 | -------------------------------------------------------------------------------- /isaac.sim.mcp_extension/isaac_sim_mcp_extension/usd.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import os 26 | import carb 27 | import omni.usd 28 | import omni 29 | from pathlib import Path 30 | from pxr import UsdShade, Sdf, UsdGeom, Gf 31 | from omni.isaac.core.utils.stage import add_reference_to_stage 32 | import json 33 | import requests 34 | 35 | 36 | class USDLoader: 37 | """ 38 | A class to load USD models and textures from local directories, 39 | with methods to bind materials and transform models. 40 | """ 41 | 42 | def __init__(self): 43 | """ 44 | Initialize the USDLoader class with default values. 45 | """ 46 | self.usd_prim = None 47 | self.material = None 48 | self.working_dir = Path(os.environ.get("USD_WORKING_DIR", "/tmp/usd")) 49 | self.stage = omni.usd.get_context().get_stage() 50 | 51 | def load_usd_model(self, abs_path=None, task_id=None): 52 | """ 53 | Load a USD model from either an absolute path or by task_id. 54 | 55 | Args: 56 | abs_path (str, optional): Absolute path to the USD file 57 | task_id (str, optional): Task ID to load model from working_dir 58 | 59 | Returns: 60 | str: Path to the loaded USD prim 61 | """ 62 | if not (abs_path or task_id): 63 | raise ValueError("Either abs_path or task_id must be provided") 64 | 65 | if task_id: 66 | usd_path = self.working_dir / task_id / "output.usd" 67 | else: 68 | usd_path = Path(abs_path) 69 | 70 | if not usd_path.exists(): 71 | raise FileNotFoundError(f"USD file not found at: {usd_path}") 72 | 73 | # Create a unique prim path name based on task_id or file name 74 | if task_id: 75 | prim_id = task_id[-5:] # Last 5 chars of task ID 76 | else: 77 | prim_id = usd_path.stem[:5] # First 5 chars of filename 78 | 79 | usd_prim_path = f"/World/model_{prim_id}" 80 | 81 | # Add the USD to the stage 82 | self.usd_prim = add_reference_to_stage(str(usd_path), usd_prim_path) 83 | 84 | print(f"Loaded USD model from {usd_path} at {usd_prim_path}") 85 | return usd_prim_path 86 | 87 | def load_texture_and_create_material(self, abs_path=None, task_id=None): 88 | """ 89 | Load a texture from either an absolute path or by task_id and create a material with it. 90 | 91 | Args: 92 | abs_path (str, optional): Absolute path to the texture file 93 | task_id (str, optional): Task ID to load texture from working_dir 94 | 95 | Returns: 96 | tuple: (str, UsdShade.Material) - Path to the loaded texture and the created material 97 | """ 98 | if not (abs_path or task_id): 99 | raise ValueError("Either abs_path or task_id must be provided") 100 | 101 | # Load texture 102 | if task_id: 103 | texture_path = self.working_dir / task_id / "textures" / "material_0.png" 104 | material_id = task_id[-5:] # Last 5 chars of task ID 105 | else: 106 | texture_path = Path(abs_path) 107 | material_id = texture_path.stem[:5] # First 5 chars of filename 108 | 109 | if not texture_path.exists(): 110 | raise FileNotFoundError(f"Texture file not found at: {texture_path}") 111 | 112 | # Create a unique material name 113 | material_path = f"/World/SimpleMaterial_{material_id}" 114 | 115 | # Create material 116 | material = UsdShade.Material.Define(self.stage, material_path) 117 | 118 | # Create shader 119 | shader = UsdShade.Shader.Define(self.stage, f"{material_path}/Shader") 120 | shader.CreateIdAttr("UsdPreviewSurface") 121 | 122 | # Connect shader to material 123 | material.CreateSurfaceOutput().ConnectToSource(shader.CreateOutput("surface", Sdf.ValueTypeNames.Token)) 124 | 125 | # Create texture 126 | texture = UsdShade.Shader.Define(self.stage, f"{material_path}/Texture") 127 | texture.CreateIdAttr("UsdUVTexture") 128 | texture.CreateInput("file", Sdf.ValueTypeNames.Asset).Set(str(texture_path)) 129 | 130 | # Connect texture to shader's diffuse color 131 | shader.CreateInput("diffuseColor", Sdf.ValueTypeNames.Color3f).ConnectToSource( 132 | texture.CreateOutput("rgb", Sdf.ValueTypeNames.Float3) 133 | ) 134 | 135 | print(f"Created material with texture at {material_path}") 136 | self.material = material 137 | return str(texture_path), material 138 | 139 | def bind_texture_to_model(self, prim=None, material=None): 140 | """ 141 | Bind a texture to a USD model. 142 | 143 | Args: 144 | prim (UsdGeom.Xformable, optional): The USD prim to bind the material to. 145 | If None, uses self.usd_prim 146 | material (UsdShade.Material, optional): The material to bind. 147 | If None, uses self.material 148 | 149 | Returns: 150 | bool: True if binding succeeded, False otherwise 151 | """ 152 | if prim is None: 153 | prim = self.usd_prim 154 | 155 | if material is None: 156 | material = self.material 157 | 158 | if prim is None or material is None: 159 | raise ValueError("Both prim and material must be provided or previously set") 160 | 161 | try: 162 | binding_api = UsdShade.MaterialBindingAPI(prim) 163 | binding_api.Bind(material) 164 | print(f"Successfully bound material to {prim.GetPath()}") 165 | return True 166 | except Exception as e: 167 | print(f"Failed to bind material: {str(e)}") 168 | return False 169 | 170 | def transform(self, prim=None, position=(0, 0, 50), scale=(10, 10, 10)): 171 | """ 172 | Transform a USD model by applying position and scale. 173 | 174 | Args: 175 | prim (UsdGeom.Xformable, optional): The USD prim to transform. 176 | If None, uses self.usd_prim 177 | position (tuple, optional): The position to set (x, y, z) 178 | scale (tuple, optional): The scale to set (x, y, z) 179 | 180 | Returns: 181 | UsdGeom.Xformable: The transformed prim 182 | """ 183 | if prim is None: 184 | prim = self.usd_prim 185 | 186 | if prim is None: 187 | raise ValueError("Prim must be provided or previously set") 188 | 189 | # Get the Xformable interface 190 | xformable = UsdGeom.Xformable(prim) 191 | 192 | # Check if transform operations already exist and use them 193 | xform_ops = xformable.GetOrderedXformOps() 194 | 195 | # Handle translation 196 | translate_op = None 197 | for op in xform_ops: 198 | if op.GetOpType() == UsdGeom.XformOp.TypeTranslate: 199 | translate_op = op 200 | break 201 | 202 | if translate_op: 203 | translate_op.Set(Gf.Vec3d(position[0],position[1],position[2])) 204 | else: 205 | xformable.AddTranslateOp().Set(Gf.Vec3d(position[0],position[1],position[2])) 206 | print(f"Model positioned at {position}") 207 | 208 | # Handle scaling 209 | scale_op = None 210 | for op in xform_ops: 211 | if op.GetOpType() == UsdGeom.XformOp.TypeScale: 212 | scale_op = op 213 | break 214 | 215 | if scale_op: 216 | scale_op.Set(Gf.Vec3d(scale[0],scale[1],scale[2])) 217 | else: 218 | xformable.AddScaleOp().Set(Gf.Vec3d(scale[0],scale[1],scale[2])) 219 | print(f"Model scaled to {scale}") 220 | 221 | return xformable 222 | 223 | def _load_prim(self, url, path:str="/World/my_usd", size=1.0): 224 | """Helper method to load a USD prim from url to specific scene path.""" 225 | try: 226 | stage = omni.usd.get_context().get_stage() 227 | 228 | # check path exists or not 229 | prim = stage.GetPrimAtPath(path) 230 | if prim: 231 | # Generate unique name based on path 232 | for p in stage.Traverse(): 233 | carb.log_info(f"Error in _load_prim: {p.GetPrimPath()}") 234 | count = len([p for p in stage.Traverse()]) 235 | path = f"{path}_{count+1}" 236 | 237 | carb.log_info(f"loading from {url} to {path}") 238 | # Create a new prim 239 | prim = stage.DefinePrim(path) 240 | carb.log_info(f"prim loaded: {path, prim}") 241 | 242 | # Add a reference to the external USD file 243 | prim.GetReferences().AddReference(url) 244 | 245 | return prim 246 | except Exception as e: 247 | carb.log_info(f"Error in _load_prim: {str(e)}") 248 | return {"error": str(e)} 249 | 250 | def _set_transform(self, prim, location=None, rotation=None, scale=None): 251 | """Set transform operations on a USD prim.""" 252 | if not prim.IsA(UsdGeom.Xformable): 253 | return 254 | 255 | xformable = UsdGeom.Xformable(prim) 256 | 257 | # Reset transform stack 258 | xformable.ClearXformOpOrder() 259 | 260 | # Build transform operations in order: scale, rotation, translation 261 | ops = [] 262 | 263 | # Add scale operation if provided 264 | if scale is not None: 265 | scale_op = xformable.AddXformOp(UsdGeom.XformOp.TypeScale, UsdGeom.XformOp.PrecisionFloat) 266 | scale_op.Set(Gf.Vec3d(scale[0], scale[1], scale[2])) 267 | ops.append(scale_op) 268 | # Add rotation operation if provided 269 | if rotation is not None: 270 | rot_op = xformable.AddXformOp(UsdGeom.XformOp.TypeRotateXYZ, UsdGeom.XformOp.PrecisionDouble) 271 | rot_op.Set(Gf.Vec3d(rotation[0], rotation[1], rotation[2])) 272 | ops.append(rot_op) 273 | # Add translation operation if provided 274 | if location is not None: 275 | trans_op = xformable.AddXformOp(UsdGeom.XformOp.TypeTranslate, UsdGeom.XformOp.PrecisionDouble) 276 | trans_op.Set(Gf.Vec3d(location[0], location[1], location[2])) 277 | ops.append(trans_op) 278 | # Apply transform operations in order 279 | xformable.SetXformOpOrder(ops) 280 | 281 | def _set_color(self, prim, color): 282 | """Set a simple color material on a USD prim.""" 283 | stage = omni.usd.get_context().get_stage() 284 | 285 | # Only apply to geometric prims 286 | if not UsdGeom.Gprim(prim): 287 | return 288 | 289 | # Create material with unique path 290 | material_path = str(prim.GetPath()) + "_material" 291 | material = UsdShade.Material.Define(stage, material_path) 292 | 293 | # Create preview surface shader 294 | shader_path = material_path + "/PreviewSurface" 295 | pbrShader = UsdShade.Shader.Define(stage, shader_path) 296 | pbrShader.CreateIdAttr("UsdPreviewSurface") 297 | 298 | # Set diffuse color 299 | pbrShader.CreateInput("diffuseColor", Sdf.ValueTypeNames.Color3f).Set(Gf.Vec3f(color[0], color[1], color[2])) 300 | 301 | # Connect shader to material surface 302 | material.CreateSurfaceOutput().ConnectToSource(pbrShader.ConnectableAPI(), "surface") 303 | 304 | # Bind material to prim 305 | UsdShade.MaterialBindingAPI(prim).Bind(material) 306 | 307 | def load_usd_from_url(self, 308 | url_path=None, 309 | target_path=None, 310 | location=None, 311 | rotation=None, 312 | scale=None, 313 | size=1.0, 314 | color=None): 315 | """ 316 | Load a usd object from url and insert into current scene at target path. 317 | 318 | Args: 319 | url (str): url of the usd object to be loaded 320 | target_path (str, optional): target path for the loaded object 321 | location (list, optional): [x, y, z] position 322 | rotation (list, optional): [x, y, z] rotation 323 | scale (list, optional): [x, y, z] scale 324 | size (float, optional): Size parameter 325 | color (list, optional): [r, g, b] color 326 | 327 | Returns: 328 | dict: Information about the loaded object 329 | """ 330 | try: 331 | # Create the prim based on type 332 | #url = "https://omniverse-content-production.s3.us-west-2.amazonaws.com/Assets/DigitalTwin/Assets/Warehouse/Storage/Drums/Plastic_A/PlasticDrum_A04_PR_V_NVD_01.usd" 333 | prim = self._load_prim(url_path, path=target_path) 334 | path = str(prim.GetPath()) 335 | 336 | # Apply transform if provided 337 | if location or rotation or scale: 338 | self._set_transform(prim, location, rotation, scale) 339 | 340 | # Apply color if provided 341 | if color: 342 | self._set_color(prim, color) 343 | 344 | # Return object info 345 | prim_info = { 346 | "target_path": path, 347 | } 348 | 349 | # Add transform information 350 | if prim.IsA(UsdGeom.Xformable): 351 | xformable = UsdGeom.Xformable(prim) 352 | trans = xformable.GetLocalTransformation() 353 | translation = trans.ExtractTranslation() 354 | 355 | prim_info["transform"] = { 356 | "translation": [translation[0], translation[1], translation[2]], 357 | } 358 | 359 | details = json.dumps(prim_info, indent=2) 360 | 361 | 362 | print(f"Loaded USD model from {url_path} at {path}") 363 | return path 364 | 365 | except Exception as e: 366 | carb.log_info(f"Error in load_usd_from_url: {str(e)}") 367 | return {"error": str(e)} 368 | 369 | @staticmethod 370 | def test_tasks_load(): 371 | """ 372 | Test method to load and process multiple task IDs. 373 | """ 374 | loader = USDLoader() 375 | 376 | # List of task IDs to test 377 | task_ids = ["cgt-20250408202506-6tdc4", "cgt-20250408202622-rzcgg", "cgt-20250408202753-44b9f"] 378 | 379 | for task_id in task_ids: 380 | try: 381 | # Load model 382 | prim_path = loader.load_usd_model(task_id=task_id) 383 | 384 | # # Load texture 385 | # texture_path = loader.load_texture(task_id=task_id) 386 | 387 | # Create material with texture 388 | texture_path, material = loader.load_texture_and_create_material(task_id=task_id) 389 | 390 | # Bind texture to model 391 | loader.bind_texture_to_model() 392 | 393 | # Transform model 394 | loader.transform(position=(0, 0, 50 + task_ids.index(task_id) * 20)) 395 | 396 | print(f"Successfully processed task {task_id}") 397 | except Exception as e: 398 | print(f"Error processing task {task_id}: {str(e)}") 399 | 400 | @staticmethod 401 | def test_absolute_paths(): 402 | """ 403 | Test method for loading from absolute paths. 404 | """ 405 | loader = USDLoader() 406 | 407 | # Test with absolute paths 408 | model_path = "/tmp/usd/cgt-20250408202506-6tdc4/output.usd" 409 | texture_path = "/tmp/usd/cgt-20250408202506-6tdc4/textures/material_0.png" 410 | 411 | try: 412 | # Load model 413 | prim_path = loader.load_usd_model(abs_path=model_path) 414 | 415 | # Load texture 416 | texture_path, material = loader.load_texture_and_create_material(abs_path=texture_path) 417 | 418 | # Create material with texture 419 | # material = loader.create_material_with_texture(texture_path) 420 | 421 | # Bind texture to model 422 | loader.bind_texture_to_model() 423 | 424 | # Transform model 425 | loader.transform() 426 | 427 | print("Successfully tested absolute paths") 428 | except Exception as e: 429 | print(f"Error testing absolute paths: {str(e)}") 430 | 431 | class USDSearch3d: 432 | def __init__(self): 433 | """Initialize Beaver3d with model name and API key from environment variables""" 434 | self.api_key = os.environ.get("NVIDIA_API_KEY") 435 | self.usd_search_server = 'https://ai.api.nvidia.com/v1/omniverse/nvidia/usdsearch' 436 | if not self.api_key: 437 | raise Exception("NVIDIA_API_KEY environment variable not set, USD Search service is not available untill NVIDIA_API_KEY is set") 438 | 439 | def search(self, text_prompt:str): 440 | #get your own NVIDIA_API_KEY from build.nvidia.com 441 | response = requests.post( 442 | url=self.usd_search_server, 443 | headers={ 444 | 'Authorization': f'Bearer {self.api_key}', 445 | 'Accept': 'application/json', 446 | 'Content-Type': 'application/json', 447 | }, 448 | data=json.dumps( 449 | dict( 450 | description = text_prompt, 451 | file_extension_include ='usd*', 452 | return_images ='true', 453 | return_metadata = 'true', 454 | return_vision_generated_metadata = 'true', 455 | cutoff_threshold = '1.05', 456 | limit = '50' 457 | ) 458 | ) 459 | ) 460 | carb.log_info(f"usd_search_3d_from_text return code: {response.status_code}") 461 | details = json.dumps(response.json(), indent=2) 462 | details = json.loads(details) 463 | url = details[0]['url'] 464 | 465 | # Convert S3 URL to HTTPS URL if needed 466 | if url.startswith("s3://deepsearch-demo-content"): 467 | url = url.replace("s3://deepsearch-demo-content", "https://omniverse-content-production.s3.us-west-2.amazonaws.com") 468 | return url 469 | 470 | @staticmethod 471 | def test_search_and_load(): 472 | text_prompt = "a rusty desk" 473 | searcher3d = USDSearch3d() 474 | url = searcher3d.search(text_prompt) 475 | target_path = "/World/search_usd" 476 | loader = USDLoader() 477 | prim_path = loader.load_usd_from_url( url, target_path ) 478 | 479 | @staticmethod 480 | def usd_search_3d_from_text(text_prompt:str, target_path:str, position=(0, 0, 50), scale=(10, 10, 10)): 481 | """ 482 | search a USD assets in USD Search service, load it into the scene and transform it. 483 | 484 | Args: 485 | text_prompt (str, ): Text prompt for 3D generation 486 | target_path (str, ): target path in current scene stage 487 | position (tuple, optional): Position to place the model 488 | scale (tuple, optional): Scale of the model 489 | 490 | Returns: 491 | dict: Dictionary with the task_id and prim_path 492 | """ 493 | try: 494 | searcher3d = USDSearch3d() 495 | url = searcher3d.search(text_prompt) 496 | loader = USDLoader() 497 | #TODO： need to validate transform location 498 | prim_path = loader.load_usd_from_url(url, target_path, location=position, scale=scale) 499 | stage = omni.usd.get_context().get_stage() 500 | prim = stage.GetPrimAtPath(prim_path) 501 | loader.transform(prim=prim, position=position, scale=scale) 502 | 503 | return {"url": url, "prim_path": prim_path} 504 | except Exception as e: 505 | carb.log_error(f"Error in usd_search_3d_from_text: {str(e)}") 506 | return {"error": str(e)} 507 | 508 | 509 | if __name__ == "__main__": 510 | # USDLoader.main() 511 | #USDLoader.test_tasks_load() 512 | #USDLoader.test_absolute_paths() 513 | #USDSearch3d.test_search_and_load() 514 | 515 | USDSearch3d.usd_search_3d_from_text(text_prompt="a rusty desk", target_path="/World/search_desk", position=(0, 0, 0), scale=(3, 3, 3)) 516 | #USDSearch3d.usd_search_3d_from_text(text_prompt="a rusty apple", target_path="/World/search_apple", position=(0, 0, 0), scale=(3, 3, 3)) 517 | USDSearch3d.usd_search_3d_from_text(text_prompt="a rusty chair", target_path="/World/search_chair", position=(10, 0, 0), scale=(3, 3, 3)) 518 | USDSearch3d.usd_search_3d_from_text(text_prompt="a rusty sofa", target_path="/World/search_chair", position=(20, 0, 0), scale=(3, 3, 3)) 519 | 520 | 521 | 522 | -------------------------------------------------------------------------------- /isaac_mcp/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | """Isaac Sim MCP Server package. 26 | 27 | This package provides an MCP (Model Context Protocol) interface for Isaac Sim, 28 | allowing AI assistants to control Isaac Sim through a WebSocket server. 29 | """ 30 | 31 | __version__ = "0.1.0" 32 | 33 | 34 | 35 | # The mcp_server module can be imported directly without Isaac Sim dependencies 36 | # For the direct connection mode 37 | try: 38 | from . import server 39 | __all__.append("server") 40 | except (ImportError, ModuleNotFoundError): 41 | pass 42 | -------------------------------------------------------------------------------- /isaac_mcp/server.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2023-2025 omni-mcp 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | # isaac_sim_mcp_server.py 26 | import time 27 | from mcp.server.fastmcp import FastMCP, Context, Image 28 | import socket 29 | import json 30 | import asyncio 31 | import logging 32 | from dataclasses import dataclass 33 | from contextlib import asynccontextmanager 34 | from typing import AsyncIterator, Dict, Any, List 35 | import os 36 | from pathlib import Path 37 | import base64 38 | from urllib.parse import urlparse 39 | 40 | # Configure logging 41 | logging.basicConfig(level=logging.INFO, 42 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 43 | logger = logging.getLogger("IsaacMCPServer") 44 | 45 | @dataclass 46 | class IsaacConnection: 47 | host: str 48 | port: int 49 | sock: socket.socket = None # Changed from 'socket' to 'sock' to avoid naming conflict 50 | 51 | def connect(self) -> bool: 52 | """Connect to the Isaac addon socket server""" 53 | if self.sock: 54 | return True 55 | 56 | try: 57 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 58 | self.sock.connect((self.host, self.port)) 59 | logger.info(f"Connected to Isaac at {self.host}:{self.port}") 60 | return True 61 | except Exception as e: 62 | logger.error(f"Failed to connect to Isaac: {str(e)}") 63 | self.sock = None 64 | return False 65 | 66 | def disconnect(self): 67 | """Disconnect from the Isaac addon""" 68 | if self.sock: 69 | try: 70 | self.sock.close() 71 | except Exception as e: 72 | logger.error(f"Error disconnecting from Isaac: {str(e)}") 73 | finally: 74 | self.sock = None 75 | 76 | def receive_full_response(self, sock, buffer_size=16384): 77 | """Receive the complete response, potentially in multiple chunks""" 78 | chunks = [] 79 | # Use a consistent timeout value that matches the addon's timeout 80 | sock.settimeout(300.0) # Match the extension's timeout 81 | 82 | try: 83 | while True: 84 | try: 85 | logger.info("Waiting for data from Isaac") 86 | #time.sleep(0.5) 87 | chunk = sock.recv(buffer_size) 88 | if not chunk: 89 | # If we get an empty chunk, the connection might be closed 90 | if not chunks: # If we haven't received anything yet, this is an error 91 | raise Exception("Connection closed before receiving any data") 92 | break 93 | 94 | chunks.append(chunk) 95 | 96 | # Check if we've received a complete JSON object 97 | try: 98 | data = b''.join(chunks) 99 | json.loads(data.decode('utf-8')) 100 | # If we get here, it parsed successfully 101 | logger.info(f"Received complete response ({len(data)} bytes)") 102 | return data 103 | except json.JSONDecodeError: 104 | # Incomplete JSON, continue receiving 105 | continue 106 | except socket.timeout: 107 | # If we hit a timeout during receiving, break the loop and try to use what we have 108 | logger.warning("Socket timeout during chunked receive") 109 | break 110 | except (ConnectionError, BrokenPipeError, ConnectionResetError) as e: 111 | logger.error(f"Socket connection error during receive: {str(e)}") 112 | raise # Re-raise to be handled by the caller 113 | except socket.timeout: 114 | logger.warning("Socket timeout during chunked receive") 115 | except Exception as e: 116 | logger.error(f"Error during receive: {str(e)}") 117 | raise 118 | 119 | # If we get here, we either timed out or broke out of the loop 120 | # Try to use what we have 121 | if chunks: 122 | data = b''.join(chunks) 123 | logger.info(f"Returning data after receive completion ({len(data)} bytes)") 124 | try: 125 | # Try to parse what we have 126 | json.loads(data.decode('utf-8')) 127 | return data 128 | except json.JSONDecodeError: 129 | # If we can't parse it, it's incomplete 130 | raise Exception("Incomplete JSON response received") 131 | else: 132 | raise Exception("No data received") 133 | 134 | def send_command(self, command_type: str, params: Dict[str, Any] = None) -> Dict[str, Any]: 135 | """Send a command to Isaac and return the response""" 136 | if not self.sock and not self.connect(): 137 | raise ConnectionError("Not connected to Isaac") 138 | 139 | command = { 140 | "type": command_type, 141 | "params": params or {} 142 | } 143 | 144 | try: 145 | # Log the command being sent 146 | logger.info(f"Sending command: {command_type} with params: {params}") 147 | 148 | # Send the command 149 | self.sock.sendall(json.dumps(command).encode('utf-8')) 150 | logger.info(f"Command sent, waiting for response...") 151 | 152 | # Set a timeout for receiving - use the same timeout as in receive_full_response 153 | self.sock.settimeout(300.0) # Match the extension's timeout 154 | 155 | # Receive the response using the improved receive_full_response method 156 | response_data = self.receive_full_response(self.sock) 157 | logger.info(f"Received {len(response_data)} bytes of data") 158 | 159 | response = json.loads(response_data.decode('utf-8')) 160 | logger.info(f"Response parsed, status: {response.get('status', 'unknown')}") 161 | 162 | if response.get("status") == "error": 163 | logger.error(f"Isaac error: {response.get('message')}") 164 | raise Exception(response.get("message", "Unknown error from Isaac")) 165 | 166 | return response.get("result", {}) 167 | except socket.timeout: 168 | logger.error("Socket timeout while waiting for response from Isaac") 169 | # Don't try to reconnect here - let the get_isaac_connection handle reconnection 170 | # Just invalidate the current socket so it will be recreated next time 171 | self.sock = None 172 | raise Exception("Timeout waiting for Isaac response - try simplifying your request") 173 | except (ConnectionError, BrokenPipeError, ConnectionResetError) as e: 174 | logger.error(f"Socket connection error: {str(e)}") 175 | self.sock = None 176 | raise Exception(f"Connection to Isaac lost: {str(e)}") 177 | except json.JSONDecodeError as e: 178 | logger.error(f"Invalid JSON response from Isaac: {str(e)}") 179 | # Try to log what was received 180 | if 'response_data' in locals() and response_data: 181 | logger.error(f"Raw response (first 200 bytes): {response_data[:200]}") 182 | raise Exception(f"Invalid response from Isaac: {str(e)}") 183 | except Exception as e: 184 | logger.error(f"Error communicating with Isaac: {str(e)}") 185 | # Don't try to reconnect here - let the get_isaac_connection handle reconnection 186 | self.sock = None 187 | raise Exception(f"Communication error with Isaac: {str(e)}") 188 | 189 | @asynccontextmanager 190 | async def server_lifespan(server: FastMCP) -> AsyncIterator[Dict[str, Any]]: 191 | """Manage server startup and shutdown lifecycle""" 192 | # We don't need to create a connection here since we're using the global connection 193 | # for resources and tools 194 | 195 | try: 196 | # Just log that we're starting up 197 | logger.info("IsaacMCP server starting up") 198 | 199 | # Try to connect to Isaac on startup to verify it's available 200 | try: 201 | # This will initialize the global connection if needed 202 | isaac = get_isaac_connection() 203 | logger.info("Successfully connected to Isaac on startup") 204 | except Exception as e: 205 | logger.warning(f"Could not connect to Isaac on startup: {str(e)}") 206 | logger.warning("Make sure the Isaac addon is running before using Isaac resources or tools") 207 | 208 | # Return an empty context - we're using the global connection 209 | yield {} 210 | finally: 211 | # Clean up the global connection on shutdown 212 | global _isaac_connection 213 | if _isaac_connection: 214 | logger.info("Disconnecting from Isaac Sim on shutdown") 215 | _isaac_connection.disconnect() 216 | _isaac_connection = None 217 | logger.info("Isaac SimMCP server shut down") 218 | 219 | # Create the MCP server with lifespan support 220 | mcp = FastMCP( 221 | "IsaacSimMCP", 222 | description="Isaac Sim integration through the Model Context Protocol", 223 | lifespan=server_lifespan 224 | ) 225 | 226 | # Resource endpoints 227 | 228 | # Global connection for resources (since resources can't access context) 229 | _isaac_connection = None 230 | # _polyhaven_enabled = False # Add this global variable 231 | 232 | def get_isaac_connection(): 233 | """Get or create a persistent Isaac connection""" 234 | global _isaac_connection, _polyhaven_enabled # Add _polyhaven_enabled to globals 235 | 236 | # If we have an existing connection, check if it's still valid 237 | if _isaac_connection is not None: 238 | try: 239 | 240 | return _isaac_connection 241 | except Exception as e: 242 | # Connection is dead, close it and create a new one 243 | logger.warning(f"Existing connection is no longer valid: {str(e)}") 244 | try: 245 | _isaac_connection.disconnect() 246 | except: 247 | pass 248 | _isaac_connection = None 249 | 250 | # Create a new connection if needed 251 | if _isaac_connection is None: 252 | _isaac_connection = IsaacConnection(host="localhost", port=8766) 253 | if not _isaac_connection.connect(): 254 | logger.error("Failed to connect to Isaac") 255 | _isaac_connection = None 256 | raise Exception("Could not connect to Isaac. Make sure the Isaac addon is running.") 257 | logger.info("Created new persistent connection to Isaac") 258 | 259 | return _isaac_connection 260 | 261 | 262 | @mcp.tool() 263 | def get_scene_info(ctx: Context) -> str: 264 | """Ping status of Isaac Sim Extension Server""" 265 | try: 266 | isaac = get_isaac_connection() 267 | result = isaac.send_command("get_scene_info") 268 | print("result: ", result) 269 | 270 | # Just return the JSON representation of what Isaac sent us 271 | return json.dumps(result, indent=2) 272 | # return json.dumps(result) 273 | # return result 274 | except Exception as e: 275 | logger.error(f"Error getting scene info from Isaac: {str(e)}") 276 | # return f"Error getting scene info: {str(e)}" 277 | return {"status": "error", "error": str(e), "message": "Error getting scene info"} 278 | 279 | # @mcp.tool() 280 | # def get_object_info(ctx: Context, object_name: str) -> str: 281 | # """ 282 | # Get detailed information about a specific object in the Isaac scene. 283 | 284 | # Parameters: 285 | # - object_name: The name of the object to get information about 286 | # """ 287 | # try: 288 | # isaac = get_isaac_connection() 289 | # result = isaac.send_command("get_object_info", {"name": object_name}) 290 | 291 | # # Just return the JSON representation of what Isaac sent us 292 | # return json.dumps(result, indent=2) 293 | # except Exception as e: 294 | # logger.error(f"Error getting object info from Isaac: {str(e)}") 295 | # return f"Error getting object info: {str(e)}" 296 | 297 | @mcp.tool("create_physics_scene") 298 | def create_physics_scene( 299 | objects: List[Dict[str, Any]] = [], 300 | floor: bool = True, 301 | gravity: List[float] = [0, -0.981, 0], 302 | scene_name: str = "physics_scene" 303 | ) -> Dict[str, Any]: 304 | """Create a physics scene with multiple objects. Before create physics scene, you need to call get_scene_info() first to verify availability of connection. 305 | 306 | Args: 307 | objects: List of objects to create. Each object should have at least 'type' and 'position'. 308 | objects = [ 309 | {"path": "/World/Cube", "type": "Cube", "size": 20, "position": [0, 100, 0]}, 310 | {"path": "/World/Sphere", "type": "Sphere", "radius": 5, "position": [5, 200, 0]}, 311 | {"path": "/World/Cone", "type": "Cone", "height": 8, "radius": 3, "position": [-5, 150, 0]} 312 | ] 313 | floor: Whether to create a floor. deafult is True 314 | gravity: The gravity vector. Default is [0, 0, -981.0] (cm/s^2). 315 | scene_name: The name of the scene. deafult is "physics_scene" 316 | 317 | Returns: 318 | Dictionary with result information. 319 | """ 320 | params = {"objects": objects, "floor": floor} 321 | 322 | if gravity is not None: 323 | params["gravity"] = gravity 324 | if scene_name is not None: 325 | params["scene_name"] = scene_name 326 | try: 327 | # Get the global connection 328 | isaac = get_isaac_connection() 329 | 330 | result = isaac.send_command("create_physics_scene", params) 331 | return f"create_physics_scene successfully: {result.get('result', '')}, {result.get('message', '')}" 332 | except Exception as e: 333 | logger.error(f"Error create_physics_scene: {str(e)}") 334 | # return f"Error create_physics_scene: {str(e)}" 335 | return {"status": "error", "error": str(e), "message": "Error create_physics_scene"} 336 | 337 | @mcp.tool("create_robot") 338 | def create_robot(robot_type: str = "g1", position: List[float] = [0, 0, 0]) -> str: 339 | """Create a robot in the Isaac scene. Directly create robot prim in stage at the right position. For any creation of robot, you need to call create_physics_scene() first. call create_robot() as first attmpt beofre call execute_script(). 340 | 341 | Args: 342 | robot_type: The type of robot to create. Available options: 343 | - "franka": Franka Emika Panda robot 344 | - "jetbot": NVIDIA JetBot robot 345 | - "carter": Carter delivery robot 346 | - "g1": Unitree G1 quadruped robot (default) 347 | - "go1": Unitree Go1 quadruped robot 348 | 349 | Returns: 350 | String with result information. 351 | """ 352 | isaac = get_isaac_connection() 353 | result = isaac.send_command("create_robot", {"robot_type": robot_type, "position": position}) 354 | return f"create_robot successfully: {result.get('result', '')}, {result.get('message', '')}" 355 | 356 | @mcp.tool("omni_kit_command") 357 | def omni_kit_command(command: str = "CreatePrim", prim_type: str = "Sphere") -> str: 358 | """Execute an Omni Kit command. 359 | 360 | Args: 361 | command: The Omni Kit command to execute. 362 | prim_type: The primitive type for the command. 363 | 364 | Returns: 365 | String with result information. 366 | """ 367 | try: 368 | # Get the global connection 369 | isaac = get_isaac_connection() 370 | 371 | result = isaac.send_command("omini_kit_command", { 372 | "command": command, 373 | "prim_type": prim_type 374 | }) 375 | return f"Omni Kit command executed successfully: {result.get('message', '')}" 376 | except Exception as e: 377 | logger.error(f"Error executing Omni Kit command: {str(e)}") 378 | # return f"Error executing Omni Kit command: {str(e)}" 379 | return {"status": "error", "error": str(e), "message": "Error executing Omni Kit command"} 380 | 381 | 382 | @mcp.tool() 383 | def execute_script(ctx: Context, code: str) -> str: 384 | """ 385 | Before execute script pls check prompt from asset_creation_strategy() to ensure the scene is properly initialized. 386 | Execute arbitrary Python code in Isaac Sim. Before executing any code, first verify if get_scene_info() has been called to ensure the scene is properly initialized. Always print the formatted code into chat to confirm before execution to confirm its correctness. 387 | Before execute script pls check if create_physics_scene() has been called to ensure the physics scene is properly initialized. 388 | When working with robots, always try using the create_robot() function first before resorting to execute_script(). The create_robot() function provides a simpler, more reliable way to add robots to your scene with proper initialization and positioning. Only use execute_script() for robot creation when you need custom configurations or behaviors not supported by create_robot(). 389 | 390 | For physics simulation, avoid using simulation_context to run simulations in the main thread as this can cause blocking. Instead, use the World class with async methods for initializing physics and running simulations. For example, use my_world = World(physics_dt=1.0/60.0) and my_world.step_async() in a loop, which allows for better performance and responsiveness. If you need to wait for physics to stabilize, consider using my_world.play() followed by multiple step_async() calls. 391 | To create an simulation of Franka robot, the code should be like this: 392 | from omni.isaac.core import SimulationContext 393 | from omni.isaac.core.utils.prims import create_prim 394 | from omni.isaac.core.utils.stage import add_reference_to_stage, is_stage_loading 395 | from omni.isaac.nucleus import get_assets_root_path 396 | 397 | assets_root_path = get_assets_root_path() 398 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 399 | simulation_context = SimulationContext() 400 | add_reference_to_stage(asset_path, "/Franka") 401 | #create_prim("/DistantLight", "DistantLight") 402 | 403 | 404 | 405 | 406 | To control the Franka robot, the code should be like this: 407 | 408 | from omni.isaac.core import SimulationContext 409 | from omni.isaac.core.articulations import Articulation 410 | from omni.isaac.core.utils.stage import add_reference_to_stage 411 | from omni.isaac.nucleus import get_assets_root_path 412 | 413 | my_world = World(stage_units_in_meters=1.0) 414 | 415 | assets_root_path = get_assets_root_path() 416 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 417 | 418 | simulation_context = SimulationContext() 419 | add_reference_to_stage(asset_path, "/Franka") 420 | 421 | # need to initialize physics getting any articulation..etc 422 | simulation_context.initialize_physics() 423 | art = Articulation("/Franka") 424 | art.initialize(my_world.physics_sim_view) 425 | dof_ptr = art.get_dof_index("panda_joint2") 426 | 427 | simulation_context.play() 428 | # NOTE: before interacting with dc directly you need to step physics for one step at least 429 | # simulation_context.step(render=True) which happens inside .play() 430 | for i in range(1000): 431 | art.set_joint_positions([-1.5], [dof_ptr]) 432 | simulation_context.step(render=True) 433 | 434 | simulation_context.stop() 435 | 436 | 437 | 438 | Parameters: 439 | - code: The Python code to execute, e.g. "omni.kit.commands.execute("CreatePrim", prim_type="Sphere")" 440 | """ 441 | try: 442 | # Get the global connection 443 | isaac = get_isaac_connection() 444 | print("code: ", code) 445 | 446 | result = isaac.send_command("execute_script", {"code": code}) 447 | print("result: ", result) 448 | return result 449 | # return f"Code executed successfully: {result.get('result', '')}" 450 | except Exception as e: 451 | logger.error(f"Error executing code: {str(e)}") 452 | # return f"Error executing code: {str(e)}" 453 | return {"status": "error", "error": str(e), "message": "Error executing code"} 454 | 455 | @mcp.prompt() 456 | def asset_creation_strategy() -> str: 457 | """Defines the preferred strategy for creating assets in Isaac Sim""" 458 | return """ 459 | 0. Before anything, always check the scene from get_scene_info(), retrive rool path of assset through return value of assets_root_path. 460 | 1. If the scene is empty, create a physics scene with create_physics_scene() 461 | 2. if execute script due to communication error, then retry 3 times at most 462 | 463 | 3. For Franka robot simulation, the code should be like this: 464 | from omni.isaac.core import SimulationContext 465 | from omni.isaac.core.utils.prims import create_prim 466 | from omni.isaac.core.utils.stage import add_reference_to_stage, is_stage_loading 467 | from omni.isaac.nucleus import get_assets_root_path 468 | 469 | assets_root_path = get_assets_root_path() 470 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 471 | add_reference_to_stage(asset_path, "/Franka") 472 | #create_prim("/DistantLight", "DistantLight") 473 | 474 | 475 | # need to initialize physics getting any articulation..etc 476 | simulation_context = SimulationContext() 477 | simulation_context.initialize_physics() 478 | simulation_context.play() 479 | 480 | for i in range(1000): 481 | simulation_context.step(render=True) 482 | 483 | simulation_context.stop() 484 | 485 | 4. For Franka robot control, the code should be like this: 486 | 487 | from omni.isaac.core import SimulationContext 488 | from omni.isaac.core.utils.prims import create_prim 489 | from omni.isaac.core.utils.stage import add_reference_to_stage, is_stage_loading 490 | from omni.isaac.nucleus import get_assets_root_path 491 | from pxr import UsdPhysics 492 | 493 | def create_physics_scene(stage, scene_path="/World/PhysicsScene"): 494 | if not stage.GetPrimAtPath(scene_path): 495 | UsdPhysics.Scene.Define(stage, scene_path) 496 | 497 | return stage.GetPrimAtPath(scene_path) 498 | 499 | stage = omni.usd.get_context().get_stage() 500 | physics_scene = create_physics_scene(stage) 501 | if not physics_scene: 502 | raise RuntimeError("Failed to create or find physics scene") 503 | import omni.physics.tensors as physx 504 | 505 | def create_simulation_view(stage): 506 | sim_view = physx.create_simulation_view(stage) 507 | if not sim_view: 508 | carb.log_error("Failed to create simulation view") 509 | return None 510 | 511 | return sim_view 512 | 513 | sim_view = create_simulation_view(stage) 514 | if not sim_view: 515 | raise RuntimeError("Failed to create simulation view") 516 | 517 | simulation_context = SimulationContext() 518 | assets_root_path = get_assets_root_path() 519 | asset_path = assets_root_path + "/Isaac/Robots/Franka/franka_alt_fingers.usd" 520 | add_reference_to_stage(asset_path, "/Franka") 521 | #create_prim("/DistantLight", "DistantLight") 522 | 523 | # need to initialize physics getting any articulation..etc 524 | simulation_context.initialize_physics() 525 | art = Articulation("/Franka") 526 | art.initialize() 527 | dof_ptr = art.get_dof_index("panda_joint2") 528 | 529 | simulation_context.play() 530 | # NOTE: before interacting with dc directly you need to step physics for one step at least 531 | # simulation_context.step(render=True) which happens inside .play() 532 | for i in range(1000): 533 | art.set_joint_positions([-1.5], [dof_ptr]) 534 | simulation_context.step(render=True) 535 | 536 | simulation_context.stop() 537 | 538 | 5. For Jetbot simulation, the code should be like this: 539 | import carb 540 | import numpy as np 541 | from omni.isaac.core import World 542 | from omni.isaac.core import SimulationContext 543 | from omni.isaac.core.utils.prims import create_prim 544 | from omni.isaac.nucleus import get_assets_root_path 545 | from omni.isaac.wheeled_robots.controllers.differential_controller import DifferentialController 546 | from omni.isaac.wheeled_robots.robots import WheeledRobot 547 | 548 | simulation_context = SimulationContext() 549 | simulation_context.initialize_physics() 550 | 551 | my_world = World(stage_units_in_meters=1.0) 552 | 553 | assets_root_path = get_assets_root_path() 554 | if assets_root_path is None: 555 | carb.log_error("Could not find Isaac Sim assets folder") 556 | jetbot_asset_path = assets_root_path + "/Isaac/Robots/Jetbot/jetbot.usd" 557 | my_jetbot = my_world.scene.add( 558 | WheeledRobot( 559 | prim_path="/World/Jetbot", 560 | name="my_jetbot", 561 | wheel_dof_names=["left_wheel_joint", "right_wheel_joint"], 562 | create_robot=True, 563 | usd_path=jetbot_asset_path, 564 | position=np.array([0, 0.0, 2.0]), 565 | ) 566 | ) 567 | 568 | 569 | create_prim("/DistantLight", "DistantLight") 570 | # need to initialize physics getting any articulation..etc 571 | 572 | 573 | my_world.scene.add_default_ground_plane() 574 | my_controller = DifferentialController(name="simple_control", wheel_radius=0.03, wheel_base=0.1125) 575 | my_world.reset() 576 | 577 | simulation_context.play() 578 | for i in range(10): 579 | simulation_context.step(render=True) 580 | 581 | i = 0 582 | reset_needed = False 583 | while i < 2000: 584 | my_world.step(render=True) 585 | if my_world.is_stopped() and not reset_needed: 586 | reset_needed = True 587 | if my_world.is_playing(): 588 | if reset_needed: 589 | my_world.reset() 590 | my_controller.reset() 591 | reset_needed = False 592 | if i >= 0 and i < 1000: 593 | # forward 594 | my_jetbot.apply_wheel_actions(my_controller.forward(command=[0.05, 0])) 595 | print(my_jetbot.get_linear_velocity()) 596 | elif i >= 1000 and i < 1300: 597 | # rotate 598 | my_jetbot.apply_wheel_actions(my_controller.forward(command=[0.0, np.pi / 12])) 599 | print(my_jetbot.get_angular_velocity()) 600 | elif i >= 1300 and i < 2000: 601 | # forward 602 | my_jetbot.apply_wheel_actions(my_controller.forward(command=[0.05, 0])) 603 | elif i == 2000: 604 | i = 0 605 | i += 1 606 | simulation_context.stop() 607 | 608 | 6. For G1 simulation, the code should be like this see g1_ok.py 609 | 610 | 611 | """ 612 | 613 | 614 | def _process_bbox(original_bbox: list[float] | list[int] | None) -> list[int] | None: 615 | if original_bbox is None: 616 | return None 617 | if all(isinstance(i, int) for i in original_bbox): 618 | return original_bbox 619 | if any(i<=0 for i in original_bbox): 620 | raise ValueError("Incorrect number range: bbox must be bigger than zero!") 621 | return [int(float(i) / max(original_bbox) * 100) for i in original_bbox] if original_bbox else None 622 | 623 | 624 | #@mcp.tool() 625 | def get_beaver3d_status(ctx: Context) -> str: 626 | """ 627 | TODO: Get the status of Beaver3D. 628 | """ 629 | return "Beaver3D service is Available" 630 | 631 | 632 | 633 | @mcp.tool("generate_3d_from_text_or_image") 634 | def generate_3d_from_text_or_image( 635 | ctx: Context, 636 | text_prompt: str = None, 637 | image_url: str = None, 638 | position: List[float] = [0, 0, 50], 639 | scale: List[float] = [10, 10, 10] 640 | ) -> str: 641 | """ 642 | Generate a 3D model from text or image, load it into the scene and transform it. 643 | 644 | Args: 645 | text_prompt (str, optional): Text prompt for 3D generation 646 | image_url (str, optional): URL of image for 3D generation 647 | position (list, optional): Position to place the model [x, y, z] 648 | scale (list, optional): Scale of the model [x, y, z] 649 | 650 | Returns: 651 | String with the task_id and prim_path information 652 | """ 653 | if not (text_prompt or image_url): 654 | return "Error: Either text_prompt or image_url must be provided" 655 | 656 | try: 657 | # Get the global connection 658 | isaac = get_isaac_connection() 659 | 660 | result = isaac.send_command("generate_3d_from_text_or_image", { 661 | "text_prompt": text_prompt, 662 | "image_url": image_url, 663 | "position": position, 664 | "scale": scale 665 | }) 666 | 667 | if result.get("status") == "success": 668 | task_id = result.get("task_id") 669 | prim_path = result.get("prim_path") 670 | return f"Successfully generated 3D model with task ID: {task_id}, loaded at prim path: {prim_path}" 671 | else: 672 | return f"Error generating 3D model: {result.get('message', 'Unknown error')}" 673 | except Exception as e: 674 | logger.error(f"Error generating 3D model: {str(e)}") 675 | return f"Error generating 3D model: {str(e)}" 676 | 677 | @mcp.tool("search_3d_usd_by_text") 678 | def search_3d_usd_by_text( 679 | ctx: Context, 680 | text_prompt: str = None, 681 | target_path: str = "/World/my_usd", 682 | position: List[float] = [0, 0, 50], 683 | scale: List[float] = [10, 10, 10] 684 | ) -> str: 685 | """ 686 | Search for a 3D model using text prompt in USD libraries, then load and position it in the scene. 687 | 688 | Args: 689 | text_prompt (str): Text description to search for matching 3D models 690 | target_path (str, optional): Path where the USD model will be placed in the scene 691 | position (list, optional): Position coordinates [x, y, z] for placing the model 692 | scale (list, optional): Scale factors [x, y, z] to resize the model 693 | 694 | Returns: 695 | String with search results including task_id and prim_path of the loaded model 696 | """ 697 | if not text_prompt: 698 | return "Error: Either text_prompt or image_url must be provided" 699 | 700 | try: 701 | # Get the global connection 702 | isaac = get_isaac_connection() 703 | params = {"text_prompt": text_prompt, 704 | "target_path": target_path} 705 | 706 | result = isaac.send_command("search_3d_usd_by_text", params) 707 | if result.get("status") == "success": 708 | task_id = result.get("task_id") 709 | prim_path = result.get("prim_path") 710 | return f"Successfully generated 3D model with task ID: {task_id}, loaded at prim path: {prim_path}" 711 | else: 712 | return f"Error generating 3D model: {result.get('message', 'Unknown error')}" 713 | except Exception as e: 714 | logger.error(f"Error generating 3D model: {str(e)}") 715 | return f"Error generating 3D model: {str(e)}" 716 | 717 | @mcp.tool("transform") 718 | def transform( 719 | ctx: Context, 720 | prim_path: str, 721 | position: List[float] = [0, 0, 50], 722 | scale: List[float] = [10, 10, 10] 723 | ) -> str: 724 | """ 725 | Transform a USD model by applying position and scale. 726 | 727 | Args: 728 | prim_path (str): Path to the USD prim to transform 729 | position (list, optional): The position to set [x, y, z] 730 | scale (list, optional): The scale to set [x, y, z] 731 | 732 | Returns: 733 | String with transformation result 734 | """ 735 | try: 736 | # Get the global connection 737 | isaac = get_isaac_connection() 738 | 739 | result = isaac.send_command("transform", { 740 | "prim_path": prim_path, 741 | "position": position, 742 | "scale": scale 743 | }) 744 | 745 | if result.get("status") == "success": 746 | return f"Successfully transformed model at {prim_path} to position {position} and scale {scale}" 747 | else: 748 | return f"Error transforming model: {result.get('message', 'Unknown error')}" 749 | except Exception as e: 750 | logger.error(f"Error transforming model: {str(e)}") 751 | return f"Error transforming model: {str(e)}" 752 | 753 | # Main execution 754 | 755 | def main(): 756 | """Run the MCP server""" 757 | mcp.run() 758 | 759 | if __name__ == "__main__": 760 | main() -------------------------------------------------------------------------------- /media/add_more_robot_into_party.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/omni-mcp/isaac-sim-mcp/6653138244ebe1f1b6b098528061a4659aeec839/media/add_more_robot_into_party.gif -------------------------------------------------------------------------------- /media/add_more_robot_into_party.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/omni-mcp/isaac-sim-mcp/6653138244ebe1f1b6b098528061a4659aeec839/media/add_more_robot_into_party.mp4 --------------------------------------------------------------------------------