├── .clang-format ├── .coderabbit.yaml ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── pull_request_template.md └── workflows │ ├── licensechecker.yaml │ ├── lint_pr.yml │ ├── mkdocs-build.yml │ ├── mkdocs-publish.yml │ ├── poetry-test.yml │ ├── poetry-update.yml │ ├── pre-commit.yml │ └── require-version-bump.yml ├── .gitignore ├── .licenserc.json ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── codecov.yml ├── config.toml ├── demos.repos ├── docker └── Dockerfile ├── docs ├── API_documentation │ ├── agents │ │ └── overview.md │ ├── aggregators │ │ ├── ROS_2_Aggregators.md │ │ └── overview.md │ ├── connectors │ │ ├── ROS_2_Connectors.md │ │ └── overview.md │ ├── langchain_integration │ │ ├── ROS_2_tools.md │ │ ├── multimodal_messages.md │ │ └── overview.md │ ├── overview.md │ └── runners │ │ └── overview.md ├── ROS_2 │ └── ros_packages.md ├── demos │ ├── agriculture.md │ ├── debugging_assistant.md │ ├── manipulation.md │ ├── overview.md │ └── rosbot_xl.md ├── extensions │ ├── nomad.md │ └── perception.md ├── faq │ ├── ROS_2_Overview.md │ ├── contributing │ │ ├── CODE_OF_CONDUCT.md │ │ ├── CONTRIBUTING.md │ │ └── SECURITY.md │ └── faq.md ├── imgs │ ├── HRI_interface.png │ ├── HRI_text_interface.png │ ├── HRI_voice_interface.png │ ├── RAI_simple_diagram_long.png │ ├── RAI_simple_diagram_medium.png │ ├── agriculture_demo.gif │ ├── connectors.png │ ├── demos.png │ ├── langchain_extensions.png │ ├── manipulation_benchmark.png │ ├── manipulation_demo.gif │ ├── o3deSimulation.png │ ├── rai_api.png │ ├── rai_packages.png │ ├── rai_sim │ │ └── o3de │ │ │ ├── image 1.png │ │ │ ├── image 2.png │ │ │ ├── image 3.png │ │ │ ├── image 4.png │ │ │ ├── image 5.png │ │ │ ├── image 6.png │ │ │ ├── image 7.png │ │ │ ├── image 8.png │ │ │ └── image.png │ ├── rai_steps.png │ ├── rosbot-xl-example.gif │ ├── sponsor.png │ ├── talk.png │ ├── tool_calling_agent_benchmark.png │ └── walkthrough.png ├── index.md ├── intro │ └── what_is_rai.md ├── setup │ ├── install.md │ ├── setup_docker.md │ ├── tracing.md │ └── vendors.md ├── simulation_and_benchmarking │ ├── overview.md │ ├── rai_bench.md │ ├── rai_sim.md │ └── simulators.md ├── speech_to_speech │ ├── agents │ │ ├── asr.md │ │ ├── overview.md │ │ └── tts.md │ ├── models │ │ └── overview.md │ ├── overview.md │ └── sounddevice.md ├── stylesheets │ └── extra.css └── tutorials │ ├── benchmarking.md │ ├── create_robots_whoami.md │ ├── overview.md │ ├── tools.md │ ├── voice_interface.md │ └── walkthrough.md ├── examples ├── agents │ ├── react.py │ ├── react_ros2.py │ ├── state_based.py │ └── streamlit_react.py ├── agriculture-demo.py ├── debugging_assistant.py ├── embodiments │ ├── agriculture_embodiment.json │ ├── manipulation_embodiment.json │ ├── ros2_debugging_assistant_embodiment.json │ └── rosbotxl_embodiment.json ├── manipulation-demo-no-binary.launch.py ├── manipulation-demo-streamlit.py ├── manipulation-demo.launch.py ├── manipulation-demo.py ├── rosbot-xl-demo.py ├── rosbot-xl.launch.py └── s2s │ ├── asr.py │ ├── conversational.py │ ├── run.sh │ └── tts.py ├── mkdocs.yml ├── poetry.lock ├── pyproject.toml ├── ros_deps.repos ├── scripts └── download_demo.sh ├── setup_shell.sh ├── src ├── rai_bench │ ├── README.md │ ├── pyproject.toml │ └── rai_bench │ │ ├── __init__.py │ │ ├── base_benchmark.py │ │ ├── docs │ │ ├── imgs │ │ │ └── tool_calling_agent_valid_schema.png │ │ └── tool_calling_agent_benchmark.md │ │ ├── examples │ │ ├── benchmarking_models.py │ │ ├── custom_scenario.py │ │ ├── custom_task.py │ │ ├── manipulation_o3de.py │ │ ├── tool_calling_agent.py │ │ ├── tool_calling_custom_agent.py │ │ ├── visualise_streamlit.py │ │ └── vlm_benchmark.py │ │ ├── manipulation_o3de │ │ ├── __init__.py │ │ ├── benchmark.py │ │ ├── interfaces.py │ │ ├── predefined │ │ │ ├── configs │ │ │ │ ├── 1a.yaml │ │ │ │ ├── 1a_1t.yaml │ │ │ │ ├── 1a_2bc.yaml │ │ │ │ ├── 1bc_1rc_1yc.yaml │ │ │ │ ├── 1carrot.yaml │ │ │ │ ├── 1carrot_1a_1t_1bc_1corn.yaml │ │ │ │ ├── 1carrot_1a_2t_1bc_1rc_3yc_stacked.yaml │ │ │ │ ├── 1carrot_1bc.yaml │ │ │ │ ├── 1carrot_1corn.yaml │ │ │ │ ├── 1carrot_1t_1rc.yaml │ │ │ │ ├── 1rc.yaml │ │ │ │ ├── 1rc_2bc_3yc.yaml │ │ │ │ ├── 1t.yaml │ │ │ │ ├── 1yc.yaml │ │ │ │ ├── 1yc_1rc.yaml │ │ │ │ ├── 2a_1bc.yaml │ │ │ │ ├── 2a_1c_2rc.yaml │ │ │ │ ├── 2carrots_1a_1t_1bc_1rc_1yc_1corn.yaml │ │ │ │ ├── 2carrots_2a.yaml │ │ │ │ ├── 2rc.yaml │ │ │ │ ├── 2rc_3bc_4yc_stacked.yaml │ │ │ │ ├── 2t.yaml │ │ │ │ ├── 2t_3a_1corn_2rc.yaml │ │ │ │ ├── 2yc_1bc_1rc.yaml │ │ │ │ ├── 3a_4t_2bc.yaml │ │ │ │ ├── 3carrots_1a_1t_2bc_2yc.yaml │ │ │ │ ├── 3carrots_1a_2bc_1rc_1yc_1corn.yaml │ │ │ │ ├── 3rc_3bc_stacked.yaml │ │ │ │ ├── 4bc.yaml │ │ │ │ ├── 4carrots.yaml │ │ │ │ ├── 4carrots_rotated.yaml │ │ │ │ └── o3de_config.yaml │ │ │ └── scenarios.py │ │ ├── results_tracking.py │ │ └── tasks │ │ │ ├── __init__.py │ │ │ ├── build_tower_task.py │ │ │ ├── group_objects_task.py │ │ │ ├── move_object_to_left_task.py │ │ │ ├── place_at_coord_task.py │ │ │ ├── place_cubes_task.py │ │ │ └── rotate_object_task.py │ │ ├── results_processing │ │ ├── __init__.py │ │ ├── data_loading.py │ │ ├── data_processing.py │ │ ├── langfuse_scores_tracing.py │ │ └── visualise │ │ │ ├── __init__.py │ │ │ ├── charts.py │ │ │ ├── display.py │ │ │ ├── manipulaiton_o3de_display.py │ │ │ └── tool_calling_agent_display.py │ │ ├── test_models.py │ │ ├── tool_calling_agent │ │ ├── __init__.py │ │ ├── benchmark.py │ │ ├── interfaces.py │ │ ├── messages │ │ │ ├── __init__.py │ │ │ ├── actions.py │ │ │ ├── base.py │ │ │ ├── services.py │ │ │ └── topics.py │ │ ├── mocked_ros2_interfaces.py │ │ ├── mocked_tools.py │ │ ├── predefined │ │ │ ├── __init__.py │ │ │ ├── basic_tasks.py │ │ │ ├── custom_interfaces_tasks.py │ │ │ ├── manipulation_tasks.py │ │ │ └── tasks.py │ │ ├── results_tracking.py │ │ ├── subtasks.py │ │ ├── tasks │ │ │ ├── __init__.py │ │ │ ├── basic.py │ │ │ ├── custom_interfaces.py │ │ │ ├── manipulation.py │ │ │ └── warehouse.py │ │ └── validators.py │ │ ├── utils.py │ │ └── vlm_benchmark │ │ ├── __init__.py │ │ ├── benchmark.py │ │ ├── interfaces.py │ │ ├── predefined │ │ ├── images │ │ │ ├── image_1.jpg │ │ │ ├── image_2.jpg │ │ │ ├── image_3.jpg │ │ │ ├── image_4.jpg │ │ │ ├── image_5.jpg │ │ │ ├── image_6.jpg │ │ │ └── image_7.jpg │ │ └── tasks.py │ │ ├── results_tracking.py │ │ └── tasks │ │ └── tasks.py ├── rai_bringup │ ├── launch │ │ ├── hri.launch.py │ │ ├── openset.launch.py │ │ ├── sim_whoami_demo.launch.py │ │ └── voice.launch.py │ ├── package.xml │ ├── resource │ │ └── rai_bringup │ └── setup.py ├── rai_core │ ├── README.md │ ├── pyproject.toml │ └── rai │ │ ├── __init__.py │ │ ├── agents │ │ ├── __init__.py │ │ ├── base.py │ │ ├── integrations │ │ │ ├── __init__.py │ │ │ └── streamlit.py │ │ ├── langchain │ │ │ ├── __init__.py │ │ │ ├── agent.py │ │ │ ├── callback.py │ │ │ ├── core │ │ │ │ ├── __init__.py │ │ │ │ ├── conversational_agent.py │ │ │ │ ├── megamind.py │ │ │ │ ├── plan_agent.py │ │ │ │ ├── react_agent.py │ │ │ │ ├── state_based_agent.py │ │ │ │ ├── structured_output_agent.py │ │ │ │ └── tool_runner.py │ │ │ ├── invocation_helpers.py │ │ │ ├── react_agent.py │ │ │ └── state_based_agent.py │ │ ├── ros2 │ │ │ ├── __init__.py │ │ │ └── state_based_agent.py │ │ └── runner.py │ │ ├── aggregators │ │ ├── __init__.py │ │ ├── base.py │ │ └── ros2 │ │ │ ├── __init__.py │ │ │ └── aggregators.py │ │ ├── communication │ │ ├── __init__.py │ │ ├── base_connector.py │ │ ├── hri_connector.py │ │ └── ros2 │ │ │ ├── __init__.py │ │ │ ├── api │ │ │ ├── __init__.py │ │ │ ├── action.py │ │ │ ├── base.py │ │ │ ├── conversion.py │ │ │ ├── service.py │ │ │ └── topic.py │ │ │ ├── connectors │ │ │ ├── __init__.py │ │ │ ├── action_mixin.py │ │ │ ├── base.py │ │ │ ├── hri_connector.py │ │ │ ├── ros2_connector.py │ │ │ └── service_mixin.py │ │ │ ├── context.py │ │ │ ├── messages.py │ │ │ ├── ros_async.py │ │ │ ├── ros_logs.py │ │ │ └── waiters.py │ │ ├── frontend │ │ ├── __init__.py │ │ ├── configurator.py │ │ └── streamlit.py │ │ ├── initialization │ │ ├── __init__.py │ │ ├── config_initialization.py │ │ └── model_initialization.py │ │ ├── messages │ │ ├── __init__.py │ │ ├── artifacts.py │ │ ├── conversion.py │ │ └── multimodal.py │ │ ├── tools │ │ ├── __init__.py │ │ ├── ros2 │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── cli.py │ │ │ ├── generic │ │ │ │ ├── __init__.py │ │ │ │ ├── actions.py │ │ │ │ ├── interface_parser.py │ │ │ │ ├── services.py │ │ │ │ ├── toolkit.py │ │ │ │ └── topics.py │ │ │ ├── manipulation │ │ │ │ ├── __init__.py │ │ │ │ └── custom.py │ │ │ ├── navigation │ │ │ │ ├── __init__.py │ │ │ │ ├── nav2.py │ │ │ │ └── nav2_blocking.py │ │ │ └── simple.py │ │ └── time.py │ │ └── types │ │ ├── __init__.py │ │ ├── base.py │ │ ├── geometry.py │ │ ├── rai_interfaces.py │ │ ├── ros2 │ │ ├── __init__.py │ │ └── convert.py │ │ ├── sensor.py │ │ ├── std.py │ │ └── vision.py ├── rai_extensions │ ├── rai_nomad │ │ ├── LICENSE │ │ ├── README.md │ │ ├── package.xml │ │ ├── rai_nomad │ │ │ ├── __init__.py │ │ │ └── nomad.py │ │ ├── resource │ │ │ ├── nomad_params.yaml │ │ │ └── rai_nomad │ │ ├── setup.cfg │ │ └── setup.py │ └── rai_perception │ │ ├── .gitignore │ │ ├── README.md │ │ ├── images │ │ └── sample.jpg │ │ ├── pyproject.toml │ │ ├── rai_perception │ │ ├── NOTICE │ │ ├── __init__.py │ │ ├── agents │ │ │ ├── __init__.py │ │ │ ├── base_vision_agent.py │ │ │ ├── grounded_sam.py │ │ │ └── grounding_dino.py │ │ ├── configs │ │ │ ├── __init__.py │ │ │ ├── gdino_config.py │ │ │ └── seg_config.yml │ │ ├── examples │ │ │ ├── __init__.py │ │ │ └── talker.py │ │ ├── scripts │ │ │ ├── __init__.py │ │ │ └── run_perception_agents.py │ │ ├── tools │ │ │ ├── __init__.py │ │ │ ├── gdino_tools.py │ │ │ └── segmentation_tools.py │ │ └── vision_markup │ │ │ ├── __init__.py │ │ │ ├── boxer.py │ │ │ └── segmenter.py │ │ ├── resource │ │ └── rai_perception │ │ └── setup.cfg ├── rai_finetune │ ├── README.md │ ├── imgs │ │ └── rai-fine-tune-system-components.png │ ├── poetry.lock │ ├── pyproject.toml │ ├── rai_finetune │ │ ├── __init__.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── extractors │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ └── langfuse_data.py │ │ │ └── formatters │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ └── tool_calling.py │ │ ├── data_cli.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── chat_template.py │ │ │ └── templates │ │ │ └── chatml.jinja │ └── setup_finetune_shell.sh ├── rai_s2s │ ├── README.md │ ├── pyproject.toml │ └── rai_s2s │ │ ├── __init__.py │ │ ├── asr │ │ ├── __init__.py │ │ ├── agents │ │ │ ├── __init__.py │ │ │ ├── asr_agent.py │ │ │ └── initialization.py │ │ └── models │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── local_whisper.py │ │ │ ├── open_ai_whisper.py │ │ │ ├── open_wake_word.py │ │ │ └── silero_vad.py │ │ ├── s2s │ │ ├── __init__.py │ │ └── agents │ │ │ ├── __init__.py │ │ │ ├── ros2s2s_agent.py │ │ │ └── s2s_agent.py │ │ ├── sound_device │ │ ├── __init__.py │ │ ├── api.py │ │ └── connector.py │ │ └── tts │ │ ├── __init__.py │ │ ├── agents │ │ ├── __init__.py │ │ ├── initialization.py │ │ └── tts_agent.py │ │ └── models │ │ ├── __init__.py │ │ ├── base.py │ │ ├── elevenlabs_tts.py │ │ ├── kokoro_tts.py │ │ └── open_tts.py ├── rai_sim │ ├── README.md │ ├── pyproject.toml │ └── rai_sim │ │ ├── __init__.py │ │ ├── launch_manager.py │ │ ├── o3de │ │ └── o3de_bridge.py │ │ ├── simulation_bridge.py │ │ └── tools.py └── rai_whoami │ ├── README.md │ ├── docs │ └── images │ │ └── rai_whoami.png │ ├── pyproject.toml │ └── rai_whoami │ ├── __init__.py │ ├── agents │ ├── __init__.py │ └── ros2 │ │ ├── __init__.py │ │ ├── embodiment_info_agent.py │ │ └── vector_store_retrieval_agent.py │ ├── build_whoami.py │ ├── initialize_docs_directory.py │ ├── models │ ├── __init__.py │ └── models.py │ ├── pipeline │ ├── __init__.py │ ├── pipeline.py │ └── pipeline_builder.py │ ├── processors │ ├── __init__.py │ ├── defaults.py │ ├── postprocessors │ │ ├── __init__.py │ │ ├── base.py │ │ ├── compressor.py │ │ └── style.py │ └── preprocessors │ │ ├── __init__.py │ │ ├── base.py │ │ ├── docs.py │ │ └── image.py │ ├── scripts │ ├── ros2_embodiment_service.py │ └── ros2_vector_store_retrieval_service.py │ ├── tools │ ├── __init__.py │ └── vector_db.py │ └── vector_db │ ├── __init__.py │ ├── builder.py │ └── faiss.py └── tests ├── __init__.py ├── agents ├── langchain │ ├── test_langchain_agent.py │ └── test_tool_runner.py └── test_agent_runner.py ├── aggregators └── test_ros2_logs_aggregator.py ├── communication ├── ros2 │ ├── __init__.py │ ├── helpers.py │ ├── test_api.py │ ├── test_connectors.py │ ├── test_context.py │ ├── test_ros2_async.py │ └── test_waiters.py ├── sounds_device │ ├── __init__.py │ ├── test_api.py │ └── test_connector.py ├── test_base_connector.py ├── test_hri_connector.py └── test_hri_message.py ├── conftest.py ├── initialization ├── test_config_initialization_script.py ├── test_model_initialization.py └── test_tracing.py ├── messages ├── test_multimodal_message.py ├── test_transport.py └── test_utils.py ├── rai_bench ├── conftest.py ├── manipulation_o3de │ └── tasks │ │ ├── test_build_tower_task.py │ │ ├── test_group_objects_task.py │ │ ├── test_move_to_left_task.py │ │ ├── test_place_at_coords_task.py │ │ ├── test_place_cubes_task.py │ │ ├── test_rotate_objects_task.py │ │ └── test_task.py └── tool_calling_agent │ ├── test_mock_tools.py │ ├── test_predefined_basic_tasks.py │ ├── test_predefined_custom_interfaces_tasks.py │ ├── test_predefined_manipulation_tasks.py │ ├── test_subtasks.py │ └── test_validators.py ├── rai_bringup ├── __init__.py └── test_openset_launch.py ├── rai_perception ├── conftest.py ├── test_base_vision_agent.py ├── test_boxer.py ├── test_gdino_tools.py ├── test_grounded_sam.py ├── test_grounding_dino.py ├── test_run_perception_agents.py └── test_segmentation_tools.py ├── rai_sim ├── conftest.py ├── test_o3de_bridge.py └── test_simulation_bridge.py ├── resources ├── image.png └── sine_wave.wav ├── smoke └── import_test.py ├── tools ├── ros2 │ ├── test_action_tools.py │ ├── test_manipulation_custom.py │ ├── test_service_tools.py │ └── test_topic_tools.py ├── test_tool_input_args_compatibility.py ├── test_tool_utils.py └── test_wait_for_seconds_tool.py └── types └── test_ros2_convert.py /.clang-format: -------------------------------------------------------------------------------- 1 | Language: Cpp 2 | BasedOnStyle: Google 3 | 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: AlwaysBreak 6 | BraceWrapping: 7 | AfterClass: true 8 | AfterFunction: true 9 | AfterNamespace: true 10 | AfterStruct: true 11 | AfterEnum: true 12 | BreakBeforeBraces: Custom 13 | ColumnLimit: 100 14 | ConstructorInitializerIndentWidth: 0 15 | ContinuationIndentWidth: 2 16 | DerivePointerAlignment: false 17 | PointerAlignment: Middle 18 | ReflowComments: false 19 | -------------------------------------------------------------------------------- /.coderabbit.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json 2 | language: "en-US" 3 | early_access: false 4 | reviews: 5 | profile: "chill" 6 | request_changes_workflow: false 7 | high_level_summary: true 8 | poem: false 9 | review_status: false 10 | collapse_walkthrough: false 11 | auto_review: 12 | enabled: false 13 | drafts: false 14 | chat: 15 | auto_reply: true 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "" 5 | labels: bug 6 | assignees: "" 7 | --- 8 | 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **To Reproduce** 13 | Steps to reproduce the behavior: 14 | 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Platform** 27 | 28 | - OS: [e.g. Ubuntu 24.04] 29 | - ROS 2 Version [e.g. Jazzy] 30 | - Other information you think might be relevant such as graphic card, drivers etc. 31 | 32 | **Version** 33 | Release number or commit hash. 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "" 5 | labels: enhancement 6 | assignees: "" 7 | --- 8 | 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Additional context** 19 | Add any other context or screenshots about the feature request here. 20 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | - Clear purpose of this PR 4 | 5 | ## Proposed Changes 6 | 7 | What does this PR add, remove or fix? 8 | 9 | ## Issues 10 | 11 | - Links to relevant issues 12 | 13 | ## Testing 14 | 15 | - How was it tested, what were the results? 16 | -------------------------------------------------------------------------------- /.github/workflows/licensechecker.yaml: -------------------------------------------------------------------------------- 1 | name: Check License Lines 2 | on: [pull_request] 3 | jobs: 4 | check-license-lines: 5 | runs-on: ubuntu-24.04 6 | steps: 7 | - uses: actions/checkout@master 8 | - name: Check License Lines 9 | uses: kt3k/license_checker@v1.0.6 10 | -------------------------------------------------------------------------------- /.github/workflows/lint_pr.yml: -------------------------------------------------------------------------------- 1 | name: "Lint PR" 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - reopened 9 | branches: 10 | - main 11 | - development 12 | 13 | jobs: 14 | main: 15 | name: Validate PR title 16 | runs-on: ubuntu-latest 17 | permissions: 18 | pull-requests: read 19 | steps: 20 | - uses: amannn/action-semantic-pull-request@v5 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | -------------------------------------------------------------------------------- /.github/workflows/mkdocs-build.yml: -------------------------------------------------------------------------------- 1 | name: build mkdocs 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - development 7 | - main 8 | 9 | jobs: 10 | build-mkdocs: 11 | if: github.event.pull_request.draft == false 12 | 13 | runs-on: 14 | - ubuntu-24.04 15 | container: 16 | image: python:3.12-slim 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | clean: true 22 | 23 | - name: Cache Poetry and pip 24 | uses: actions/cache@v4 25 | with: 26 | path: | 27 | ~/.cache/pypoetry 28 | ~/.cache/pip 29 | key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} 30 | restore-keys: | 31 | ${{ runner.os }}-poetry- 32 | 33 | - name: Install Poetry 34 | uses: snok/install-poetry@v1 35 | with: 36 | version: 2.1.1 37 | 38 | - name: Install python dependencies 39 | run: poetry install --with docs 40 | 41 | - name: Install rai packages 42 | run: poetry run pip install src/rai_core src/rai_sim src/rai_bench src/rai_s2s --no-deps 43 | 44 | - name: Build docs 45 | shell: bash 46 | run: | 47 | poetry run mkdocs build --strict 48 | -------------------------------------------------------------------------------- /.github/workflows/mkdocs-publish.yml: -------------------------------------------------------------------------------- 1 | name: publish mkdocs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | publish-mkdocs: 13 | runs-on: 14 | - ubuntu-24.04 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Configure Git Credentials 19 | run: | 20 | git config user.name github-actions[bot] 21 | git config user.email 41898282+github-actions[bot]@users.noreply.github.com 22 | 23 | - name: Cache Poetry and pip 24 | uses: actions/cache@v4 25 | with: 26 | path: | 27 | ~/.cache/pypoetry 28 | ~/.cache/pip 29 | key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} 30 | restore-keys: | 31 | ${{ runner.os }}-poetry- 32 | 33 | - name: Install Poetry 34 | uses: snok/install-poetry@v1 35 | with: 36 | version: 2.1.1 37 | 38 | - name: Install python dependencies 39 | run: poetry install --with docs 40 | 41 | - name: Install rai packages 42 | run: poetry run pip install src/rai_core src/rai_sim src/rai_bench src/rai_s2s --no-deps 43 | 44 | - name: Build and Deploy 45 | shell: bash 46 | run: | 47 | poetry run mkdocs gh-deploy --force 48 | -------------------------------------------------------------------------------- /.github/workflows/poetry-update.yml: -------------------------------------------------------------------------------- 1 | name: Poetry Update 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | schedule: 7 | - cron: "0 0 * * 1" 8 | 9 | jobs: 10 | upgrade-dependencies: 11 | runs-on: ubuntu-24.04 12 | 13 | steps: 14 | - uses: fuzzylabs/gha-poetry-update@v1 15 | with: 16 | python-version: 3.12 17 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: Pre-commit Checks 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | - development 8 | workflow_dispatch: 9 | 10 | jobs: 11 | pre-commit: 12 | runs-on: 13 | - self-hosted 14 | 15 | container: 16 | image: osrf/ros:humble-desktop-full 17 | 18 | steps: 19 | - name: Install pre-commit 20 | run: | 21 | apt update && apt install -y python3-pip shellcheck python-is-python3 22 | 23 | - name: Checkout code 24 | uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | 28 | - run: | 29 | git config --global --add safe.directory /__w/rai/rai 30 | 31 | - name: Import ros2 dependencies 32 | shell: bash 33 | run: | 34 | vcs import . < ros_deps.repos 35 | 36 | - name: Run pre-commit 37 | uses: pre-commit/action@v3.0.1 38 | with: 39 | extra_args: "--all-files" 40 | -------------------------------------------------------------------------------- /.github/workflows/require-version-bump.yml: -------------------------------------------------------------------------------- 1 | name: Require Version Bump 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | paths: 8 | - 'src/rai_core/pyproject.toml' 9 | - 'src/rai_core/**' 10 | 11 | jobs: 12 | require-version-bump: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | 21 | - name: Get rai_core version from BASE and HEAD 22 | id: require-version-bump 23 | run: | 24 | BASE_SHA=${{ github.event.pull_request.base.sha }} 25 | HEAD_SHA=${{ github.event.pull_request.head.sha }} 26 | FILE_PATH="src/rai_core/pyproject.toml" 27 | 28 | git show $BASE_SHA:$FILE_PATH > base.txt || echo "BASE file not found" 29 | git show $HEAD_SHA:$FILE_PATH > head.txt || echo "HEAD file not found" 30 | 31 | - name: Compare version lines 32 | run: | 33 | BASE_VERSION=$(grep '^version =' base.txt || echo "MISSING") 34 | HEAD_VERSION=$(grep '^version =' head.txt || echo "MISSING") 35 | 36 | echo "BASE: $BASE_VERSION" 37 | echo "HEAD: $HEAD_VERSION" 38 | 39 | if [ "$BASE_VERSION" = "$HEAD_VERSION" ]; then 40 | echo "❌ Version has not changed. Please bump the version in the pyproject.toml file according to Semantic Versioning." 41 | exit 1 42 | else 43 | echo "✅ Version changed." 44 | fi 45 | -------------------------------------------------------------------------------- /.licenserc.json: -------------------------------------------------------------------------------- 1 | { 2 | "**/*.py": [ 3 | "# Copyright ", 4 | "#", 5 | "# Licensed under the Apache License, Version 2.0 (the \"License\");" 6 | ], 7 | "**/*.{h,cpp,hpp}": [ 8 | "// Copyright ", 9 | "//", 10 | "// Licensed under the Apache License, Version 2.0 (the \"License\");" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: check-json 6 | - id: check-merge-conflict 7 | - id: check-toml 8 | - id: check-xml 9 | - id: check-yaml 10 | - id: detect-private-key 11 | - id: end-of-file-fixer 12 | - id: mixed-line-ending 13 | - id: trailing-whitespace 14 | args: [--markdown-linebreak-ext=md] 15 | 16 | - repo: https://github.com/pre-commit/mirrors-prettier 17 | rev: v2.7.1 18 | hooks: 19 | - id: prettier 20 | args: [--tab-width=4] 21 | files: \.md$ 22 | 23 | - repo: https://github.com/gruntwork-io/pre-commit 24 | rev: v0.1.23 25 | hooks: 26 | - id: shellcheck 27 | 28 | - repo: https://github.com/astral-sh/ruff-pre-commit 29 | rev: v0.9.4 30 | hooks: 31 | - id: ruff 32 | args: [--extend-select, "I,RUF022", --fix, --ignore, E731] 33 | exclude: src/rai_core/rai/frontend/configurator.py 34 | - id: ruff-format 35 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Only the newest release for Ubuntu 22.04, Ubuntu 24.04; ROS 2 Humble, and ROS 2 Jazzy are supported. The policy will evolve as the project matures. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | 0.9.x | :white_check_mark: | 10 | | 1.0.x | :white_check_mark: | 11 | | 1.1.x | :white_check_mark: | 12 | 13 | ## Reporting a Vulnerability 14 | 15 | Treat and report vulnerability as any other bug. 16 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - src/rai_core/rai/frontend/* 3 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | [vendor] 2 | simple_model = "openai" 3 | complex_model = "openai" 4 | embeddings_model = "openai" 5 | 6 | [aws] 7 | simple_model = "anthropic.claude-3-haiku-20240307-v1:0" 8 | complex_model = "anthropic.claude-3-5-sonnet-20240620-v1:0" 9 | embeddings_model = "amazon.titan-embed-text-v1" 10 | region_name = "us-east-1" 11 | 12 | [openai] 13 | simple_model = "gpt-4o-mini" 14 | complex_model = "gpt-4o" 15 | embeddings_model = "text-embedding-ada-002" 16 | base_url = "https://api.openai.com/v1/" 17 | 18 | [ollama] 19 | simple_model = "llama3.2" 20 | complex_model = "llama3.1:70b" 21 | embeddings_model = "llama3.2" 22 | base_url = "http://localhost:11434" 23 | 24 | [tracing] 25 | project = "rai" 26 | 27 | [tracing.langfuse] 28 | use_langfuse = false 29 | host = "http://localhost:3000" 30 | 31 | [tracing.langsmith] 32 | use_langsmith = false 33 | host = "https://api.smith.langchain.com" 34 | 35 | [asr] 36 | recording_device_name = "default" 37 | transcription_model = "LocalWhisper" 38 | language = "en" 39 | vad_model = "SileroVAD" 40 | silence_grace_period = 0.3 41 | vad_threshold = 0.3 42 | use_wake_word = false 43 | wake_word_model = "" 44 | wake_word_threshold = 0.5 45 | wake_word_model_name = "" 46 | transcription_model_name = "tiny" 47 | 48 | [tts] 49 | vendor = "ElevenLabs" 50 | voice = "" 51 | speaker_device_name = "default" 52 | -------------------------------------------------------------------------------- /demos.repos: -------------------------------------------------------------------------------- 1 | repositories: 2 | src/examples/rai-rosbot-xl-demo: 3 | type: git 4 | url: https://github.com/RobotecAI/rai-rosbot-xl-demo.git 5 | version: development 6 | src/examples/rai-manipulation-demo: 7 | type: git 8 | url: https://github.com/RobotecAI/rai-manipulation-demo.git 9 | version: development 10 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARG ROS_DISTRO=jazzy 16 | 17 | FROM osrf/ros:${ROS_DISTRO}-desktop-full 18 | ENV DEBIAN_FRONTEND=noninteractive 19 | 20 | # Install dependencies 21 | RUN apt-get update && apt-get install -y \ 22 | python3 \ 23 | python3-pip \ 24 | git \ 25 | wget 26 | 27 | # Install Poetry 28 | RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.1 29 | ENV PATH="/root/.local/bin:$PATH" 30 | 31 | # Clone and setup RAI 32 | WORKDIR /rai 33 | RUN git clone https://github.com/RobotecAI/rai.git . 34 | 35 | # Install Python dependencies with Poetry 36 | RUN poetry install --with nomad,perception 37 | 38 | # Install ROS dependencies 39 | RUN /bin/bash -c '. /opt/ros/${ROS_DISTRO}/setup.bash && \ 40 | rosdep install --from-paths src --ignore-src -r -y' 41 | 42 | # Build the workspace 43 | RUN /bin/bash -c '. /opt/ros/${ROS_DISTRO}/setup.bash && colcon build --symlink-install' 44 | -------------------------------------------------------------------------------- /docs/ROS_2/ros_packages.md: -------------------------------------------------------------------------------- 1 | # ROS Packages 2 | 3 | RAI includes multiple configurable ROS 2 packages. 4 | 5 | | Package | Description | Documentation | 6 | | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------- | 7 | | **rai_perception** | Object detection tools based on open-set models and machine learning techniques. Integrates [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO) and [GroundedSAM](https://github.com/IDEA-Research/Grounded-SAM-2) with ROS 2. | [rai_perception](../extensions/perception.md) | 8 | | **rai_nomad** | Package integrating [NoMaD](https://general-navigation-models.github.io/nomad/index.html) -- an exploration model with ROS2. | [rai_nomad](../extensions/nomad.md) | 9 | | **rai_interfaces** | Definition of custom messages and services used in RAI. | | 10 | | **rai_bringup** | Launch files to run RAI. | | 11 | -------------------------------------------------------------------------------- /docs/extensions/nomad.md: -------------------------------------------------------------------------------- 1 | --8<-- "src/rai_extensions/rai_nomad/README.md" 2 | -------------------------------------------------------------------------------- /docs/extensions/perception.md: -------------------------------------------------------------------------------- 1 | --8<-- "src/rai_extensions/rai_perception/README.md:sec1" 2 | 3 | --8<-- "src/rai_extensions/rai_perception/README.md:sec4" 4 | 5 | --8<-- "src/rai_extensions/rai_perception/README.md:sec5" 6 | -------------------------------------------------------------------------------- /docs/faq/contributing/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ../../../CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /docs/faq/contributing/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ../../../CONTRIBUTING.md -------------------------------------------------------------------------------- /docs/faq/contributing/SECURITY.md: -------------------------------------------------------------------------------- 1 | ../../../SECURITY.md -------------------------------------------------------------------------------- /docs/imgs/HRI_interface.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/HRI_interface.png -------------------------------------------------------------------------------- /docs/imgs/HRI_text_interface.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/HRI_text_interface.png -------------------------------------------------------------------------------- /docs/imgs/HRI_voice_interface.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/HRI_voice_interface.png -------------------------------------------------------------------------------- /docs/imgs/RAI_simple_diagram_long.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/RAI_simple_diagram_long.png -------------------------------------------------------------------------------- /docs/imgs/RAI_simple_diagram_medium.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/RAI_simple_diagram_medium.png -------------------------------------------------------------------------------- /docs/imgs/agriculture_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/agriculture_demo.gif -------------------------------------------------------------------------------- /docs/imgs/connectors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/connectors.png -------------------------------------------------------------------------------- /docs/imgs/demos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/demos.png -------------------------------------------------------------------------------- /docs/imgs/langchain_extensions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/langchain_extensions.png -------------------------------------------------------------------------------- /docs/imgs/manipulation_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/manipulation_benchmark.png -------------------------------------------------------------------------------- /docs/imgs/manipulation_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/manipulation_demo.gif -------------------------------------------------------------------------------- /docs/imgs/o3deSimulation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/o3deSimulation.png -------------------------------------------------------------------------------- /docs/imgs/rai_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_api.png -------------------------------------------------------------------------------- /docs/imgs/rai_packages.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_packages.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 1.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 2.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 3.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 4.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 5.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 6.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 7.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image 8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image 8.png -------------------------------------------------------------------------------- /docs/imgs/rai_sim/o3de/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_sim/o3de/image.png -------------------------------------------------------------------------------- /docs/imgs/rai_steps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rai_steps.png -------------------------------------------------------------------------------- /docs/imgs/rosbot-xl-example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/rosbot-xl-example.gif -------------------------------------------------------------------------------- /docs/imgs/sponsor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/sponsor.png -------------------------------------------------------------------------------- /docs/imgs/talk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/talk.png -------------------------------------------------------------------------------- /docs/imgs/tool_calling_agent_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/tool_calling_agent_benchmark.png -------------------------------------------------------------------------------- /docs/imgs/walkthrough.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/docs/imgs/walkthrough.png -------------------------------------------------------------------------------- /docs/setup/setup_docker.md: -------------------------------------------------------------------------------- 1 | # Setup RAI with docker 2 | 3 | !!! warning "Docker images are experimental" 4 | 5 | Docker images are experimental. For tested setup, see the 6 | [local setup](install.md). 7 | 8 | ## 1. Build the docker image 9 | 10 | Choose the docker image based on your preferred ROS 2 version. 11 | 12 | ### 1.1. Humble 13 | 14 | ```bash 15 | docker build -t rai:humble --build-arg ROS_DISTRO=humble -f docker/Dockerfile . 16 | ``` 17 | 18 | ### 1.2. Jazzy 19 | 20 | ```bash 21 | docker build -t rai:jazzy --build-arg ROS_DISTRO=jazzy -f docker/Dockerfile . 22 | ``` 23 | 24 | ## 2. Run the docker container 25 | 26 | !!! tip "ROS 2 communication" 27 | 28 | If you intend to run demos on the host machine, ensure the docker container can communicate 29 | with it. Test this by running the standard ROS 2 example with one node in docker and one on the 30 | host: 31 | [link](https://docs.ros.org/en/jazzy/Installation/Ubuntu-Install-Debs.html#try-some-examples). If 32 | topics are not visible or cannot be subscribed to, try using 33 | [rmw_cyclone_dds](https://github.com/ros2/rmw_cyclonedds) instead of the default 34 | rmw_fastrtps_cpp. 35 | 36 | ### 2.1. Humble 37 | 38 | ```bash 39 | docker run --net=host --ipc=host --pid=host -it rai:humble 40 | ``` 41 | 42 | ### 2.2. Jazzy 43 | 44 | ```bash 45 | docker run --net=host --ipc=host --pid=host -it rai:jazzy 46 | ``` 47 | 48 | ## 3. Run the tests to confirm the setup 49 | 50 | ```sh 51 | cd /rai 52 | source setup_shell.sh 53 | poetry run pytest 54 | ``` 55 | -------------------------------------------------------------------------------- /docs/simulation_and_benchmarking/simulators.md: -------------------------------------------------------------------------------- 1 | TBD 2 | -------------------------------------------------------------------------------- /docs/speech_to_speech/overview.md: -------------------------------------------------------------------------------- 1 | # Speech To Speech 2 | 3 | ## Introduction 4 | 5 | `rai s2s` provides tools and components for voice interaction with the system. This package contains plug-and-play Agents which can be easily integrated with Agents provided by `rai core`, as well as custom ones. It also provides integration with host sound system, which can be used for low level sound manipulation. 6 | 7 | ## Core Components 8 | 9 | | Component | Description | 10 | | ---------------------------- | --------------------------------------------------------------------------------------------------------- | 11 | | [Agents](agents/overview.md) | Agents in `rai s2s` provide functionality for voice interaction with the rest of the system. | 12 | | [Models](models/overview.md) | `rai s2s` provides a models which can be optionally installed and utilized by the Agents. | 13 | | [Connector](sounddevice.md) | The `sounddevice` connector allows for interfacing directly with sound devices for asynchronous sound IO. | 14 | 15 | ## Best Practices 16 | 17 | When utilizing S2S features: 18 | 19 | 1. Deployment of `SpeechToSpeechAgent` is meant for local setup, while the `SpeechRecognition` and `TextToSpeech` Agents are meant to be ran on separate hosts. 20 | 2. Note that `sounddevice` python API has notable issues in multi-threaded environment - this can lead to issues when developing Agents using the `SoundDeviceConnector` 21 | -------------------------------------------------------------------------------- /docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | .center-badges { 2 | text-align: center; 3 | } 4 | :root > * { 5 | --md-primary-fg-color: #1A88AB; 6 | } 7 | 8 | table td code { 9 | white-space: nowrap; 10 | word-break: keep-all; 11 | } 12 | 13 | pre, pre code, .highlight pre, .highlight code { 14 | white-space: pre-wrap !important; 15 | word-break: normal !important; 16 | overflow-x: auto; 17 | } 18 | -------------------------------------------------------------------------------- /examples/agents/react.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language goveself.rning permissions and 13 | # limitations under the License. 14 | 15 | 16 | from rai.agents.langchain import ReActAgent 17 | from rai.communication.hri_connector import HRIMessage 18 | from rai.communication.ros2 import ROS2Connector, ROS2Context, ROS2HRIConnector 19 | from rai.tools.ros2 import ROS2Toolkit 20 | 21 | 22 | @ROS2Context() 23 | def main(): 24 | ros2_connector = ROS2Connector() 25 | hri_connector = ROS2HRIConnector() 26 | 27 | agent = ReActAgent( 28 | target_connectors={ 29 | "/to_human": hri_connector, 30 | }, # agent's output is sent to /to_human ros2 topic 31 | tools=ROS2Toolkit(connector=ros2_connector).get_tools(), 32 | ) 33 | agent.run() 34 | agent(HRIMessage(text="What do you see?")) 35 | agent.wait() # wait for agent to finish 36 | agent.stop() 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /examples/agents/react_ros2.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language goveself.rning permissions and 13 | # limitations under the License. 14 | 15 | 16 | from rai.agents import AgentRunner 17 | from rai.agents.langchain.react_agent import ReActAgent 18 | from rai.communication.ros2 import ROS2Connector, ROS2Context 19 | from rai.communication.ros2.connectors.hri_connector import ROS2HRIConnector 20 | from rai.tools.ros2 import ROS2Toolkit 21 | 22 | 23 | @ROS2Context() 24 | def main(): 25 | ros2_connector = ROS2Connector() 26 | hri_connector = ROS2HRIConnector() 27 | 28 | agent = ReActAgent( 29 | target_connectors={ 30 | "/to_human": hri_connector, 31 | }, 32 | tools=ROS2Toolkit(connector=ros2_connector).get_tools(), 33 | ) 34 | # Agent will wait for messages published to /from_human ros2 topic 35 | agent.subscribe_source("/from_human", hri_connector) 36 | runner = AgentRunner([agent]) 37 | runner.run_and_wait_for_shutdown() 38 | 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /examples/agents/state_based.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language goveself.rning permissions and 13 | # limitations under the License. 14 | 15 | from rai.agents import wait_for_shutdown 16 | from rai.agents.langchain import StateBasedConfig 17 | from rai.agents.ros2 import ROS2StateBasedAgent 18 | from rai.aggregators.ros2 import ( 19 | ROS2ImgVLMDiffAggregator, 20 | ROS2LogsAggregator, 21 | ) 22 | from rai.communication.ros2 import ( 23 | ROS2Connector, 24 | ROS2Context, 25 | ROS2HRIConnector, 26 | ) 27 | from rai.tools.ros2.generic.toolkit import ROS2Toolkit 28 | 29 | 30 | @ROS2Context() 31 | def main(): 32 | hri_connector = ROS2HRIConnector() 33 | ros2_connector = ROS2Connector() 34 | 35 | config = StateBasedConfig( 36 | aggregators={ 37 | ("/camera/camera/color/image_raw", "sensor_msgs/msg/Image"): [ 38 | ROS2ImgVLMDiffAggregator() 39 | ], 40 | "/rosout": [ 41 | ROS2LogsAggregator() 42 | ], # if msg_type is not provided, topic has to exist 43 | } 44 | ) 45 | 46 | agent = ROS2StateBasedAgent( 47 | config=config, 48 | target_connectors={"to_human": hri_connector}, 49 | tools=ROS2Toolkit(connector=ros2_connector).get_tools(), 50 | ) 51 | agent.subscribe_source("/from_human", hri_connector) 52 | agent.run() 53 | wait_for_shutdown([agent]) 54 | 55 | 56 | if __name__ == "__main__": 57 | main() 58 | -------------------------------------------------------------------------------- /examples/debugging_assistant.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import streamlit as st 16 | from rai import get_llm_model 17 | from rai.agents.langchain.core import create_conversational_agent 18 | from rai.frontend import run_streamlit_app 19 | from rai.tools.ros2 import ROS2CLIToolkit 20 | 21 | from rai_whoami import EmbodimentInfo 22 | 23 | 24 | @st.cache_resource 25 | def initialize_agent(): 26 | llm = get_llm_model(model_type="complex_model", streaming=True) 27 | embodiment_info = EmbodimentInfo.from_file( 28 | "examples/embodiments/ros2_debugging_assistant_embodiment.json" 29 | ) 30 | agent = create_conversational_agent( 31 | llm, 32 | ROS2CLIToolkit().get_tools(), 33 | system_prompt=embodiment_info.to_langchain(), 34 | ) 35 | return agent 36 | 37 | 38 | st.set_page_config( 39 | page_title="ROS 2 Debugging Assistant", 40 | page_icon=":robot:", 41 | ) 42 | 43 | 44 | def main(): 45 | run_streamlit_app( 46 | initialize_agent(), 47 | page_title="ROS 2 Debugging Assistant", 48 | initial_message="Hi! I am a ROS 2 assistant. How can I help you?", 49 | ) 50 | 51 | 52 | if __name__ == "__main__": 53 | main() 54 | -------------------------------------------------------------------------------- /examples/embodiments/agriculture_embodiment.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": [ 3 | "You must call only one service per stop event.", 4 | "The tractor can only handle one service call at a time." 5 | ], 6 | "capabilities": [ 7 | "Assess the situation when the tractor stops unexpectedly.", 8 | "Call one of the following services based on your assessment: continue, current_state, flash, or replan.", 9 | "Ability to signal, replan, or check state as needed." 10 | ], 11 | "behaviors": [ 12 | "If the stop was unnecessary, call the continue service.", 13 | "If you want to check the current state of the tractor, call the current_state service.", 14 | "If you want to flash the lights to signal possible animals to move out of the way, call the flash service.", 15 | "If you want to replan the path due to an obstacle, call the replan service.", 16 | "Make sure to list available services before calling one of them." 17 | ], 18 | "description": "You are an autonomous tractor operating in an agricultural field. You are activated whenever the tractor stops due to an unexpected situation. Your primary task is to assess the situation and call the most appropriate service. The system may stop you unnecessarily at times, so your judgment is required to determine the correct action." 19 | } 20 | -------------------------------------------------------------------------------- /examples/embodiments/manipulation_embodiment.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": [ 3 | "Always follow the specified payload limit; do not exceed 3 kg." 4 | ], 5 | "capabilities": [ 6 | "7 degrees of freedom for complex motion and task execution.", 7 | "Ability to carry up to 3 kg of payload.", 8 | "High precision with 0.1 mm repeatability.", 9 | "850 mm reach suitable for a variety of industrial applications.", 10 | "Interfaces to detect and manipulate objects.", 11 | "Coordinate system: x - front to back (positive is forward), y - left to right (positive is right), z - up to down (positive is up)." 12 | ], 13 | "behaviors": [ 14 | "Performing precise dispensing tasks.", 15 | "Conducting welding tasks with precision and ease.", 16 | "Before starting the task, grab the camera image to understand the environment." 17 | ], 18 | "description": "You are an AI agent deployed on Franka Emika Panda, a versatile 7-axis robotic arm designed for industrial tasks like dispensing, remote tool handling, and welding, known for its high precision and adaptability in complex operations. It is designed for collaborative use with humans and is versatile and lightweight. The robot has a sleek design with smooth, articulated joints allowing for flexible and precise movements. It features seven axes or degrees of freedom, enabling it to perform complex tasks. The overall appearance is streamlined, with a clean white finish and some black accents. You are a robotic arm with interfaces to detect and manipulate objects.", 19 | "images": [] 20 | } 21 | -------------------------------------------------------------------------------- /examples/embodiments/ros2_debugging_assistant_embodiment.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": [ 3 | "Always use the available tools to gather information from the ROS 2 system before answering user questions.", 4 | "Be proactive in diagnosing and troubleshooting ROS 2 issues by querying the system state, available nodes, topics, and services." 5 | ], 6 | "capabilities": [ 7 | "Query ROS 2 nodes, topics, services, and parameters.", 8 | "Diagnose issues and provide solutions based on the current state of the ROS 2 system.", 9 | "Guide the user through debugging and configuration tasks." 10 | ], 11 | "behaviors": [ 12 | "When asked a question, use the tools to retrieve relevant information from the ROS 2 system before responding.", 13 | "If you do not have enough information to answer, suggest or perform additional queries.", 14 | "Explain your reasoning and steps to the user as you work through their issue.", 15 | "Summarize findings and provide actionable advice based on the data you retrieve." 16 | ], 17 | "description": "You are a ROS 2 expert helping a user with their ROS 2 questions. You have access to various tools that allow you to query the ROS 2 system. Be proactive and use the tools to answer questions. Retrieve as much information from the ROS 2 system as possible. Your primary goal is to help users with their ROS 2 questions by leveraging available tools to retrieve and analyze information from the ROS 2 system. Always aim to provide detailed, accurate, and helpful responses." 18 | } 19 | -------------------------------------------------------------------------------- /examples/rosbot-xl.launch.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from launch import LaunchDescription 16 | from launch.actions import ExecuteProcess, IncludeLaunchDescription 17 | from launch.launch_description_sources import PythonLaunchDescriptionSource 18 | from launch.substitutions import LaunchConfiguration 19 | from launch_ros.substitutions import FindPackageShare 20 | 21 | 22 | def generate_launch_description(): 23 | return LaunchDescription( 24 | [ 25 | ExecuteProcess( 26 | cmd=["bash", "run-nav.bash"], 27 | cwd="src/examples/rai-rosbot-xl-demo", 28 | output="screen", 29 | ), 30 | IncludeLaunchDescription( 31 | PythonLaunchDescriptionSource( 32 | [ 33 | FindPackageShare("rai_bringup"), 34 | "/launch/openset.launch.py", 35 | ] 36 | ), 37 | ), 38 | ExecuteProcess( 39 | cmd=[ 40 | LaunchConfiguration("game_launcher"), 41 | "-bg_ConnectToAssetProcessor=0", 42 | ], 43 | output="screen", 44 | ), 45 | ] 46 | ) 47 | -------------------------------------------------------------------------------- /ros_deps.repos: -------------------------------------------------------------------------------- 1 | repositories: 2 | src/rai_interfaces: 3 | type: git 4 | url: https://github.com/RobotecAI/rai_interfaces.git 5 | version: main 6 | -------------------------------------------------------------------------------- /setup_shell.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Suppress ShellCheck warning about not following external file 4 | # shellcheck disable=SC1091 5 | . "$(poetry env info --path)"/bin/activate 6 | 7 | # Suppress ShellCheck warning about not following external file 8 | # shellcheck disable=SC1091 9 | 10 | case "$SHELL" in 11 | *bash) 12 | . install/setup.bash 13 | echo "Sourced bash install" 14 | ;; 15 | *zsh) 16 | . install/setup.zsh 17 | echo "Sourced zsh install" 18 | ;; 19 | *fish) 20 | echo "fish is not supported" 21 | ;; 22 | *sh) 23 | . install/setup.sh 24 | echo "Sourced sh install." 25 | ;; 26 | *) 27 | echo "Unknown shell: $0" 28 | ;; 29 | esac 30 | 31 | export PYTHONPATH 32 | PYTHONPATH="$(dirname "$(dirname "$(poetry run which python)")")/lib/python$(poetry run python --version | awk '{print $2}' | cut -d. -f1,2)/site-packages:$PYTHONPATH" 33 | PYTHONPATH="src/rai_core:$PYTHONPATH" 34 | PYTHONPATH="src/rai_sim:$PYTHONPATH" 35 | PYTHONPATH="src/rai_s2s:$PYTHONPATH" 36 | PYTHONPATH="src/rai_bench:$PYTHONPATH" 37 | -------------------------------------------------------------------------------- /src/rai_bench/README.md: -------------------------------------------------------------------------------- 1 | # RAI Bench 2 | 3 | For tutorial see [RAI Bench Tutorial](../../docs/tutorials/benchmarking.md) 4 | For understanding the structure of the package visit [RAI Bench Overview](../../docs/simulation_and_benchmarking) 5 | -------------------------------------------------------------------------------- /src/rai_bench/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "rai-bench" 3 | version = "0.2.0" 4 | description = "Package for running and creating benchmarks." 5 | authors = ["Jakub Matejczyk ", "Magdalena Kotynia "] 6 | readme = "README.md" 7 | 8 | packages = [ 9 | { include = "rai_bench", from = "." }, 10 | ] 11 | [tool.poetry.dependencies] 12 | python = "^3.10" 13 | inflect = "7.5.0" 14 | plotly = "^6.0.1" 15 | rai_core = ">=2.0.0.a2,<3.0.0" 16 | rai_sim = ">=0.0.2,<1.0.0" 17 | 18 | [build-system] 19 | requires = ["poetry-core"] 20 | build-backend = "poetry.core.masonry.api" 21 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from .test_models import ( 15 | ManipulationO3DEBenchmarkConfig, 16 | ToolCallingAgentBenchmarkConfig, 17 | test_models, 18 | ) 19 | from .utils import ( 20 | define_benchmark_logger, 21 | get_llm_for_benchmark, 22 | parse_manipulation_o3de_benchmark_args, 23 | parse_tool_calling_benchmark_args, 24 | parse_vlm_benchmark_args, 25 | ) 26 | 27 | __all__ = [ 28 | "ManipulationO3DEBenchmarkConfig", 29 | "ToolCallingAgentBenchmarkConfig", 30 | "define_benchmark_logger", 31 | "get_llm_for_benchmark", 32 | "parse_manipulation_o3de_benchmark_args", 33 | "parse_tool_calling_benchmark_args", 34 | "parse_vlm_benchmark_args", 35 | "test_models", 36 | ] 37 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/docs/imgs/tool_calling_agent_valid_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/docs/imgs/tool_calling_agent_valid_schema.png -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/docs/tool_calling_agent_benchmark.md: -------------------------------------------------------------------------------- 1 | # Tool Calling Agent Benchmark 2 | 3 | ## Validaiton 4 | 5 | Our validation flow consists of: 6 | 7 | - SubTask - smallest block, responsible for validating single tool call (ex. ListTopics) 8 | - Validator - consists of subtasks. Based on the validator type, checks whether all subtasks were completed in a certain way 9 | - Task - consists of validators. Every Validator can be treated as single step that is scored atomically. Visit examples/tool_calling_agent/tasks.py for more intuition. Every Task has always the same prompt and available tools, only the validation methods can be parametrized. On top of validator, you can pass extra_tool_calls param to allow model to correct itself. 10 | 11 | ![alt text](imgs/tool_calling_agent_valid_schema.png) 12 | 13 | Implementations can be found: 14 | 15 | - Validators [Validators](../tool_calling_agent/validators.py) 16 | - Subtasks [Validators](../tool_calling_agent/tasks/subtasks.py) 17 | - Tasks, including basic, custom interfaces and manipulation [Tasks](../tool_calling_agent/tasks/) 18 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/examples/manipulation_o3de.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from rai_bench import define_benchmark_logger, parse_manipulation_o3de_benchmark_args 18 | from rai_bench.manipulation_o3de import get_scenarios, run_benchmark 19 | from rai_bench.utils import get_llm_for_benchmark 20 | 21 | if __name__ == "__main__": 22 | args = parse_manipulation_o3de_benchmark_args() 23 | experiment_dir = Path(args.out_dir) 24 | experiment_dir.mkdir(parents=True, exist_ok=True) 25 | bench_logger = define_benchmark_logger(out_dir=experiment_dir) 26 | 27 | # import ready scenarios 28 | scenarios = get_scenarios(logger=bench_logger, levels=args.levels) 29 | 30 | llm = get_llm_for_benchmark( 31 | model_name=args.model_name, 32 | vendor=args.vendor, 33 | ) 34 | run_benchmark( 35 | llm=llm, 36 | out_dir=experiment_dir, 37 | o3de_config_path=args.o3de_config_path, 38 | scenarios=scenarios, 39 | bench_logger=bench_logger, 40 | ) 41 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/examples/tool_calling_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from rai_bench import ( 18 | define_benchmark_logger, 19 | parse_tool_calling_benchmark_args, 20 | ) 21 | from rai_bench.tool_calling_agent import ( 22 | get_tasks, 23 | run_benchmark, 24 | ) 25 | from rai_bench.utils import get_llm_for_benchmark 26 | 27 | if __name__ == "__main__": 28 | args = parse_tool_calling_benchmark_args() 29 | experiment_dir = Path(args.out_dir) 30 | experiment_dir.mkdir(parents=True, exist_ok=True) 31 | bench_logger = define_benchmark_logger(out_dir=experiment_dir) 32 | 33 | tasks = get_tasks( 34 | extra_tool_calls=args.extra_tool_calls, 35 | complexities=args.complexities, 36 | task_types=args.task_types, 37 | n_shots=args.n_shots, 38 | prompt_detail=args.prompt_detail, 39 | ) 40 | for task in tasks: 41 | task.set_logger(bench_logger) 42 | 43 | llm = get_llm_for_benchmark( 44 | model_name=args.model_name, 45 | vendor=args.vendor, 46 | ) 47 | 48 | run_benchmark( 49 | llm=llm, 50 | out_dir=experiment_dir, 51 | tasks=tasks, 52 | bench_logger=bench_logger, 53 | ) 54 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/examples/vlm_benchmark.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | from rai_bench import ( 18 | define_benchmark_logger, 19 | parse_vlm_benchmark_args, 20 | ) 21 | from rai_bench.utils import get_llm_for_benchmark 22 | from rai_bench.vlm_benchmark import get_spatial_tasks, run_benchmark 23 | 24 | if __name__ == "__main__": 25 | args = parse_vlm_benchmark_args() 26 | experiment_dir = Path(args.out_dir) 27 | experiment_dir.mkdir(parents=True, exist_ok=True) 28 | bench_logger = define_benchmark_logger(out_dir=experiment_dir) 29 | try: 30 | tasks = get_spatial_tasks() 31 | for task in tasks: 32 | task.set_logger(bench_logger) 33 | 34 | llm = get_llm_for_benchmark( 35 | model_name=args.model_name, 36 | vendor=args.vendor, 37 | ) 38 | run_benchmark( 39 | llm=llm, 40 | out_dir=experiment_dir, 41 | tasks=tasks, 42 | bench_logger=bench_logger, 43 | ) 44 | except Exception as e: 45 | bench_logger.critical( 46 | msg=f"Benchmark failed with error: {e}", 47 | exc_info=True, 48 | ) 49 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .benchmark import run_benchmark 16 | from .predefined.scenarios import get_scenarios 17 | 18 | __all__ = ["get_scenarios", "run_benchmark"] 19 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1a.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: apple1 3 | prefab_name: apple 4 | pose: 5 | translation: 6 | x: 0.3 7 | y: -0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1a_1t.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: apple1 3 | prefab_name: apple 4 | pose: 5 | translation: 6 | x: 0.3 7 | y: -0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: tomato1 15 | prefab_name: tomato 16 | pose: 17 | translation: 18 | x: 0.2 19 | y: 0.4 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1a_2bc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: apple1 3 | prefab_name: apple 4 | pose: 5 | translation: 6 | x: 0.3 7 | y: -0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: blue_cube1 15 | prefab_name: blue_cube 16 | pose: 17 | translation: 18 | x: 0.1 19 | y: 0.5 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: blue_cube2 27 | prefab_name: blue_cube 28 | pose: 29 | translation: 30 | x: 0.2 31 | y: -0.4 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1bc_1rc_1yc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: blue_cube1 3 | prefab_name: blue_cube 4 | pose: 5 | translation: 6 | x: 0.15 7 | y: 0.35 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: red_cube1 15 | prefab_name: red_cube 16 | pose: 17 | translation: 18 | x: 0.45 19 | y: 0.1 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: yellow_cube1 27 | prefab_name: yellow_cube 28 | pose: 29 | translation: 30 | x: 0.25 31 | y: 0.5 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1carrot.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.4 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1carrot_1a_1t_1bc_1corn.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.1 7 | y: -0.2 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: apple1 15 | prefab_name: tomato 16 | pose: 17 | translation: 18 | x: 0.2 19 | y: -0.1 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: tomato1 27 | prefab_name: tomato 28 | pose: 29 | translation: 30 | x: 0.3 31 | y: 0.4 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | - name: blue_cube1 39 | prefab_name: blue_cube 40 | pose: 41 | translation: 42 | x: 0.4 43 | y: 0.5 44 | z: 0.05 45 | rotation: 46 | x: 0.0 47 | y: 0.0 48 | z: 0.0 49 | w: 1.0 50 | - name: corn1 51 | prefab_name: corn 52 | pose: 53 | translation: 54 | x: 0.55 55 | y: 0.5 56 | z: 0.05 57 | rotation: 58 | x: 0.0 59 | y: 0.0 60 | z: 0.0 61 | w: 1.0 62 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1carrot_1bc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.3 7 | y: -0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: blue_cube1 15 | prefab_name: blue_cube 16 | pose: 17 | translation: 18 | x: 0.2 19 | y: 0.4 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1carrot_1corn.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: corn2 15 | prefab_name: corn 16 | pose: 17 | translation: 18 | x: 0.5 19 | y: 0.4 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1carrot_1t_1rc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.4 7 | y: 0.0 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: tomato1 15 | prefab_name: tomato 16 | pose: 17 | translation: 18 | x: 0.45 19 | y: 0.0 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: red_cube1 27 | prefab_name: red_cube 28 | pose: 29 | translation: 30 | x: 0.35 31 | y: 0.0 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1rc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: red_cube1 3 | prefab_name: red_cube 4 | pose: 5 | translation: 6 | x: 0.2 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1rc_2bc_3yc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: cube1 3 | prefab_name: red_cube 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: cube2 15 | prefab_name: blue_cube 16 | pose: 17 | translation: 18 | x: 0.4 19 | y: -0.3 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: cube3 27 | prefab_name: yellow_cube 28 | pose: 29 | translation: 30 | x: 0.5 31 | y: -0.4 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | - name: cube4 39 | prefab_name: yellow_cube 40 | pose: 41 | translation: 42 | x: 0.5 43 | y: 0.3 44 | z: 0.05 45 | rotation: 46 | x: 0.0 47 | y: 0.0 48 | z: 0.0 49 | w: 1.0 50 | - name: cube5 51 | prefab_name: yellow_cube 52 | pose: 53 | translation: 54 | x: 0.6 55 | y: 0.4 56 | z: 0.05 57 | rotation: 58 | x: 0.0 59 | y: 0.0 60 | z: 0.0 61 | w: 1.0 62 | - name: cube6 63 | prefab_name: blue_cube 64 | pose: 65 | translation: 66 | x: 0.6 67 | y: -0.5 68 | z: 0.05 69 | rotation: 70 | x: 0.0 71 | y: 0.0 72 | z: 0.0 73 | w: 1.0 74 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1t.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: tomato1 3 | prefab_name: tomato 4 | pose: 5 | translation: 6 | x: 0.2 7 | y: 0.4 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1yc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: yellow_cube1 3 | prefab_name: yellow_cube 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: 0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/1yc_1rc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: yellow_cube1 3 | prefab_name: yellow_cube 4 | pose: 5 | translation: 6 | x: 0.2 7 | y: 0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: red_cube1 15 | prefab_name: red_cube 16 | pose: 17 | translation: 18 | x: 0.4 19 | y: 0.2 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/2a_1bc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: apple1 3 | prefab_name: apple 4 | pose: 5 | translation: 6 | x: 0.3 7 | y: -0.1 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: apple2 15 | prefab_name: apple 16 | pose: 17 | translation: 18 | x: 0.3 19 | y: -0.15 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: blue_cube1 27 | prefab_name: blue_cube 28 | pose: 29 | translation: 30 | x: 0.3 31 | y: -0.2 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/2a_1c_2rc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: apple1 3 | prefab_name: apple 4 | pose: 5 | translation: 6 | x: 0.1 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: apple2 15 | prefab_name: apple 16 | pose: 17 | translation: 18 | x: 0.35 19 | y: 0.1 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | 27 | - name: corn1 28 | prefab_name: corn 29 | pose: 30 | translation: 31 | x: 0.2 32 | y: 0.0 33 | z: 0.05 34 | rotation: 35 | x: 0.0 36 | y: 0.0 37 | z: 0.0 38 | w: 1.0 39 | 40 | - name: corn2 41 | prefab_name: corn 42 | pose: 43 | translation: 44 | x: 0.1 45 | y: -0.45 46 | z: 0.05 47 | rotation: 48 | x: 0.0 49 | y: 0.0 50 | z: 0.0 51 | w: 1.0 52 | - name: red_cube1 53 | prefab_name: red_cube 54 | pose: 55 | translation: 56 | x: 0.05 57 | y: 0.25 58 | z: 0.05 59 | rotation: 60 | x: 0.0 61 | y: 0.0 62 | z: 0.0 63 | w: 1.0 64 | - name: red_cube2 65 | prefab_name: red_cube 66 | pose: 67 | translation: 68 | x: 0.25 69 | y: -0.5 70 | z: 0.05 71 | rotation: 72 | x: 0.0 73 | y: 0.0 74 | z: 0.0 75 | w: 1.0 76 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/2carrots_2a.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: corn1 15 | prefab_name: apple 16 | pose: 17 | translation: 18 | x: 0.5 19 | y: 0.4 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | 27 | - name: carrot2 28 | prefab_name: carrot 29 | pose: 30 | translation: 31 | x: 0.1 32 | y: -0.3 33 | z: 0.05 34 | rotation: 35 | x: 0.0 36 | y: 0.0 37 | z: 0.0 38 | w: 1.0 39 | - name: corn2 40 | prefab_name: apple 41 | pose: 42 | translation: 43 | x: 0.1 44 | y: 0.4 45 | z: 0.05 46 | rotation: 47 | x: 0.0 48 | y: 0.0 49 | z: 0.0 50 | w: 1.0 51 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/2rc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: cube1 3 | prefab_name: red_cube 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: cube2 15 | prefab_name: red_cube 16 | pose: 17 | translation: 18 | x: 0.5 19 | y: 0.3 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/2t.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: tomato1 3 | prefab_name: tomato 4 | pose: 5 | translation: 6 | x: 0.3 7 | y: 0.4 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: tomato2 15 | prefab_name: tomato 16 | pose: 17 | translation: 18 | x: 0.4 19 | y: 0.5 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/2yc_1bc_1rc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: cube1 3 | prefab_name: red_cube 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: cube2 15 | prefab_name: blue_cube 16 | pose: 17 | translation: 18 | x: 0.4 19 | y: -0.3 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | 27 | - name: cube3 28 | prefab_name: yellow_cube 29 | pose: 30 | translation: 31 | x: 0.5 32 | y: -0.4 33 | z: 0.05 34 | rotation: 35 | x: 0.0 36 | y: 0.0 37 | z: 0.0 38 | w: 1.0 39 | - name: cube4 40 | prefab_name: yellow_cube 41 | pose: 42 | translation: 43 | x: 0.5 44 | y: 0.3 45 | z: 0.05 46 | rotation: 47 | x: 0.0 48 | y: 0.0 49 | z: 0.0 50 | w: 1.0 51 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/3rc_3bc_stacked.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: cube1 3 | prefab_name: red_cube 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | 15 | - name: cube2 16 | prefab_name: blue_cube 17 | pose: 18 | translation: 19 | x: 0.4 20 | y: -0.3 21 | z: 0.05 22 | rotation: 23 | x: 0.0 24 | y: 0.0 25 | z: 0.0 26 | w: 1.0 27 | 28 | - name: cube3 29 | prefab_name: red_cube 30 | pose: 31 | translation: 32 | x: 0.5 33 | y: -0.3 34 | z: 0.15 35 | rotation: 36 | x: 0.0 37 | y: 0.0 38 | z: 0.0 39 | w: 1.0 40 | 41 | - name: cube4 42 | prefab_name: red_cube 43 | pose: 44 | translation: 45 | x: 0.5 46 | y: -0.3 47 | z: 0.25 48 | rotation: 49 | x: 0.0 50 | y: 0.0 51 | z: 0.0 52 | w: 1.0 53 | 54 | - name: cube5 55 | prefab_name: blue_cube 56 | pose: 57 | translation: 58 | x: 0.3 59 | y: -0.3 60 | z: 0.05 61 | rotation: 62 | x: 0.0 63 | y: 0.0 64 | z: 0.0 65 | w: 1.0 66 | 67 | - name: cube6 68 | prefab_name: blue_cube 69 | pose: 70 | translation: 71 | x: 0.3 72 | y: -0.3 73 | z: 0.15 74 | rotation: 75 | x: 0.0 76 | y: 0.0 77 | z: 0.0 78 | w: 1.0 79 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/4bc.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: blue_cube1 3 | prefab_name: blue_cube 4 | pose: 5 | translation: 6 | x: 0.1 7 | y: -0.2 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | - name: blue_cube2 15 | prefab_name: blue_cube 16 | pose: 17 | translation: 18 | x: 0.2 19 | y: -0.1 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.0 25 | w: 1.0 26 | - name: blue_cube3 27 | prefab_name: blue_cube 28 | pose: 29 | translation: 30 | x: 0.3 31 | y: 0.4 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.0 37 | w: 1.0 38 | - name: blue_cube4 39 | prefab_name: blue_cube 40 | pose: 41 | translation: 42 | x: 0.4 43 | y: 0.5 44 | z: 0.05 45 | rotation: 46 | x: 0.0 47 | y: 0.0 48 | z: 0.0 49 | w: 1.0 50 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/4carrots.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.0 13 | w: 1.0 14 | 15 | - name: carrot2 16 | prefab_name: carrot 17 | pose: 18 | translation: 19 | x: 0.5 20 | y: 0.3 21 | z: 0.05 22 | rotation: 23 | x: 0.0 24 | y: 0.0 25 | z: 0.0 26 | w: 1.0 27 | - name: carrot3 28 | prefab_name: carrot 29 | pose: 30 | translation: 31 | x: 0.5 32 | y: 0.4 33 | z: 0.05 34 | rotation: 35 | x: 0.0 36 | y: 0.0 37 | z: 0.0 38 | w: 1.0 39 | 40 | - name: carrot4 41 | prefab_name: carrot 42 | pose: 43 | translation: 44 | x: 0.5 45 | y: -0.4 46 | z: 0.05 47 | rotation: 48 | x: 0.0 49 | y: 0.0 50 | z: 0.0 51 | w: 1.0 52 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/4carrots_rotated.yaml: -------------------------------------------------------------------------------- 1 | entities: 2 | - name: carrot1 3 | prefab_name: carrot 4 | pose: 5 | translation: 6 | x: 0.5 7 | y: -0.3 8 | z: 0.05 9 | rotation: 10 | x: 0.0 11 | y: 0.0 12 | z: 0.7071 # sin(45°) 13 | w: 0.7071 # cos(45°) -> 90° rotation 14 | - name: carrot2 15 | prefab_name: carrot 16 | pose: 17 | translation: 18 | x: 0.5 19 | y: 0.3 20 | z: 0.05 21 | rotation: 22 | x: 0.0 23 | y: 0.0 24 | z: 0.3827 # sin(22.5°) 25 | w: 0.9239 # cos(22.5°) -> 45° rotation 26 | - name: carrot3 27 | prefab_name: carrot 28 | pose: 29 | translation: 30 | x: 0.5 31 | y: 0.4 32 | z: 0.05 33 | rotation: 34 | x: 0.0 35 | y: 0.0 36 | z: 0.9239 # sin(67.5°) 37 | w: 0.3827 # cos(67.5°) -> 135° rotation 38 | - name: carrot4 39 | prefab_name: carrot 40 | pose: 41 | translation: 42 | x: 0.5 43 | y: -0.4 44 | z: 0.05 45 | rotation: 46 | x: 0.0 47 | y: 0.0 48 | z: 1.0 # sin(90°) 49 | w: 0.0 # cos(90°) -> 180° rotation 50 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/predefined/configs/o3de_config.yaml: -------------------------------------------------------------------------------- 1 | binary_path: demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher 2 | level: RoboticManipulationBenchmark 3 | required_simulation_ros2_interfaces: 4 | services: 5 | - /spawn_entity 6 | - /delete_entity 7 | topics: 8 | - /color_image5 9 | - /depth_image5 10 | - /color_camera_info5 11 | actions: [] 12 | required_robotic_ros2_interfaces: 13 | services: 14 | - /grounding_dino_classify 15 | - /grounded_sam_segment 16 | - /manipulator_move_to 17 | topics: [] 18 | actions: [] 19 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/results_tracking.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pydantic import BaseModel, Field 16 | 17 | 18 | class ScenarioResult(BaseModel): 19 | """Result of a single scenario execution.""" 20 | 21 | task_prompt: str = Field(..., description="The task prompt.") 22 | system_prompt: str = Field(..., description="The system prompt.") 23 | model_name: str = Field(..., description="Name of the LLM.") 24 | scene_config_path: str = Field( 25 | ..., description="Path to the scene configuration file." 26 | ) 27 | score: float = Field( 28 | ..., description="Value between 0 and 1, describing the task score." 29 | ) 30 | level: str = Field(..., description="Difficulty of the scenario") 31 | total_time: float = Field(..., description="Total time taken to complete the task.") 32 | number_of_tool_calls: int = Field( 33 | ..., description="Number of tool calls made during the task." 34 | ) 35 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/manipulation_o3de/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from rai_bench.manipulation_o3de.tasks.build_tower_task import ( 15 | BuildCubeTowerTask, 16 | ) 17 | from rai_bench.manipulation_o3de.tasks.group_objects_task import ( 18 | GroupObjectsTask, 19 | ) 20 | from rai_bench.manipulation_o3de.tasks.move_object_to_left_task import ( 21 | MoveObjectsToLeftTask, 22 | ) 23 | from rai_bench.manipulation_o3de.tasks.place_at_coord_task import ( 24 | PlaceObjectAtCoordTask, 25 | ) 26 | from rai_bench.manipulation_o3de.tasks.place_cubes_task import ( 27 | PlaceCubesTask, 28 | ) 29 | from rai_bench.manipulation_o3de.tasks.rotate_object_task import ( 30 | RotateObjectTask, 31 | ) 32 | 33 | __all__ = [ 34 | "BuildCubeTowerTask", 35 | "GroupObjectsTask", 36 | "MoveObjectsToLeftTask", 37 | "PlaceCubesTask", 38 | "PlaceObjectAtCoordTask", 39 | "RotateObjectTask", 40 | ] 41 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/results_processing/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/results_processing/visualise/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/results_processing/visualise/manipulaiton_o3de_display.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import streamlit as st 16 | 17 | from rai_bench.results_processing.data_loading import ( 18 | BenchmarkResults, 19 | ) 20 | from rai_bench.results_processing.data_processing import ( 21 | create_model_summary_dataframe, 22 | ) 23 | from rai_bench.results_processing.visualise.display import display_models_summ_data 24 | 25 | 26 | def render_manipulation_model_performance_tab(bench_results: BenchmarkResults): 27 | st.header("Model Performance") 28 | 29 | summ_df = create_model_summary_dataframe(bench_results) 30 | display_models_summ_data(summ_df) 31 | 32 | 33 | def render_manipulation_o3de(bench_results: BenchmarkResults): 34 | tabs = st.tabs( 35 | [ 36 | "Model Performance", 37 | ] 38 | ) 39 | with tabs[0]: 40 | render_manipulation_model_performance_tab(bench_results) 41 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .benchmark import run_benchmark 16 | from .predefined.tasks import get_tasks 17 | 18 | __all__ = ["get_tasks", "run_benchmark"] 19 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/messages/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/messages/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from rai.types import ROS2BaseModel, Time 16 | 17 | 18 | class Clock(ROS2BaseModel): 19 | _prefix: str = "rosgraph_msgs/msg" 20 | clock: Time = Time() 21 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/messages/services.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | from rai.types import Image, Pose, ROS2BaseModel 18 | 19 | 20 | class StringListRequest(ROS2BaseModel): 21 | pass 22 | 23 | 24 | class StringListResponse(ROS2BaseModel): 25 | success: bool = False 26 | string_list: List[str] = [] 27 | 28 | 29 | class VectorStoreRetrievalRequest(ROS2BaseModel): 30 | query: str = "" 31 | 32 | 33 | class VectorStoreRetrievalResponse(ROS2BaseModel): 34 | success: bool = False 35 | message: str = "" 36 | documents: List[str] = [] 37 | scores: List[float] = [] 38 | 39 | 40 | class WhatISeeRequest(ROS2BaseModel): 41 | pass 42 | 43 | 44 | class WhatISeeResponse(ROS2BaseModel): 45 | observations: List[str] = [] 46 | perception_source: str = "" 47 | image: Image = Image() 48 | pose: Pose = Pose() 49 | 50 | 51 | class PlannerInterfaceDescription(ROS2BaseModel): 52 | name: str = "" 53 | pipeline_id: str = "" 54 | planner_ids: List[str] = [] 55 | 56 | 57 | class QueryPlannerInterfaceResponse(ROS2BaseModel): 58 | planner_interfaces: List[PlannerInterfaceDescription] = [] 59 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/messages/topics.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | from rai.types import Header, Image, ROS2BaseModel 18 | 19 | 20 | class AudioMessage(ROS2BaseModel): 21 | _prefix: str = "rai_interfaces/msg" 22 | audio: List[int] = [] 23 | sample_rate: int = 0 24 | channels: int = 0 25 | 26 | 27 | # NOTE(boczekbartek): this message is duplicated here only for benchmarking purposes. 28 | # for communication in rai please use rai.communication.ros2.ROS2HRIMessage 29 | class HRIMessage(ROS2BaseModel): 30 | _prefix: str = "rai_interfaces/msg" 31 | header: Header = Header() 32 | text: str = "" 33 | images: List[Image] = [] 34 | audios: List[AudioMessage] = [] 35 | communication_id: str = "" 36 | seq_no: int = 0 37 | seq_end: bool = False 38 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/predefined/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .basic_tasks import get_basic_tasks 16 | from .custom_interfaces_tasks import get_custom_interfaces_tasks 17 | from .manipulation_tasks import get_manipulation_tasks 18 | 19 | __all__ = [ 20 | "get_basic_tasks", 21 | "get_custom_interfaces_tasks", 22 | "get_manipulation_tasks", 23 | ] 24 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/tool_calling_agent/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .benchmark import run_benchmark 16 | from .predefined.tasks import get_spatial_tasks 17 | 18 | __all__ = ["get_spatial_tasks", "run_benchmark"] 19 | -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_1.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_2.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_3.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_4.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_5.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_6.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_bench/rai_bench/vlm_benchmark/predefined/images/image_7.jpg -------------------------------------------------------------------------------- /src/rai_bench/rai_bench/vlm_benchmark/results_tracking.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from uuid import UUID 17 | 18 | from pydantic import BaseModel, Field 19 | 20 | 21 | class TaskResult(BaseModel): 22 | task_prompt: str = Field(..., description="The task prompt.") 23 | system_prompt: str = Field(..., description="The system prompt.") 24 | complexity: str = Field(..., description="Complexity of the task.") 25 | type: str = Field( 26 | ..., description="Type of task, for example: bool_response_image_task" 27 | ) 28 | model_name: str = Field(..., description="Name of the LLM.") 29 | score: float = Field( 30 | ..., 31 | description="Value between 0 and 1.", 32 | ) 33 | 34 | total_time: float = Field(..., description="Total time taken to complete the task.") 35 | run_id: UUID = Field(..., description="UUID of the task run.") 36 | -------------------------------------------------------------------------------- /src/rai_bringup/launch/openset.launch.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from launch import LaunchDescription 17 | from launch.actions import ExecuteProcess 18 | 19 | 20 | def generate_launch_description(): 21 | return LaunchDescription( 22 | [ 23 | ExecuteProcess( 24 | cmd=["python", "run_perception_agents.py"], 25 | cwd="src/rai_extensions/rai_perception/rai_perception/scripts", 26 | output="screen", 27 | ), 28 | ] 29 | ) 30 | -------------------------------------------------------------------------------- /src/rai_bringup/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | rai_bringup 4 | 0.0.1 5 | Bringup scripts and configurations for the RAI. 6 | Maciej Majek 7 | Apache-2.0 8 | 9 | python3-pytest 10 | 11 | 12 | ament_python 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/rai_bringup/resource/rai_bringup: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/rai_bringup/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | from glob import glob 18 | 19 | from setuptools import find_packages, setup 20 | 21 | package_name = "rai_bringup" 22 | 23 | setup( 24 | name=package_name, 25 | version="0.0.1", 26 | packages=find_packages(exclude=["test"]), 27 | data_files=[ 28 | ("share/ament_index/resource_index/packages", ["resource/" + package_name]), 29 | ("share/" + package_name, ["package.xml"]), 30 | (os.path.join("share", package_name, "launch"), glob("launch/*.launch.py")), 31 | ], 32 | install_requires=["setuptools"], 33 | zip_safe=True, 34 | maintainer="maciejmajek", 35 | maintainer_email="maciej.majek@robotec.ai", 36 | description="Bringup scripts and configurations for the RAI.", 37 | license="Apache-2.0", 38 | tests_require=["pytest"], 39 | entry_points={ 40 | "console_scripts": [], 41 | }, 42 | ) 43 | -------------------------------------------------------------------------------- /src/rai_core/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["poetry-core>=1.0.0"] 3 | build-backend = "poetry.core.masonry.api" 4 | 5 | [tool.poetry] 6 | name = "rai_core" 7 | version = "2.5.10" 8 | description = "Core functionality for RAI framework" 9 | authors = ["Maciej Majek ", "Bartłomiej Boczek ", "Kajetan Rachwał "] 10 | readme = "README.md" 11 | classifiers = [ 12 | "Programming Language :: Python :: 3", 13 | "Development Status :: 4 - Beta", 14 | "License :: OSI Approved :: Apache Software License", 15 | ] 16 | packages = [ 17 | { include = "rai", from = "." }, 18 | ] 19 | 20 | [tool.poetry.dependencies] 21 | python = "^3.10, <3.13" 22 | langchain-core = "^0.3" 23 | langgraph = "*" 24 | langgraph-prebuilt = "*" 25 | langchain = "*" 26 | langchain-aws = "*" 27 | langchain-openai = "*" 28 | langchain-ollama = "^0.3.4" 29 | langchain-community = "*" 30 | 31 | requests = "^2.32.2" 32 | coloredlogs = "^15.0.1" 33 | tqdm = "^4.66.4" 34 | deprecated = "^1.2.14" 35 | tomli = "^2.0.1" 36 | tomli-w = "^1.1.0" 37 | opencv-python = "^4.9.0.80" 38 | lark = "^1.1.9" 39 | transforms3d = "^0.4.1" 40 | pillow = "^11.0.0" 41 | langfuse = "^2.60.2" 42 | pydub = "^0.25.1" 43 | streamlit = "^1.44" 44 | numpy = "<2.0" 45 | 46 | [tool.poetry.scripts] 47 | rai-config-init = "rai.initialization.config_initialization:main" 48 | -------------------------------------------------------------------------------- /src/rai_core/rai/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .agents import AgentRunner, ReActAgent 16 | from .initialization import ( 17 | get_embeddings_model, 18 | get_llm_model, 19 | get_llm_model_config_and_vendor, 20 | get_llm_model_direct, 21 | get_tracing_callbacks, 22 | ) 23 | 24 | __all__ = [ 25 | "AgentRunner", 26 | "ReActAgent", 27 | "get_embeddings_model", 28 | "get_llm_model", 29 | "get_llm_model_config_and_vendor", 30 | "get_llm_model_direct", 31 | "get_tracing_callbacks", 32 | ] 33 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from rai.agents.base import BaseAgent 16 | from rai.agents.langchain import ( 17 | BaseStateBasedAgent, 18 | ReActAgent, 19 | ) 20 | from rai.agents.runner import AgentRunner, wait_for_shutdown 21 | 22 | __all__ = [ 23 | "AgentRunner", 24 | "BaseAgent", 25 | "BaseStateBasedAgent", 26 | "ReActAgent", 27 | "wait_for_shutdown", 28 | ] 29 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | from abc import ABC, abstractmethod 17 | 18 | 19 | class BaseAgent(ABC): 20 | def __init__(self): 21 | """Initializes a new agent instance and sets up logging with the class name.""" 22 | self.logger = logging.getLogger(self.__class__.__name__) 23 | 24 | @abstractmethod 25 | def run(self): 26 | """Starts the agent's main execution loop. 27 | In some cases, concrete run implementation may not be needed. 28 | In that case use pass as a placeholder.""" 29 | pass 30 | 31 | @abstractmethod 32 | def stop(self): 33 | """Gracefully terminates the agent's execution and cleans up resources.""" 34 | pass 35 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/integrations/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/langchain/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .agent import BaseState, LangChainAgent, newMessageBehaviorType 16 | from .callback import HRICallbackHandler 17 | from .core import ( 18 | ReActAgentState, 19 | create_react_runnable, 20 | create_state_based_runnable, 21 | ) 22 | from .invocation_helpers import invoke_llm_with_tracing 23 | from .react_agent import ReActAgent 24 | from .state_based_agent import BaseStateBasedAgent, StateBasedConfig 25 | 26 | __all__ = [ 27 | "BaseState", 28 | "BaseStateBasedAgent", 29 | "HRICallbackHandler", 30 | "LangChainAgent", 31 | "ReActAgent", 32 | "ReActAgentState", 33 | "StateBasedConfig", 34 | "create_react_runnable", 35 | "create_state_based_runnable", 36 | "invoke_llm_with_tracing", 37 | "newMessageBehaviorType", 38 | ] 39 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/langchain/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .conversational_agent import State as ConversationalAgentState 16 | from .conversational_agent import create_conversational_agent 17 | from .megamind import ( 18 | ContextProvider, 19 | Executor, 20 | create_megamind, 21 | get_initial_megamind_state, 22 | ) 23 | from .react_agent import ( 24 | ReActAgentState, 25 | create_react_runnable, 26 | ) 27 | from .state_based_agent import create_state_based_runnable 28 | from .structured_output_agent import create_structured_output_runnable 29 | from .tool_runner import SubAgentToolRunner, ToolRunner 30 | 31 | __all__ = [ 32 | "ContextProvider", 33 | "ConversationalAgentState", 34 | "Executor", 35 | "ReActAgentState", 36 | "SubAgentToolRunner", 37 | "ToolRunner", 38 | "create_conversational_agent", 39 | "create_megamind", 40 | "create_react_runnable", 41 | "create_state_based_runnable", 42 | "create_structured_output_runnable", 43 | "get_initial_megamind_state", 44 | ] 45 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/langchain/react_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Dict, List, Optional 16 | 17 | from langchain_core.language_models import BaseChatModel 18 | from langchain_core.tools import BaseTool 19 | 20 | from rai.agents.langchain.agent import LangChainAgent 21 | from rai.agents.langchain.core import ( 22 | ReActAgentState, 23 | create_react_runnable, 24 | ) 25 | from rai.communication.hri_connector import HRIConnector, HRIMessage 26 | from rai.messages.multimodal import SystemMultimodalMessage 27 | 28 | 29 | class ReActAgent(LangChainAgent): 30 | def __init__( 31 | self, 32 | target_connectors: Dict[str, HRIConnector[HRIMessage]], 33 | llm: Optional[BaseChatModel] = None, 34 | tools: Optional[List[BaseTool]] = None, 35 | state: Optional[ReActAgentState] = None, 36 | system_prompt: Optional[str | SystemMultimodalMessage] = None, 37 | stream_response: bool = True, 38 | ): 39 | runnable = create_react_runnable( 40 | llm=llm, tools=tools, system_prompt=system_prompt 41 | ) 42 | super().__init__( 43 | target_connectors=target_connectors, 44 | runnable=runnable, 45 | state=state, 46 | stream_response=stream_response, 47 | ) 48 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/ros2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from .state_based_agent import ROS2StateBasedAgent 15 | 16 | __all__ = ["ROS2StateBasedAgent"] 17 | -------------------------------------------------------------------------------- /src/rai_core/rai/agents/ros2/state_based_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from rai.agents.langchain import BaseStateBasedAgent 16 | from rai.communication.ros2 import ROS2Connector 17 | 18 | 19 | class ROS2StateBasedAgent(BaseStateBasedAgent): 20 | def setup_connector(self): 21 | return ROS2Connector() 22 | -------------------------------------------------------------------------------- /src/rai_core/rai/aggregators/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base import BaseAggregator 16 | 17 | __all__ = ["BaseAggregator"] 18 | -------------------------------------------------------------------------------- /src/rai_core/rai/aggregators/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from collections import deque 17 | from typing import Deque, Generic, List, TypeVar 18 | 19 | from langchain_core.messages import BaseMessage 20 | 21 | T = TypeVar("T") 22 | 23 | 24 | class BaseAggregator(ABC, Generic[T]): 25 | """ 26 | Interface for aggregators. 27 | """ 28 | 29 | def __init__(self, max_size: int | None = None) -> None: 30 | super().__init__() 31 | self._buffer: Deque[T] = deque() 32 | self.max_size = max_size 33 | 34 | def __call__(self, msg: T) -> None: 35 | if self.max_size is not None and len(self._buffer) >= self.max_size: 36 | self._buffer.popleft() 37 | self._buffer.append(msg) 38 | 39 | @abstractmethod 40 | def get(self) -> BaseMessage | None: 41 | """Returns the outcome of processing the aggregated message""" 42 | 43 | def clear_buffer(self) -> None: 44 | """Clears the buffer of messages""" 45 | self._buffer.clear() 46 | 47 | def get_buffer(self) -> List[T]: 48 | """Returns a copy of the buffer of messages""" 49 | return list(self._buffer) 50 | 51 | def __str__(self) -> str: 52 | return f"{self.__class__.__name__}(len={len(self._buffer)})" 53 | -------------------------------------------------------------------------------- /src/rai_core/rai/aggregators/ros2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .aggregators import ( 16 | ROS2GetLastImageAggregator, 17 | ROS2ImgVLMDescriptionAggregator, 18 | ROS2ImgVLMDiffAggregator, 19 | ROS2LogsAggregator, 20 | ) 21 | 22 | __all__ = [ 23 | "ROS2GetLastImageAggregator", 24 | "ROS2ImgVLMDescriptionAggregator", 25 | "ROS2ImgVLMDiffAggregator", 26 | "ROS2LogsAggregator", 27 | ] 28 | -------------------------------------------------------------------------------- /src/rai_core/rai/communication/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base_connector import BaseConnector, BaseMessage 16 | from .hri_connector import HRIConnector, HRIMessage 17 | 18 | __all__ = [ 19 | "BaseConnector", 20 | "BaseMessage", 21 | "HRIConnector", 22 | "HRIMessage", 23 | ] 24 | -------------------------------------------------------------------------------- /src/rai_core/rai/communication/ros2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib.util 16 | 17 | if importlib.util.find_spec("rclpy") is None: 18 | raise ImportError( 19 | "This is a ROS2 feature. Make sure ROS2 is installed and sourced." 20 | ) 21 | 22 | from .api import ( 23 | IROS2Message, # TODO: IROS2Message should not be a part of the public API 24 | ) 25 | from .connectors import ROS2Connector, ROS2HRIConnector 26 | from .context import ROS2Context 27 | from .messages import ROS2HRIMessage, ROS2Message 28 | from .waiters import wait_for_ros2_actions, wait_for_ros2_services, wait_for_ros2_topics 29 | 30 | __all__ = [ 31 | "IROS2Message", 32 | "ROS2Connector", 33 | "ROS2Context", 34 | "ROS2HRIConnector", 35 | "ROS2HRIMessage", 36 | "ROS2Message", 37 | "wait_for_ros2_actions", 38 | "wait_for_ros2_services", 39 | "wait_for_ros2_topics", 40 | ] 41 | -------------------------------------------------------------------------------- /src/rai_core/rai/communication/ros2/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .action import ROS2ActionAPI 16 | from .base import IROS2Message 17 | from .conversion import ( 18 | convert_ros_img_to_base64, 19 | convert_ros_img_to_cv2mat, 20 | convert_ros_img_to_ndarray, 21 | import_message_from_str, 22 | ros2_message_to_dict, 23 | ) 24 | from .service import ROS2ServiceAPI 25 | from .topic import ROS2TopicAPI 26 | 27 | __all__ = [ 28 | "IROS2Message", 29 | "ROS2ActionAPI", 30 | "ROS2ServiceAPI", 31 | "ROS2TopicAPI", 32 | "convert_ros_img_to_base64", 33 | "convert_ros_img_to_cv2mat", 34 | "convert_ros_img_to_ndarray", 35 | "import_message_from_str", 36 | "ros2_message_to_dict", 37 | ] 38 | -------------------------------------------------------------------------------- /src/rai_core/rai/communication/ros2/connectors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .action_mixin import ROS2ActionMixin 16 | from .base import ROS2BaseConnector 17 | from .hri_connector import ROS2HRIConnector 18 | from .ros2_connector import ROS2Connector 19 | from .service_mixin import ROS2ServiceMixin 20 | 21 | __all__ = [ 22 | "ROS2ActionMixin", 23 | "ROS2BaseConnector", 24 | "ROS2Connector", 25 | "ROS2HRIConnector", 26 | "ROS2ServiceMixin", 27 | ] 28 | -------------------------------------------------------------------------------- /src/rai_core/rai/communication/ros2/connectors/ros2_connector.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from rai.communication.ros2.connectors.base import ROS2BaseConnector 16 | from rai.communication.ros2.messages import ROS2Message 17 | 18 | 19 | class ROS2Connector(ROS2BaseConnector[ROS2Message]): 20 | pass 21 | -------------------------------------------------------------------------------- /src/rai_core/rai/communication/ros2/ros_async.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import threading 17 | from typing import Any 18 | 19 | import rclpy 20 | import rclpy.task 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def get_future_result( 26 | future: rclpy.task.Future, timeout_sec: float = 5.0 27 | ) -> Any | None: 28 | """Replaces rclpy.spin_until_future_complete""" 29 | result = None 30 | event = threading.Event() 31 | 32 | def callback(future: rclpy.task.Future) -> None: 33 | nonlocal result 34 | result = future.result() 35 | event.set() 36 | 37 | future.add_done_callback(callback) 38 | 39 | timed_out = not event.wait(timeout=timeout_sec) 40 | if timed_out: 41 | logger.warning(f"Future timed out after {timeout_sec}s") 42 | 43 | return result 44 | -------------------------------------------------------------------------------- /src/rai_core/rai/frontend/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..frontend.streamlit import run_streamlit_app 16 | 17 | __all__ = ["run_streamlit_app"] 18 | -------------------------------------------------------------------------------- /src/rai_core/rai/initialization/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .model_initialization import ( 16 | get_embeddings_model, 17 | get_llm_model, 18 | get_llm_model_config_and_vendor, 19 | get_llm_model_direct, 20 | get_tracing_callbacks, 21 | ) 22 | 23 | __all__ = [ 24 | "get_embeddings_model", 25 | "get_llm_model", 26 | "get_llm_model_config_and_vendor", 27 | "get_llm_model_direct", 28 | "get_tracing_callbacks", 29 | ] 30 | -------------------------------------------------------------------------------- /src/rai_core/rai/messages/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from langchain_core.messages import ( 17 | AIMessage, 18 | HumanMessage, 19 | SystemMessage, 20 | ToolMessage, 21 | ) 22 | 23 | from .artifacts import MultimodalArtifact, get_stored_artifacts, store_artifacts 24 | from .conversion import preprocess_image 25 | from .multimodal import ( 26 | AIMultimodalMessage, 27 | HumanMultimodalMessage, 28 | MultimodalMessage, 29 | SystemMultimodalMessage, 30 | ToolMultimodalMessage, 31 | ) 32 | 33 | __all__ = [ 34 | "AIMessage", 35 | "AIMultimodalMessage", 36 | "HumanMessage", 37 | "HumanMultimodalMessage", 38 | "MultimodalArtifact", 39 | "MultimodalMessage", 40 | "SystemMessage", 41 | "SystemMultimodalMessage", 42 | "ToolMessage", 43 | "ToolMultimodalMessage", 44 | "get_stored_artifacts", 45 | "preprocess_image", 46 | "store_artifacts", 47 | ] 48 | -------------------------------------------------------------------------------- /src/rai_core/rai/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_core/rai/tools/ros2/generic/toolkit.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | from langchain_core.tools import BaseTool 18 | 19 | from rai.tools.ros2.base import BaseROS2Toolkit 20 | 21 | 22 | class ROS2Toolkit(BaseROS2Toolkit): 23 | def get_tools(self) -> List[BaseTool]: 24 | # lazy import to avoid circular import 25 | from rai.tools.ros2.generic import ( 26 | ROS2ActionToolkit, 27 | ROS2ServicesToolkit, 28 | ROS2TopicsToolkit, 29 | ) 30 | 31 | return [ 32 | *ROS2TopicsToolkit( 33 | connector=self.connector, 34 | readable=self.readable, 35 | writable=self.writable, 36 | forbidden=self.forbidden, 37 | ).get_tools(), 38 | *ROS2ServicesToolkit( 39 | connector=self.connector, 40 | readable=self.readable, 41 | writable=self.writable, 42 | forbidden=self.forbidden, 43 | ).get_tools(), 44 | *ROS2ActionToolkit( 45 | connector=self.connector, 46 | readable=self.readable, 47 | writable=self.writable, 48 | forbidden=self.forbidden, 49 | ).get_tools(), 50 | ] 51 | -------------------------------------------------------------------------------- /src/rai_core/rai/tools/ros2/manipulation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib.util 16 | 17 | if importlib.util.find_spec("rclpy") is None: 18 | raise ImportError( 19 | "This is a ROS2 feature. Make sure ROS2 is installed and sourced." 20 | ) 21 | 22 | from .custom import ( 23 | GetObjectPositionsTool, 24 | MoveObjectFromToTool, 25 | MoveObjectFromToToolInput, 26 | MoveToPointTool, 27 | MoveToPointToolInput, 28 | ResetArmTool, 29 | ) 30 | 31 | __all__ = [ 32 | "GetObjectPositionsTool", 33 | "MoveObjectFromToTool", 34 | "MoveObjectFromToToolInput", 35 | "MoveToPointTool", 36 | "MoveToPointToolInput", 37 | "ResetArmTool", 38 | ] 39 | -------------------------------------------------------------------------------- /src/rai_core/rai/tools/ros2/navigation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .nav2 import ( 16 | CancelNavigateToPoseTool, 17 | GetNavigateToPoseFeedbackTool, 18 | GetNavigateToPoseResultTool, 19 | GetOccupancyGridTool, 20 | Nav2Toolkit, 21 | NavigateToPoseTool, 22 | ) 23 | from .nav2_blocking import NavigateToPoseBlockingTool 24 | 25 | __all__ = [ 26 | "CancelNavigateToPoseTool", 27 | "GetNavigateToPoseFeedbackTool", 28 | "GetNavigateToPoseResultTool", 29 | "GetOccupancyGridTool", 30 | "Nav2Toolkit", 31 | "NavigateToPoseBlockingTool", 32 | "NavigateToPoseTool", 33 | ] 34 | -------------------------------------------------------------------------------- /src/rai_core/rai/tools/time.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import time 17 | from typing import Type 18 | 19 | from langchain_core.tools import BaseTool 20 | from pydantic import BaseModel, Field 21 | 22 | 23 | class WaitForSecondsToolInput(BaseModel): 24 | """Input for the WaitForSecondsTool tool.""" 25 | 26 | seconds: int = Field(..., description="The number of seconds to wait") 27 | 28 | 29 | class WaitForSecondsTool(BaseTool): 30 | """Wait for a specified number of seconds""" 31 | 32 | name: str = "WaitForSecondsTool" 33 | description: str = ( 34 | "A tool for waiting. " 35 | "Useful for pausing execution for a specified number of seconds. " 36 | "Input should be the number of seconds to wait." 37 | "Maximum allowed time is 10 seconds" 38 | ) 39 | 40 | args_schema: Type[WaitForSecondsToolInput] = WaitForSecondsToolInput 41 | 42 | def _run(self, seconds: int): 43 | """Waits for the specified number of seconds.""" 44 | if seconds > 10: 45 | seconds = 10 46 | time.sleep(seconds) 47 | return f"Waited for {seconds} seconds." 48 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from .base import RaiBaseModel, ROS2BaseModel 17 | from .geometry import ( 18 | Path, 19 | Point, 20 | Pose, 21 | Pose2D, 22 | PoseStamped, 23 | PoseWithCovariance, 24 | Quaternion, 25 | ) 26 | from .sensor import CameraInfo, Image 27 | from .std import Header, Time 28 | from .vision import ( 29 | BoundingBox2D, 30 | Detection2D, 31 | ObjectHypothesis, 32 | ObjectHypothesisWithPose, 33 | RegionOfInterest, 34 | ) 35 | 36 | __all__ = [ 37 | "BoundingBox2D", 38 | "CameraInfo", 39 | "Detection2D", 40 | "Header", 41 | "Image", 42 | "ObjectHypothesis", 43 | "ObjectHypothesisWithPose", 44 | "Path", 45 | "Point", 46 | "Pose", 47 | "Pose2D", 48 | "PoseStamped", 49 | "PoseWithCovariance", 50 | "Quaternion", 51 | "ROS2BaseModel", 52 | "RaiBaseModel", 53 | "RegionOfInterest", 54 | "Time", 55 | ] 56 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pydantic import BaseModel, ConfigDict 16 | 17 | 18 | class RaiBaseModel(BaseModel): 19 | model_config = ConfigDict(extra="forbid") 20 | 21 | 22 | class ROS2BaseModel(RaiBaseModel): 23 | """ 24 | Base model for messages replicated from ros2 25 | 26 | Attributes: 27 | _prefix: str 28 | Prefix of the ros2 message, for example std_msgs/msg 29 | """ 30 | 31 | _prefix: str 32 | 33 | def get_msg_name(self) -> str: 34 | return f"{self._prefix}/{self.__class__.__name__}" 35 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/geometry.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | from rai.types.base import ROS2BaseModel 18 | from rai.types.std import Header 19 | 20 | 21 | class BaseGeometryModel(ROS2BaseModel): 22 | _prefix: str = "geometry_msgs/msg" 23 | 24 | 25 | class Point(BaseGeometryModel): 26 | x: float = 0.0 27 | y: float = 0.0 28 | z: float = 0.0 29 | 30 | 31 | class Quaternion(BaseGeometryModel): 32 | x: float = 0.0 33 | y: float = 0.0 34 | z: float = 0.0 35 | w: float = 1.0 36 | 37 | 38 | class Pose(BaseGeometryModel): 39 | position: Point = Point() 40 | orientation: Quaternion = Quaternion() 41 | 42 | 43 | class PoseStamped(BaseGeometryModel): 44 | header: Header = Header() 45 | pose: Pose = Pose() 46 | 47 | 48 | class PoseWithCovariance(BaseGeometryModel): 49 | pose: Pose = Pose() 50 | covariance: List[float] = [0.0] * 36 51 | 52 | 53 | class Pose2D(BaseGeometryModel): 54 | x: float = 0.0 55 | y: float = 0.0 56 | theta: float = 0.0 57 | 58 | 59 | class Path(ROS2BaseModel): 60 | _prefix: str = "nav_msgs/msg" 61 | header: Header = Header() 62 | poses: List[Pose] = [] 63 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/ros2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib.util 16 | 17 | if importlib.util.find_spec("rosidl_runtime_py") is None: 18 | raise ImportError( 19 | "This is a ROS2 feature. Make sure ROS2 is installed and sourced." 20 | ) 21 | 22 | from .convert import from_ros2_msg, to_ros2_msg 23 | 24 | __all__ = ["from_ros2_msg", "to_ros2_msg"] 25 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/ros2/convert.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib 16 | from typing import Any 17 | 18 | import rosidl_runtime_py 19 | 20 | from rai.communication.ros2.api.conversion import import_message_from_str 21 | 22 | from ..base import ROS2BaseModel 23 | 24 | 25 | def to_ros2_msg(base_model: ROS2BaseModel) -> Any: 26 | if not isinstance(base_model, ROS2BaseModel): 27 | raise TypeError(f"Expected Ros2BaseModel, got {type(base_model)}") 28 | msg_name = base_model.get_msg_name() 29 | ros2_msg_cls = import_message_from_str(msg_name) 30 | msg_args = base_model.model_dump() 31 | ros2_msg = ros2_msg_cls() 32 | rosidl_runtime_py.set_message.set_message_fields(ros2_msg, msg_args) 33 | return ros2_msg 34 | 35 | 36 | def from_ros2_msg(msg: Any) -> ROS2BaseModel: 37 | if not hasattr(msg, "__class__"): 38 | raise TypeError(f"Expected ROS2 message, got {type(msg)}") 39 | msg_name = msg.__class__.__name__ 40 | types_module = importlib.import_module("rai.types") 41 | try: 42 | base_model_cls: ROS2BaseModel = getattr(types_module, msg_name) 43 | except AttributeError: 44 | raise ImportError( 45 | f"Could not find ROS2BaseModel class for {msg_name} in rai.types module" 46 | ) 47 | msg_dict = rosidl_runtime_py.message_to_ordereddict(msg) 48 | return base_model_cls.model_validate(msg_dict) 49 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/sensor.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from typing import List 15 | 16 | from rai.types.base import ROS2BaseModel 17 | from rai.types.std import Header 18 | from rai.types.vision import RegionOfInterest 19 | 20 | 21 | class BaseSensorModel(ROS2BaseModel): 22 | _prefix: str = "sensor_msgs/msg" 23 | header: Header = Header() 24 | 25 | 26 | class CameraInfo(BaseSensorModel): 27 | height: int = 0 28 | width: int = 0 29 | distortion_model: str = "" 30 | d: List[float] = [] 31 | k: List[float] = [0.0] * 9 32 | r: List[float] = [0.0] * 9 33 | p: List[float] = [0.0] * 12 34 | binning_x: int = 0 35 | binning_y: int = 0 36 | roi: RegionOfInterest = RegionOfInterest() 37 | 38 | 39 | class Image(BaseSensorModel): 40 | height: int = 0 41 | width: int = 0 42 | encoding: str = "" 43 | is_bigendian: int = 0 44 | step: int = 0 45 | data: List[int] = [] 46 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/std.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pydantic import Field 16 | 17 | from rai.types.base import ROS2BaseModel 18 | 19 | 20 | class BaseStdModel(ROS2BaseModel): 21 | _prefix: str = "std_msgs/msg" 22 | 23 | 24 | class Time(BaseStdModel): 25 | sec: int = 0 26 | nanosec: int = 0 27 | 28 | 29 | class Header(BaseStdModel): 30 | frame_id: str = Field(default="", description="Reference frame of the message") 31 | stamp: Time = Time() 32 | -------------------------------------------------------------------------------- /src/rai_core/rai/types/vision.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | from rai.types.base import ROS2BaseModel 18 | from rai.types.geometry import Pose2D, PoseWithCovariance 19 | from rai.types.std import Header 20 | 21 | 22 | class BaseVisionModel(ROS2BaseModel): 23 | _prefix: str = "vision_msgs/msg" 24 | 25 | 26 | class ObjectHypothesis(BaseVisionModel): 27 | class_id: str = "" 28 | score: float = 0.0 29 | 30 | 31 | class ObjectHypothesisWithPose(BaseVisionModel): 32 | hypothesis: ObjectHypothesis = ObjectHypothesis() 33 | pose: PoseWithCovariance = PoseWithCovariance() 34 | 35 | 36 | class BoundingBox2D(BaseVisionModel): 37 | center: Pose2D = Pose2D() 38 | size_x: float = 0.0 39 | size_y: float = 0.0 40 | 41 | 42 | class Detection2D(BaseVisionModel): 43 | header: Header = Header() 44 | results: List[ObjectHypothesisWithPose] = [] 45 | bbox: BoundingBox2D = BoundingBox2D() 46 | id: str = "" 47 | 48 | 49 | class RegionOfInterest(BaseVisionModel): 50 | x_offset: int = 0 51 | y_offset: int = 0 52 | height: int = 0 53 | width: int = 0 54 | do_rectify: bool = False 55 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_nomad/README.md: -------------------------------------------------------------------------------- 1 | # RAI NoMaD 2 | 3 | This package provides a ROS2 Node which loads and runs the [NoMaD](https://general-navigation-models.github.io/nomad/index.html) model, that can be dynamically activated and deactivated using ROS2 messages. 4 | 5 | ## Running instructions 6 | 7 | ### 1. Setup the ROS2 workspace 8 | 9 | In the base directory of the `RAI` package install dependencies: 10 | 11 | ``` 12 | poetry install --with nomad 13 | ``` 14 | 15 | Source the ros installation 16 | 17 | ``` 18 | source /opt/ros/${ROS_DISTRO}/setup.bash 19 | ``` 20 | 21 | Run the build process: 22 | 23 | ``` 24 | colcon build 25 | ``` 26 | 27 | Setup your shell: 28 | 29 | ``` 30 | source ./setup_shell.sh 31 | ``` 32 | 33 | ### 2. Run the node 34 | 35 | Run the ROS2 node using `ros2 run`: 36 | 37 | ``` 38 | ros2 run rai_nomad nomad --ros-args -p image_topic:= 39 | ``` 40 | 41 | The model will be loaded and ready, but it will not run until you call the `/rai_nomad/start` service. Then it will start outputting velocity commands and your robot should start moving. You can then stop the model by calling the `/rai_nomad/stop` service. 42 | 43 | ### Parameters 44 | 45 | - `model_path`: Path to where the model should be downloaded. By default it will be downloaded into the package's `share` directory. 46 | - `image_topic`: The topic where the camera images are being published. 47 | - `cmd_vel_topic`: The topic where the velocity commands are being published. Default: `/cmd_vel`. 48 | - `linear_vel`: Linear velocity scaling of the model output. 49 | - `angular_vel`: Angular velocity scaling of the model output. 50 | - `max_v` and `max_w`: Maximum linear and angular velocities that the model can output. 51 | - `rate`: The rate at which the model will run. 52 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_nomad/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | rai_nomad 5 | 0.0.0 6 | Package enabling exploration using NoMaD model in RAI 7 | kdabrowski 8 | Apache-2.0 9 | 10 | ament_copyright 11 | ament_pep257 12 | python3-pytest 13 | 14 | std_msgs 15 | geometry_msgs 16 | sensor_msgs 17 | 18 | 19 | ament_python 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_nomad/rai_nomad/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_nomad/resource/rai_nomad: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_nomad/setup.cfg: -------------------------------------------------------------------------------- 1 | [develop] 2 | script_dir=$base/lib/rai_nomad 3 | [install] 4 | install_scripts=$base/lib/rai_nomad 5 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_nomad/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from setuptools import find_packages, setup 17 | 18 | package_name = "rai_nomad" 19 | 20 | setup( 21 | name=package_name, 22 | version="0.0.0", 23 | packages=find_packages(exclude=["test"]), 24 | data_files=[ 25 | ("share/ament_index/resource_index/packages", ["resource/" + package_name]), 26 | ("share/" + package_name, ["package.xml"]), 27 | ("share/" + package_name, ["resource/nomad_params.yaml"]), 28 | ], 29 | install_requires=["setuptools"], 30 | zip_safe=True, 31 | maintainer="kdabrowski", 32 | maintainer_email="kacper.dabrowski@robotec.ai", 33 | description="Package enabling exploration using NoMaD model in RAI", 34 | license="Apache-2.0", 35 | tests_require=["pytest"], 36 | entry_points={ 37 | "console_scripts": ["nomad = rai_nomad.nomad:main"], 38 | }, 39 | ) 40 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/.gitignore: -------------------------------------------------------------------------------- 1 | install/ 2 | build/ 3 | log/ 4 | .mypy_cache/ 5 | __pycache__/ 6 | *.pyc 7 | *.pyo 8 | *.pyd 9 | *.so 10 | *.dylib 11 | *.dll 12 | *.exe 13 | *.out 14 | *.log 15 | *.tmp 16 | *.tmp.* 17 | *.swp 18 | *.bak 19 | *.cache 20 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/images/sample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_extensions/rai_perception/images/sample.jpg -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "rai-perception" 3 | # TODO, update the version once it is published to PyPi 4 | version = "0.1.5" 5 | description = "Package for object detection, segmentation and gripping point detection." 6 | authors = ["Kajetan Rachwał "] 7 | readme = "README.md" 8 | packages = [ 9 | { include = "rai_perception", from = "." }, 10 | ] 11 | 12 | [tool.poetry.dependencies] 13 | python = "^3.8" 14 | torch = "^2.3.1" 15 | torchvision = "^0.18.1" 16 | rf-groundingdino = "^0.2.0" 17 | rai-gsam2 = "^1.1.2" 18 | rai_core = ">=2.0.0.a2,<3.0.0" 19 | 20 | [build-system] 21 | requires = ["poetry-core>=1.0.0"] 22 | build-backend = "poetry.core.masonry.api" 23 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/NOTICE: -------------------------------------------------------------------------------- 1 | Contains 3rd party code for configuration file for the groundingdino model. 2 | File: config.py 3 | Copyright: IDEA-Research 4 | License: Apache License Version 2.0 5 | Retrieved from: https://github.com/IDEA-Research/GroundingDINO/commit/3e7a8ca2dcef5b71910c4a38f586b995b34e6c3f#diff-c1cb8937ca1377cea0a1d4c3bb39325cd7dc4403f1874247a7b8b68b6b303bab 6 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from .agents.grounded_sam import GSAM_NODE_NAME, GSAM_SERVICE_NAME, GroundedSamAgent 17 | from .agents.grounding_dino import ( 18 | GDINO_NODE_NAME, 19 | GDINO_SERVICE_NAME, 20 | GroundingDinoAgent, 21 | ) 22 | from .tools import GetDetectionTool, GetDistanceToObjectsTool 23 | 24 | __all__ = [ 25 | "GDINO_NODE_NAME", 26 | "GDINO_SERVICE_NAME", 27 | "GSAM_NODE_NAME", 28 | "GSAM_SERVICE_NAME", 29 | "GetDetectionTool", 30 | "GetDistanceToObjectsTool", 31 | "GroundedSamAgent", 32 | "GroundingDinoAgent", 33 | ] 34 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base_vision_agent import BaseVisionAgent 16 | from .grounded_sam import GroundedSamAgent 17 | from .grounding_dino import GroundingDinoAgent 18 | 19 | __all__ = [ 20 | "BaseVisionAgent", 21 | "GroundedSamAgent", 22 | "GroundingDinoAgent", 23 | ] 24 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/configs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/scripts/run_perception_agents.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import rclpy 17 | from rai.agents import wait_for_shutdown 18 | 19 | from rai_perception.agents import GroundedSamAgent, GroundingDinoAgent 20 | 21 | 22 | def main(): 23 | rclpy.init() 24 | agent1 = GroundingDinoAgent() 25 | agent2 = GroundedSamAgent() 26 | agent1.run() 27 | agent2.run() 28 | wait_for_shutdown([agent1, agent2]) 29 | rclpy.shutdown() 30 | 31 | 32 | if __name__ == "__main__": 33 | main() 34 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .gdino_tools import DistanceMeasurement, GetDetectionTool, GetDistanceToObjectsTool 16 | from .segmentation_tools import GetGrabbingPointTool, GetSegmentationTool 17 | 18 | __all__ = [ 19 | "DistanceMeasurement", 20 | "GetDetectionTool", 21 | "GetDistanceToObjectsTool", 22 | "GetGrabbingPointTool", 23 | "GetSegmentationTool", 24 | ] 25 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/rai_perception/vision_markup/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/resource/rai_perception: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/rai_extensions/rai_perception/setup.cfg: -------------------------------------------------------------------------------- 1 | [build_scripts] 2 | executable = /usr/bin/env python3 3 | [develop] 4 | script_dir=$base/lib/rai_perception 5 | [install] 6 | install_scripts=$base/lib/rai_perception 7 | -------------------------------------------------------------------------------- /src/rai_finetune/imgs/rai-fine-tune-system-components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_finetune/imgs/rai-fine-tune-system-components.png -------------------------------------------------------------------------------- /src/rai_finetune/rai_finetune/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Julia Jia 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | RAI Fine-tuning utilities using Unsloth 17 | """ 18 | 19 | from . import data, utils 20 | 21 | __all__ = ["data", "utils"] 22 | -------------------------------------------------------------------------------- /src/rai_finetune/rai_finetune/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Julia Jia 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Data processing for RAI fine-tuning. 17 | 18 | This package provides data extraction, formatting, and validation capabilities 19 | for fine-tuning various model types. 20 | """ 21 | 22 | from .extractors import BaseDataExtractor, LangfuseDataExtractor 23 | from .formatters import BaseDataFormatter, ToolCallingDataFormatter 24 | 25 | __all__ = [ 26 | "BaseDataExtractor", 27 | "BaseDataFormatter", 28 | "LangfuseDataExtractor", 29 | "ToolCallingDataFormatter", 30 | ] 31 | -------------------------------------------------------------------------------- /src/rai_finetune/rai_finetune/data/extractors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Julia Jia 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Data extractors for RAI fine-tuning. 17 | 18 | This package contains classes for extracting raw data from various sources 19 | for fine-tuning purposes. 20 | """ 21 | 22 | from .base import BaseDataExtractor 23 | from .langfuse_data import LangfuseDataExtractor 24 | 25 | __all__ = [ 26 | "BaseDataExtractor", 27 | "LangfuseDataExtractor", 28 | ] 29 | -------------------------------------------------------------------------------- /src/rai_finetune/rai_finetune/data/formatters/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Julia Jia 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Data formatters for RAI fine-tuning. 17 | 18 | This package contains classes for formatting data for different model types 19 | and training scenarios. 20 | """ 21 | 22 | from .base import BaseDataFormatter 23 | from .tool_calling import ToolCallingDataFormatter 24 | 25 | __all__ = [ 26 | "BaseDataFormatter", 27 | "ToolCallingDataFormatter", 28 | ] 29 | -------------------------------------------------------------------------------- /src/rai_finetune/rai_finetune/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Julia Jia 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Utility modules for RAI fine-tuning. 17 | 18 | This package provides common utilities like chat templates and other shared functionality. 19 | """ 20 | 21 | from .chat_template import ChatTemplate 22 | 23 | __all__ = [ 24 | "ChatTemplate", 25 | ] 26 | -------------------------------------------------------------------------------- /src/rai_finetune/rai_finetune/utils/templates/chatml.jinja: -------------------------------------------------------------------------------- 1 | {% if messages[0]['role'] == 'system' %} 2 | {% set offset = 1 %} 3 | {% else %} 4 | {% set offset = 0 %} 5 | {% endif %} 6 | 7 | {{ bos_token }} 8 | {% for message in messages %} 9 | {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %} 10 | {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} 11 | {% endif %} 12 | 13 | {{ '<|im_start|>' + message['role'] + '\n' + message['content'] | trim + '<|im_end|>\n' }} 14 | {% endfor %} 15 | 16 | {% if add_generation_prompt %} 17 | {{ '<|im_start|>assistant\n' }} 18 | {% endif %} 19 | -------------------------------------------------------------------------------- /src/rai_finetune/setup_finetune_shell.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Suppress ShellCheck warning about not following external file 4 | # shellcheck disable=SC1091 5 | 6 | cd src/rai_finetune || { 7 | echo "Error: Failed to change to src/rai_finetune directory" >&2 8 | exit 1 9 | } 10 | . "$(poetry env info --path)"/bin/activate 11 | 12 | # go back to the root directory 13 | cd - || { 14 | echo "Error: Failed to return to previous directory" >&2 15 | exit 1 16 | } 17 | 18 | export PYTHONPATH 19 | PYTHONPATH="$(dirname "$(dirname "$(poetry run which python)")")/lib/python$(poetry run python --version | awk '{print $2}' | cut -d. -f1,2)/site-packages:$PYTHONPATH" 20 | PYTHONPATH="src/rai_finetune:$PYTHONPATH" 21 | -------------------------------------------------------------------------------- /src/rai_s2s/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "rai_s2s" 3 | version = "1.0.0" 4 | description = "Speech-to-Speech module for RAI framework" 5 | authors = ["Kajetan Rachwał "] 6 | readme = "README.md" 7 | classifiers = [ 8 | "Programming Language :: Python :: 3", 9 | "Development Status :: 4 - Beta", 10 | "License :: OSI Approved :: Apache Software License", 11 | ] 12 | packages = [ 13 | { include = "rai_s2s", from = "." }, 14 | ] 15 | 16 | [build-system] 17 | requires = ["poetry-core>=1.0.0"] 18 | build-backend = "poetry.core.masonry.api" 19 | 20 | [tool.poetry.dependencies] 21 | python = "^3.10, <3.13" 22 | sounddevice = "^0.4.7" 23 | scipy = "^1.14.0" 24 | pydub = "^0.25.1" 25 | torchaudio = "2.3.1" 26 | # Optional dependencies (won't be installed unless requested) 27 | elevenlabs = { version = "^1.4.1", optional = true } 28 | openai-whisper = { version = "^20231117", optional = true } 29 | faster-whisper = { version = "^1.1.1", optional = true } 30 | openwakeword = { git = "https://github.com/maciejmajek/openWakeWord.git", branch = "chore/remove-tflite-backend", optional = true } 31 | kokoro-onnx = { version = "0.3.3", optional = true } 32 | onnxruntime-gpu = { version = ">=1.20.1", optional = true, markers = "platform_machine == 'x86_64' and sys_platform != 'darwin'" } 33 | 34 | [tool.poetry.extras] 35 | elevenlabs = ["elevenlabs"] 36 | whisper = ["openai-whisper"] 37 | fasterwhisper = ["faster-whisper"] 38 | wakeword = ["openwakeword"] 39 | kokoro = ["kokoro-onnx"] 40 | gpu = ["onnxruntime-gpu"] 41 | all = ["elevenlabs", "openai-whisper", "faster-whisper", "openwakeword", "kokoro-onnx", "onnxruntime-gpu"] 42 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .tts.agents import TextToSpeechAgent 16 | from .tts.models import ElevenLabsTTS, KokoroTTS, OpenTTS 17 | 18 | __all__ = ["ElevenLabsTTS", "KokoroTTS", "OpenTTS", "TextToSpeechAgent"] 19 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/asr/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .agents import SpeechRecognitionAgent 16 | from .agents.initialization import ( 17 | TRANSCRIBE_MODELS, 18 | ASRAgentConfig, 19 | MicrophoneConfig, 20 | TranscribeConfig, 21 | VADConfig, 22 | WWConfig, 23 | load_config, 24 | ) 25 | from .models import FasterWhisper, LocalWhisper, OpenAIWhisper, OpenWakeWord, SileroVAD 26 | 27 | __all__ = [ 28 | "TRANSCRIBE_MODELS", 29 | "ASRAgentConfig", 30 | "FasterWhisper", 31 | "LocalWhisper", 32 | "MicrophoneConfig", 33 | "OpenAIWhisper", 34 | "OpenWakeWord", 35 | "SileroVAD", 36 | "SpeechRecognitionAgent", 37 | "SpeechRecognitionAgent", 38 | "TranscribeConfig", 39 | "VADConfig", 40 | "WWConfig", 41 | "load_config", 42 | ] 43 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/asr/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .asr_agent import SpeechRecognitionAgent 16 | from .initialization import ( 17 | TRANSCRIBE_MODELS, 18 | ASRAgentConfig, 19 | MicrophoneConfig, 20 | TranscribeConfig, 21 | VADConfig, 22 | WWConfig, 23 | load_config, 24 | ) 25 | 26 | __all__ = [ 27 | "TRANSCRIBE_MODELS", 28 | "ASRAgentConfig", 29 | "MicrophoneConfig", 30 | "SpeechRecognitionAgent", 31 | "TranscribeConfig", 32 | "VADConfig", 33 | "WWConfig", 34 | "load_config", 35 | ] 36 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/asr/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base import BaseTranscriptionModel, BaseVoiceDetectionModel 16 | from .local_whisper import FasterWhisper, LocalWhisper 17 | from .open_ai_whisper import OpenAIWhisper 18 | from .open_wake_word import OpenWakeWord 19 | from .silero_vad import SileroVAD 20 | 21 | __all__ = [ 22 | "BaseTranscriptionModel", 23 | "BaseVoiceDetectionModel", 24 | "FasterWhisper", 25 | "LocalWhisper", 26 | "OpenAIWhisper", 27 | "OpenWakeWord", 28 | "SileroVAD", 29 | ] 30 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/s2s/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/s2s/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .ros2s2s_agent import ROS2S2SAgent 16 | from .s2s_agent import SpeechToSpeechAgent 17 | 18 | __all__ = ["ROS2S2SAgent", "SpeechToSpeechAgent"] 19 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/sound_device/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib.util 16 | 17 | if importlib.util.find_spec("sounddevice") is None: 18 | raise ImportError( 19 | "This feature is based on sounddevice. Make sure sounddevice is installed." 20 | ) 21 | 22 | from .api import SoundDeviceAPI, SoundDeviceConfig, SoundDeviceError 23 | from .connector import SoundDeviceConnector, SoundDeviceMessage 24 | 25 | __all__ = [ 26 | "SoundDeviceAPI", 27 | "SoundDeviceConfig", 28 | "SoundDeviceConnector", 29 | "SoundDeviceError", 30 | "SoundDeviceMessage", 31 | ] 32 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/tts/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .agents import TTS_MODELS, TextToSpeechAgent, TTSAgentConfig, load_config 16 | from .models import ElevenLabsTTS, KokoroTTS, OpenTTS 17 | 18 | __all__ = [ 19 | "TTS_MODELS", 20 | "ElevenLabsTTS", 21 | "KokoroTTS", 22 | "OpenTTS", 23 | "TTSAgentConfig", 24 | "TextToSpeechAgent", 25 | "TextToSpeechAgent", 26 | "load_config", 27 | ] 28 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/tts/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .initialization import TTS_MODELS, TTSAgentConfig, load_config 16 | from .tts_agent import TextToSpeechAgent 17 | 18 | __all__ = ["TTS_MODELS", "TTSAgentConfig", "TextToSpeechAgent", "load_config"] 19 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/tts/agents/initialization.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from dataclasses import dataclass 16 | from typing import Optional 17 | 18 | import tomli 19 | 20 | 21 | @dataclass 22 | class SpeakerConfig: 23 | device_name: str 24 | 25 | 26 | TTS_MODELS = ["KokoroTTS", "OpenTTS", "ElevenLabs"] 27 | 28 | 29 | @dataclass 30 | class TTSConfig: 31 | model_type: str = TTS_MODELS[0] 32 | voice: str = "" 33 | 34 | 35 | @dataclass 36 | class TTSAgentConfig: 37 | text_to_speech: TTSConfig 38 | speaker: SpeakerConfig 39 | 40 | 41 | def load_config(config_path: Optional[str] = None) -> TTSAgentConfig: 42 | if config_path is None: 43 | with open("config.toml", "rb") as f: 44 | config_dict = tomli.load(f) 45 | else: 46 | with open(config_path, "rb") as f: 47 | config_dict = tomli.load(f) 48 | return TTSAgentConfig( 49 | text_to_speech=TTSConfig( 50 | model_type=config_dict["tts"]["vendor"], voice=config_dict["tts"]["voice"] 51 | ), 52 | speaker=SpeakerConfig(device_name=config_dict["tts"]["speaker_device_name"]), 53 | ) 54 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/tts/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base import TTSModel, TTSModelError 16 | from .elevenlabs_tts import ElevenLabsTTS 17 | from .kokoro_tts import KokoroTTS 18 | from .open_tts import OpenTTS 19 | 20 | __all__ = ["ElevenLabsTTS", "KokoroTTS", "OpenTTS", "TTSModel", "TTSModelError"] 21 | -------------------------------------------------------------------------------- /src/rai_s2s/rai_s2s/tts/models/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from typing import Tuple 17 | 18 | from pydub import AudioSegment 19 | 20 | 21 | class TTSModelError(Exception): 22 | pass 23 | 24 | 25 | class TTSModel(ABC): 26 | sample_rate: int = -1 27 | channels: int = 1 28 | 29 | @abstractmethod 30 | def get_speech(self, text: str) -> AudioSegment: 31 | pass 32 | 33 | @abstractmethod 34 | def get_tts_params(self) -> Tuple[int, int]: 35 | pass 36 | 37 | def set_tts_params(self, target_sample_rate: int, channels: int): 38 | self.sample_rate = target_sample_rate 39 | self.channels = channels 40 | 41 | def _resample(self, audio: AudioSegment) -> AudioSegment: 42 | """ 43 | Resample an AudioSegment to a specified sample rate and number of channels. 44 | 45 | :param audio: The input AudioSegment. 46 | :param target_sample_rate: The desired sample rate in Hz. 47 | :param channels: The desired number of audio channels. 48 | :return: A new AudioSegment with the specified sample rate and channels. 49 | """ 50 | return audio.set_frame_rate(self.sample_rate) 51 | -------------------------------------------------------------------------------- /src/rai_sim/README.md: -------------------------------------------------------------------------------- 1 | ## RAI Sim 2 | 3 | ## Description 4 | 5 | The RAI Sim is a package providing interface to implement connection with a specific simulation. 6 | 7 | ### Components 8 | 9 | - `SimulationBridge` - An interface for connecting with a specific simulation. It manages scene setup, spawning, despawning objects, getting current state of the scene. 10 | 11 | - `SimulationConfig` - base config class to specify the entities to be spawned. For each simulation bridge there should be specified custom simulation config specifying additional parameters needed to run and connect with the simulation. 12 | 13 | - `SceneState` - stores the current info about spawned entities 14 | 15 | ### Example implementation 16 | 17 | - `O3DExROS2Bridge` - An implementation of SimulationBridge for working with simulation based on O3DE and ROS2. 18 | - `O3DExROS2SimulationConfig` - config class for `O3DExROS2Bridge` 19 | -------------------------------------------------------------------------------- /src/rai_sim/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "rai-sim" 3 | version = "0.0.2" 4 | description = "Package to run simulations" 5 | authors = ["Jakub Matejczyk ", "Magdalena Kotynia ", "Kacper Dąbrowski "] 6 | readme = "README.md" 7 | classifiers = [ 8 | "Programming Language :: Python :: 3", 9 | "Development Status :: 4 - Beta", 10 | "License :: OSI Approved :: Apache Software License", 11 | ] 12 | packages = [ 13 | { include = "rai_sim", from = "." }, 14 | ] 15 | 16 | [tool.poetry.dependencies] 17 | python = "^3.10, <3.13" 18 | PyYAML = "*" 19 | rai_core = ">=2.0.0.a2,<3.0.0" 20 | 21 | [build-system] 22 | requires = ["poetry-core"] 23 | build-backend = "poetry.core.masonry.api" 24 | -------------------------------------------------------------------------------- /src/rai_sim/rai_sim/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """RAI Simulations package.""" 16 | -------------------------------------------------------------------------------- /src/rai_whoami/docs/images/rai_whoami.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/src/rai_whoami/docs/images/rai_whoami.png -------------------------------------------------------------------------------- /src/rai_whoami/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "rai_whoami" 3 | version = "0.0.5" 4 | description = "Package to extract embodiment information from robot documentation" 5 | authors = ["Maciej Majek "] 6 | readme = "README.md" 7 | classifiers = [ 8 | "Programming Language :: Python :: 3", 9 | "Development Status :: 4 - Beta", 10 | "License :: OSI Approved :: Apache Software License", 11 | ] 12 | packages = [ 13 | { include = "rai_whoami", from = "." }, 14 | ] 15 | 16 | [tool.poetry.dependencies] 17 | python = "^3.10, <3.13" 18 | PyYAML = "*" 19 | langchain = "*" 20 | PyPDF2 = "*" 21 | Pillow = "*" 22 | faiss-cpu = "*" 23 | pypdf = "^5.4.0" 24 | 25 | [tool.poetry.scripts] 26 | build-whoami = "rai_whoami.build_whoami:main" 27 | initialize-docs = "rai_whoami.initialize_docs_directory:main" 28 | 29 | 30 | [build-system] 31 | requires = ["poetry-core"] 32 | build-backend = "poetry.core.masonry.api" 33 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .models import EmbodimentInfo, EmbodimentSource 16 | from .pipeline import Pipeline, PipelineBuilder 17 | from .processors import get_default_postprocessors, get_default_preprocessors 18 | from .tools import QueryDatabaseTool 19 | 20 | __all__ = [ 21 | "EmbodimentInfo", 22 | "EmbodimentSource", 23 | "Pipeline", 24 | "PipelineBuilder", 25 | "QueryDatabaseTool", 26 | "get_default_postprocessors", 27 | "get_default_preprocessors", 28 | ] 29 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/agents/ros2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .embodiment_info_agent import ROS2EmbodimentInfoAgent 16 | from .vector_store_retrieval_agent import ROS2VectorStoreRetrievalAgent 17 | 18 | __all__ = ["ROS2EmbodimentInfoAgent", "ROS2VectorStoreRetrievalAgent"] 19 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/initialize_docs_directory.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | from pathlib import Path 17 | 18 | SUBDIRECTORIES = [ 19 | "images", 20 | "documentation", 21 | "urdf", 22 | ] 23 | 24 | 25 | def initialize_docs_directory(args: argparse.Namespace) -> None: 26 | documentation_dir = Path(args.documentation_dir) 27 | if documentation_dir.exists(): 28 | print( 29 | f"Directory {documentation_dir} already exists. Remove it or use different folder." 30 | ) 31 | return 32 | 33 | for subdirectory in SUBDIRECTORIES: 34 | (documentation_dir / subdirectory).mkdir(parents=True) 35 | print( 36 | f"Initialized subdirectory {subdirectory} at {documentation_dir / subdirectory}" 37 | ) 38 | 39 | 40 | def main(): 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("documentation_dir", type=Path) 43 | args = parser.parse_args() 44 | initialize_docs_directory(args) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .models import ( 16 | DocumentLoader, 17 | EmbodimentInfo, 18 | EmbodimentInfoDirectoryStructure, 19 | EmbodimentSource, 20 | EmbodimentSourceDirectoryStructure, 21 | ) 22 | 23 | __all__ = [ 24 | "DocumentLoader", 25 | "EmbodimentInfo", 26 | "EmbodimentInfoDirectoryStructure", 27 | "EmbodimentSource", 28 | "EmbodimentSourceDirectoryStructure", 29 | ] 30 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .pipeline import Pipeline 16 | from .pipeline_builder import PipelineBuilder 17 | 18 | __all__ = ["Pipeline", "PipelineBuilder"] 19 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/processors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .defaults import get_default_postprocessors, get_default_preprocessors 16 | from .postprocessors import CompressorPostProcessor, StylePostProcessor 17 | from .preprocessors import DocsPreProcessor, ImagePreProcessor 18 | 19 | __all__ = [ 20 | "CompressorPostProcessor", 21 | "DocsPreProcessor", 22 | "ImagePreProcessor", 23 | "StylePostProcessor", 24 | "get_default_postprocessors", 25 | "get_default_preprocessors", 26 | ] 27 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/processors/defaults.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | from rai_whoami.processors.postprocessors import ( 18 | CompressorPostProcessor, 19 | StylePostProcessor, 20 | ) 21 | from rai_whoami.processors.postprocessors.base import DataPostProcessor 22 | from rai_whoami.processors.preprocessors import ( 23 | DocsPreProcessor, 24 | ImagePreProcessor, 25 | ) 26 | from rai_whoami.processors.preprocessors.base import DataPreProcessor 27 | 28 | 29 | def get_default_preprocessors() -> List[DataPreProcessor]: 30 | return [ 31 | ImagePreProcessor(), 32 | DocsPreProcessor(), 33 | ] 34 | 35 | 36 | def get_default_postprocessors() -> List[DataPostProcessor]: 37 | return [ 38 | CompressorPostProcessor(), 39 | StylePostProcessor(), 40 | ] 41 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/processors/postprocessors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .compressor import CompressorPostProcessor 16 | from .style import StylePostProcessor 17 | 18 | __all__ = ["CompressorPostProcessor", "StylePostProcessor"] 19 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/processors/postprocessors/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | 17 | from rai_whoami.models import EmbodimentInfo 18 | 19 | 20 | class DataPostProcessor(ABC): 21 | @abstractmethod 22 | def process(self, input: EmbodimentInfo) -> EmbodimentInfo: 23 | pass 24 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/processors/preprocessors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .docs import DocsPreProcessor 16 | from .image import ImagePreProcessor 17 | 18 | __all__ = ["DocsPreProcessor", "ImagePreProcessor"] 19 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/processors/preprocessors/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | 17 | from rai_whoami.models import EmbodimentInfo, EmbodimentSource 18 | 19 | 20 | class DataPreProcessor(ABC): 21 | @abstractmethod 22 | def process(self, input: EmbodimentSource) -> EmbodimentInfo: 23 | pass 24 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/scripts/ros2_embodiment_service.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | 17 | from rai.agents import wait_for_shutdown 18 | from rai.communication.ros2 import ROS2Context 19 | 20 | from rai_whoami.agents.ros2 import ROS2EmbodimentInfoAgent 21 | 22 | 23 | @ROS2Context() 24 | def main(): 25 | parser = argparse.ArgumentParser() 26 | parser.add_argument("root_dir", type=str) 27 | args = parser.parse_args() 28 | 29 | agent = ROS2EmbodimentInfoAgent( 30 | service_name="rai_whoami_embodiment_info_service", 31 | root_dir=args.root_dir, 32 | ) 33 | agent.run() 34 | wait_for_shutdown([agent]) 35 | 36 | 37 | if __name__ == "__main__": 38 | main() 39 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/scripts/ros2_vector_store_retrieval_service.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | 17 | from rai.agents import wait_for_shutdown 18 | from rai.communication.ros2 import ROS2Context 19 | 20 | from rai_whoami.agents.ros2 import ROS2VectorStoreRetrievalAgent 21 | 22 | 23 | @ROS2Context() 24 | def main(): 25 | parser = argparse.ArgumentParser() 26 | parser.add_argument("root_dir", type=str) 27 | args = parser.parse_args() 28 | 29 | agent = ROS2VectorStoreRetrievalAgent( 30 | service_name="rai_whoami_documentation_service", 31 | root_dir=args.root_dir, 32 | ) 33 | agent.run() 34 | wait_for_shutdown([agent]) 35 | 36 | 37 | if __name__ == "__main__": 38 | main() 39 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from rai_whoami.tools.vector_db import QueryDatabaseTool 16 | 17 | __all__ = ["QueryDatabaseTool"] 18 | -------------------------------------------------------------------------------- /src/rai_whoami/rai_whoami/vector_db/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .builder import VectorDBBuilder 16 | from .faiss import FAISSBuilder 17 | 18 | __all__ = ["FAISSBuilder", "VectorDBBuilder"] 19 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/communication/ros2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .helpers import ( 16 | ActionServer, 17 | MessagePublisher, 18 | MessageSubscriber, 19 | multi_threaded_spinner, 20 | shutdown_executors_and_threads, 21 | ) 22 | 23 | __all__ = [ 24 | "ActionServer", 25 | "MessagePublisher", 26 | "MessageSubscriber", 27 | "multi_threaded_spinner", 28 | "shutdown_executors_and_threads", 29 | ] 30 | -------------------------------------------------------------------------------- /tests/communication/sounds_device/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/communication/test_hri_connector.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from langchain_core.messages import AIMessage 17 | from rai.communication import HRIMessage 18 | from rai.communication.hri_connector import HRIConnector 19 | 20 | 21 | def test_build_message(): 22 | class DummyHRIMessage(HRIMessage): 23 | pass 24 | 25 | class DummyHRIConnector(HRIConnector[DummyHRIMessage]): 26 | pass 27 | 28 | connector = DummyHRIConnector() 29 | message = connector.build_message( 30 | AIMessage(content="Hello"), "test_conversation_id", 0, False 31 | ) 32 | assert message.text == "Hello" 33 | assert message.message_author == "ai" 34 | assert message.images == [] 35 | assert message.audios == [] 36 | assert isinstance(message, DummyHRIMessage) 37 | -------------------------------------------------------------------------------- /tests/initialization/test_config_initialization_script.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | from rai.initialization import config_initialization 18 | 19 | 20 | def run_main(monkeypatch, args): 21 | monkeypatch.setattr(sys, "argv", ["config_init"] + args) 22 | config_initialization.main() 23 | 24 | 25 | def test_config_initialization_main_creates_and_respects_force( 26 | tmp_path, monkeypatch, capsys 27 | ): 28 | monkeypatch.chdir(tmp_path) 29 | 30 | run_main(monkeypatch, []) 31 | created = tmp_path / "config.toml" 32 | assert created.exists() 33 | default_content = created.read_bytes() 34 | 35 | created.write_text("custom-config", encoding="utf-8") 36 | run_main(monkeypatch, []) 37 | 38 | output = capsys.readouterr().out 39 | assert "config.toml already exists" in output 40 | assert created.read_text(encoding="utf-8") == "custom-config" 41 | 42 | run_main(monkeypatch, ["--force"]) 43 | output = capsys.readouterr().out 44 | assert "config.toml created successfully." in output 45 | assert created.read_bytes() == default_content 46 | -------------------------------------------------------------------------------- /tests/messages/test_multimodal_message.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from rai.messages import HumanMultimodalMessage 17 | 18 | 19 | class TestMultimodalMessage: 20 | """Test the MultimodalMessage class and expected behaviors.""" 21 | 22 | def test_human_multimodal_message_text_simple(self): 23 | """Test text() method with simple text content.""" 24 | msg = HumanMultimodalMessage(content="Hello world") 25 | assert msg.text() == "Hello world" 26 | assert isinstance(msg.text(), str) 27 | 28 | def test_human_multimodal_message_text_with_images(self): 29 | """Test text() method with text and images.""" 30 | # Use a small valid base64 image (1x1 pixel PNG) 31 | valid_base64_image = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" 32 | msg = HumanMultimodalMessage( 33 | content="Look at this image", images=[valid_base64_image] 34 | ) 35 | assert msg.text() == "Look at this image" 36 | # Should only return text type blocks, not image content 37 | assert valid_base64_image not in msg.text() 38 | -------------------------------------------------------------------------------- /tests/rai_bench/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import pytest 15 | from rai.types import Header, Point, Pose, PoseStamped, Quaternion 16 | 17 | from rai_bench.tool_calling_agent.interfaces import TaskArgs 18 | from rai_sim.simulation_bridge import Entity 19 | 20 | 21 | def create_entity( 22 | name: str, 23 | prefab: str, 24 | x: float, 25 | y: float, 26 | z: float, 27 | orientation: Quaternion | None = None, 28 | ) -> Entity: 29 | if orientation is None: 30 | orientation = Quaternion() 31 | return Entity( 32 | name=name, 33 | prefab_name=prefab, 34 | pose=PoseStamped( 35 | pose=Pose(position=Point(x=x, y=y, z=z), orientation=orientation), 36 | header=Header(frame_id="/test_frame"), 37 | ), 38 | ) 39 | 40 | 41 | @pytest.fixture 42 | def task_args() -> TaskArgs: 43 | """Create basic task arguments for testing.""" 44 | return TaskArgs( 45 | extra_tool_calls=0, 46 | prompt_detail="brief", 47 | examples_in_system_prompt=0, 48 | ) 49 | -------------------------------------------------------------------------------- /tests/rai_bringup/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/resources/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/tests/resources/image.png -------------------------------------------------------------------------------- /tests/resources/sine_wave.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RobotecAI/rai/HEAD/tests/resources/sine_wave.wav -------------------------------------------------------------------------------- /tests/tools/test_tool_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import pytest 17 | from geometry_msgs.msg import Point, TransformStamped 18 | from nav2_msgs.action import NavigateToPose 19 | from rai.communication.ros2.api import ros2_message_to_dict 20 | from sensor_msgs.msg import Image 21 | from tf2_msgs.msg import TFMessage 22 | 23 | 24 | # TODO(`maciejmajek`): Add custom RAI messages? 25 | @pytest.mark.parametrize( 26 | "message", 27 | [ 28 | Point(), 29 | Image(), 30 | TFMessage(), 31 | TransformStamped(), 32 | NavigateToPose.Goal(), 33 | NavigateToPose.Result(), 34 | NavigateToPose.Feedback(), 35 | ], 36 | ids=lambda x: x.__class__.__name__, 37 | ) 38 | def test_ros2_message_to_dict(message): 39 | assert ros2_message_to_dict(message) 40 | -------------------------------------------------------------------------------- /tests/tools/test_wait_for_seconds_tool.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2025 Robotec.AI 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from unittest.mock import patch 16 | 17 | from rai.tools.time import WaitForSecondsTool 18 | 19 | 20 | def test_wait_for_seconds_tool_caps_duration(): 21 | tool = WaitForSecondsTool() 22 | 23 | with patch("rai.tools.time.time.sleep") as mock_sleep: 24 | result = tool._run(15) 25 | 26 | mock_sleep.assert_called_once_with(10) 27 | assert result == "Waited for 10 seconds." 28 | --------------------------------------------------------------------------------