├── .gitattributes ├── addons └── godot_rl_agents │ ├── icon.png │ ├── rewards │ ├── RewardFunction2D.gd │ ├── RewardFunction3D.gd │ ├── ApproachNodeReward2D.gd │ └── ApproachNodeReward3D.gd │ ├── plugin.cfg │ ├── sensors │ ├── sensors_3d │ │ ├── ExampleRaycastSensor3D.tscn │ │ ├── ISensor3D.gd │ │ ├── RaycastSensor3D.tscn │ │ ├── RGBCameraSensor3D.tscn │ │ ├── RGBCameraSensor3D.gd │ │ ├── PositionSensor3D.gd │ │ ├── RaycastSensor3D.gd │ │ └── GridSensor3D.gd │ └── sensors_2d │ │ ├── RaycastSensor2D.tscn │ │ ├── ISensor2D.gd │ │ ├── RGBCameraSensor2D.tscn │ │ ├── ExampleRaycastSensor2D.tscn │ │ ├── PositionSensor2D.gd │ │ ├── RaycastSensor2D.gd │ │ ├── RGBCameraSensor2D.gd │ │ └── GridSensor2D.gd │ ├── godot_rl_agents.gd │ ├── onnx │ ├── csharp │ │ ├── docs │ │ │ ├── SessionConfigurator.xml │ │ │ └── ONNXInference.xml │ │ ├── ONNXInference.cs │ │ └── SessionConfigurator.cs │ └── wrapper │ │ └── ONNX_wrapper.gd │ ├── controller │ ├── ai_controller_2d.gd │ └── ai_controller_3d.gd │ └── sync.gd ├── .gitignore ├── Godot RL Agents.csproj ├── project.godot ├── LICENSE ├── README.md ├── Godot RL Agents.sln └── script_templates └── AIController └── controller_template.gd /.gitattributes: -------------------------------------------------------------------------------- 1 | # Normalize line endings for all files that Git considers text files. 2 | * text=auto eol=lf 3 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edbeeching/godot_rl_agents_plugin/HEAD/addons/godot_rl_agents/icon.png -------------------------------------------------------------------------------- /addons/godot_rl_agents/rewards/RewardFunction2D.gd: -------------------------------------------------------------------------------- 1 | extends Node2D 2 | class_name RewardFunction2D 3 | 4 | 5 | func get_reward(): 6 | return 0.0 7 | 8 | 9 | func reset(): 10 | return 11 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/rewards/RewardFunction3D.gd: -------------------------------------------------------------------------------- 1 | extends Node3D 2 | class_name RewardFunction3D 3 | 4 | 5 | func get_reward(): 6 | return 0.0 7 | 8 | 9 | func reset(): 10 | return 11 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/plugin.cfg: -------------------------------------------------------------------------------- 1 | [plugin] 2 | 3 | name="GodotRLAgents" 4 | description="Custom nodes for the godot rl agents toolkit " 5 | author="Edward Beeching" 6 | version="0.1" 7 | script="godot_rl_agents.gd" 8 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/ExampleRaycastSensor3D.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene format=3 uid="uid://biu787qh4woik"] 2 | 3 | [node name="ExampleRaycastSensor3D" type="Node3D"] 4 | 5 | [node name="Camera3D" type="Camera3D" parent="."] 6 | transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0.804183, 0, 2.70146) 7 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=2 format=3 uid="uid://drvfihk5esgmv"] 2 | 3 | [ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="1"] 4 | 5 | [node name="RaycastSensor2D" type="Node2D"] 6 | script = ExtResource("1") 7 | n_rays = 17.0 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Godot 4+ specific ignores 2 | .godot/ 3 | 4 | # Godot-specific ignores 5 | .import/ 6 | export.cfg 7 | export_presets.cfg 8 | 9 | # Imported translations (automatically generated from CSV files) 10 | *.translation 11 | 12 | # Mono-specific ignores 13 | .mono/ 14 | data_*/ 15 | mono_crash.*.json 16 | .vs/ 17 | *.import 18 | -------------------------------------------------------------------------------- /Godot RL Agents.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | net6.0 4 | true 5 | GodotRLAgents 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/ISensor2D.gd: -------------------------------------------------------------------------------- 1 | extends Node2D 2 | class_name ISensor2D 3 | 4 | var _obs: Array = [] 5 | var _active := false 6 | 7 | 8 | func get_observation(): 9 | pass 10 | 11 | 12 | func activate(): 13 | _active = true 14 | 15 | 16 | func deactivate(): 17 | _active = false 18 | 19 | 20 | func _update_observation(): 21 | pass 22 | 23 | 24 | func reset(): 25 | pass 26 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/ISensor3D.gd: -------------------------------------------------------------------------------- 1 | extends Node3D 2 | class_name ISensor3D 3 | 4 | var _obs: Array = [] 5 | var _active := false 6 | 7 | 8 | func get_observation(): 9 | pass 10 | 11 | 12 | func activate(): 13 | _active = true 14 | 15 | 16 | func deactivate(): 17 | _active = false 18 | 19 | 20 | func _update_observation(): 21 | pass 22 | 23 | 24 | func reset(): 25 | pass 26 | -------------------------------------------------------------------------------- /project.godot: -------------------------------------------------------------------------------- 1 | ; Engine configuration file. 2 | ; It's best edited using the editor UI and not directly, 3 | ; since the parameters that go here are not all obvious. 4 | ; 5 | ; Format: 6 | ; [section] ; section goes between [] 7 | ; param=value ; assign values to parameters 8 | 9 | config_version=5 10 | 11 | [application] 12 | 13 | config/name="Godot RL Agents" 14 | config/features=PackedStringArray("4.0") 15 | 16 | [dotnet] 17 | 18 | project/assembly_name="Godot RL Agents" 19 | 20 | [editor_plugins] 21 | 22 | enabled=PackedStringArray("res://addons/godot_rl_agents/plugin.cfg") 23 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/godot_rl_agents.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends EditorPlugin 3 | 4 | 5 | func _enter_tree(): 6 | # Initialization of the plugin goes here. 7 | # Add the new type with a name, a parent type, a script and an icon. 8 | add_custom_type("Sync", "Node", preload("sync.gd"), preload("icon.png")) 9 | #add_custom_type("RaycastSensor2D2", "Node", preload("raycast_sensor_2d.gd"), preload("icon.png")) 10 | 11 | 12 | func _exit_tree(): 13 | # Clean-up of the plugin goes here. 14 | # Always remember to remove it from the engine when deactivated. 15 | remove_custom_type("Sync") 16 | #remove_custom_type("RaycastSensor2D2") 17 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/rewards/ApproachNodeReward2D.gd: -------------------------------------------------------------------------------- 1 | extends RewardFunction2D 2 | class_name ApproachNodeReward2D 3 | 4 | ## Calculates the reward for approaching node 5 | ## a reward is only added when the agent reaches a new 6 | ## best distance to the target object. 7 | 8 | ## Best distance reward will be calculated for this object 9 | @export var target_node: Node2D 10 | 11 | ## Scales the reward, 1.0 means the reward is equal to 12 | ## how much closer the agent is than the previous best. 13 | @export_range(0.0, 1.0, 0.0001, "or_greater") var reward_scale: float = 1.0 14 | 15 | var _best_distance 16 | 17 | 18 | func get_reward() -> float: 19 | var reward := 0.0 20 | var current_distance := global_position.distance_to(target_node.global_position) 21 | if not _best_distance: 22 | _best_distance = current_distance 23 | if current_distance < _best_distance: 24 | reward = (_best_distance - current_distance) * reward_scale 25 | _best_distance = current_distance 26 | return reward 27 | 28 | 29 | func reset(): 30 | _best_distance = null 31 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/rewards/ApproachNodeReward3D.gd: -------------------------------------------------------------------------------- 1 | extends RewardFunction3D 2 | class_name ApproachNodeReward3D 3 | 4 | ## Calculates the reward for approaching node 5 | ## a reward is only added when the agent reaches a new 6 | ## best distance to the target object. 7 | 8 | ## Best distance reward will be calculated for this object 9 | @export var target_node: Node3D 10 | 11 | ## Scales the reward, 1.0 means the reward is equal to 12 | ## how much closer the agent is than the previous best. 13 | @export_range(0.0, 1.0, 0.0001, "or_greater") var reward_scale: float = 1.0 14 | 15 | var _best_distance 16 | 17 | 18 | func get_reward() -> float: 19 | var reward := 0.0 20 | var current_distance := global_position.distance_to(target_node.global_position) 21 | if not _best_distance: 22 | _best_distance = current_distance 23 | if current_distance < _best_distance: 24 | reward = (_best_distance - current_distance) * reward_scale 25 | _best_distance = current_distance 26 | return reward 27 | 28 | 29 | func reset(): 30 | _best_distance = null 31 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=2 format=3 uid="uid://b803cbh1fmy66"] 2 | 3 | [ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd" id="1"] 4 | 5 | [node name="RaycastSensor3D" type="Node3D"] 6 | script = ExtResource("1") 7 | n_rays_width = 4.0 8 | n_rays_height = 2.0 9 | ray_length = 11.0 10 | 11 | [node name="node_1 0" type="RayCast3D" parent="."] 12 | target_position = Vector3(-1.38686, -2.84701, 10.5343) 13 | 14 | [node name="node_1 1" type="RayCast3D" parent="."] 15 | target_position = Vector3(-1.38686, 2.84701, 10.5343) 16 | 17 | [node name="node_2 0" type="RayCast3D" parent="."] 18 | target_position = Vector3(1.38686, -2.84701, 10.5343) 19 | 20 | [node name="node_2 1" type="RayCast3D" parent="."] 21 | target_position = Vector3(1.38686, 2.84701, 10.5343) 22 | 23 | [node name="node_3 0" type="RayCast3D" parent="."] 24 | target_position = Vector3(4.06608, -2.84701, 9.81639) 25 | 26 | [node name="node_3 1" type="RayCast3D" parent="."] 27 | target_position = Vector3(4.06608, 2.84701, 9.81639) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Edward Beeching 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/onnx/csharp/docs/SessionConfigurator.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | The main SessionConfigurator Class that handles the execution options and providers for the inference process. 6 | 7 | 8 | 9 | 10 | Creates a SessionOptions with all available execution providers. 11 | 12 | SessionOptions with all available execution providers. 13 | 14 | 15 | 16 | Appends any execution provider available in the current system. 17 | 18 | 19 | This function is mainly verbose for tracking implementation progress of different compute APIs. 20 | 21 | 22 | 23 | 24 | Checks for available GPUs. 25 | 26 | An integer identifier for each compute platform. 27 | 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Godot RL Agents 3 | 4 | This repository contains the Godot 4 asset / plugin for the Godot RL Agents library, you can find out more about the library on its Github page [here](https://github.com/edbeeching/godot_rl_agents). 5 | 6 | The Godot RL Agents is a fully Open Source package that allows video game creators, AI researchers and hobbyists the opportunity to learn complex behaviors for their Non Player Characters or agents. 7 | This libary provided this following functionaly: 8 | * An interface between games created in the [Godot Engine](https://godotengine.org/) and Machine Learning algorithms running in Python 9 | * Wrappers for three well known rl frameworks: StableBaselines3, Sample Factory and [Ray RLLib](https://docs.ray.io/en/latest/rllib/index.html) 10 | * Support for memory-based agents, with LSTM or attention based interfaces 11 | * Support for 2D and 3D games 12 | * A suite of AI sensors to augment your agent's capacity to observe the game world 13 | * Godot and Godot RL Agents are completely free and open source under the very permissive MIT license. No strings attached, no royalties, nothing. 14 | 15 | You can find out more about Godot RL agents in our AAAI-2022 Workshop [paper](https://arxiv.org/abs/2112.03636). 16 | 17 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=3 format=3 uid="uid://baaywi3arsl2m"] 2 | 3 | [ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd" id="1"] 4 | 5 | [sub_resource type="ViewportTexture" id="ViewportTexture_y72s3"] 6 | viewport_path = NodePath("SubViewport") 7 | 8 | [node name="RGBCameraSensor3D" type="Node3D"] 9 | script = ExtResource("1") 10 | 11 | [node name="RemoteTransform" type="RemoteTransform3D" parent="."] 12 | remote_path = NodePath("../SubViewport/Camera") 13 | 14 | [node name="SubViewport" type="SubViewport" parent="."] 15 | size = Vector2i(36, 36) 16 | render_target_update_mode = 3 17 | 18 | [node name="Camera" type="Camera3D" parent="SubViewport"] 19 | near = 0.5 20 | 21 | [node name="Control" type="Control" parent="."] 22 | layout_mode = 3 23 | anchors_preset = 15 24 | anchor_right = 1.0 25 | anchor_bottom = 1.0 26 | grow_horizontal = 2 27 | grow_vertical = 2 28 | metadata/_edit_use_anchors_ = true 29 | 30 | [node name="CameraTexture" type="Sprite2D" parent="Control"] 31 | texture = SubResource("ViewportTexture_y72s3") 32 | centered = false 33 | 34 | [node name="ProcessedTexture" type="Sprite2D" parent="Control"] 35 | centered = false 36 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/onnx/csharp/docs/ONNXInference.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | The main ONNXInference Class that handles the inference process. 6 | 7 | 8 | 9 | 10 | Starts the inference process. 11 | 12 | Path to the ONNX model, expects a path inside resources. 13 | How many observations will the model recieve. 14 | 15 | 16 | 17 | Runs the given input through the model and returns the output. 18 | 19 | Dictionary containing all observations. 20 | How many different agents are creating these observations. 21 | A Dictionary of arrays, containing instructions based on the observations. 22 | 23 | 24 | 25 | Loads the given model into the inference process, using the best Execution provider available. 26 | 27 | Path to the ONNX model, expects a path inside resources. 28 | InferenceSession ready to run. 29 | 30 | 31 | -------------------------------------------------------------------------------- /Godot RL Agents.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.33530.505 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Godot RL Agents", "Godot RL Agents.csproj", "{055E8CBC-A3EC-41A8-BC53-EC3010682AE4}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | ExportDebug|Any CPU = ExportDebug|Any CPU 12 | ExportRelease|Any CPU = ExportRelease|Any CPU 13 | EndGlobalSection 14 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 15 | {055E8CBC-A3EC-41A8-BC53-EC3010682AE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 16 | {055E8CBC-A3EC-41A8-BC53-EC3010682AE4}.Debug|Any CPU.Build.0 = Debug|Any CPU 17 | {055E8CBC-A3EC-41A8-BC53-EC3010682AE4}.ExportDebug|Any CPU.ActiveCfg = ExportDebug|Any CPU 18 | {055E8CBC-A3EC-41A8-BC53-EC3010682AE4}.ExportDebug|Any CPU.Build.0 = ExportDebug|Any CPU 19 | {055E8CBC-A3EC-41A8-BC53-EC3010682AE4}.ExportRelease|Any CPU.ActiveCfg = ExportRelease|Any CPU 20 | {055E8CBC-A3EC-41A8-BC53-EC3010682AE4}.ExportRelease|Any CPU.Build.0 = ExportRelease|Any CPU 21 | EndGlobalSection 22 | GlobalSection(SolutionProperties) = preSolution 23 | HideSolutionNode = FALSE 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=3 format=3 uid="uid://bav1cl8uwc45c"] 2 | 3 | [ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd" id="1_txpo2"] 4 | 5 | [sub_resource type="ViewportTexture" id="ViewportTexture_jks1s"] 6 | viewport_path = NodePath("SubViewport") 7 | 8 | [node name="RGBCameraSensor2D" type="Node2D"] 9 | script = ExtResource("1_txpo2") 10 | displayed_image_scale_factor = Vector2(3, 3) 11 | 12 | [node name="RemoteTransform" type="RemoteTransform2D" parent="."] 13 | remote_path = NodePath("../SubViewport/Camera") 14 | 15 | [node name="SubViewport" type="SubViewport" parent="."] 16 | canvas_item_default_texture_filter = 0 17 | size = Vector2i(36, 36) 18 | render_target_update_mode = 4 19 | 20 | [node name="Camera" type="Camera2D" parent="SubViewport"] 21 | position_smoothing_speed = 2.0 22 | 23 | [node name="Control" type="Window" parent="."] 24 | canvas_item_default_texture_filter = 0 25 | title = "CameraSensor" 26 | position = Vector2i(20, 40) 27 | size = Vector2i(64, 64) 28 | theme_override_font_sizes/title_font_size = 12 29 | metadata/_edit_use_anchors_ = true 30 | 31 | [node name="CameraTexture" type="Sprite2D" parent="Control"] 32 | texture = SubResource("ViewportTexture_jks1s") 33 | centered = false 34 | 35 | [node name="ProcessedTexture" type="Sprite2D" parent="Control"] 36 | centered = false 37 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/ExampleRaycastSensor2D.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=5 format=3 uid="uid://ddeq7mn1ealyc"] 2 | 3 | [ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="1"] 4 | 5 | [sub_resource type="GDScript" id="2"] 6 | script/source = "extends Node2D 7 | 8 | 9 | 10 | func _physics_process(delta: float) -> void: 11 | print(\"step start\") 12 | 13 | " 14 | 15 | [sub_resource type="GDScript" id="1"] 16 | script/source = "extends RayCast2D 17 | 18 | var steps = 1 19 | 20 | func _physics_process(delta: float) -> void: 21 | print(\"processing raycast\") 22 | steps += 1 23 | if steps % 2: 24 | force_raycast_update() 25 | 26 | print(is_colliding()) 27 | " 28 | 29 | [sub_resource type="CircleShape2D" id="3"] 30 | 31 | [node name="ExampleRaycastSensor2D" type="Node2D"] 32 | script = SubResource("2") 33 | 34 | [node name="ExampleAgent" type="Node2D" parent="."] 35 | position = Vector2(573, 314) 36 | rotation = 0.286234 37 | 38 | [node name="RaycastSensor2D" type="Node2D" parent="ExampleAgent"] 39 | script = ExtResource("1") 40 | 41 | [node name="TestRayCast2D" type="RayCast2D" parent="."] 42 | script = SubResource("1") 43 | 44 | [node name="StaticBody2D" type="StaticBody2D" parent="."] 45 | position = Vector2(1, 52) 46 | 47 | [node name="CollisionShape2D" type="CollisionShape2D" parent="StaticBody2D"] 48 | shape = SubResource("3") 49 | -------------------------------------------------------------------------------- /script_templates/AIController/controller_template.gd: -------------------------------------------------------------------------------- 1 | # meta-name: AI Controller Logic 2 | # meta-description: Methods that need implementing for AI controllers 3 | # meta-default: true 4 | extends _BASE_ 5 | 6 | #-- Methods that need implementing using the "extend script" option in Godot --# 7 | 8 | func get_obs() -> Dictionary: 9 | assert(false, "the get_obs method is not implemented when extending from ai_controller") 10 | return {"obs":[]} 11 | 12 | func get_reward() -> float: 13 | assert(false, "the get_reward method is not implemented when extending from ai_controller") 14 | return 0.0 15 | 16 | func get_action_space() -> Dictionary: 17 | assert(false, "the get get_action_space method is not implemented when extending from ai_controller") 18 | return { 19 | "example_actions_continous" : { 20 | "size": 2, 21 | "action_type": "continuous" 22 | }, 23 | "example_actions_discrete" : { 24 | "size": 2, 25 | "action_type": "discrete" 26 | }, 27 | } 28 | 29 | func set_action(action) -> void: 30 | assert(false, "the get set_action method is not implemented when extending from ai_controller") 31 | # -----------------------------------------------------------------------------# 32 | 33 | #-- Methods that can be overridden if needed --# 34 | 35 | #func get_obs_space() -> Dictionary: 36 | # May need overriding if the obs space is complex 37 | # var obs = get_obs() 38 | # return { 39 | # "obs": { 40 | # "size": [len(obs["obs"])], 41 | # "space": "box" 42 | # }, 43 | # } -------------------------------------------------------------------------------- /addons/godot_rl_agents/onnx/wrapper/ONNX_wrapper.gd: -------------------------------------------------------------------------------- 1 | extends Resource 2 | class_name ONNXModel 3 | var inferencer_script = load("res://addons/godot_rl_agents/onnx/csharp/ONNXInference.cs") 4 | 5 | var inferencer = null 6 | 7 | ## How many action values the model outputs 8 | var action_output_size: int 9 | 10 | ## Used to differentiate models 11 | ## that only output continuous action mean (e.g. sb3, cleanrl export) 12 | ## versus models that output mean and logstd (e.g. rllib export) 13 | var action_means_only: bool 14 | 15 | ## Whether action_means_value has been set already for this model 16 | var action_means_only_set: bool 17 | 18 | # Must provide the path to the model and the batch size 19 | func _init(model_path, batch_size): 20 | inferencer = inferencer_script.new() 21 | action_output_size = inferencer.Initialize(model_path, batch_size) 22 | 23 | # This function is the one that will be called from the game, 24 | # requires the observations as an Dictionary and the state_ins as an int 25 | # returns a Dictionary containing the action the model takes. 26 | func run_inference(obs: Dictionary, state_ins: int) -> Dictionary: 27 | if inferencer == null: 28 | printerr("Inferencer not initialized") 29 | return {} 30 | return inferencer.RunInference(obs, state_ins) 31 | 32 | 33 | func _notification(what): 34 | if what == NOTIFICATION_PREDELETE: 35 | inferencer.FreeDisposables() 36 | inferencer.free() 37 | 38 | # Check whether agent uses a continuous actions model with only action means or not 39 | func set_action_means_only(agent_action_space): 40 | action_means_only_set = true 41 | var continuous_only: bool = true 42 | var continuous_actions: int 43 | for action in agent_action_space: 44 | if not agent_action_space[action]["action_type"] == "continuous": 45 | continuous_only = false 46 | break 47 | else: 48 | continuous_actions += agent_action_space[action]["size"] 49 | if continuous_only: 50 | if continuous_actions == action_output_size: 51 | action_means_only = true 52 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/PositionSensor2D.gd: -------------------------------------------------------------------------------- 1 | extends ISensor2D 2 | class_name PositionSensor2D 3 | 4 | @export var objects_to_observe: Array[Node2D] 5 | 6 | ## Whether to include relative x position in obs 7 | @export var include_x := true 8 | ## Whether to include relative y position in obs 9 | @export var include_y := true 10 | 11 | ## Max distance, values in obs will be normalized, 12 | ## 0 will represent the closest distance possible, and 1 the farthest. 13 | ## Do not use a much larger value than needed, as it would make the obs 14 | ## very small after normalization. 15 | @export_range(0.01, 20_000) var max_distance := 1.0 16 | 17 | @export var use_separate_direction: bool = false 18 | 19 | @export var debug_lines: bool = true 20 | @export var debug_color: Color = Color.GREEN 21 | 22 | @onready var line: Line2D 23 | 24 | 25 | func _ready() -> void: 26 | if debug_lines: 27 | line = Line2D.new() 28 | add_child(line) 29 | line.width = 1 30 | line.default_color = debug_color 31 | 32 | func get_observation(): 33 | var observations: Array[float] 34 | 35 | if debug_lines: 36 | line.clear_points() 37 | 38 | for obj in objects_to_observe: 39 | var relative_position := Vector2.ZERO 40 | 41 | ## If object has been removed, keep the zeroed position 42 | if is_instance_valid(obj): relative_position = to_local(obj.global_position) 43 | 44 | if debug_lines: 45 | line.add_point(Vector2.ZERO) 46 | line.add_point(relative_position) 47 | 48 | var direction := Vector2.ZERO 49 | var distance := 0.0 50 | if use_separate_direction: 51 | direction = relative_position.normalized() 52 | distance = min(relative_position.length() / max_distance, 1.0) 53 | if include_x: 54 | observations.append(direction.x) 55 | if include_y: 56 | observations.append(direction.y) 57 | observations.append(distance) 58 | else: 59 | relative_position = relative_position.limit_length(max_distance) / max_distance 60 | if include_x: 61 | observations.append(relative_position.x) 62 | if include_y: 63 | observations.append(relative_position.y) 64 | 65 | return observations 66 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd: -------------------------------------------------------------------------------- 1 | extends Node3D 2 | class_name RGBCameraSensor3D 3 | var camera_pixels = null 4 | 5 | @onready var camera_texture := $Control/CameraTexture as Sprite2D 6 | @onready var processed_texture := $Control/ProcessedTexture as Sprite2D 7 | @onready var sub_viewport := $SubViewport as SubViewport 8 | @onready var displayed_image: ImageTexture 9 | 10 | @export var render_image_resolution := Vector2i(36, 36) 11 | ## Display size does not affect rendered or sent image resolution. 12 | ## Scale is relative to either render image or downscale image resolution 13 | ## depending on which mode is set. 14 | @export var displayed_image_scale_factor := Vector2i(8, 8) 15 | 16 | @export_group("Downscale image options") 17 | ## Enable to downscale the rendered image before sending the obs. 18 | @export var downscale_image: bool = false 19 | ## If downscale_image is true, will display the downscaled image instead of rendered image. 20 | @export var display_downscaled_image: bool = true 21 | ## This is the resolution of the image that will be sent after downscaling 22 | @export var resized_image_resolution := Vector2i(36, 36) 23 | 24 | 25 | func _ready(): 26 | sub_viewport.size = render_image_resolution 27 | camera_texture.scale = displayed_image_scale_factor 28 | 29 | if downscale_image and display_downscaled_image: 30 | camera_texture.visible = false 31 | processed_texture.scale = displayed_image_scale_factor 32 | else: 33 | processed_texture.visible = false 34 | 35 | 36 | func get_camera_pixel_encoding(): 37 | var image := camera_texture.get_texture().get_image() as Image 38 | 39 | if downscale_image: 40 | image.resize( 41 | resized_image_resolution.x, resized_image_resolution.y, Image.INTERPOLATE_NEAREST 42 | ) 43 | if display_downscaled_image: 44 | if not processed_texture.texture: 45 | displayed_image = ImageTexture.create_from_image(image) 46 | processed_texture.texture = displayed_image 47 | else: 48 | displayed_image.update(image) 49 | 50 | return image.get_data().hex_encode() 51 | 52 | 53 | func get_camera_shape() -> Array: 54 | var size = resized_image_resolution if downscale_image else render_image_resolution 55 | 56 | assert( 57 | size.x >= 36 and size.y >= 36, 58 | "Camera sensor sent image resolution must be 36x36 or larger." 59 | ) 60 | if sub_viewport.transparent_bg: 61 | return [4, size.y, size.x] 62 | else: 63 | return [3, size.y, size.x] 64 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/PositionSensor3D.gd: -------------------------------------------------------------------------------- 1 | extends ISensor3D 2 | class_name PositionSensor3D 3 | 4 | @export var objects_to_observe: Array[Node3D] 5 | 6 | ## Whether to include relative x position in obs 7 | @export var include_x := true 8 | ## Whether to include relative y position in obs 9 | @export var include_y := true 10 | ## Whether to include relative z position in obs 11 | @export var include_z := true 12 | 13 | ## Max distance, values in obs will be normalized, 14 | ## 0 will represent the closest distance possible, and 1 the farthest. 15 | ## Do not use a much larger value than needed, as it would make the obs 16 | ## very small after normalization. 17 | @export_range(0.01, 2_500) var max_distance := 1.0 18 | 19 | @export var use_separate_direction: bool = false 20 | 21 | @export var debug_lines: bool = true 22 | @export var debug_color: Color = Color.GREEN 23 | 24 | @onready var mesh: ImmediateMesh 25 | 26 | 27 | func _ready() -> void: 28 | if debug_lines: 29 | var debug_mesh = MeshInstance3D.new() 30 | add_child(debug_mesh) 31 | var line_material := StandardMaterial3D.new() 32 | line_material.albedo_color = debug_color 33 | debug_mesh.material_override = line_material 34 | debug_mesh.mesh = ImmediateMesh.new() 35 | mesh = debug_mesh.mesh 36 | 37 | 38 | func get_observation(): 39 | var observations: Array[float] 40 | 41 | if debug_lines: 42 | mesh.clear_surfaces() 43 | mesh.surface_begin(Mesh.PRIMITIVE_LINES) 44 | mesh.surface_set_color(debug_color) 45 | 46 | for obj in objects_to_observe: 47 | var relative_position := Vector3.ZERO 48 | 49 | ## If object has been removed, keep the zeroed position 50 | if is_instance_valid(obj): relative_position = to_local(obj.global_position) 51 | 52 | if debug_lines: 53 | mesh.surface_add_vertex(Vector3.ZERO) 54 | mesh.surface_add_vertex(relative_position) 55 | 56 | var direction := Vector3.ZERO 57 | var distance := 0.0 58 | if use_separate_direction: 59 | direction = relative_position.normalized() 60 | distance = min(relative_position.length() / max_distance, 1.0) 61 | if include_x: 62 | observations.append(direction.x) 63 | if include_y: 64 | observations.append(direction.y) 65 | if include_z: 66 | observations.append(direction.z) 67 | observations.append(distance) 68 | else: 69 | relative_position = relative_position.limit_length(max_distance) / max_distance 70 | if include_x: 71 | observations.append(relative_position.x) 72 | if include_y: 73 | observations.append(relative_position.y) 74 | if include_z: 75 | observations.append(relative_position.z) 76 | 77 | if debug_lines: 78 | mesh.surface_end() 79 | return observations 80 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends ISensor2D 3 | class_name RaycastSensor2D 4 | 5 | @export_flags_2d_physics var collision_mask := 1: 6 | get: 7 | return collision_mask 8 | set(value): 9 | collision_mask = value 10 | _update() 11 | 12 | @export var collide_with_areas := false: 13 | get: 14 | return collide_with_areas 15 | set(value): 16 | collide_with_areas = value 17 | _update() 18 | 19 | @export var collide_with_bodies := true: 20 | get: 21 | return collide_with_bodies 22 | set(value): 23 | collide_with_bodies = value 24 | _update() 25 | 26 | @export var n_rays := 16.0: 27 | get: 28 | return n_rays 29 | set(value): 30 | n_rays = value 31 | _update() 32 | 33 | @export_range(5, 3000, 5.0) var ray_length := 200: 34 | get: 35 | return ray_length 36 | set(value): 37 | ray_length = value 38 | _update() 39 | @export_range(5, 360, 5.0) var cone_width := 360.0: 40 | get: 41 | return cone_width 42 | set(value): 43 | cone_width = value 44 | _update() 45 | 46 | @export var debug_draw := true: 47 | get: 48 | return debug_draw 49 | set(value): 50 | debug_draw = value 51 | _update() 52 | 53 | var _angles = [] 54 | var rays := [] 55 | 56 | 57 | func _update(): 58 | if Engine.is_editor_hint(): 59 | if debug_draw: 60 | _spawn_nodes() 61 | else: 62 | for ray in get_children(): 63 | if ray is RayCast2D: 64 | remove_child(ray) 65 | 66 | 67 | func _ready() -> void: 68 | _spawn_nodes() 69 | 70 | 71 | func _spawn_nodes(): 72 | for ray in rays: 73 | ray.queue_free() 74 | rays = [] 75 | 76 | _angles = [] 77 | var step = cone_width / (n_rays) 78 | var start = step / 2 - cone_width / 2 79 | 80 | for i in n_rays: 81 | var angle = start + i * step 82 | var ray = RayCast2D.new() 83 | ray.set_target_position( 84 | Vector2(ray_length * cos(deg_to_rad(angle)), ray_length * sin(deg_to_rad(angle))) 85 | ) 86 | ray.set_name("node_" + str(i)) 87 | ray.enabled = false 88 | ray.collide_with_areas = collide_with_areas 89 | ray.collide_with_bodies = collide_with_bodies 90 | ray.collision_mask = collision_mask 91 | add_child(ray) 92 | rays.append(ray) 93 | 94 | _angles.append(start + i * step) 95 | 96 | 97 | func get_observation() -> Array: 98 | return self.calculate_raycasts() 99 | 100 | 101 | func calculate_raycasts() -> Array: 102 | var result = [] 103 | for ray in rays: 104 | ray.enabled = true 105 | ray.force_raycast_update() 106 | var distance = _get_raycast_distance(ray) 107 | result.append(distance) 108 | ray.enabled = false 109 | return result 110 | 111 | 112 | func _get_raycast_distance(ray: RayCast2D) -> float: 113 | if !ray.is_colliding(): 114 | return 0.0 115 | 116 | var distance = (global_position - ray.get_collision_point()).length() 117 | distance = clamp(distance, 0.0, ray_length) 118 | return (ray_length - distance) / ray_length 119 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd: -------------------------------------------------------------------------------- 1 | extends Node2D 2 | class_name RGBCameraSensor2D 3 | var camera_pixels = null 4 | 5 | @export var camera_zoom_factor := Vector2(0.1, 0.1) 6 | @onready var camera := $SubViewport/Camera 7 | @onready var preview_window := $Control 8 | @onready var camera_texture := $Control/CameraTexture as Sprite2D 9 | @onready var processed_texture := $Control/ProcessedTexture as Sprite2D 10 | @onready var sub_viewport := $SubViewport as SubViewport 11 | @onready var displayed_image: ImageTexture 12 | 13 | @export var render_image_resolution := Vector2i(36, 36) 14 | ## Display size does not affect rendered or sent image resolution. 15 | ## Scale is relative to either render image or downscale image resolution 16 | ## depending on which mode is set. 17 | @export var displayed_image_scale_factor := Vector2i(8, 8) 18 | 19 | @export_group("Downscale image options") 20 | ## Enable to downscale the rendered image before sending the obs. 21 | @export var downscale_image: bool = false 22 | ## If downscale_image is true, will display the downscaled image instead of rendered image. 23 | @export var display_downscaled_image: bool = true 24 | ## This is the resolution of the image that will be sent after downscaling 25 | @export var resized_image_resolution := Vector2i(36, 36) 26 | 27 | 28 | func _ready(): 29 | DisplayServer.register_additional_output(self) 30 | 31 | camera.zoom = camera_zoom_factor 32 | 33 | var preview_size: Vector2 34 | 35 | sub_viewport.world_2d = get_tree().get_root().get_world_2d() 36 | sub_viewport.size = render_image_resolution 37 | camera_texture.scale = displayed_image_scale_factor 38 | 39 | if downscale_image and display_downscaled_image: 40 | camera_texture.visible = false 41 | processed_texture.scale = displayed_image_scale_factor 42 | preview_size = displayed_image_scale_factor * resized_image_resolution 43 | else: 44 | processed_texture.visible = false 45 | preview_size = displayed_image_scale_factor * render_image_resolution 46 | 47 | preview_window.size = preview_size 48 | 49 | 50 | func get_camera_pixel_encoding(): 51 | var image := camera_texture.get_texture().get_image() as Image 52 | 53 | if downscale_image: 54 | image.resize( 55 | resized_image_resolution.x, resized_image_resolution.y, Image.INTERPOLATE_NEAREST 56 | ) 57 | if display_downscaled_image: 58 | if not processed_texture.texture: 59 | displayed_image = ImageTexture.create_from_image(image) 60 | processed_texture.texture = displayed_image 61 | else: 62 | displayed_image.update(image) 63 | 64 | return image.get_data().hex_encode() 65 | 66 | 67 | func get_camera_shape() -> Array: 68 | var size = resized_image_resolution if downscale_image else render_image_resolution 69 | 70 | assert( 71 | size.x >= 36 and size.y >= 36, 72 | "Camera sensor sent image resolution must be 36x36 or larger." 73 | ) 74 | if sub_viewport.transparent_bg: 75 | return [4, size.y, size.x] 76 | else: 77 | return [3, size.y, size.x] 78 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/controller/ai_controller_2d.gd: -------------------------------------------------------------------------------- 1 | extends Node2D 2 | class_name AIController2D 3 | 4 | enum ControlModes { 5 | INHERIT_FROM_SYNC, ## Inherit setting from sync node 6 | HUMAN, ## Test the environment manually 7 | TRAINING, ## Train a model 8 | ONNX_INFERENCE, ## Load a pretrained model using an .onnx file 9 | RECORD_EXPERT_DEMOS ## Record observations and actions for expert demonstrations 10 | } 11 | @export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC 12 | ## The path to a trained .onnx model file to use for inference (overrides the path set in sync node). 13 | @export var onnx_model_path := "" 14 | ## Once the number of steps has passed, the flag 'needs_reset' will be set to 'true' for this instance. 15 | @export var reset_after := 1000 16 | 17 | @export_group("Record expert demos mode options") 18 | ## Path where the demos will be saved. The file can later be used for imitation learning. 19 | @export var expert_demo_save_path: String 20 | ## The action that erases the last recorded episode from the currently recorded data. 21 | @export var remove_last_episode_key: InputEvent 22 | ## Action will be repeated for n frames. Will introduce control lag if larger than 1. 23 | ## Can be used to ensure that action_repeat on inference and training matches 24 | ## the recorded demonstrations. 25 | @export var action_repeat: int = 1 26 | 27 | @export_group("Multi-policy mode options") 28 | ## Allows you to set certain agents to use different policies. 29 | ## Changing has no effect with default SB3 training. Works with Rllib example. 30 | ## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md 31 | @export var policy_name: String = "shared_policy" 32 | 33 | var onnx_model: ONNXModel 34 | 35 | var heuristic := "human" 36 | var done := false 37 | var reward := 0.0 38 | var n_steps := 0 39 | var needs_reset := false 40 | 41 | var _player: Node2D 42 | 43 | 44 | func _ready(): 45 | add_to_group("AGENT") 46 | 47 | 48 | func init(player: Node2D): 49 | _player = player 50 | 51 | 52 | #region Methods that need implementing using the "extend script" option in Godot 53 | func get_obs() -> Dictionary: 54 | assert(false, "the get_obs method is not implemented when extending from ai_controller") 55 | return {"obs": []} 56 | 57 | 58 | func get_reward() -> float: 59 | assert(false, "the get_reward method is not implemented when extending from ai_controller") 60 | return 0.0 61 | 62 | 63 | func get_action_space() -> Dictionary: 64 | assert( 65 | false, "the get_action_space method is not implemented when extending from ai_controller" 66 | ) 67 | return { 68 | "example_actions_continous": {"size": 2, "action_type": "continuous"}, 69 | "example_actions_discrete": {"size": 2, "action_type": "discrete"}, 70 | } 71 | 72 | 73 | func set_action(action) -> void: 74 | assert(false, "the set_action method is not implemented when extending from ai_controller") 75 | 76 | 77 | #endregion 78 | 79 | 80 | #region Methods that sometimes need implementing using the "extend script" option in Godot 81 | # Only needed if you are recording expert demos with this AIController 82 | func get_action() -> Array: 83 | assert( 84 | false, 85 | "the get_action method is not implemented in extended AIController but demo_recorder is used" 86 | ) 87 | return [] 88 | 89 | 90 | # For providing additional info (e.g. `is_success` for SB3 training) 91 | func get_info() -> Dictionary: 92 | return {} 93 | 94 | 95 | #endregion 96 | 97 | 98 | func _physics_process(delta): 99 | n_steps += 1 100 | if n_steps > reset_after: 101 | needs_reset = true 102 | 103 | 104 | func get_obs_space(): 105 | # may need overriding if the obs space is complex 106 | var obs = get_obs() 107 | return { 108 | "obs": {"size": [len(obs["obs"])], "space": "box"}, 109 | } 110 | 111 | 112 | func reset(): 113 | n_steps = 0 114 | needs_reset = false 115 | 116 | 117 | func reset_if_done(): 118 | if done: 119 | reset() 120 | 121 | 122 | func set_heuristic(h): 123 | # sets the heuristic from "human" or "model" nothing to change here 124 | heuristic = h 125 | 126 | 127 | func get_done(): 128 | return done 129 | 130 | 131 | func set_done_false(): 132 | done = false 133 | 134 | 135 | func zero_reward(): 136 | reward = 0.0 137 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/controller/ai_controller_3d.gd: -------------------------------------------------------------------------------- 1 | extends Node3D 2 | class_name AIController3D 3 | 4 | enum ControlModes { 5 | INHERIT_FROM_SYNC, ## Inherit setting from sync node 6 | HUMAN, ## Test the environment manually 7 | TRAINING, ## Train a model 8 | ONNX_INFERENCE, ## Load a pretrained model using an .onnx file 9 | RECORD_EXPERT_DEMOS ## Record observations and actions for expert demonstrations 10 | } 11 | @export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC 12 | ## The path to a trained .onnx model file to use for inference (overrides the path set in sync node). 13 | @export var onnx_model_path := "" 14 | ## Once the number of steps has passed, the flag 'needs_reset' will be set to 'true' for this instance. 15 | @export var reset_after := 1000 16 | 17 | @export_group("Record expert demos mode options") 18 | ## Path where the demos will be saved. The file can later be used for imitation learning. 19 | @export var expert_demo_save_path: String 20 | ## The action that erases the last recorded episode from the currently recorded data. 21 | @export var remove_last_episode_key: InputEvent 22 | ## Action will be repeated for n frames. Will introduce control lag if larger than 1. 23 | ## Can be used to ensure that action_repeat on inference and training matches 24 | ## the recorded demonstrations. 25 | @export var action_repeat: int = 1 26 | 27 | @export_group("Multi-policy mode options") 28 | ## Allows you to set certain agents to use different policies. 29 | ## Changing has no effect with default SB3 training. Works with Rllib example. 30 | ## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md 31 | @export var policy_name: String = "shared_policy" 32 | 33 | var onnx_model: ONNXModel 34 | 35 | var heuristic := "human" 36 | var done := false 37 | var reward := 0.0 38 | var n_steps := 0 39 | var needs_reset := false 40 | 41 | var _player: Node3D 42 | 43 | 44 | func _ready(): 45 | add_to_group("AGENT") 46 | 47 | 48 | func init(player: Node3D): 49 | _player = player 50 | 51 | 52 | #region Methods that need implementing using the "extend script" option in Godot 53 | func get_obs() -> Dictionary: 54 | assert(false, "the get_obs method is not implemented when extending from ai_controller") 55 | return {"obs": []} 56 | 57 | 58 | func get_reward() -> float: 59 | assert(false, "the get_reward method is not implemented when extending from ai_controller") 60 | return 0.0 61 | 62 | 63 | func get_action_space() -> Dictionary: 64 | assert( 65 | false, "the get_action_space method is not implemented when extending from ai_controller" 66 | ) 67 | return { 68 | "example_actions_continous": {"size": 2, "action_type": "continuous"}, 69 | "example_actions_discrete": {"size": 2, "action_type": "discrete"}, 70 | } 71 | 72 | 73 | func set_action(action) -> void: 74 | assert(false, "the set_action method is not implemented when extending from ai_controller") 75 | 76 | 77 | #endregion 78 | 79 | 80 | #region Methods that sometimes need implementing using the "extend script" option in Godot 81 | # Only needed if you are recording expert demos with this AIController 82 | func get_action() -> Array: 83 | assert( 84 | false, 85 | "the get_action method is not implemented in extended AIController but demo_recorder is used" 86 | ) 87 | return [] 88 | 89 | 90 | # For providing additional info (e.g. `is_success` for SB3 training) 91 | func get_info() -> Dictionary: 92 | return {} 93 | 94 | 95 | #endregion 96 | 97 | 98 | func _physics_process(delta): 99 | n_steps += 1 100 | if n_steps > reset_after: 101 | needs_reset = true 102 | 103 | 104 | func get_obs_space(): 105 | # may need overriding if the obs space is complex 106 | var obs = get_obs() 107 | return { 108 | "obs": {"size": [len(obs["obs"])], "space": "box"}, 109 | } 110 | 111 | 112 | func reset(): 113 | n_steps = 0 114 | needs_reset = false 115 | 116 | 117 | func reset_if_done(): 118 | if done: 119 | reset() 120 | 121 | 122 | func set_heuristic(h): 123 | # sets the heuristic from "human" or "model" nothing to change here 124 | heuristic = h 125 | 126 | 127 | func get_done(): 128 | return done 129 | 130 | 131 | func set_done_false(): 132 | done = false 133 | 134 | 135 | func zero_reward(): 136 | reward = 0.0 137 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/onnx/csharp/ONNXInference.cs: -------------------------------------------------------------------------------- 1 | using Godot; 2 | using Microsoft.ML.OnnxRuntime; 3 | using Microsoft.ML.OnnxRuntime.Tensors; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | 7 | namespace GodotONNX 8 | { 9 | /// 10 | public partial class ONNXInference : GodotObject 11 | { 12 | 13 | private InferenceSession session; 14 | /// 15 | /// Path to the ONNX model. Use Initialize to change it. 16 | /// 17 | private string modelPath; 18 | private int batchSize; 19 | 20 | private SessionOptions SessionOpt; 21 | 22 | /// 23 | /// init function 24 | /// 25 | /// 26 | /// 27 | /// Returns the output size of the model 28 | public int Initialize(string Path, int BatchSize) 29 | { 30 | modelPath = Path; 31 | batchSize = BatchSize; 32 | SessionOpt = SessionConfigurator.MakeConfiguredSessionOptions(); 33 | session = LoadModel(modelPath); 34 | return session.OutputMetadata["output"].Dimensions[1]; 35 | } 36 | 37 | 38 | /// 39 | public Godot.Collections.Dictionary> RunInference(Godot.Collections.Dictionary> obs, int state_ins) 40 | { 41 | //Current model: Any (Godot Rl Agents) 42 | //Expects a tensor of shape [batch_size, input_size] type float for any output of the agents observation dictionary and a tensor of shape [batch_size] type float named state_ins 43 | 44 | var modelInputsList = new List 45 | { 46 | NamedOnnxValue.CreateFromTensor("state_ins", new DenseTensor(new float[] { state_ins }, new int[] { batchSize })) 47 | }; 48 | foreach (var key in obs.Keys) 49 | { 50 | var subObs = obs[key]; 51 | // Fill the input tensors for each key of the observation 52 | // create span of observation from specific inputSize 53 | var obsData = new float[subObs.Count]; //There's probably a better way to do this 54 | for (int i = 0; i < subObs.Count; i++) 55 | { 56 | obsData[i] = subObs[i]; 57 | } 58 | modelInputsList.Add( 59 | NamedOnnxValue.CreateFromTensor(key, new DenseTensor(obsData, new int[] { batchSize, subObs.Count })) 60 | ); 61 | } 62 | 63 | IReadOnlyCollection outputNames = new List { "output", "state_outs" }; //ONNX is sensible to these names, as well as the input names 64 | 65 | IDisposableReadOnlyCollection results; 66 | //We do not use "using" here so we get a better exception explaination later 67 | try 68 | { 69 | results = session.Run(modelInputsList, outputNames); 70 | } 71 | catch (OnnxRuntimeException e) 72 | { 73 | //This error usually means that the model is not compatible with the input, beacause of the input shape (size) 74 | GD.Print("Error at inference: ", e); 75 | return null; 76 | } 77 | //Can't convert IEnumerable to Variant, so we have to convert it to an array or something 78 | Godot.Collections.Dictionary> output = new Godot.Collections.Dictionary>(); 79 | DisposableNamedOnnxValue output1 = results.First(); 80 | DisposableNamedOnnxValue output2 = results.Last(); 81 | Godot.Collections.Array output1Array = new Godot.Collections.Array(); 82 | Godot.Collections.Array output2Array = new Godot.Collections.Array(); 83 | 84 | foreach (float f in output1.AsEnumerable()) 85 | { 86 | output1Array.Add(f); 87 | } 88 | 89 | foreach (float f in output2.AsEnumerable()) 90 | { 91 | output2Array.Add(f); 92 | } 93 | 94 | output.Add(output1.Name, output1Array); 95 | output.Add(output2.Name, output2Array); 96 | 97 | //Output is a dictionary of arrays, ex: { "output" : [0.1, 0.2, 0.3, 0.4, ...], "state_outs" : [0.5, ...]} 98 | results.Dispose(); 99 | return output; 100 | } 101 | /// 102 | public InferenceSession LoadModel(string Path) 103 | { 104 | using Godot.FileAccess file = FileAccess.Open(Path, Godot.FileAccess.ModeFlags.Read); 105 | byte[] model = file.GetBuffer((int)file.GetLength()); 106 | //file.Close(); file.Dispose(); //Close the file, then dispose the reference. 107 | return new InferenceSession(model, SessionOpt); //Load the model 108 | } 109 | public void FreeDisposables() 110 | { 111 | session.Dispose(); 112 | SessionOpt.Dispose(); 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends ISensor3D 3 | class_name RayCastSensor3D 4 | @export_flags_3d_physics var collision_mask = 1: 5 | get: 6 | return collision_mask 7 | set(value): 8 | collision_mask = value 9 | _update() 10 | @export_flags_3d_physics var boolean_class_mask = 1: 11 | get: 12 | return boolean_class_mask 13 | set(value): 14 | boolean_class_mask = value 15 | _update() 16 | 17 | @export var n_rays_width := 6.0: 18 | get: 19 | return n_rays_width 20 | set(value): 21 | n_rays_width = value 22 | _update() 23 | 24 | @export var n_rays_height := 6.0: 25 | get: 26 | return n_rays_height 27 | set(value): 28 | n_rays_height = value 29 | _update() 30 | 31 | @export var ray_length := 10.0: 32 | get: 33 | return ray_length 34 | set(value): 35 | ray_length = value 36 | _update() 37 | 38 | @export var cone_width := 60.0: 39 | get: 40 | return cone_width 41 | set(value): 42 | cone_width = value 43 | _update() 44 | 45 | @export var cone_height := 60.0: 46 | get: 47 | return cone_height 48 | set(value): 49 | cone_height = value 50 | _update() 51 | 52 | @export var collide_with_areas := false: 53 | get: 54 | return collide_with_areas 55 | set(value): 56 | collide_with_areas = value 57 | _update() 58 | 59 | @export var collide_with_bodies := true: 60 | get: 61 | return collide_with_bodies 62 | set(value): 63 | collide_with_bodies = value 64 | _update() 65 | 66 | @export var class_sensor := false 67 | 68 | var rays := [] 69 | var geo = null 70 | 71 | 72 | func _update(): 73 | if Engine.is_editor_hint(): 74 | if is_node_ready(): 75 | _spawn_nodes() 76 | 77 | 78 | func _ready() -> void: 79 | if Engine.is_editor_hint(): 80 | if get_child_count() == 0: 81 | _spawn_nodes() 82 | else: 83 | _spawn_nodes() 84 | 85 | 86 | func _spawn_nodes(): 87 | print("spawning nodes") 88 | for ray in get_children(): 89 | ray.queue_free() 90 | if geo: 91 | geo.clear() 92 | #$Lines.remove_points() 93 | rays = [] 94 | 95 | var horizontal_step = cone_width / (n_rays_width) 96 | var vertical_step = cone_height / (n_rays_height) 97 | 98 | var horizontal_start = horizontal_step / 2 - cone_width / 2 99 | var vertical_start = vertical_step / 2 - cone_height / 2 100 | 101 | var points = [] 102 | 103 | for i in n_rays_width: 104 | for j in n_rays_height: 105 | var angle_w = horizontal_start + i * horizontal_step 106 | var angle_h = vertical_start + j * vertical_step 107 | #angle_h = 0.0 108 | var ray = RayCast3D.new() 109 | var cast_to = to_spherical_coords(ray_length, angle_w, angle_h) 110 | ray.set_target_position(cast_to) 111 | 112 | points.append(cast_to) 113 | 114 | ray.set_name("node_" + str(i) + " " + str(j)) 115 | ray.enabled = true 116 | ray.collide_with_bodies = collide_with_bodies 117 | ray.collide_with_areas = collide_with_areas 118 | ray.collision_mask = collision_mask 119 | add_child(ray) 120 | ray.set_owner(get_tree().edited_scene_root) 121 | rays.append(ray) 122 | ray.force_raycast_update() 123 | 124 | 125 | # if Engine.editor_hint: 126 | # _create_debug_lines(points) 127 | 128 | 129 | func _create_debug_lines(points): 130 | if not geo: 131 | geo = ImmediateMesh.new() 132 | add_child(geo) 133 | 134 | geo.clear() 135 | geo.begin(Mesh.PRIMITIVE_LINES) 136 | for point in points: 137 | geo.set_color(Color.AQUA) 138 | geo.add_vertex(Vector3.ZERO) 139 | geo.add_vertex(point) 140 | geo.end() 141 | 142 | 143 | func display(): 144 | if geo: 145 | geo.display() 146 | 147 | 148 | func to_spherical_coords(r, inc, azimuth) -> Vector3: 149 | return Vector3( 150 | r * sin(deg_to_rad(inc)) * cos(deg_to_rad(azimuth)), 151 | r * sin(deg_to_rad(azimuth)), 152 | r * cos(deg_to_rad(inc)) * cos(deg_to_rad(azimuth)) 153 | ) 154 | 155 | 156 | func get_observation() -> Array: 157 | return self.calculate_raycasts() 158 | 159 | 160 | func calculate_raycasts() -> Array: 161 | var result = [] 162 | for ray in rays: 163 | ray.set_enabled(true) 164 | ray.force_raycast_update() 165 | var distance = _get_raycast_distance(ray) 166 | 167 | result.append(distance) 168 | if class_sensor: 169 | var hit_class: float = 0 170 | if ray.get_collider(): 171 | var hit_collision_layer = ray.get_collider().collision_layer 172 | hit_collision_layer = hit_collision_layer & collision_mask 173 | hit_class = (hit_collision_layer & boolean_class_mask) > 0 174 | result.append(float(hit_class)) 175 | ray.set_enabled(false) 176 | return result 177 | 178 | 179 | func _get_raycast_distance(ray: RayCast3D) -> float: 180 | if !ray.is_colliding(): 181 | return 0.0 182 | 183 | var distance = (global_transform.origin - ray.get_collision_point()).length() 184 | distance = clamp(distance, 0.0, ray_length) 185 | return (ray_length - distance) / ray_length 186 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/onnx/csharp/SessionConfigurator.cs: -------------------------------------------------------------------------------- 1 | using Godot; 2 | using Microsoft.ML.OnnxRuntime; 3 | 4 | namespace GodotONNX 5 | { 6 | /// 7 | 8 | public static class SessionConfigurator 9 | { 10 | public enum ComputeName 11 | { 12 | CUDA, 13 | ROCm, 14 | DirectML, 15 | CoreML, 16 | CPU 17 | } 18 | 19 | /// 20 | public static SessionOptions MakeConfiguredSessionOptions() 21 | { 22 | SessionOptions sessionOptions = new(); 23 | SetOptions(sessionOptions); 24 | return sessionOptions; 25 | } 26 | 27 | private static void SetOptions(SessionOptions sessionOptions) 28 | { 29 | sessionOptions.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING; 30 | ApplySystemSpecificOptions(sessionOptions); 31 | } 32 | 33 | /// 34 | static public void ApplySystemSpecificOptions(SessionOptions sessionOptions) 35 | { 36 | //Most code for this function is verbose only, the only reason it exists is to track 37 | //implementation progress of the different compute APIs. 38 | 39 | //December 2022: CUDA is not working. 40 | 41 | string OSName = OS.GetName(); //Get OS Name 42 | 43 | //ComputeName ComputeAPI = ComputeCheck(); //Get Compute API 44 | // //TODO: Get CPU architecture 45 | 46 | //Linux can use OpenVINO (C#) on x64 and ROCm on x86 (GDNative/C++) 47 | //Windows can use OpenVINO (C#) on x64 48 | //TODO: try TensorRT instead of CUDA 49 | //TODO: Use OpenVINO for Intel Graphics 50 | 51 | // Temporarily using CPU on all platforms to avoid errors detected with DML 52 | ComputeName ComputeAPI = ComputeName.CPU; 53 | 54 | //match OS and Compute API 55 | GD.Print($"OS: {OSName} Compute API: {ComputeAPI}"); 56 | 57 | // CPU is set by default without appending necessary 58 | // sessionOptions.AppendExecutionProvider_CPU(0); 59 | 60 | /* 61 | switch (OSName) 62 | { 63 | case "Windows": //Can use CUDA, DirectML 64 | if (ComputeAPI is ComputeName.CUDA) 65 | { 66 | //CUDA 67 | //sessionOptions.AppendExecutionProvider_CUDA(0); 68 | //sessionOptions.AppendExecutionProvider_DML(0); 69 | } 70 | else if (ComputeAPI is ComputeName.DirectML) 71 | { 72 | //DirectML 73 | //sessionOptions.AppendExecutionProvider_DML(0); 74 | } 75 | break; 76 | case "X11": //Can use CUDA, ROCm 77 | if (ComputeAPI is ComputeName.CUDA) 78 | { 79 | //CUDA 80 | //sessionOptions.AppendExecutionProvider_CUDA(0); 81 | } 82 | if (ComputeAPI is ComputeName.ROCm) 83 | { 84 | //ROCm, only works on x86 85 | //Research indicates that this has to be compiled as a GDNative plugin 86 | //GD.Print("ROCm not supported yet, using CPU."); 87 | //sessionOptions.AppendExecutionProvider_CPU(0); 88 | } 89 | break; 90 | case "macOS": //Can use CoreML 91 | if (ComputeAPI is ComputeName.CoreML) 92 | { //CoreML 93 | //TODO: Needs testing 94 | //sessionOptions.AppendExecutionProvider_CoreML(0); 95 | //CoreML on ARM64, out of the box, on x64 needs .tar file from GitHub 96 | } 97 | break; 98 | default: 99 | GD.Print("OS not Supported."); 100 | break; 101 | } 102 | */ 103 | } 104 | 105 | 106 | /// 107 | public static ComputeName ComputeCheck() 108 | { 109 | string adapterName = Godot.RenderingServer.GetVideoAdapterName(); 110 | //string adapterVendor = Godot.RenderingServer.GetVideoAdapterVendor(); 111 | adapterName = adapterName.ToUpper(new System.Globalization.CultureInfo("")); 112 | //TODO: GPU vendors for MacOS, what do they even use these days? 113 | 114 | if (adapterName.Contains("INTEL")) 115 | { 116 | return ComputeName.DirectML; 117 | } 118 | if (adapterName.Contains("AMD") || adapterName.Contains("RADEON")) 119 | { 120 | return ComputeName.DirectML; 121 | } 122 | if (adapterName.Contains("NVIDIA")) 123 | { 124 | return ComputeName.CUDA; 125 | } 126 | 127 | GD.Print("Graphics Card not recognized."); //Should use CPU 128 | return ComputeName.CPU; 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_2d/GridSensor2D.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends ISensor2D 3 | class_name GridSensor2D 4 | 5 | @export var debug_view := false: 6 | get: 7 | return debug_view 8 | set(value): 9 | debug_view = value 10 | _update() 11 | 12 | @export_flags_2d_physics var detection_mask := 0: 13 | get: 14 | return detection_mask 15 | set(value): 16 | detection_mask = value 17 | _update() 18 | 19 | @export var collide_with_areas := false: 20 | get: 21 | return collide_with_areas 22 | set(value): 23 | collide_with_areas = value 24 | _update() 25 | 26 | @export var collide_with_bodies := true: 27 | get: 28 | return collide_with_bodies 29 | set(value): 30 | collide_with_bodies = value 31 | _update() 32 | 33 | @export_range(1, 200, 0.1) var cell_width := 20.0: 34 | get: 35 | return cell_width 36 | set(value): 37 | cell_width = value 38 | _update() 39 | 40 | @export_range(1, 200, 0.1) var cell_height := 20.0: 41 | get: 42 | return cell_height 43 | set(value): 44 | cell_height = value 45 | _update() 46 | 47 | @export_range(1, 21, 2, "or_greater") var grid_size_x := 3: 48 | get: 49 | return grid_size_x 50 | set(value): 51 | grid_size_x = value 52 | _update() 53 | 54 | @export_range(1, 21, 2, "or_greater") var grid_size_y := 3: 55 | get: 56 | return grid_size_y 57 | set(value): 58 | grid_size_y = value 59 | _update() 60 | 61 | var _obs_buffer: PackedFloat64Array 62 | var _rectangle_shape: RectangleShape2D 63 | var _collision_mapping: Dictionary 64 | var _n_layers_per_cell: int 65 | 66 | var _highlighted_cell_color: Color 67 | var _standard_cell_color: Color 68 | 69 | 70 | func get_observation(): 71 | return _obs_buffer 72 | 73 | 74 | func _update(): 75 | if Engine.is_editor_hint(): 76 | if is_node_ready(): 77 | _spawn_nodes() 78 | 79 | 80 | func _ready() -> void: 81 | _set_colors() 82 | 83 | if Engine.is_editor_hint(): 84 | if get_child_count() == 0: 85 | _spawn_nodes() 86 | else: 87 | _spawn_nodes() 88 | 89 | 90 | func _set_colors() -> void: 91 | _standard_cell_color = Color(100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0) 92 | _highlighted_cell_color = Color(255.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0) 93 | 94 | 95 | func _get_collision_mapping() -> Dictionary: 96 | # defines which layer is mapped to which cell obs index 97 | var total_bits = 0 98 | var collision_mapping = {} 99 | for i in 32: 100 | var bit_mask = 2 ** i 101 | if (detection_mask & bit_mask) > 0: 102 | collision_mapping[i] = total_bits 103 | total_bits += 1 104 | 105 | return collision_mapping 106 | 107 | 108 | func _spawn_nodes(): 109 | for cell in get_children(): 110 | cell.name = "_%s" % cell.name # Otherwise naming below will fail 111 | cell.queue_free() 112 | 113 | _collision_mapping = _get_collision_mapping() 114 | #prints("collision_mapping", _collision_mapping, len(_collision_mapping)) 115 | # allocate memory for the observations 116 | _n_layers_per_cell = len(_collision_mapping) 117 | _obs_buffer = PackedFloat64Array() 118 | _obs_buffer.resize(grid_size_x * grid_size_y * _n_layers_per_cell) 119 | _obs_buffer.fill(0) 120 | #prints(len(_obs_buffer), _obs_buffer ) 121 | 122 | _rectangle_shape = RectangleShape2D.new() 123 | _rectangle_shape.set_size(Vector2(cell_width, cell_height)) 124 | 125 | var shift := Vector2( 126 | -(grid_size_x / 2) * cell_width, 127 | -(grid_size_y / 2) * cell_height, 128 | ) 129 | 130 | for i in grid_size_x: 131 | for j in grid_size_y: 132 | var cell_position = Vector2(i * cell_width, j * cell_height) + shift 133 | _create_cell(i, j, cell_position) 134 | 135 | 136 | func _create_cell(i: int, j: int, position: Vector2): 137 | var cell := Area2D.new() 138 | cell.position = position 139 | cell.name = "GridCell %s %s" % [i, j] 140 | cell.modulate = _standard_cell_color 141 | 142 | if collide_with_areas: 143 | cell.area_entered.connect(_on_cell_area_entered.bind(i, j)) 144 | cell.area_exited.connect(_on_cell_area_exited.bind(i, j)) 145 | 146 | if collide_with_bodies: 147 | cell.body_entered.connect(_on_cell_body_entered.bind(i, j)) 148 | cell.body_exited.connect(_on_cell_body_exited.bind(i, j)) 149 | 150 | cell.collision_layer = 0 151 | cell.collision_mask = detection_mask 152 | cell.monitorable = true 153 | add_child(cell) 154 | cell.set_owner(get_tree().edited_scene_root) 155 | 156 | var col_shape := CollisionShape2D.new() 157 | col_shape.shape = _rectangle_shape 158 | col_shape.name = "CollisionShape2D" 159 | cell.add_child(col_shape) 160 | col_shape.set_owner(get_tree().edited_scene_root) 161 | 162 | if debug_view: 163 | var quad = MeshInstance2D.new() 164 | quad.name = "MeshInstance2D" 165 | var quad_mesh = QuadMesh.new() 166 | 167 | quad_mesh.set_size(Vector2(cell_width, cell_height)) 168 | 169 | quad.mesh = quad_mesh 170 | cell.add_child(quad) 171 | quad.set_owner(get_tree().edited_scene_root) 172 | 173 | 174 | func _update_obs(cell_i: int, cell_j: int, collision_layer: int, entered: bool): 175 | for key in _collision_mapping: 176 | var bit_mask = 2 ** key 177 | if (collision_layer & bit_mask) > 0: 178 | var collison_map_index = _collision_mapping[key] 179 | 180 | var obs_index = ( 181 | (cell_i * grid_size_y * _n_layers_per_cell) 182 | + (cell_j * _n_layers_per_cell) 183 | + collison_map_index 184 | ) 185 | #prints(obs_index, cell_i, cell_j) 186 | if entered: 187 | _obs_buffer[obs_index] += 1 188 | else: 189 | _obs_buffer[obs_index] -= 1 190 | 191 | 192 | func _toggle_cell(cell_i: int, cell_j: int): 193 | var cell = get_node_or_null("GridCell %s %s" % [cell_i, cell_j]) 194 | 195 | if cell == null: 196 | print("cell not found, returning") 197 | 198 | var n_hits = 0 199 | var start_index = (cell_i * grid_size_y * _n_layers_per_cell) + (cell_j * _n_layers_per_cell) 200 | for i in _n_layers_per_cell: 201 | n_hits += _obs_buffer[start_index + i] 202 | 203 | if n_hits > 0: 204 | cell.modulate = _highlighted_cell_color 205 | else: 206 | cell.modulate = _standard_cell_color 207 | 208 | 209 | func _on_cell_area_entered(area: Area2D, cell_i: int, cell_j: int): 210 | #prints("_on_cell_area_entered", cell_i, cell_j) 211 | _update_obs(cell_i, cell_j, area.collision_layer, true) 212 | if debug_view: 213 | _toggle_cell(cell_i, cell_j) 214 | #print(_obs_buffer) 215 | 216 | 217 | func _on_cell_area_exited(area: Area2D, cell_i: int, cell_j: int): 218 | #prints("_on_cell_area_exited", cell_i, cell_j) 219 | _update_obs(cell_i, cell_j, area.collision_layer, false) 220 | if debug_view: 221 | _toggle_cell(cell_i, cell_j) 222 | 223 | 224 | func _on_cell_body_entered(body: Node2D, cell_i: int, cell_j: int): 225 | #prints("_on_cell_body_entered", cell_i, cell_j) 226 | _update_obs(cell_i, cell_j, body.collision_layer, true) 227 | if debug_view: 228 | _toggle_cell(cell_i, cell_j) 229 | 230 | 231 | func _on_cell_body_exited(body: Node2D, cell_i: int, cell_j: int): 232 | #prints("_on_cell_body_exited", cell_i, cell_j) 233 | _update_obs(cell_i, cell_j, body.collision_layer, false) 234 | if debug_view: 235 | _toggle_cell(cell_i, cell_j) 236 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sensors/sensors_3d/GridSensor3D.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends ISensor3D 3 | class_name GridSensor3D 4 | 5 | @export var debug_view := false: 6 | get: 7 | return debug_view 8 | set(value): 9 | debug_view = value 10 | _update() 11 | 12 | @export_flags_3d_physics var detection_mask := 0: 13 | get: 14 | return detection_mask 15 | set(value): 16 | detection_mask = value 17 | _update() 18 | 19 | @export var collide_with_areas := false: 20 | get: 21 | return collide_with_areas 22 | set(value): 23 | collide_with_areas = value 24 | _update() 25 | 26 | @export var collide_with_bodies := false: 27 | # NOTE! The sensor will not detect StaticBody3D, add an area to static bodies to detect them 28 | get: 29 | return collide_with_bodies 30 | set(value): 31 | collide_with_bodies = value 32 | _update() 33 | 34 | @export_range(0.1, 2, 0.1) var cell_width := 1.0: 35 | get: 36 | return cell_width 37 | set(value): 38 | cell_width = value 39 | _update() 40 | 41 | @export_range(0.1, 2, 0.1) var cell_height := 1.0: 42 | get: 43 | return cell_height 44 | set(value): 45 | cell_height = value 46 | _update() 47 | 48 | @export_range(1, 21, 1, "or_greater") var grid_size_x := 3: 49 | get: 50 | return grid_size_x 51 | set(value): 52 | grid_size_x = value 53 | _update() 54 | 55 | @export_range(1, 21, 1, "or_greater") var grid_size_z := 3: 56 | get: 57 | return grid_size_z 58 | set(value): 59 | grid_size_z = value 60 | _update() 61 | 62 | var _obs_buffer: PackedFloat64Array 63 | var _box_shape: BoxShape3D 64 | var _collision_mapping: Dictionary 65 | var _n_layers_per_cell: int 66 | 67 | var _highlighted_box_material: StandardMaterial3D 68 | var _standard_box_material: StandardMaterial3D 69 | 70 | 71 | func get_observation(): 72 | return _obs_buffer 73 | 74 | 75 | func reset(): 76 | _obs_buffer.fill(0) 77 | 78 | 79 | func _update(): 80 | if Engine.is_editor_hint(): 81 | if is_node_ready(): 82 | _spawn_nodes() 83 | 84 | 85 | func _ready() -> void: 86 | _make_materials() 87 | 88 | if Engine.is_editor_hint(): 89 | if get_child_count() == 0: 90 | _spawn_nodes() 91 | else: 92 | _spawn_nodes() 93 | 94 | 95 | func _make_materials() -> void: 96 | if _highlighted_box_material != null and _standard_box_material != null: 97 | return 98 | 99 | _standard_box_material = StandardMaterial3D.new() 100 | _standard_box_material.set_transparency(1) # ALPHA 101 | _standard_box_material.albedo_color = Color( 102 | 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0 103 | ) 104 | 105 | _highlighted_box_material = StandardMaterial3D.new() 106 | _highlighted_box_material.set_transparency(1) # ALPHA 107 | _highlighted_box_material.albedo_color = Color( 108 | 255.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0 109 | ) 110 | 111 | 112 | func _get_collision_mapping() -> Dictionary: 113 | # defines which layer is mapped to which cell obs index 114 | var total_bits = 0 115 | var collision_mapping = {} 116 | for i in 32: 117 | var bit_mask = 2 ** i 118 | if (detection_mask & bit_mask) > 0: 119 | collision_mapping[i] = total_bits 120 | total_bits += 1 121 | 122 | return collision_mapping 123 | 124 | 125 | func _spawn_nodes(): 126 | for cell in get_children(): 127 | cell.name = "_%s" % cell.name # Otherwise naming below will fail 128 | cell.queue_free() 129 | 130 | _collision_mapping = _get_collision_mapping() 131 | #prints("collision_mapping", _collision_mapping, len(_collision_mapping)) 132 | # allocate memory for the observations 133 | _n_layers_per_cell = len(_collision_mapping) 134 | _obs_buffer = PackedFloat64Array() 135 | _obs_buffer.resize(grid_size_x * grid_size_z * _n_layers_per_cell) 136 | _obs_buffer.fill(0) 137 | #prints(len(_obs_buffer), _obs_buffer ) 138 | 139 | _box_shape = BoxShape3D.new() 140 | _box_shape.set_size(Vector3(cell_width, cell_height, cell_width)) 141 | 142 | var shift := Vector3( 143 | -(grid_size_x / 2) * cell_width, 144 | 0, 145 | -(grid_size_z / 2) * cell_width, 146 | ) 147 | 148 | for i in grid_size_x: 149 | for j in grid_size_z: 150 | var cell_position = Vector3(i * cell_width, 0.0, j * cell_width) + shift 151 | _create_cell(i, j, cell_position) 152 | 153 | 154 | func _create_cell(i: int, j: int, position: Vector3): 155 | var cell := Area3D.new() 156 | cell.position = position 157 | cell.name = "GridCell %s %s" % [i, j] 158 | 159 | if collide_with_areas: 160 | cell.area_entered.connect(_on_cell_area_entered.bind(i, j)) 161 | cell.area_exited.connect(_on_cell_area_exited.bind(i, j)) 162 | 163 | if collide_with_bodies: 164 | cell.body_entered.connect(_on_cell_body_entered.bind(i, j)) 165 | cell.body_exited.connect(_on_cell_body_exited.bind(i, j)) 166 | 167 | # cell.body_shape_entered.connect(_on_cell_body_shape_entered.bind(i, j)) 168 | # cell.body_shape_exited.connect(_on_cell_body_shape_exited.bind(i, j)) 169 | 170 | cell.collision_layer = 0 171 | cell.collision_mask = detection_mask 172 | cell.monitorable = true 173 | cell.input_ray_pickable = false 174 | add_child(cell) 175 | cell.set_owner(get_tree().edited_scene_root) 176 | 177 | var col_shape := CollisionShape3D.new() 178 | col_shape.shape = _box_shape 179 | col_shape.name = "CollisionShape3D" 180 | cell.add_child(col_shape) 181 | col_shape.set_owner(get_tree().edited_scene_root) 182 | 183 | if debug_view: 184 | var box = MeshInstance3D.new() 185 | box.name = "MeshInstance3D" 186 | var box_mesh = BoxMesh.new() 187 | 188 | box_mesh.set_size(Vector3(cell_width, cell_height, cell_width)) 189 | box_mesh.material = _standard_box_material 190 | 191 | box.mesh = box_mesh 192 | cell.add_child(box) 193 | box.set_owner(get_tree().edited_scene_root) 194 | 195 | 196 | func _update_obs(cell_i: int, cell_j: int, collision_layer: int, entered: bool): 197 | for key in _collision_mapping: 198 | var bit_mask = 2 ** key 199 | if (collision_layer & bit_mask) > 0: 200 | var collison_map_index = _collision_mapping[key] 201 | 202 | var obs_index = ( 203 | (cell_i * grid_size_z * _n_layers_per_cell) 204 | + (cell_j * _n_layers_per_cell) 205 | + collison_map_index 206 | ) 207 | #prints(obs_index, cell_i, cell_j) 208 | if entered: 209 | _obs_buffer[obs_index] += 1 210 | else: 211 | _obs_buffer[obs_index] -= 1 212 | 213 | 214 | func _toggle_cell(cell_i: int, cell_j: int): 215 | var cell = get_node_or_null("GridCell %s %s" % [cell_i, cell_j]) 216 | 217 | if cell == null: 218 | print("cell not found, returning") 219 | 220 | var n_hits = 0 221 | var start_index = (cell_i * grid_size_z * _n_layers_per_cell) + (cell_j * _n_layers_per_cell) 222 | for i in _n_layers_per_cell: 223 | n_hits += _obs_buffer[start_index + i] 224 | 225 | var cell_mesh = cell.get_node_or_null("MeshInstance3D") 226 | if n_hits > 0: 227 | cell_mesh.mesh.material = _highlighted_box_material 228 | else: 229 | cell_mesh.mesh.material = _standard_box_material 230 | 231 | 232 | func _on_cell_area_entered(area: Area3D, cell_i: int, cell_j: int): 233 | #prints("_on_cell_area_entered", cell_i, cell_j) 234 | _update_obs(cell_i, cell_j, area.collision_layer, true) 235 | if debug_view: 236 | _toggle_cell(cell_i, cell_j) 237 | #print(_obs_buffer) 238 | 239 | 240 | func _on_cell_area_exited(area: Area3D, cell_i: int, cell_j: int): 241 | #prints("_on_cell_area_exited", cell_i, cell_j) 242 | _update_obs(cell_i, cell_j, area.collision_layer, false) 243 | if debug_view: 244 | _toggle_cell(cell_i, cell_j) 245 | 246 | 247 | func _on_cell_body_entered(body: Node3D, cell_i: int, cell_j: int): 248 | #prints("_on_cell_body_entered", cell_i, cell_j) 249 | _update_obs(cell_i, cell_j, body.collision_layer, true) 250 | if debug_view: 251 | _toggle_cell(cell_i, cell_j) 252 | 253 | 254 | func _on_cell_body_exited(body: Node3D, cell_i: int, cell_j: int): 255 | #prints("_on_cell_body_exited", cell_i, cell_j) 256 | _update_obs(cell_i, cell_j, body.collision_layer, false) 257 | if debug_view: 258 | _toggle_cell(cell_i, cell_j) 259 | -------------------------------------------------------------------------------- /addons/godot_rl_agents/sync.gd: -------------------------------------------------------------------------------- 1 | extends Node 2 | class_name Sync 3 | 4 | # --fixed-fps 2000 --disable-render-loop 5 | 6 | enum ControlModes { 7 | HUMAN, ## Test the environment manually 8 | TRAINING, ## Train a model 9 | ONNX_INFERENCE ## Load a pretrained model using an .onnx file 10 | } 11 | @export var control_mode: ControlModes = ControlModes.TRAINING 12 | ## Action will be repeated for n frames (Godot physics steps). 13 | @export_range(1, 10, 1, "or_greater") var action_repeat := 8 14 | ## Speeds up the physics in the environment to enable faster training. 15 | @export_range(0, 10, 0.1, "or_greater") var speed_up := 1.0 16 | ## The path to a trained .onnx model file to use for inference (only needed for the 'Onnx Inference' control mode). 17 | @export var onnx_model_path := "" 18 | ## Whether the inference will be deterministic (NOTE: Only applies to discrete actions in onnx inference mode) 19 | @export var deterministic_inference := true 20 | 21 | # Onnx model stored for each requested path 22 | var onnx_models: Dictionary 23 | 24 | @onready var start_time = Time.get_ticks_msec() 25 | 26 | const MAJOR_VERSION := "0" 27 | const MINOR_VERSION := "7" 28 | const DEFAULT_PORT := "11008" 29 | const DEFAULT_SEED := "1" 30 | var stream: StreamPeerTCP = null 31 | var connected = false 32 | var message_center 33 | var should_connect = true 34 | 35 | var all_agents: Array 36 | var agents_training: Array 37 | ## Policy name of each agent, for use with multi-policy multi-agent RL cases 38 | var agents_training_policy_names: Array[String] = ["shared_policy"] 39 | var agents_inference: Array 40 | var agents_heuristic: Array 41 | 42 | ## For recording expert demos 43 | var agent_demo_record: Node 44 | ## File path for writing recorded trajectories 45 | var expert_demo_save_path: String 46 | ## Stores recorded trajectories 47 | var demo_trajectories: Array 48 | ## A trajectory includes obs: Array, acts: Array, terminal (set in Python env instead) 49 | var current_demo_trajectory: Array 50 | 51 | var need_to_send_obs = false 52 | var args = null 53 | var initialized = false 54 | var just_reset = false 55 | var onnx_model = null 56 | var n_action_steps = 0 57 | 58 | var _action_space_training: Array[Dictionary] = [] 59 | var _action_space_inference: Array[Dictionary] = [] 60 | var _obs_space_training: Array[Dictionary] = [] 61 | 62 | 63 | # Called when the node enters the scene tree for the first time. 64 | func _ready(): 65 | await get_parent().ready 66 | get_tree().set_pause(true) 67 | _initialize() 68 | await get_tree().create_timer(1.0).timeout 69 | get_tree().set_pause(false) 70 | 71 | 72 | func _initialize(): 73 | _get_agents() 74 | args = _get_args() 75 | Engine.physics_ticks_per_second = _get_speedup() * 60 # Replace with function body. 76 | Engine.time_scale = _get_speedup() * 1.0 77 | prints( 78 | "physics ticks", 79 | Engine.physics_ticks_per_second, 80 | Engine.time_scale, 81 | _get_speedup(), 82 | speed_up 83 | ) 84 | 85 | _set_heuristic("human", all_agents) 86 | 87 | _initialize_training_agents() 88 | _initialize_inference_agents() 89 | _initialize_demo_recording() 90 | 91 | _set_seed() 92 | _set_action_repeat() 93 | initialized = true 94 | 95 | 96 | func _initialize_training_agents(): 97 | if agents_training.size() > 0: 98 | _obs_space_training.resize(agents_training.size()) 99 | _action_space_training.resize(agents_training.size()) 100 | for agent_idx in range(0, agents_training.size()): 101 | _obs_space_training[agent_idx] = agents_training[agent_idx].get_obs_space() 102 | _action_space_training[agent_idx] = agents_training[agent_idx].get_action_space() 103 | connected = connect_to_server() 104 | if connected: 105 | _set_heuristic("model", agents_training) 106 | _handshake() 107 | _send_env_info() 108 | else: 109 | push_warning( 110 | "Couldn't connect to Python server, using human controls instead. ", 111 | "Did you start the training server using e.g. `gdrl` from the console?" 112 | ) 113 | 114 | 115 | func _initialize_inference_agents(): 116 | if agents_inference.size() > 0: 117 | if control_mode == ControlModes.ONNX_INFERENCE: 118 | assert( 119 | FileAccess.file_exists(onnx_model_path), 120 | "Onnx Model Path set on Sync node does not exist: %s" % onnx_model_path 121 | ) 122 | onnx_models[onnx_model_path] = ONNXModel.new(onnx_model_path, 1) 123 | 124 | for agent in agents_inference: 125 | var action_space = agent.get_action_space() 126 | _action_space_inference.append(action_space) 127 | 128 | var agent_onnx_model: ONNXModel 129 | if agent.onnx_model_path.is_empty(): 130 | assert( 131 | onnx_models.has(onnx_model_path), 132 | ( 133 | "Node %s has no onnx model path set " % agent.get_path() 134 | + "and sync node's control mode is not set to OnnxInference. " 135 | + "Either add the path to the AIController, " 136 | + "or if you want to use the path set on sync node instead, " 137 | + "set control mode to OnnxInference." 138 | ) 139 | ) 140 | prints( 141 | "Info: AIController %s" % agent.get_path(), 142 | "has no onnx model path set.", 143 | "Using path set on the sync node instead." 144 | ) 145 | agent_onnx_model = onnx_models[onnx_model_path] 146 | else: 147 | if not onnx_models.has(agent.onnx_model_path): 148 | assert( 149 | FileAccess.file_exists(agent.onnx_model_path), 150 | ( 151 | "Onnx Model Path set on %s node does not exist: %s" 152 | % [agent.get_path(), agent.onnx_model_path] 153 | ) 154 | ) 155 | onnx_models[agent.onnx_model_path] = ONNXModel.new(agent.onnx_model_path, 1) 156 | agent_onnx_model = onnx_models[agent.onnx_model_path] 157 | 158 | agent.onnx_model = agent_onnx_model 159 | if not agent_onnx_model.action_means_only_set: 160 | agent_onnx_model.set_action_means_only(action_space) 161 | 162 | _set_heuristic("model", agents_inference) 163 | 164 | 165 | func _initialize_demo_recording(): 166 | if agent_demo_record: 167 | expert_demo_save_path = agent_demo_record.expert_demo_save_path 168 | assert( 169 | not expert_demo_save_path.is_empty(), 170 | "Expert demo save path set in %s is empty." % agent_demo_record.get_path() 171 | ) 172 | 173 | InputMap.add_action("RemoveLastDemoEpisode") 174 | InputMap.action_add_event( 175 | "RemoveLastDemoEpisode", agent_demo_record.remove_last_episode_key 176 | ) 177 | current_demo_trajectory.resize(2) 178 | current_demo_trajectory[0] = [] 179 | current_demo_trajectory[1] = [] 180 | agent_demo_record.heuristic = "demo_record" 181 | 182 | 183 | func _physics_process(_delta): 184 | # two modes, human control, agent control 185 | # pause tree, send obs, get actions, set actions, unpause tree 186 | 187 | _demo_record_process() 188 | 189 | if n_action_steps % action_repeat != 0: 190 | n_action_steps += 1 191 | return 192 | 193 | n_action_steps += 1 194 | 195 | _training_process() 196 | _inference_process() 197 | _heuristic_process() 198 | 199 | 200 | func _training_process(): 201 | if connected: 202 | get_tree().set_pause(true) 203 | 204 | var obs = _get_obs_from_agents(agents_training) 205 | var info = _get_info_from_agents(agents_training) 206 | 207 | if just_reset: 208 | just_reset = false 209 | 210 | var reply = {"type": "reset", "obs": obs, "info": info} 211 | _send_dict_as_json_message(reply) 212 | # this should go straight to getting the action and setting it checked the agent, no need to perform one phyics tick 213 | get_tree().set_pause(false) 214 | return 215 | 216 | if need_to_send_obs: 217 | need_to_send_obs = false 218 | var reward = _get_reward_from_agents() 219 | var done = _get_done_from_agents() 220 | #_reset_agents_if_done() # this ensures the new observation is from the next env instance : NEEDS REFACTOR 221 | 222 | var reply = {"type": "step", "obs": obs, "reward": reward, "done": done, "info": info} 223 | _send_dict_as_json_message(reply) 224 | 225 | var handled = handle_message() 226 | 227 | 228 | func _inference_process(): 229 | if agents_inference.size() > 0: 230 | var obs: Array = _get_obs_from_agents(agents_inference) 231 | var actions = [] 232 | 233 | for agent_id in range(0, agents_inference.size()): 234 | var model: ONNXModel = agents_inference[agent_id].onnx_model 235 | var action = model.run_inference(obs[agent_id], 1.0) 236 | var action_dict = _extract_action_dict( 237 | action["output"], _action_space_inference[agent_id], model.action_means_only 238 | ) 239 | actions.append(action_dict) 240 | 241 | _set_agent_actions(actions, agents_inference) 242 | _reset_agents_if_done(agents_inference) 243 | get_tree().set_pause(false) 244 | 245 | 246 | func _demo_record_process(): 247 | if not agent_demo_record: 248 | return 249 | 250 | if Input.is_action_just_pressed("RemoveLastDemoEpisode"): 251 | print("[Sync script][Demo recorder] Removing last recorded episode.") 252 | demo_trajectories.remove_at(demo_trajectories.size() - 1) 253 | print("Remaining episode count: %d" % demo_trajectories.size()) 254 | 255 | if n_action_steps % agent_demo_record.action_repeat != 0: 256 | return 257 | 258 | var obs_dict: Dictionary = agent_demo_record.get_obs() 259 | 260 | # Get the current obs from the agent 261 | assert( 262 | obs_dict.has("obs"), 263 | "Demo recorder needs an 'obs' key in get_obs() returned dictionary to record obs from." 264 | ) 265 | current_demo_trajectory[0].append(obs_dict.obs) 266 | 267 | # Get the action applied for the current obs from the agent 268 | agent_demo_record.set_action() 269 | var acts = agent_demo_record.get_action() 270 | 271 | var terminal = agent_demo_record.get_done() 272 | # Record actions only for non-terminal states 273 | if terminal: 274 | agent_demo_record.set_done_false() 275 | else: 276 | current_demo_trajectory[1].append(acts) 277 | 278 | if terminal: 279 | #current_demo_trajectory[2].append(true) 280 | demo_trajectories.append(current_demo_trajectory.duplicate(true)) 281 | print("[Sync script][Demo recorder] Recorded episode count: %d" % demo_trajectories.size()) 282 | current_demo_trajectory[0].clear() 283 | current_demo_trajectory[1].clear() 284 | 285 | 286 | func _heuristic_process(): 287 | for agent in agents_heuristic: 288 | _reset_agents_if_done(agents_heuristic) 289 | 290 | 291 | func _extract_action_dict(action_array: Array, action_space: Dictionary, action_means_only: bool): 292 | var index = 0 293 | var result = {} 294 | for key in action_space.keys(): 295 | var size = action_space[key]["size"] 296 | var action_type = action_space[key]["action_type"] 297 | 298 | if action_type == "discrete": 299 | var largest_logit: float = -INF # Value of the largest logit for this action in the actions array 300 | var largest_logit_idx: int # Index of the largest logit for this action in the actions array 301 | for logit_idx in range(0, size): 302 | var logit_value = action_array[index + logit_idx] 303 | if logit_value > largest_logit: 304 | largest_logit = logit_value 305 | largest_logit_idx = logit_idx 306 | if deterministic_inference: 307 | result[key] = largest_logit_idx # Index of the largest logit is the discrete action value 308 | else: 309 | var exp_logit_sum: float # Sum of exp of each logit 310 | var exp_logits: Array[float] 311 | 312 | for logit_idx in range(0, size): 313 | # Normalize using the max logit to add stability in case a logit would be huge after exp 314 | exp_logits.append(exp(action_array[index + logit_idx] - largest_logit)) 315 | exp_logit_sum += exp_logits[logit_idx] 316 | 317 | # Choose a random number, will be used to select an action 318 | var random_value = randf_range(0, exp_logit_sum) 319 | 320 | # Select the first index at which the sum is larger than the random number 321 | var sum: float 322 | for exp_logit_idx in exp_logits.size(): 323 | sum += exp_logits[exp_logit_idx] 324 | if sum > random_value: 325 | result[key] = exp_logit_idx 326 | break 327 | index += size 328 | elif action_type == "continuous": 329 | # For continous actions, we only take the action mean values 330 | result[key] = clamp_array(action_array.slice(index, index + size), -1.0, 1.0) 331 | if action_means_only: 332 | index += size # model only outputs action means, so we move index by size 333 | else: 334 | index += size * 2 # model outputs logstd after action mean, we skip the logstd part 335 | 336 | else: 337 | assert( 338 | false, 339 | ( 340 | 'Only "discrete" and "continuous" action types supported. Found: %s action type set.' 341 | % action_type 342 | ) 343 | ) 344 | 345 | return result 346 | 347 | 348 | ## For AIControllers that inherit mode from sync, sets the correct mode. 349 | func _set_agent_mode(agent: Node): 350 | var agent_inherits_mode: bool = agent.control_mode == agent.ControlModes.INHERIT_FROM_SYNC 351 | 352 | if agent_inherits_mode: 353 | match control_mode: 354 | ControlModes.HUMAN: 355 | agent.control_mode = agent.ControlModes.HUMAN 356 | ControlModes.TRAINING: 357 | agent.control_mode = agent.ControlModes.TRAINING 358 | ControlModes.ONNX_INFERENCE: 359 | agent.control_mode = agent.ControlModes.ONNX_INFERENCE 360 | 361 | 362 | func _get_agents(): 363 | all_agents = get_tree().get_nodes_in_group("AGENT") 364 | for agent in all_agents: 365 | _set_agent_mode(agent) 366 | 367 | if agent.control_mode == agent.ControlModes.TRAINING: 368 | agents_training.append(agent) 369 | elif agent.control_mode == agent.ControlModes.ONNX_INFERENCE: 370 | agents_inference.append(agent) 371 | elif agent.control_mode == agent.ControlModes.HUMAN: 372 | agents_heuristic.append(agent) 373 | elif agent.control_mode == agent.ControlModes.RECORD_EXPERT_DEMOS: 374 | assert( 375 | not agent_demo_record, 376 | "Currently only a single AIController can be used for recording expert demos." 377 | ) 378 | agent_demo_record = agent 379 | 380 | var training_agent_count = agents_training.size() 381 | agents_training_policy_names.resize(training_agent_count) 382 | for i in range(0, training_agent_count): 383 | agents_training_policy_names[i] = agents_training[i].policy_name 384 | 385 | 386 | func _set_heuristic(heuristic, agents: Array): 387 | for agent in agents: 388 | agent.set_heuristic(heuristic) 389 | 390 | 391 | func _handshake(): 392 | print("performing handshake") 393 | 394 | var json_dict = _get_dict_json_message() 395 | assert(json_dict["type"] == "handshake") 396 | var major_version = json_dict["major_version"] 397 | var minor_version = json_dict["minor_version"] 398 | if major_version != MAJOR_VERSION: 399 | print("WARNING: major verison mismatch ", major_version, " ", MAJOR_VERSION) 400 | if minor_version != MINOR_VERSION: 401 | print("WARNING: minor verison mismatch ", minor_version, " ", MINOR_VERSION) 402 | 403 | print("handshake complete") 404 | 405 | 406 | func _get_dict_json_message(): 407 | # returns a dictionary from of the most recent message 408 | # this is not waiting 409 | while stream.get_available_bytes() == 0: 410 | stream.poll() 411 | if stream.get_status() != 2: 412 | print("server disconnected status, closing") 413 | get_tree().quit() 414 | return null 415 | 416 | OS.delay_usec(10) 417 | 418 | var message = stream.get_string() 419 | var json_data = JSON.parse_string(message) 420 | 421 | return json_data 422 | 423 | 424 | func _send_dict_as_json_message(dict): 425 | stream.put_string(JSON.stringify(dict, "", false)) 426 | 427 | 428 | func _send_env_info(): 429 | var json_dict = _get_dict_json_message() 430 | assert(json_dict["type"] == "env_info") 431 | 432 | var message = { 433 | "type": "env_info", 434 | "observation_space": _obs_space_training, 435 | "action_space": _action_space_training, 436 | "n_agents": len(agents_training), 437 | "agent_policy_names": agents_training_policy_names 438 | } 439 | _send_dict_as_json_message(message) 440 | 441 | 442 | func connect_to_server(): 443 | print("Waiting for one second to allow server to start") 444 | OS.delay_msec(1000) 445 | print("trying to connect to server") 446 | stream = StreamPeerTCP.new() 447 | 448 | # "localhost" was not working on windows VM, had to use the IP 449 | var ip = "127.0.0.1" 450 | var port = _get_port() 451 | var connect = stream.connect_to_host(ip, port) 452 | stream.set_no_delay(true) # TODO check if this improves performance or not 453 | stream.poll() 454 | # Fetch the status until it is either connected (2) or failed to connect (3) 455 | while stream.get_status() < 2: 456 | stream.poll() 457 | return stream.get_status() == 2 458 | 459 | 460 | func _get_args(): 461 | print("getting command line arguments") 462 | var arguments = {} 463 | for argument in OS.get_cmdline_args(): 464 | print(argument) 465 | if argument.find("=") > -1: 466 | var key_value = argument.split("=") 467 | arguments[key_value[0].lstrip("--")] = key_value[1] 468 | else: 469 | # Options without an argument will be present in the dictionary, 470 | # with the value set to an empty string. 471 | arguments[argument.lstrip("--")] = "" 472 | 473 | return arguments 474 | 475 | 476 | func _get_speedup(): 477 | print(args) 478 | return args.get("speedup", str(speed_up)).to_float() 479 | 480 | 481 | func _get_port(): 482 | return args.get("port", DEFAULT_PORT).to_int() 483 | 484 | 485 | func _set_seed(): 486 | var _seed = args.get("env_seed", DEFAULT_SEED).to_int() 487 | seed(_seed) 488 | 489 | 490 | func _set_action_repeat(): 491 | action_repeat = args.get("action_repeat", str(action_repeat)).to_int() 492 | 493 | 494 | func disconnect_from_server(): 495 | stream.disconnect_from_host() 496 | 497 | 498 | func handle_message() -> bool: 499 | # get json message: reset, step, close 500 | var message = _get_dict_json_message() 501 | if message["type"] == "close": 502 | print("received close message, closing game") 503 | get_tree().quit() 504 | get_tree().set_pause(false) 505 | return true 506 | 507 | if message["type"] == "reset": 508 | print("resetting all agents") 509 | _reset_agents() 510 | just_reset = true 511 | get_tree().set_pause(false) 512 | #print("resetting forcing draw") 513 | # RenderingServer.force_draw() 514 | # var obs = _get_obs_from_agents() 515 | # print("obs ", obs) 516 | # var reply = { 517 | # "type": "reset", 518 | # "obs": obs 519 | # } 520 | # _send_dict_as_json_message(reply) 521 | return true 522 | 523 | if message["type"] == "call": 524 | var method = message["method"] 525 | var returns = _call_method_on_agents(method) 526 | var reply = {"type": "call", "returns": returns} 527 | print("calling method from Python") 528 | _send_dict_as_json_message(reply) 529 | return handle_message() 530 | 531 | if message["type"] == "action": 532 | var action = message["action"] 533 | _set_agent_actions(action, agents_training) 534 | need_to_send_obs = true 535 | get_tree().set_pause(false) 536 | return true 537 | 538 | print("message was not handled") 539 | return false 540 | 541 | 542 | func _call_method_on_agents(method): 543 | var returns = [] 544 | for agent in all_agents: 545 | returns.append(agent.call(method)) 546 | 547 | return returns 548 | 549 | 550 | func _reset_agents_if_done(agents = all_agents): 551 | for agent in agents: 552 | if agent.get_done(): 553 | agent.set_done_false() 554 | 555 | 556 | func _reset_agents(agents = all_agents): 557 | for agent in agents: 558 | agent.needs_reset = true 559 | #agent.reset() 560 | 561 | 562 | func _get_obs_from_agents(agents: Array = all_agents): 563 | var obs = [] 564 | for agent in agents: 565 | obs.append(agent.get_obs()) 566 | return obs 567 | 568 | 569 | func _get_reward_from_agents(agents: Array = agents_training): 570 | var rewards = [] 571 | for agent in agents: 572 | rewards.append(agent.get_reward()) 573 | agent.zero_reward() 574 | return rewards 575 | 576 | 577 | func _get_info_from_agents(agents: Array = all_agents): 578 | var info = [] 579 | for agent in agents: 580 | info.append(agent.get_info()) 581 | return info 582 | 583 | 584 | func _get_done_from_agents(agents: Array = agents_training): 585 | var dones = [] 586 | for agent in agents: 587 | var done = agent.get_done() 588 | if done: 589 | agent.set_done_false() 590 | dones.append(done) 591 | return dones 592 | 593 | 594 | func _set_agent_actions(actions, agents: Array = all_agents): 595 | for i in range(len(actions)): 596 | agents[i].set_action(actions[i]) 597 | 598 | 599 | func clamp_array(arr: Array, min: float, max: float): 600 | var output: Array = [] 601 | for a in arr: 602 | output.append(clamp(a, min, max)) 603 | return output 604 | 605 | 606 | ## Save recorded export demos on window exit (Close game window instead of "Stop" button in Godot Editor) 607 | func _notification(what): 608 | if demo_trajectories.size() == 0 or expert_demo_save_path.is_empty(): 609 | return 610 | 611 | if what == NOTIFICATION_PREDELETE: 612 | var json_string = JSON.stringify(demo_trajectories, "", false) 613 | var file = FileAccess.open(expert_demo_save_path, FileAccess.WRITE) 614 | 615 | if not file: 616 | var error: Error = FileAccess.get_open_error() 617 | assert(not error, "There was an error opening the file: %d" % error) 618 | 619 | file.store_line(json_string) 620 | var error = file.get_error() 621 | assert(not error, "There was an error after trying to write to the file: %d" % error) 622 | --------------------------------------------------------------------------------