├── .gitignore ├── .gitmodules ├── LICENSE ├── MachineLearningRemote.uplugin ├── README.md ├── Resources └── Icon128.png ├── Server └── ServerReadme.md └── Source ├── MLProcess ├── MLProcess.Build.cs ├── Private │ ├── MLProcess.cpp │ └── MLProcessModule.cpp └── Public │ ├── MLProcess.h │ └── MLProcessModule.h ├── MachineLearningBase ├── MachineLearningBase.Build.cs ├── Private │ ├── MachineLearningBase.cpp │ └── MachineLearningBaseComponent.cpp └── Public │ ├── MachineLearningBase.h │ └── MachineLearningBaseComponent.h └── MachineLearningRemote ├── MachineLearningRemote.Build.cs ├── Private ├── MachineLearningRemote.cpp └── MachineLearningRemoteComponent.cpp └── Public ├── MachineLearningRemote.h └── MachineLearningRemoteComponent.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Visual Studio 2015 user specific files 2 | .vs/ 3 | 4 | # Visual Studio 2015 database file 5 | *.VC.db 6 | 7 | # Compiled Object files 8 | *.slo 9 | *.lo 10 | *.o 11 | *.obj 12 | 13 | # Precompiled Headers 14 | *.gch 15 | *.pch 16 | 17 | # Compiled Dynamic libraries 18 | *.so 19 | *.dylib 20 | *.dll 21 | 22 | # Fortran module files 23 | *.mod 24 | 25 | # Compiled Static libraries 26 | *.lai 27 | *.la 28 | *.a 29 | *.lib 30 | 31 | # Executables 32 | *.exe 33 | *.out 34 | *.app 35 | *.ipa 36 | 37 | # These project files can be generated by the engine 38 | *.xcodeproj 39 | *.xcworkspace 40 | *.sln 41 | *.suo 42 | *.opensdf 43 | *.sdf 44 | *.VC.db 45 | *.VC.opendb 46 | 47 | # Precompiled Assets 48 | SourceArt/**/*.png 49 | SourceArt/**/*.tga 50 | 51 | # Binary Files 52 | Binaries/* 53 | Plugins/*/Binaries/* 54 | 55 | # Builds 56 | Build/* 57 | 58 | # Whitelist PakBlacklist-.txt files 59 | !Build/*/ 60 | Build/*/** 61 | !Build/*/PakBlacklist*.txt 62 | 63 | # Don't ignore icon files in Build 64 | !Build/**/*.ico 65 | 66 | # Built data for maps 67 | *_BuiltData.uasset 68 | 69 | # Configuration files generated by the Editor 70 | Saved/* 71 | 72 | # Compiled source files for the engine to use 73 | Intermediate/* 74 | Plugins/*/Intermediate/* 75 | 76 | # Cache files for the editor to use 77 | DerivedDataCache/* 78 | Server/python* 79 | !Server/ServerReadme.md -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "Server/ml-remote-server"] 2 | path = Server/ml-remote-server 3 | url = https://github.com/getnamo/ml-remote-server 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Jan Kaniewski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MachineLearningRemote.uplugin: -------------------------------------------------------------------------------- 1 | { 2 | "FileVersion": 3, 3 | "Version": 1, 4 | "VersionName": "0.9.0", 5 | "FriendlyName": "MachineLearningRemote", 6 | "Description": "Machine Learning plugin for the Unreal Engine, encapsulating calls to remote servers running Tensorflow or Pytorch. Contains base API for many specialized plugin variants to be easily swappable without changing developer bind.", 7 | "Category": "Computing", 8 | "CreatedBy": "Getnamo", 9 | "CreatedByURL": "http://www.getnamo.com", 10 | "DocsURL": "https://github.com/getnamo/machine-learning-remote-ue4", 11 | "MarketplaceURL": "", 12 | "SupportURL": "https://github.com/getnamo/machine-learning-remote-ue4/issues", 13 | "CanContainContent": true, 14 | "IsBetaVersion": false, 15 | "Installed": false, 16 | "Modules": [ 17 | { 18 | "Name": "MachineLearningBase", 19 | "Type": "Runtime", 20 | "LoadingPhase": "Default" 21 | }, 22 | { 23 | "Name": "MachineLearningRemote", 24 | "Type": "Runtime", 25 | "LoadingPhase": "Default" 26 | }, 27 | { 28 | "Name": "MLProcess", 29 | "Type": "Runtime", 30 | "LoadingPhase": "Default" 31 | } 32 | ], 33 | "Plugins": [ 34 | { 35 | "Name": "SocketIOClient", 36 | "Enabled": true 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MachineLearningRemote Unreal Plugin 2 | A Machine Learning (ML) plugin for the Unreal Engine, encapsulating calls to remote python servers running python ML libraries like Tensorflow or Pytorch. Depends on server complement repository: https://github.com/getnamo/ml-remote-server. 3 | 4 | [![GitHub release](https://img.shields.io/github/release/getnamo/MachineLearningRemote-Unreal.svg)](https://github.com/getnamo/MachineLearningRemote-Unreal/releases) 5 | [![Github All Releases](https://img.shields.io/github/downloads/getnamo/MachineLearningRemote-Unreal/total.svg)](https://github.com/getnamo/MachineLearningRemote-Unreal/releases) 6 | 7 | 8 | Should have the same api as [tensorflow-ue4](https://github.com/getnamo/tensorflow-ue4), but with freedom to run a host server on platform of choice (e.g. remote win/linux/mac instances) and without a hard bind to the tensorflow library. 9 | 10 | ## Unreal Machine Learning Plugin Variants 11 | 12 | Want to run tensorflow or pytorch on a remote (or local) python server? 13 | 14 | - https://github.com/getnamo/MachineLearningRemote-Unreal 15 | 16 | Want to use tensorflow python api with a python instance embedded in your unreal engine project? 17 | 18 | - https://github.com/getnamo/TensorFlow-Unreal 19 | 20 | Want native tensorflow inference? (WIP) 21 | 22 | - https://github.com/getnamo/TensorFlowNative-Unreal 23 | 24 | ## Quick Install & Setup 25 | 26 | 1. Install and setup https://github.com/getnamo/ml-remote-server on your target backend (can be a local folder), or setup the one embedded in plugin. 27 | 2. Download [Latest Release](https://github.com/getnamo/MachineLearningRemote-Unreal/releases) 28 | 3. Create new or choose project. 29 | 4. Browse to your project folder (typically found at Documents/Unreal Project/{Your Project Root}) 30 | 5. Copy Plugins folder into your Project root. 31 | 6. Plugin should be now ready to use. Remember to startup your [server](https://github.com/getnamo/ml-remote-server) when using this plugin. 32 | 33 | ## How to use 34 | 35 | ### Blueprint API 36 | 37 | Add a ```MachineLearningRemote``` component to an actor of choice 38 | 39 | ![](https://i.imgur.com/Mx3gNAi.png) 40 | 41 | Change server endpoint and ```DefaultScript``` to fit your use case. ```DefaultScript``` is the file name of your ML script which is placed in your *<[server](https://github.com/getnamo/ml-remote-server)>/scripts* folder. See https://github.com/getnamo/MachineLearningRemote-Unreal#python-api for example scripts. 42 | 43 | ![](https://i.imgur.com/R3YVPtm.png) 44 | 45 | In your script the ```on_setup``` and if ```self.should_train_on_start``` is true ```on_begin_training``` gets called. When your script has trained or it is otherwise ready, you can send inputs to it using ```SendSIOJsonInput``` or other variants (string/raw). 46 | 47 | ![](https://i.imgur.com/WjmFLAu.png) 48 | 49 | Your inputs will be processed on your script side and any value you return from there will be sent back and returned in ```ResultData``` as *USIOJsonValue* in your latent callback. 50 | 51 | #### Other input variants 52 | 53 | See https://github.com/getnamo/MachineLearningRemote-Unreal/blob/master/Source/MachineLearningRemote/Public/MachineLearningRemoteComponent.h for all variants 54 | 55 | #### Custom Function 56 | 57 | Change the ```FunctionName``` parameter in the ```SendSIOJsonInput``` to call a different function name in your script. This name will be used verbatim. 58 | 59 | ### Python API 60 | These scripts should be placed in your *<[server](https://github.com/getnamo/ml-remote-server)>/scripts* folder. If a matching script is defined in your ```MachineLearningRemote```->```DefaultScript``` property it should load on connect. 61 | 62 | Keep in mind that ```tensorflow``` is optional and used as an illustrative example of ML, you can use any other valid python library e.g. pytorch instead without issue. 63 | 64 | See https://github.com/getnamo/ml-remote-server/tree/master/scripts for additional examples. 65 | See https://github.com/getnamo/TensorFlow-Unreal#python-api for more detailed api examples. 66 | 67 | #### empty_example 68 | 69 | Bare bones API example. 70 | 71 | ```python 72 | import tensorflow as tf 73 | from mlpluginapi import MLPluginAPI 74 | 75 | class ExampleAPI(MLPluginAPI): 76 | 77 | #optional api: setup your model for training 78 | def on_setup(self): 79 | pass 80 | 81 | #optional api: parse input object and return a result object, which will be converted to json for UE4 82 | def on_json_input(self, input): 83 | result = {} 84 | return result 85 | 86 | #optional api: start training your network 87 | def on_begin_training(self): 88 | pass 89 | 90 | 91 | #NOTE: this is a module function, not a class function. Change your CLASSNAME to reflect your class 92 | #required function to get our api 93 | def get_api(): 94 | #return CLASSNAME.getInstance() 95 | return ExampleAPI.get_instance() 96 | ``` 97 | #### add_example 98 | 99 | Super basic example showing how to add using the tensorflow library. 100 | 101 | ```python 102 | import tensorflow as tf 103 | import unreal_engine as ue #for remote logging only, this is a proxy import to enable same functionality as local variants 104 | from mlpluginapi import MLPluginAPI 105 | 106 | class ExampleAPI(MLPluginAPI): 107 | 108 | #expected optional api: setup your model for training 109 | def on_setup(self): 110 | self.sess = tf.InteractiveSession() 111 | #self.graph = tf.get_default_graph() 112 | 113 | self.a = tf.placeholder(tf.float32) 114 | self.b = tf.placeholder(tf.float32) 115 | 116 | #operation 117 | self.c = self.a + self.b 118 | 119 | ue.log('setup complete') 120 | pass 121 | 122 | #expected optional api: parse input object and return a result object, which will be converted to json for UE4 123 | def on_json_input(self, json_input): 124 | 125 | ue.log(json_input) 126 | 127 | feed_dict = {self.a: json_input['a'], self.b: json_input['b']} 128 | 129 | raw_result = self.sess.run(self.c, feed_dict) 130 | 131 | ue.log('raw result: ' + str(raw_result)) 132 | 133 | return {'c':raw_result.tolist()} 134 | 135 | #custom function to change the op 136 | def change_operation(self, type): 137 | if(type == '+'): 138 | self.c = self.a + self.b 139 | 140 | elif(type == '-'): 141 | self.c = self.a - self.b 142 | ue.log('operation changed to ' + type) 143 | 144 | 145 | #expected optional api: start training your network 146 | def on_begin_training(self): 147 | pass 148 | 149 | #NOTE: this is a module function, not a class function. Change your CLASSNAME to reflect your class 150 | #required function to get our api 151 | def get_api(): 152 | #return CLASSNAME.get_instance() 153 | return ExampleAPI.get_instance() 154 | ``` 155 | 156 | #### mnist_simple 157 | 158 | One of the most basic ML examples using tensorflow to train a softmax mnist recognizer. 159 | 160 | ```python 161 | #Converted to ue4 use from: https://www.tensorflow.org/get_started/mnist/beginners 162 | #mnist_softmax.py: https://github.com/tensorflow/tensorflow/blob/r1.1/tensorflow/examples/tutorials/mnist/mnist_softmax.py 163 | 164 | # Import data 165 | from tensorflow.examples.tutorials.mnist import input_data 166 | 167 | import tensorflow as tf 168 | import unreal_engine as ue 169 | from mlpluginapi import MLPluginAPI 170 | 171 | import operator 172 | 173 | class MnistSimple(MLPluginAPI): 174 | 175 | #expected api: storedModel and session, json inputs 176 | def on_json_input(self, jsonInput): 177 | #expect an image struct in json format 178 | pixelarray = jsonInput['pixels'] 179 | ue.log('image len: ' + str(len(pixelarray))) 180 | 181 | #embedd the input image pixels as 'x' 182 | feed_dict = {self.model['x']: [pixelarray]} 183 | 184 | result = self.sess.run(self.model['y'], feed_dict) 185 | 186 | #convert our raw result to a prediction 187 | index, value = max(enumerate(result[0]), key=operator.itemgetter(1)) 188 | 189 | ue.log('max: ' + str(value) + 'at: ' + str(index)) 190 | 191 | #set the prediction result in our json 192 | jsonInput['prediction'] = index 193 | 194 | return jsonInput 195 | 196 | #expected api: no params forwarded for training? TBC 197 | def on_begin_training(self): 198 | 199 | ue.log("starting mnist simple training") 200 | 201 | self.scripts_path = ue.get_content_dir() + "Scripts" 202 | self.data_dir = self.scripts_path + '/dataset/mnist' 203 | 204 | mnist = input_data.read_data_sets(self.data_dir) 205 | 206 | # Create the model 207 | x = tf.placeholder(tf.float32, [None, 784]) 208 | W = tf.Variable(tf.zeros([784, 10])) 209 | b = tf.Variable(tf.zeros([10])) 210 | y = tf.matmul(x, W) + b 211 | 212 | # Define loss and optimizer 213 | y_ = tf.placeholder(tf.int64, [None]) 214 | 215 | # The raw formulation of cross-entropy, 216 | # 217 | # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), 218 | # reduction_indices=[1])) 219 | # 220 | # can be numerically unstable. 221 | # 222 | # So here we use tf.losses.sparse_softmax_cross_entropy on the raw 223 | # outputs of 'y', and then average across the batch. 224 | cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y) 225 | train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) 226 | 227 | #update session for this thread 228 | self.sess = tf.InteractiveSession() 229 | tf.global_variables_initializer().run() 230 | 231 | # Train 232 | for i in range(1000): 233 | batch_xs, batch_ys = mnist.train.next_batch(100) 234 | self.sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) 235 | if i % 100 == 0: 236 | ue.log(i) 237 | if(self.should_stop): 238 | ue.log('early break') 239 | break 240 | 241 | # Test trained model 242 | correct_prediction = tf.equal(tf.argmax(y, 1), y_) 243 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 244 | finalAccuracy = self.sess.run(accuracy, feed_dict={x: mnist.test.images, 245 | y_: mnist.test.labels}) 246 | ue.log('final training accuracy: ' + str(finalAccuracy)) 247 | 248 | #return trained model 249 | self.model = {'x':x, 'y':y, 'W':W,'b':b} 250 | 251 | #store optional summary information 252 | self.summary = {'x':str(x), 'y':str(y), 'W':str(W), 'b':str(b)} 253 | 254 | self.stored['summary'] = self.summary 255 | return self.stored 256 | 257 | #required function to get our api 258 | def get_api(): 259 | #return CLASSNAME.get_instance() 260 | return MnistSimple.get_instance() 261 | ``` 262 | 263 | 264 | ### C++ API 265 | 266 | Available since 0.3.1. 267 | 268 | Same as blueprint API except for one additional callback variant. Use the lambda overloaded functions e.g. assuming you have a component defined as 269 | ```c++ 270 | UMachineLearningRemoteComponent* MLComponent; //NB: this needs to be allocated with NewObject or CreateDefaultSubobject 271 | ``` 272 | 273 | #### SendRawInput 274 | ```c++ 275 | //Let's say you want to send some raw data 276 | TArray InputData; 277 | //... fill 278 | 279 | MLComponent->SendRawInput(InputData, [this](TArray& ResultData) 280 | { 281 | //Now we got our results back, do something with them here 282 | }, FunctionName); 283 | ``` 284 | 285 | #### SendStringInput 286 | 287 | Keep in mind that if you're using USIOJConvert utilities you'll need to add *SIOJson*, and *Json* as dependency modules in your project build.cs. 288 | 289 | ```c# 290 | PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "Json", "SIOJson" }); 291 | ``` 292 | 293 | Sending just a String 294 | 295 | ```c++ 296 | FString InputString = TEXT("Some Data"); 297 | 298 | MLComponent->SendStringInput(InputString, [this](const FString& ResultData) 299 | { 300 | //e.g. just print the result 301 | UE_LOG(LogTemp, Log, TEXT("Got some results: %s"), *ResultData); 302 | }, FunctionName); 303 | ``` 304 | 305 | A custom JsonObject 306 | 307 | ```c++ 308 | //Make an object {"myKey":"myValue"} 309 | TSharedPtr JsonObject = MakeShareable(new FJsonObject); 310 | JsonObject->SetStringField(TEXT("myKey"), TEXT("myValue")); 311 | FString InputString = USIOJConvert::ToJsonString(JsonObject); 312 | 313 | MLComponent->SendStringInput(InputString, [this](const FString& ResultData) 314 | { 315 | //assuming you got a json string response we could query it, e.g. assume {"someNumber":5} 316 | TSharedPtr JsonObject = USIOJConvert::ToJsonObject(ResultData); 317 | double MyNumber = JsonObject->GetNumberField("someNumber"); 318 | 319 | //do something with your number result 320 | }, FunctionName); 321 | ``` 322 | 323 | Structs via Json 324 | ```c++ 325 | //Let's say you want to send some struct data in json format 326 | 327 | USTRUCT() 328 | struct FTestCppStruct 329 | { 330 | GENERATED_BODY() 331 | 332 | UPROPERTY() 333 | int32 Index; 334 | 335 | UPROPERTY() 336 | float SomeNumber; 337 | 338 | UPROPERTY() 339 | FString Name; 340 | }; 341 | 342 | //... 343 | 344 | FTestCppStruct TestStruct; 345 | TestStruct.Name = TEXT("George"); 346 | TestStruct.Index = 5; 347 | TestStruct.SomeNumber = 5.123f; 348 | FString StructJsonString = USIOJConvert::ToJsonString(USIOJConvert::ToJsonObject(FTestCppStruct::StaticStruct(), &TestStruct)); 349 | 350 | //In this example we're using the same struct type for the result, but you could use a different one or custom Json 351 | FTestCppStruct ResultStruct; 352 | 353 | MLComponent->SendStringInput(StructJsonString, [this, &ResultStruct](const FString& ResultData) 354 | { 355 | //do something with the result, let's say we we have another struct of same type and we'd like to fill it with the results 356 | USIOJConvert::JsonObjectToUStruct(USIOJConvert::ToJsonObject(ResultData), FTestCppStruct::StaticStruct(), &ResultStruct); 357 | }, FunctionName); 358 | ``` 359 | -------------------------------------------------------------------------------- /Resources/Icon128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getnamo/MachineLearningRemote-Unreal/5e34ac2a738bf91798eba24fe83b5b92edf8bcec/Resources/Icon128.png -------------------------------------------------------------------------------- /Server/ServerReadme.md: -------------------------------------------------------------------------------- 1 | ## Server 2 | 3 | For embedded server versions there will be ```pythonv.v.v``` and ```ml-remote-server``` folder present. These are generally .gitignored otherwise -------------------------------------------------------------------------------- /Source/MLProcess/MLProcess.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | 5 | public class MLProcess: ModuleRules 6 | { 7 | public MLProcess(ReadOnlyTargetRules Target) : base(Target) 8 | { 9 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 10 | 11 | PublicIncludePaths.AddRange( 12 | new string[] { 13 | // ... add public include paths required here ... 14 | } 15 | ); 16 | 17 | 18 | PrivateIncludePaths.AddRange( 19 | new string[] { 20 | // ... add other private include paths required here ... 21 | } 22 | ); 23 | 24 | 25 | PublicDependencyModuleNames.AddRange( 26 | new string[] 27 | { 28 | "Core", 29 | // ... add other public dependencies that you statically link with here ... 30 | } 31 | ); 32 | 33 | 34 | PrivateDependencyModuleNames.AddRange( 35 | new string[] 36 | { 37 | "CoreUObject", 38 | "Engine", 39 | // ... add private dependencies that you statically link with here ... 40 | } 41 | ); 42 | 43 | 44 | DynamicallyLoadedModuleNames.AddRange( 45 | new string[] 46 | { 47 | // ... add any modules that your module loads dynamically here ... 48 | } 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Source/MLProcess/Private/MLProcess.cpp: -------------------------------------------------------------------------------- 1 | #include "MLProcess.h" 2 | #include "GenericPlatform/GenericPlatformProcess.h" 3 | 4 | #if PLATFORM_WINDOWS 5 | #include "Windows/WindowsHWrapper.h" 6 | #endif 7 | 8 | FMLProcess::FMLProcess() 9 | { 10 | 11 | } 12 | 13 | FMLProcess::~FMLProcess() 14 | { 15 | if (IsRunning()) 16 | { 17 | Terminate(); 18 | } 19 | Close(); 20 | } 21 | 22 | 23 | TSharedPtr FMLProcess::Create(const FString& URL, const FString& Parms, bool bLaunchDetached, bool bLaunchHidden, bool bLaunchReallyHidden, int32 PriorityModifier, const FString& OptionalWorkingDirectory, bool bUsePipe) 24 | { 25 | uint32 ProcessID; 26 | void* ReadPipe{ nullptr }; 27 | void* WritePipe{ nullptr }; 28 | if (bUsePipe) 29 | { 30 | FPlatformProcess::CreatePipe(ReadPipe, WritePipe); 31 | } 32 | auto Handle = FPlatformProcess::CreateProc(*URL, *Parms, bLaunchDetached, bLaunchHidden, bLaunchReallyHidden, &ProcessID, PriorityModifier, OptionalWorkingDirectory.Len() ? *OptionalWorkingDirectory : nullptr, WritePipe); 33 | if (Handle.IsValid()) 34 | { 35 | TSharedPtr Instance = MakeShareable(new FMLProcess()); 36 | Instance->ProcessHandle = Handle; 37 | Instance->ProcessID = ProcessID; 38 | Instance->ReadPipe = ReadPipe; 39 | Instance->WritePipe = WritePipe; 40 | return Instance; 41 | } 42 | else 43 | { 44 | return nullptr; 45 | } 46 | } 47 | 48 | TSharedPtr FMLProcess::Open(const FString& ProcName) 49 | { 50 | #if PLATFORM_WINDOWS 51 | FPlatformProcess::FProcEnumerator ProcEnumerator; 52 | 53 | while (ProcEnumerator.MoveNext()) 54 | { 55 | auto Current = ProcEnumerator.GetCurrent(); 56 | if (Current.GetName() == ProcName) 57 | { 58 | return Open_PID(Current.GetPID()); 59 | } 60 | } 61 | #endif 62 | return nullptr; 63 | } 64 | 65 | TSharedPtr FMLProcess::Open_PID(int32 ProcessId) 66 | { 67 | #if PLATFORM_WINDOWS 68 | auto Handle = FPlatformProcess::OpenProcess(ProcessId); 69 | if (Handle.IsValid()) 70 | { 71 | TSharedPtr Instance = MakeShareable(new FMLProcess()); 72 | Instance->ProcessHandle = Handle; 73 | Instance->ProcessID = ProcessId; 74 | /*Instance->ReadPipe = ReadPipe; 75 | Instance->WritePipe = WritePipe;*/ 76 | return Instance; 77 | } 78 | #endif 79 | return nullptr; 80 | } 81 | 82 | FString FMLProcess::GetApplicationName(int32 ProcessId) 83 | { 84 | return FPlatformProcess::GetApplicationName(ProcessId); 85 | } 86 | 87 | int32 FMLProcess::GetCurrentProcessId() 88 | { 89 | return (int32)FPlatformProcess::GetCurrentProcessId(); 90 | } 91 | 92 | bool FMLProcess::IsApplicationRunning_PID(int32 ProcessId) 93 | { 94 | return FPlatformProcess::IsApplicationRunning(ProcessId); 95 | } 96 | 97 | bool FMLProcess::IsApplicationRunning(const FString& ProcName) 98 | { 99 | return FPlatformProcess::IsApplicationRunning(*ProcName); 100 | } 101 | 102 | bool FMLProcess::IsRunning() 103 | { 104 | return FPlatformProcess::IsProcRunning(ProcessHandle); 105 | } 106 | 107 | void FMLProcess::Close() 108 | { 109 | FPlatformProcess::CloseProc(ProcessHandle); 110 | 111 | if (ReadPipe || WritePipe) 112 | { 113 | FPlatformProcess::ClosePipe(ReadPipe, WritePipe); 114 | ReadPipe = WritePipe = nullptr; 115 | } 116 | } 117 | void FMLProcess::Sleep(float Seconds) 118 | { 119 | FPlatformProcess::Sleep(Seconds); 120 | } 121 | 122 | FString FMLProcess::ReadFromPipe() 123 | { 124 | return FPlatformProcess::ReadPipe(ReadPipe); 125 | } 126 | 127 | bool FMLProcess::ReadArrayFromPipe(TArray& Array) 128 | { 129 | return FPlatformProcess::ReadPipeToArray(ReadPipe, Array); 130 | } 131 | 132 | bool FMLProcess::WriteToPipe(const FString& Message, FString& OutWritten) 133 | { 134 | return FPlatformProcess::WritePipe(WritePipe, Message, &OutWritten); 135 | } 136 | 137 | void FMLProcess::SimulateKeypress(int32 KeyEvent) 138 | { 139 | #if PLATFORM_WINDOWS 140 | INPUT input; 141 | WORD vkey = KeyEvent; 142 | input.type = INPUT_KEYBOARD; 143 | input.ki.time = 0; 144 | input.ki.dwExtraInfo = 0; 145 | input.ki.wVk = vkey; 146 | input.ki.dwFlags = KEYEVENTF_UNICODE; 147 | SendInput(1, &input, sizeof(INPUT)); 148 | 149 | input.ki.dwFlags = KEYEVENTF_UNICODE | KEYEVENTF_KEYUP; 150 | SendInput(1, &input, sizeof(INPUT)); 151 | #endif 152 | } 153 | 154 | void FMLProcess::Wait() 155 | { 156 | return FPlatformProcess::WaitForProc(ProcessHandle); 157 | } 158 | 159 | void FMLProcess::Terminate(bool KillTree) 160 | { 161 | return FPlatformProcess::TerminateProc(ProcessHandle, KillTree); 162 | } 163 | 164 | bool FMLProcess::GetReturnCode(int32& ReturnCode) 165 | { 166 | return FPlatformProcess::GetProcReturnCode(ProcessHandle, &ReturnCode); 167 | } 168 | 169 | void FMLProcess::SetEnvironmentVar(const FString& VarName, const FString& VarValue) 170 | { 171 | FPlatformMisc::SetEnvironmentVar(*VarName, *VarValue); 172 | } 173 | 174 | FString FMLProcess::GetEnvironmentVar(const FString& VarName) 175 | { 176 | return FPlatformMisc::GetEnvironmentVariable(*VarName); 177 | } 178 | 179 | void FMLProcess::LaunchURL(const FString& URL, const FString& Parms, FString& Error) 180 | { 181 | FPlatformProcess::LaunchURL(*URL, *Parms, &Error); 182 | } 183 | 184 | bool FMLProcess::CanLaunchURL(const FString& URL) 185 | { 186 | return FPlatformProcess::CanLaunchURL(*URL); 187 | } 188 | 189 | FString FMLProcess::GetString(const FString& Key, bool bFlag) 190 | { 191 | if (Key == TEXT("BaseDir")) return FPlatformProcess::BaseDir(); 192 | if (Key == TEXT("UserDir")) return FPlatformProcess::UserDir(); 193 | if (Key == TEXT("UserSettingsDir")) return FPlatformProcess::UserSettingsDir(); 194 | if (Key == TEXT("UserTempDir")) return FPlatformProcess::UserTempDir(); 195 | if (Key == TEXT("ApplicationSettingsDir")) return FPlatformProcess::ApplicationSettingsDir(); 196 | if (Key == TEXT("ComputerName")) return FPlatformProcess::ComputerName(); 197 | if (Key == TEXT("UserName")) return FPlatformProcess::UserName(bFlag); 198 | if (Key == TEXT("ShaderDir")) return FPlatformProcess::ShaderDir(); 199 | if (Key == TEXT("CurrentWorkingDirectory")) return FPlatformProcess::GetCurrentWorkingDirectory(); 200 | if (Key == TEXT("ShaderWorkingDir")) return FPlatformProcess::ShaderWorkingDir(); 201 | if (Key == TEXT("ExecutableName")) return FPlatformProcess::ExecutableName(bFlag); 202 | if (Key == TEXT("ModulePrefix")) return FPlatformProcess::GetModulePrefix(); 203 | if (Key == TEXT("ModuleExtension")) return FPlatformProcess::GetModuleExtension(); 204 | if (Key == TEXT("BinariesSubdirectory")) return FPlatformProcess::GetBinariesSubdirectory(); 205 | if (Key == TEXT("ModulesDirectory")) return FPlatformProcess::GetModulesDirectory(); 206 | return FString(); 207 | } -------------------------------------------------------------------------------- /Source/MLProcess/Private/MLProcessModule.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | #include "MLProcessModule.h" 4 | 5 | #define LOCTEXT_NAMESPACE "FMachineLearningRemoteModule" 6 | 7 | void FMLProcessModule::StartupModule() 8 | { 9 | // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module 10 | } 11 | 12 | void FMLProcessModule::ShutdownModule() 13 | { 14 | // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, 15 | // we call this function before unloading the module. 16 | } 17 | 18 | #undef LOCTEXT_NAMESPACE 19 | 20 | IMPLEMENT_MODULE(FMLProcessModule, MLProcess) -------------------------------------------------------------------------------- /Source/MLProcess/Public/MLProcess.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | //Wrapper process class largely inspired from unreal.js's JavascriptProcess which is licensed under BSD 3-clause 4 | class MLPROCESS_API FMLProcess 5 | { 6 | public: 7 | 8 | FMLProcess(); 9 | ~FMLProcess(); 10 | 11 | FProcHandle ProcessHandle; 12 | uint32 ProcessID; 13 | void* ReadPipe; 14 | void* WritePipe; 15 | 16 | static TSharedPtr Create(const FString& URL, const FString& Parms, bool bLaunchDetached, bool bLaunchHidden, bool bLaunchReallyHidden, int32 PriorityModifier, const FString& OptionalWorkingDirectory, bool bUsePipe); 17 | 18 | static TSharedPtr Open(const FString& ProcName); 19 | 20 | static TSharedPtr Open_PID(int32 ProcessId); 21 | 22 | static FString GetApplicationName(int32 ProcessId); 23 | 24 | static int32 GetCurrentProcessId(); 25 | 26 | static bool IsApplicationRunning_PID(int32 ProcessId); 27 | 28 | static bool IsApplicationRunning(const FString& ProcName); 29 | 30 | static void SetEnvironmentVar(const FString& VarName, const FString& VarValue); 31 | 32 | static FString GetEnvironmentVar(const FString& VarName); 33 | 34 | bool IsRunning(); 35 | 36 | void Close(); 37 | 38 | static void Sleep(float Seconds); 39 | 40 | void Wait(); 41 | 42 | void Terminate(bool KillTree = false); 43 | 44 | bool GetReturnCode(int32& ReturnCode); 45 | 46 | FString ReadFromPipe(); 47 | 48 | bool ReadArrayFromPipe(TArray& Array); 49 | 50 | bool WriteToPipe(const FString& Message, FString& OutWritten); 51 | 52 | static void SimulateKeypress(int32 KeyEvent); 53 | 54 | static void LaunchURL(const FString& URL, const FString& Parms, FString& Error); 55 | 56 | static bool CanLaunchURL(const FString& URL); 57 | 58 | static FString GetString(const FString& Key, bool bFlag); 59 | }; -------------------------------------------------------------------------------- /Source/MLProcess/Public/MLProcessModule.h: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Modules/ModuleManager.h" 7 | 8 | class FMLProcessModule : public IModuleInterface 9 | { 10 | public: 11 | 12 | /** IModuleInterface implementation */ 13 | virtual void StartupModule() override; 14 | virtual void ShutdownModule() override; 15 | }; 16 | -------------------------------------------------------------------------------- /Source/MachineLearningBase/MachineLearningBase.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | 5 | public class MachineLearningBase : ModuleRules 6 | { 7 | public MachineLearningBase(ReadOnlyTargetRules Target) : base(Target) 8 | { 9 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 10 | 11 | PublicIncludePaths.AddRange( 12 | new string[] { 13 | // ... add public include paths required here ... 14 | } 15 | ); 16 | 17 | 18 | PrivateIncludePaths.AddRange( 19 | new string[] { 20 | // ... add other private include paths required here ... 21 | } 22 | ); 23 | 24 | 25 | PublicDependencyModuleNames.AddRange( 26 | new string[] 27 | { 28 | "Core", 29 | // ... add other public dependencies that you statically link with here ... 30 | } 31 | ); 32 | 33 | 34 | PrivateDependencyModuleNames.AddRange( 35 | new string[] 36 | { 37 | "CoreUObject", 38 | "Engine", 39 | "Slate", 40 | "SlateCore", 41 | "Json", 42 | // ... add private dependencies that you statically link with here ... 43 | } 44 | ); 45 | 46 | 47 | DynamicallyLoadedModuleNames.AddRange( 48 | new string[] 49 | { 50 | // ... add any modules that your module loads dynamically here ... 51 | } 52 | ); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Source/MachineLearningBase/Private/MachineLearningBase.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | #include "MachineLearningBase.h" 4 | 5 | DEFINE_LOG_CATEGORY(MLBaseLog); 6 | 7 | #define LOCTEXT_NAMESPACE "FMachineLearningBaseModule" 8 | 9 | void FMachineLearningBaseModule::StartupModule() 10 | { 11 | // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module 12 | } 13 | 14 | void FMachineLearningBaseModule::ShutdownModule() 15 | { 16 | // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, 17 | // we call this function before unloading the module. 18 | } 19 | 20 | #undef LOCTEXT_NAMESPACE 21 | 22 | IMPLEMENT_MODULE(FMachineLearningBaseModule, MachineLearningBase) -------------------------------------------------------------------------------- /Source/MachineLearningBase/Private/MachineLearningBaseComponent.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #include "MachineLearningBaseComponent.h" 4 | #include "MachineLearningBase.h" 5 | #include "Engine/Engine.h" 6 | #include "Runtime/Engine/Public/LatentActions.h" 7 | 8 | UMachineLearningBaseComponent::UMachineLearningBaseComponent() 9 | { 10 | PrimaryComponentTick.bCanEverTick = false; 11 | bIsConnectedToBackend = false; 12 | } 13 | 14 | void UMachineLearningBaseComponent::SendStringInput(const FString& InputData, const FString& FunctionName /*= TEXT("Default")*/) 15 | { 16 | FString ResultData = TEXT("SendInput not implemented"); 17 | UE_LOG(MLBaseLog, Warning, TEXT("%s"), *ResultData); 18 | OnInputResult.Broadcast(ResultData, FunctionName); 19 | } 20 | 21 | void UMachineLearningBaseComponent::SendStringInput(const FString& InputData, TFunction ResultCallback, const FString& FunctionName /*= TEXT("onJsonInput")*/) 22 | { 23 | UE_LOG(MLBaseLog, Warning, TEXT("SendStringInputLambdaOverload not implemented")); 24 | if (ResultCallback) 25 | { 26 | ResultCallback(TEXT("")); 27 | } 28 | } 29 | 30 | void UMachineLearningBaseComponent::SendRawInput(const TArray& InputData, const FString& FunctionName /*= TEXT("onFloatArrayInput")*/) 31 | { 32 | UE_LOG(MLBaseLog, Warning, TEXT("SendRawInput not implemented")); 33 | 34 | TArray ResultData; 35 | OnRawInputResult.Broadcast(ResultData, FunctionName); 36 | } 37 | 38 | void UMachineLearningBaseComponent::SendRawInput(const TArray& InputData, TFunction& ResultData)> ResultCallback /*= nullptr*/, const FString& FunctionName /*= TEXT("onFloatArrayInput")*/) 39 | { 40 | UE_LOG(MLBaseLog, Warning, TEXT("SendRawInputLambdaOverload not implemented")); 41 | if (ResultCallback) 42 | { 43 | TArray ResultData; 44 | ResultCallback(ResultData); 45 | } 46 | } 47 | 48 | void UMachineLearningBaseComponent::SendStringInputGraphCallback(const FString& InputData, FString& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName /*= TEXT("Default")*/) 49 | { 50 | ResultData = TEXT("SendInputGraphResult not implemented"); 51 | 52 | UE_LOG(MLBaseLog, Warning, TEXT("%s"), *ResultData); 53 | 54 | ImmediateLatentResponse(LatentInfo); 55 | } 56 | 57 | void UMachineLearningBaseComponent::SendRawInputGraphCallback(const TArray& InputData, TArray& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName /*= TEXT("onJsonInput")*/) 58 | { 59 | UE_LOG(MLBaseLog, Warning, TEXT("SendRawInputGraphResult not implemented")); 60 | ImmediateLatentResponse(LatentInfo); 61 | } 62 | 63 | void UMachineLearningBaseComponent::ImmediateLatentResponse(struct FLatentActionInfo LatentInfo) 64 | { 65 | //Create a dummy pending latent action that returns in next tick 66 | UWorld* World = GEngine->GetWorldFromContextObject(this, EGetWorldErrorMode::LogAndReturnNull); 67 | if (!World) 68 | { 69 | return; 70 | } 71 | 72 | FLatentActionManager& LatentActionManager = World->GetLatentActionManager(); 73 | FPendingLatentAction *LatentAction = LatentActionManager.FindExistingAction(LatentInfo.CallbackTarget, LatentInfo.UUID); 74 | LatentAction = new FPendingLatentAction(); //safe to use new since latentactionmanager will delete it 75 | int32 UUID = LatentInfo.UUID; 76 | LatentActionManager.AddNewAction(LatentInfo.CallbackTarget, LatentInfo.UUID, LatentAction); 77 | } -------------------------------------------------------------------------------- /Source/MachineLearningBase/Public/MachineLearningBase.h: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Modules/ModuleManager.h" 7 | 8 | DECLARE_LOG_CATEGORY_EXTERN(MLBaseLog, Log, All); 9 | 10 | class FMachineLearningBaseModule : public IModuleInterface 11 | { 12 | public: 13 | 14 | /** IModuleInterface implementation */ 15 | virtual void StartupModule() override; 16 | virtual void ShutdownModule() override; 17 | }; 18 | -------------------------------------------------------------------------------- /Source/MachineLearningBase/Public/MachineLearningBaseComponent.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Components/ActorComponent.h" 7 | #include "Runtime/Engine/Classes/Engine/LatentActionManager.h" 8 | #include "Runtime/Json/Public/Dom/JsonValue.h" 9 | #include "MachineLearningBaseComponent.generated.h" 10 | 11 | /** Struct for combining data string and function for auto-serialized input */ 12 | USTRUCT() 13 | struct FMLSendStringObject 14 | { 15 | GENERATED_USTRUCT_BODY() 16 | 17 | UPROPERTY() 18 | FString InputData; 19 | 20 | UPROPERTY() 21 | FString TargetFunction; 22 | }; 23 | 24 | 25 | /** Wraps a float array in a UStruct for auto-serialization. May not be performant */ 26 | USTRUCT() 27 | struct FMLSendRawObject 28 | { 29 | GENERATED_USTRUCT_BODY() 30 | 31 | UPROPERTY() 32 | TArray InputData; 33 | 34 | UPROPERTY() 35 | FString TargetFunction; 36 | }; 37 | 38 | 39 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMLLogSignature, FString, Text); 40 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMLConnectionSignature, FString, Endpoint); 41 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMLResultSignature, FString, ResultData, FString, CallingFunctionName); 42 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMLRawResultSignature, TArray, ResultData, FString, CallingFunctionName); 43 | 44 | /** 45 | * Base component for both remote and standard MachineLearning components. 46 | * Considered abstract, ensures all MachineLearning components have same base API 47 | */ 48 | UCLASS(ClassGroup = Computing) 49 | class MACHINELEARNINGBASE_API UMachineLearningBaseComponent : public UActorComponent 50 | { 51 | GENERATED_BODY() 52 | 53 | public: 54 | 55 | UMachineLearningBaseComponent(); 56 | 57 | /** 58 | * When the given setup gets a connection to a backend (e.g. server for remote, library for native). 59 | * This is a good place to do initialization type calls instead of BeginPlay. 60 | */ 61 | UPROPERTY(BlueprintAssignable, Category = MachineLearningEvents) 62 | FMLConnectionSignature OnConnectedToBackend; 63 | 64 | UPROPERTY(BlueprintAssignable, Category = MachineLearningEvents) 65 | FMLConnectionSignature OnDisconnectedFromBackend; 66 | 67 | UPROPERTY(BlueprintAssignable, Category = MachineLearningEvents) 68 | FMLLogSignature OnLog; 69 | 70 | /** SendInput variant will return data to this event. */ 71 | UPROPERTY(BlueprintAssignable, Category = MachineLearningEvents) 72 | FMLResultSignature OnInputResult; 73 | 74 | UPROPERTY(BlueprintAssignable, Category = MachineLearningEvents) 75 | FMLRawResultSignature OnRawInputResult; 76 | 77 | /** What this means is different for each sub-type */ 78 | UPROPERTY(BlueprintReadOnly, Category = MachineLearningProperties) 79 | bool bIsConnectedToBackend; 80 | 81 | /** Send input to ML side result comes back to the OnResult event. Optionally re-target to another function name. */ 82 | UFUNCTION(BlueprintCallable, Category = MachineLearningFunctions) 83 | virtual void SendStringInput(const FString& InputData, const FString& FunctionName = TEXT("onJsonInput")); 84 | 85 | /** Send float array input, bypasses encoding. Useful for large data/native inference, may not work in remote context. Result comes back to the OnRawResult event*/ 86 | UFUNCTION(BlueprintCallable, Category = MachineLearningFunctions) 87 | virtual void SendRawInput(const TArray& InputData, const FString& FunctionName = TEXT("onFloatArrayInput")); 88 | 89 | /** BP Only. Send input to ML side result will come back as a latent action in the graph. Recommended method. Optionally re-target to another function name. */ 90 | UFUNCTION(BlueprintCallable, meta = (Latent, LatentInfo = "LatentInfo"), Category = MachineLearningFunctions) 91 | virtual void SendStringInputGraphCallback(const FString& InputData, FString& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName = TEXT("onJsonInput")); 92 | 93 | /** BP Only. Send float array input, bypasses encoding. Useful for large data/native inference, may not work in remote context. Result will come back as a latent action in the graph.*/ 94 | UFUNCTION(BlueprintCallable, meta = (Latent, LatentInfo = "LatentInfo"), Category = MachineLearningFunctions) 95 | virtual void SendRawInputGraphCallback(const TArray& InputData, TArray& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName = TEXT("onFloatArrayInput")); 96 | 97 | //C++ Only. Native lambda callback variant, lambda must be specified to differentiate it from blueprint variant 98 | virtual void SendStringInput(const FString& InputData, TFunction ResultCallback, const FString& FunctionName = TEXT("onJsonInput")); 99 | virtual void SendRawInput(const TArray& InputData, TFunction& ResultData)> ResultCallback, const FString& FunctionName = TEXT("onFloatArrayInput")); 100 | 101 | //convenience variant, to be added later 102 | //virtual void SendJsonInput(TSharedPtr InputData, TFunction ResultData)> ResultCallback, const FString& FunctionName = TEXT("onJsonInput")); 103 | 104 | private: 105 | void ImmediateLatentResponse(struct FLatentActionInfo LatentInfo); 106 | 107 | }; 108 | -------------------------------------------------------------------------------- /Source/MachineLearningRemote/MachineLearningRemote.Build.cs: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | using UnrealBuildTool; 4 | 5 | public class MachineLearningRemote : ModuleRules 6 | { 7 | public MachineLearningRemote(ReadOnlyTargetRules Target) : base(Target) 8 | { 9 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 10 | 11 | PublicIncludePaths.AddRange( 12 | new string[] { 13 | // ... add public include paths required here ... 14 | } 15 | ); 16 | 17 | 18 | PrivateIncludePaths.AddRange( 19 | new string[] { 20 | // ... add other private include paths required here ... 21 | } 22 | ); 23 | 24 | 25 | PublicDependencyModuleNames.AddRange( 26 | new string[] 27 | { 28 | "Core", 29 | // ... add other public dependencies that you statically link with here ... 30 | } 31 | ); 32 | 33 | 34 | PrivateDependencyModuleNames.AddRange( 35 | new string[] 36 | { 37 | "CoreUObject", 38 | "Engine", 39 | "Slate", 40 | "SlateCore", 41 | "SocketIOClient", 42 | "SocketIOLib", 43 | "Json", 44 | "SIOJson", 45 | "CoreUtility", 46 | "MachineLearningBase", 47 | "MLProcess" 48 | // ... add private dependencies that you statically link with here ... 49 | } 50 | ); 51 | 52 | 53 | DynamicallyLoadedModuleNames.AddRange( 54 | new string[] 55 | { 56 | // ... add any modules that your module loads dynamically here ... 57 | } 58 | ); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Source/MachineLearningRemote/Private/MachineLearningRemote.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | #include "MachineLearningRemote.h" 4 | #include "MachineLearningBase.h" 5 | 6 | #if WITH_EDITOR 7 | DEFINE_LOG_CATEGORY(MLBaseLog); 8 | #endif 9 | 10 | #define LOCTEXT_NAMESPACE "FMachineLearningRemoteModule" 11 | 12 | void FMachineLearningRemoteModule::StartupModule() 13 | { 14 | // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module 15 | } 16 | 17 | void FMachineLearningRemoteModule::ShutdownModule() 18 | { 19 | // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, 20 | // we call this function before unloading the module. 21 | } 22 | 23 | #undef LOCTEXT_NAMESPACE 24 | 25 | IMPLEMENT_MODULE(FMachineLearningRemoteModule, MachineLearningRemote) 26 | -------------------------------------------------------------------------------- /Source/MachineLearningRemote/Private/MachineLearningRemoteComponent.cpp: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | 4 | #include "MachineLearningRemoteComponent.h" 5 | #include "MachineLearningBase.h" 6 | #include "SocketIOClient.h" 7 | #include "CULambdaRunnable.h" 8 | 9 | bool UMachineLearningRemoteComponent::bServerIsRunning = false; 10 | TSharedPtr UMachineLearningRemoteComponent::Process = nullptr; 11 | 12 | UMachineLearningRemoteComponent::UMachineLearningRemoteComponent() 13 | { 14 | PrimaryComponentTick.bCanEverTick = true; 15 | 16 | bConnectOnBeginPlay = true; 17 | ServerType = ETFServerType::SERVER_PYTHON; 18 | ServerAddressAndPort = TEXT("http://localhost:8080"); 19 | SendInputEventName = TEXT("sendInput"); 20 | StartScriptEventName = TEXT("startScript"); 21 | ScriptStartedEventName = TEXT("scriptStarted"); 22 | LogEventName = TEXT("log"); 23 | DefaultScript = TEXT("empty_example"); 24 | bScriptRunning = false; 25 | bStartScriptOnConnection = true; 26 | 27 | bUseEmbeddedServer = false; 28 | bAutoStartServer = false; 29 | bPrintServerLog = true; 30 | EmbeddedServerRelativePath = TEXT("python3.7"); 31 | 32 | Socket = ISocketIOClientModule::Get().NewValidNativePointer(); 33 | } 34 | 35 | UMachineLearningRemoteComponent::~UMachineLearningRemoteComponent() 36 | { 37 | ISocketIOClientModule::Get().ReleaseNativePointer(Socket); 38 | } 39 | 40 | void UMachineLearningRemoteComponent::BeginPlay() 41 | { 42 | Super::BeginPlay(); 43 | 44 | /** autostart server if enabled */ 45 | if (bAutoStartServer && !bServerIsRunning) 46 | { 47 | bServerIsRunning = true; 48 | FString PluginServerFolderPath = FPaths::Combine(FPaths::ProjectPluginsDir(), TEXT("machine-learning-remote-ue4/Server")); 49 | FString ServerScriptPath = FPaths::Combine(PluginServerFolderPath, TEXT("ml-remote-server")); 50 | 51 | //UE_LOG(LogTemp, Log, TEXT("%s"), *ServerPath); 52 | if (bUseEmbeddedServer) 53 | { 54 | FString EmbeddedServerPath = FPaths::Combine(PluginServerFolderPath, EmbeddedServerRelativePath); //-p ServerScriptPath 55 | Process = FMLProcess::Create(EmbeddedServerPath + TEXT("/python.exe"), ServerScriptPath + TEXT("/server.py -e"), true, false, false, 0, ServerScriptPath, true); 56 | } 57 | else 58 | { 59 | Process = FMLProcess::Create(TEXT("python"), ServerScriptPath + TEXT("/server.py"), true, false, false, 0, TEXT(""), true); 60 | } 61 | 62 | } 63 | 64 | //Setup callbacks 65 | Socket->OnConnectedCallback = [this](const FString& SocketId, const FString& SessionId) 66 | { 67 | if (Socket.IsValid()) 68 | { 69 | bIsConnectedToBackend = true; 70 | OnConnectedToBackend.Broadcast(SessionId); 71 | 72 | if (bStartScriptOnConnection) 73 | { 74 | StartScript(DefaultScript); 75 | } 76 | } 77 | }; 78 | 79 | Socket->OnDisconnectedCallback = [this](const ESIOConnectionCloseReason Reason) 80 | { 81 | if (Socket.IsValid()) 82 | { 83 | bIsConnectedToBackend = false; 84 | OnDisconnectedFromBackend.Broadcast(Socket->LastSessionId); 85 | } 86 | }; 87 | 88 | Socket->OnNamespaceConnectedCallback = [this](const FString& Namespace) 89 | { 90 | if (Socket.IsValid()) 91 | { 92 | } 93 | }; 94 | 95 | Socket->OnNamespaceDisconnectedCallback = [this](const FString& Namespace) 96 | { 97 | if (Socket.IsValid()) 98 | { 99 | 100 | } 101 | }; 102 | Socket->OnReconnectionCallback = [this](const uint32 AttemptCount, const uint32 DelayInMs) 103 | { 104 | if (Socket.IsValid()) 105 | { 106 | } 107 | }; 108 | 109 | Socket->OnFailCallback = [this]() 110 | { 111 | if (Socket.IsValid()) 112 | { 113 | 114 | }; 115 | }; 116 | Socket->OnEvent(ScriptStartedEventName, [this](const FString& EventName, const TSharedPtr& Params) 117 | { 118 | bScriptRunning = true; 119 | OnScriptStarted.Broadcast(Params->AsString()); 120 | }); 121 | 122 | Socket->OnEvent(LogEventName, [this](const FString& EventName, const TSharedPtr& Params) 123 | { 124 | OnLog.Broadcast(Params->AsString()); 125 | }); 126 | 127 | if (bConnectOnBeginPlay) 128 | { 129 | Socket->Connect(ServerAddressAndPort); 130 | } 131 | } 132 | 133 | void UMachineLearningRemoteComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) 134 | { 135 | Socket->Disconnect(); 136 | Socket->ClearAllCallbacks(); 137 | 138 | if (bServerIsRunning) 139 | { 140 | Process->IsRunning(); 141 | Process->Terminate(); 142 | Process->Close(); 143 | Process = nullptr; 144 | bServerIsRunning = false; 145 | } 146 | Super::EndPlay(EndPlayReason); 147 | } 148 | 149 | void UMachineLearningRemoteComponent::TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) 150 | { 151 | if (bServerIsRunning && Process.IsValid()) 152 | { 153 | FString PipeData = Process->ReadFromPipe(); 154 | 155 | if (!PipeData.IsEmpty() && bPrintServerLog) 156 | { 157 | UE_LOG(LogTemp, Log, TEXT("Server: %s"), *PipeData); 158 | } 159 | 160 | if (!Process->IsRunning()) 161 | { 162 | int32 ReturnCode; 163 | Process->GetReturnCode(ReturnCode); 164 | UE_LOG(LogTemp, Log, TEXT("Server finished with: %d"), ReturnCode); 165 | 166 | Process = nullptr; 167 | bServerIsRunning = false; 168 | } 169 | } 170 | Super::TickComponent(DeltaTime, TickType, ThisTickFunction); 171 | } 172 | 173 | void UMachineLearningRemoteComponent::SendSIOJsonInput(USIOJsonValue* InputData, USIOJsonValue*& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName /*= TEXT("onJsonInput")*/) 174 | { 175 | //Wrap input data with targeting information. Cannot formalize this as a struct unfortunately. 176 | auto SendObject = USIOJConvert::MakeJsonObject(); 177 | SendObject->SetStringField(TEXT("targetFunction"), FunctionName); 178 | SendObject->SetField(TEXT("inputData"), InputData->GetRootValue()); 179 | 180 | FCULatentAction* LatentAction = FCULatentAction::CreateLatentAction(LatentInfo, this); 181 | 182 | Socket->Emit(SendInputEventName, SendObject, [this, FunctionName, LatentAction, &ResultData](auto ResponseArray) 183 | { 184 | //UE_LOG(MLBaseLog, Log, TEXT("Got callback response: %s"), *USIOJConvert::ToJsonString(ResponseArray)); 185 | if (ResponseArray.Num() == 0) 186 | { 187 | return; 188 | } 189 | 190 | //We only handle the one response for now 191 | TSharedPtr Response = ResponseArray[0]; 192 | 193 | ResultData = NewObject(); 194 | 195 | ResultData->SetRootValue(Response); 196 | 197 | LatentAction->Call(); //resume the latent action 198 | }); 199 | } 200 | 201 | void UMachineLearningRemoteComponent::StartScript(const FString& ScriptName) 202 | { 203 | Socket->Emit(StartScriptEventName, ScriptName); 204 | //todo get a callback when it has started? 205 | } 206 | 207 | void UMachineLearningRemoteComponent::SendStringInput(const FString& InputData, const FString& FunctionName /*= TEXT("onJsonInput")*/) 208 | { 209 | const FString SafeFunctionName = FunctionName; 210 | SendStringInput(InputData, [this, SafeFunctionName](const FString& ResultData) 211 | { 212 | OnInputResult.Broadcast(ResultData, SafeFunctionName); 213 | }, FunctionName); 214 | } 215 | 216 | void UMachineLearningRemoteComponent::SendStringInput(const FString& InputData, TFunction ResultCallback, const FString& FunctionName /*= TEXT("onJsonInput")*/) 217 | { 218 | //Embed data in a ustruct, this will get auto-serialized into a python/json object on other side 219 | FMLSendStringObject SendObject; 220 | SendObject.InputData = InputData; 221 | SendObject.TargetFunction = FunctionName; 222 | 223 | const auto SafeCallback = ResultCallback; 224 | 225 | Socket->Emit(SendInputEventName, FMLSendStringObject::StaticStruct(), &SendObject, [this, FunctionName, SafeCallback](auto ResponseArray) 226 | { 227 | //UE_LOG(MLBaseLog, Log, TEXT("Got callback response: %s"), *USIOJConvert::ToJsonString(ResponseArray)); 228 | if (ResponseArray.Num() == 0) 229 | { 230 | return; 231 | } 232 | 233 | //We only handle the one response for now 234 | TSharedPtr Response = ResponseArray[0]; 235 | 236 | //Grab the value as a string 237 | //Todo: support non-string encoding? 238 | FString Result; 239 | 240 | if (Response->Type == EJson::String) 241 | { 242 | Result = Response->AsString(); 243 | } 244 | else 245 | { 246 | Result = USIOJConvert::ToJsonString(Response); 247 | } 248 | 249 | if (SafeCallback) 250 | { 251 | SafeCallback(Result); 252 | } 253 | }); 254 | } 255 | 256 | void UMachineLearningRemoteComponent::SendRawInput(const TArray& InputData, const FString& FunctionName /*= TEXT("onFloatArrayInput")*/) 257 | { 258 | const FString SafeFunctionName = FunctionName; 259 | SendRawInput(InputData, [this, SafeFunctionName](TArray& ResultData) 260 | { 261 | OnRawInputResult.Broadcast(ResultData, SafeFunctionName); 262 | }, FunctionName); 263 | } 264 | 265 | void UMachineLearningRemoteComponent::SendRawInput(const TArray& InputData, TFunction& ResultData)> ResultCallback, const FString& FunctionName /*= TEXT("onFloatArrayInput")*/) 266 | { 267 | //Embed data in a ustruct, this will get auto-serialized into a python/json object on other side 268 | FMLSendRawObject SendObject; 269 | SendObject.InputData = InputData; 270 | SendObject.TargetFunction = FunctionName; 271 | 272 | const auto SafeCallback = ResultCallback; 273 | 274 | Socket->Emit(SendInputEventName, FMLSendRawObject::StaticStruct(), &SendObject, [this, FunctionName, SafeCallback](auto ResponseArray) 275 | { 276 | //UE_LOG(MLBaseLog, Log, TEXT("Got callback response: %s"), *USIOJConvert::ToJsonString(ResponseArray)); 277 | if (ResponseArray.Num() == 0) 278 | { 279 | return; 280 | } 281 | 282 | //We only handle the one response for now 283 | TSharedPtr Response = ResponseArray[0]; 284 | 285 | if (Response->Type != EJson::Object) 286 | { 287 | UE_LOG(MLBaseLog, Warning, TEXT("SendRawInput: Expected float array wrapped object, got %s"), *USIOJConvert::ToJsonString(ResponseArray)); 288 | return; 289 | } 290 | 291 | FMLSendRawObject ReceiveObject; 292 | USIOJConvert::JsonObjectToUStruct(Response->AsObject(), FMLSendRawObject::StaticStruct(), &ReceiveObject); 293 | 294 | if (SafeCallback != nullptr) 295 | { 296 | SafeCallback(ReceiveObject.InputData); 297 | } 298 | }); 299 | } 300 | 301 | void UMachineLearningRemoteComponent::SendStringInputGraphCallback(const FString& InputData, FString& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName /*= TEXT("onJsonInput")*/) 302 | { 303 | FCULatentAction* LatentAction = FCULatentAction::CreateLatentAction(LatentInfo, this); 304 | 305 | //Embed data in a ustruct, this will get auto-serialized into a python/json object on other side 306 | FMLSendStringObject SendObject; 307 | SendObject.InputData = InputData; 308 | SendObject.TargetFunction = FunctionName; 309 | 310 | Socket->Emit(SendInputEventName, FMLSendStringObject::StaticStruct(), &SendObject, [this, FunctionName, LatentAction, &ResultData](auto ResponseArray) 311 | { 312 | //UE_LOG(MLBaseLog, Log, TEXT("Got callback response: %s"), *USIOJConvert::ToJsonString(ResponseArray)); 313 | if (ResponseArray.Num() == 0) 314 | { 315 | return; 316 | } 317 | 318 | //We only handle the one response for now 319 | TSharedPtr Response = ResponseArray[0]; 320 | 321 | //Grab the value as a string 322 | //Todo: support non-string encoding? 323 | if (Response->Type == EJson::String) 324 | { 325 | ResultData = Response->AsString(); 326 | } 327 | else 328 | { 329 | ResultData = USIOJConvert::ToJsonString(Response); 330 | } 331 | 332 | LatentAction->Call(); //resume the latent action 333 | }); 334 | } 335 | 336 | void UMachineLearningRemoteComponent::SendRawInputGraphCallback(const TArray& InputData, TArray& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName /*= TEXT("onJsonInput")*/) 337 | { 338 | FCULatentAction* LatentAction = FCULatentAction::CreateLatentAction(LatentInfo, this); 339 | 340 | //Embed data in a ustruct, this will get auto-serialized into a python/json object on other side 341 | FMLSendRawObject SendObject; 342 | SendObject.InputData = InputData; 343 | SendObject.TargetFunction = FunctionName; 344 | 345 | Socket->Emit(SendInputEventName, FMLSendRawObject::StaticStruct(), &SendObject, [this, FunctionName, LatentAction, &ResultData](auto ResponseArray) 346 | { 347 | //UE_LOG(MLBaseLog, Log, TEXT("Got callback response: %s"), *USIOJConvert::ToJsonString(ResponseArray)); 348 | if (ResponseArray.Num() == 0) 349 | { 350 | return; 351 | } 352 | 353 | //We only handle the one response for now 354 | TSharedPtr Response = ResponseArray[0]; 355 | 356 | if (Response->Type != EJson::Object) 357 | { 358 | UE_LOG(MLBaseLog, Warning, TEXT("SendRawInputGraphCallback: Expected float array wrapped object, got %s"), *USIOJConvert::ToJsonString(ResponseArray)); 359 | return; 360 | } 361 | 362 | FMLSendRawObject ReceiveObject; 363 | USIOJConvert::JsonObjectToUStruct(Response->AsObject(), FMLSendRawObject::StaticStruct(), &ReceiveObject); 364 | 365 | ResultData = ReceiveObject.InputData; 366 | 367 | LatentAction->Call(); //resume the latent action 368 | }); 369 | } 370 | -------------------------------------------------------------------------------- /Source/MachineLearningRemote/Public/MachineLearningRemote.h: -------------------------------------------------------------------------------- 1 | // Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Modules/ModuleManager.h" 7 | 8 | class FMachineLearningRemoteModule : public IModuleInterface 9 | { 10 | public: 11 | 12 | /** IModuleInterface implementation */ 13 | virtual void StartupModule() override; 14 | virtual void ShutdownModule() override; 15 | }; 16 | -------------------------------------------------------------------------------- /Source/MachineLearningRemote/Public/MachineLearningRemoteComponent.h: -------------------------------------------------------------------------------- 1 | // Fill out your copyright notice in the Description page of Project Settings. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "MachineLearningBaseComponent.h" 7 | #include "SocketIONative.h" 8 | #include "SIOJsonValue.h" 9 | #include "MLProcess.h" 10 | #include "MachineLearningRemoteComponent.generated.h" 11 | 12 | UENUM(BlueprintType) 13 | enum class ETFServerType : uint8 14 | { 15 | SERVER_PYTHON, 16 | SERVER_NODEJS 17 | }; 18 | 19 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMLScriptSignature, FString, ScriptName); 20 | 21 | /** 22 | * Implements ML API via remote server calls 23 | */ 24 | UCLASS(BlueprintType, Blueprintable, ClassGroup = Computing, meta = (BlueprintSpawnableComponent)) 25 | class MACHINELEARNINGREMOTE_API UMachineLearningRemoteComponent : public UMachineLearningBaseComponent 26 | { 27 | GENERATED_BODY() 28 | public: 29 | 30 | UMachineLearningRemoteComponent(); 31 | ~UMachineLearningRemoteComponent(); 32 | 33 | UPROPERTY(BlueprintAssignable, Category = MachineLearningEvents) 34 | FMLScriptSignature OnScriptStarted; 35 | 36 | /** remote server and address, default localhost:3000 */ 37 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 38 | FString ServerAddressAndPort; 39 | 40 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 41 | FString SendInputEventName; 42 | 43 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 44 | FString StartScriptEventName; 45 | 46 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 47 | FString ScriptStartedEventName; 48 | 49 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 50 | FString LogEventName; 51 | 52 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 53 | bool bConnectOnBeginPlay; 54 | 55 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 56 | bool bStartScriptOnConnection; 57 | 58 | UPROPERTY(BlueprintReadOnly, Category = MLRemoteProperties) 59 | bool bScriptRunning; 60 | 61 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 62 | FString DefaultScript; 63 | 64 | /** Support both python and nodejs servers */ 65 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLRemoteProperties) 66 | ETFServerType ServerType; 67 | 68 | /** If true it will bootup a server on beginplay. Requires python to be installed. */ 69 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLEmbeddedProperties) 70 | bool bAutoStartServer; 71 | 72 | static bool bServerIsRunning; 73 | 74 | /** EXPERIMENTAL: If true, will launch an embedded server. Requires the embedded server type exists in third party. */ 75 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLEmbeddedProperties) 76 | bool bUseEmbeddedServer; 77 | 78 | /** EXPERIMENTAL: Relative to plugin root */ 79 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLEmbeddedProperties) 80 | FString EmbeddedServerRelativePath; 81 | 82 | UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = MLEmbeddedProperties) 83 | bool bPrintServerLog; 84 | 85 | 86 | virtual void BeginPlay() override; 87 | virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; 88 | virtual void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override; 89 | 90 | /** For remote ML components we can use socket.io protocol to communicate objects directly. Return result in graph context. */ 91 | UFUNCTION(BlueprintCallable, meta = (Latent, LatentInfo = "LatentInfo"), Category = MLFunctions) 92 | virtual void SendSIOJsonInput(USIOJsonValue* InputData, USIOJsonValue*& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName = TEXT("on_json_input")); 93 | 94 | /** Start designated script. Get's called on beginplay by default*/ 95 | UFUNCTION(BlueprintCallable, Category = MLFunctions) 96 | virtual void StartScript(const FString& ScriptName); 97 | 98 | //NB by default python pep expects lowercase_lowercase function names. This is different from BaseComponent defaults, which uses camelCase 99 | virtual void SendStringInput(const FString& InputData, const FString& FunctionName = TEXT("on_json_input")) override; 100 | virtual void SendStringInput(const FString& InputData, TFunction ResultCallback, const FString& FunctionName = TEXT("on_json_input")) override; 101 | virtual void SendRawInput(const TArray& InputData, const FString& FunctionName = TEXT("on_float_array_input")) override; 102 | virtual void SendRawInput(const TArray& InputData, TFunction& ResultData)> ResultCallback, const FString& FunctionName = TEXT("on_float_array_input")) override; 103 | virtual void SendStringInputGraphCallback(const FString& InputData, FString& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName = TEXT("on_json_input")) override; 104 | virtual void SendRawInputGraphCallback(const TArray& InputData, TArray& ResultData, struct FLatentActionInfo LatentInfo, const FString& FunctionName = TEXT("on_float_array_input")) override; 105 | 106 | protected: 107 | TSharedPtr Socket; 108 | static TSharedPtr Process; 109 | }; 110 | --------------------------------------------------------------------------------