├── .gitignore ├── Config ├── GatoGPT.Config.TextGenerationPresetsConfig │ ├── DefaultModelProfiles.json │ └── FilenamePresetMap.json ├── GodotEGP.Config.EngineConfig │ └── LoggerManager.json └── GodotEGP.Config.ResourceDefinitionConfig │ └── Resources.json ├── Dockerfile ├── GatoGPT.csproj ├── GatoGPT.http ├── Grammars └── json.gbnf ├── LICENSE ├── Program.cs ├── Properties └── launchSettings.json ├── README.md ├── appsettings.Development.json ├── appsettings.json ├── classes ├── AI │ ├── Embedding │ │ ├── Backends │ │ │ ├── BuiltinSentenceTransformerBackend.cs │ │ │ ├── EmbeddingBackend.cs │ │ │ └── IEmbeddingBackend.cs │ │ └── EmbeddingModelDefinition.cs │ ├── IModelBackend.cs │ ├── ModelBackend.cs │ ├── ModelDefinition.cs │ ├── ModelProfile.cs │ └── TextGeneration │ │ ├── Backends │ │ ├── BuiltinLlamaBackend.cs │ │ ├── ITextGenerationBackend.cs │ │ ├── LlamaCppBackend.cs │ │ ├── LlamaCppServerBackend.cs │ │ └── TextGenerationBackend.cs │ │ ├── InferenceParams.cs │ │ ├── InferenceResult.cs │ │ ├── LlamaCacheManager.cs │ │ ├── LlamaModelDefinition.cs │ │ ├── LoadParams.cs │ │ ├── ModelDefinition.cs │ │ ├── ModelProfile.cs │ │ ├── StatefulChat.cs │ │ ├── TokenFilter │ │ ├── CaptureMarkdownOutput.cs │ │ ├── ITokenFilter.cs │ │ ├── StripAntiprompt.cs │ │ ├── StripLeadingSpace.cs │ │ └── TokenFilterProcessor.cs │ │ └── TokenizedString.cs ├── CLI │ ├── GatoGPTCLI.cs │ └── ProcessRunner.cs ├── CodeTesting.cs ├── Config │ ├── EmbeddingModelManagerConfig.cs │ ├── GlobalConfig.cs │ ├── LlamaCacheManagerConfig.cs │ ├── LlamaModelManagerConfig.cs │ ├── ModelDefinitionsConfig.cs │ ├── ModelDownloadConfig.cs │ ├── SaveDataManagerConfig.cs │ └── TextGenerationPresetsConfig.cs ├── Event │ └── Events.cs ├── Handler │ ├── DownloadConfigHandler.cs │ ├── EmbeddingConfigHandler.cs │ └── LlamaConfigHandler.cs ├── Resource │ ├── EmbeddingModel.cs │ ├── LlamaGrammar.cs │ ├── LlamaModel.cs │ └── SentenceTransformerModel.cs ├── Service │ ├── EmbeddingModelManager.cs │ ├── EmbeddingService.cs │ ├── LlamaCacheService.cs │ ├── ModelDownloadManager.cs │ ├── ModelManager.cs │ ├── TextGenerationModelManager.cs │ └── TextGenerationService.cs └── WebAPI │ ├── Application.cs │ ├── ConfigureSwaggerOptions.cs │ ├── Controllers │ └── v1 │ │ ├── ChatCompletionsController.cs │ │ ├── CompletionsController.cs │ │ ├── EmbeddingsController.cs │ │ ├── ExtendedTokenizeController.cs │ │ └── ModelsController.cs │ ├── Dtos │ ├── BaseDto.cs │ ├── ChatCompletionChoiceDto.cs │ ├── ChatCompletionChunkDto.cs │ ├── ChatCompletionCreateDto.cs │ ├── ChatCompletionCreateExtendedDto.cs │ ├── ChatCompletionCreateFunctionDto.cs │ ├── ChatCompletionCreateOpenAIDto.cs │ ├── ChatCompletionCreateResponseFormatDto.cs │ ├── ChatCompletionCreateToolDto.cs │ ├── ChatCompletionDto.cs │ ├── ChatCompletionMessageCreateDto.cs │ ├── ChatCompletionMessageDto.cs │ ├── ChatCompletionToolCallsDto.cs │ ├── ChatCompletionToolChoiceDto.cs │ ├── ChatCompletionToolFunctionDto.cs │ ├── CompletionBaseDto.cs │ ├── CompletionChoiceBaseDto.cs │ ├── CompletionChoiceDto.cs │ ├── CompletionCreateBaseDto.cs │ ├── CompletionCreateDto.cs │ ├── CompletionCreateOpenAIDto.cs │ ├── CompletionDto.cs │ ├── CompletionUsageDto.cs │ ├── EmbeddingCreateDto.cs │ ├── EmbeddingDto.cs │ ├── EmbeddingUsageDto.cs │ ├── ErrorDto.cs │ ├── ExtendedTokenizeCreateDto.cs │ ├── ExtendedTokenizeDto.cs │ ├── ListBaseDto.cs │ └── ModelDto.cs │ ├── Entities │ └── ModelEntity.cs │ ├── Extensions │ ├── CustomCorsExtension.cs │ ├── ExceptionExtension.cs │ └── VersioningExtension.cs │ ├── MappingProfiles │ ├── ChatCompletionMappings.cs │ ├── CompletionMappings.cs │ ├── EmbeddingsMappings.cs │ └── ModelMappings.cs │ └── ServerSentEventManager.cs ├── docker-compose.yml └── templates └── cs.template /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from `dotnet new gitignore` 5 | 6 | # dotenv files 7 | .env 8 | 9 | # User-specific files 10 | *.rsuser 11 | *.suo 12 | *.user 13 | *.userosscache 14 | *.sln.docstates 15 | 16 | # User-specific files (MonoDevelop/Xamarin Studio) 17 | *.userprefs 18 | 19 | # Mono auto generated files 20 | mono_crash.* 21 | 22 | # Build results 23 | [Dd]ebug/ 24 | [Dd]ebugPublic/ 25 | [Rr]elease/ 26 | [Rr]eleases/ 27 | x64/ 28 | x86/ 29 | [Ww][Ii][Nn]32/ 30 | [Aa][Rr][Mm]/ 31 | [Aa][Rr][Mm]64/ 32 | bld/ 33 | [Bb]in/ 34 | [Oo]bj/ 35 | [Ll]og/ 36 | [Ll]ogs/ 37 | 38 | # Visual Studio 2015/2017 cache/options directory 39 | .vs/ 40 | # Uncomment if you have tasks that create the project's static files in wwwroot 41 | #wwwroot/ 42 | 43 | # Visual Studio 2017 auto generated files 44 | Generated\ Files/ 45 | 46 | # MSTest test Results 47 | [Tt]est[Rr]esult*/ 48 | [Bb]uild[Ll]og.* 49 | 50 | # NUnit 51 | *.VisualState.xml 52 | TestResult.xml 53 | nunit-*.xml 54 | 55 | # Build Results of an ATL Project 56 | [Dd]ebugPS/ 57 | [Rr]eleasePS/ 58 | dlldata.c 59 | 60 | # Benchmark Results 61 | BenchmarkDotNet.Artifacts/ 62 | 63 | # .NET 64 | project.lock.json 65 | project.fragment.lock.json 66 | artifacts/ 67 | 68 | # Tye 69 | .tye/ 70 | 71 | # ASP.NET Scaffolding 72 | ScaffoldingReadMe.txt 73 | 74 | # StyleCop 75 | StyleCopReport.xml 76 | 77 | # Files built by Visual Studio 78 | *_i.c 79 | *_p.c 80 | *_h.h 81 | *.ilk 82 | *.meta 83 | *.obj 84 | *.iobj 85 | *.pch 86 | *.pdb 87 | *.ipdb 88 | *.pgc 89 | *.pgd 90 | *.rsp 91 | *.sbr 92 | *.tlb 93 | *.tli 94 | *.tlh 95 | *.tmp 96 | *.tmp_proj 97 | *_wpftmp.csproj 98 | *.log 99 | *.tlog 100 | *.vspscc 101 | *.vssscc 102 | .builds 103 | *.pidb 104 | *.svclog 105 | *.scc 106 | 107 | # Chutzpah Test files 108 | _Chutzpah* 109 | 110 | # Visual C++ cache files 111 | ipch/ 112 | *.aps 113 | *.ncb 114 | *.opendb 115 | *.opensdf 116 | *.sdf 117 | *.cachefile 118 | *.VC.db 119 | *.VC.VC.opendb 120 | 121 | # Visual Studio profiler 122 | *.psess 123 | *.vsp 124 | *.vspx 125 | *.sap 126 | 127 | # Visual Studio Trace Files 128 | *.e2e 129 | 130 | # TFS 2012 Local Workspace 131 | $tf/ 132 | 133 | # Guidance Automation Toolkit 134 | *.gpState 135 | 136 | # ReSharper is a .NET coding add-in 137 | _ReSharper*/ 138 | *.[Rr]e[Ss]harper 139 | *.DotSettings.user 140 | 141 | # TeamCity is a build add-in 142 | _TeamCity* 143 | 144 | # DotCover is a Code Coverage Tool 145 | *.dotCover 146 | 147 | # AxoCover is a Code Coverage Tool 148 | .axoCover/* 149 | !.axoCover/settings.json 150 | 151 | # Coverlet is a free, cross platform Code Coverage Tool 152 | coverage*.json 153 | coverage*.xml 154 | coverage*.info 155 | 156 | # Visual Studio code coverage results 157 | *.coverage 158 | *.coveragexml 159 | 160 | # NCrunch 161 | _NCrunch_* 162 | .*crunch*.local.xml 163 | nCrunchTemp_* 164 | 165 | # MightyMoose 166 | *.mm.* 167 | AutoTest.Net/ 168 | 169 | # Web workbench (sass) 170 | .sass-cache/ 171 | 172 | # Installshield output folder 173 | [Ee]xpress/ 174 | 175 | # DocProject is a documentation generator add-in 176 | DocProject/buildhelp/ 177 | DocProject/Help/*.HxT 178 | DocProject/Help/*.HxC 179 | DocProject/Help/*.hhc 180 | DocProject/Help/*.hhk 181 | DocProject/Help/*.hhp 182 | DocProject/Help/Html2 183 | DocProject/Help/html 184 | 185 | # Click-Once directory 186 | publish/ 187 | 188 | # Publish Web Output 189 | *.[Pp]ublish.xml 190 | *.azurePubxml 191 | # Note: Comment the next line if you want to checkin your web deploy settings, 192 | # but database connection strings (with potential passwords) will be unencrypted 193 | *.pubxml 194 | *.publishproj 195 | 196 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 197 | # checkin your Azure Web App publish settings, but sensitive information contained 198 | # in these scripts will be unencrypted 199 | PublishScripts/ 200 | 201 | # NuGet Packages 202 | *.nupkg 203 | # NuGet Symbol Packages 204 | *.snupkg 205 | # The packages folder can be ignored because of Package Restore 206 | **/[Pp]ackages/* 207 | # except build/, which is used as an MSBuild target. 208 | !**/[Pp]ackages/build/ 209 | # Uncomment if necessary however generally it will be regenerated when needed 210 | #!**/[Pp]ackages/repositories.config 211 | # NuGet v3's project.json files produces more ignorable files 212 | *.nuget.props 213 | *.nuget.targets 214 | 215 | # Microsoft Azure Build Output 216 | csx/ 217 | *.build.csdef 218 | 219 | # Microsoft Azure Emulator 220 | ecf/ 221 | rcf/ 222 | 223 | # Windows Store app package directories and files 224 | AppPackages/ 225 | BundleArtifacts/ 226 | Package.StoreAssociation.xml 227 | _pkginfo.txt 228 | *.appx 229 | *.appxbundle 230 | *.appxupload 231 | 232 | # Visual Studio cache files 233 | # files ending in .cache can be ignored 234 | *.[Cc]ache 235 | # but keep track of directories ending in .cache 236 | !?*.[Cc]ache/ 237 | 238 | # Others 239 | ClientBin/ 240 | ~$* 241 | *~ 242 | *.dbmdl 243 | *.dbproj.schemaview 244 | *.jfm 245 | *.pfx 246 | *.publishsettings 247 | orleans.codegen.cs 248 | 249 | # Including strong name files can present a security risk 250 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 251 | #*.snk 252 | 253 | # Since there are multiple workflows, uncomment next line to ignore bower_components 254 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 255 | #bower_components/ 256 | 257 | # RIA/Silverlight projects 258 | Generated_Code/ 259 | 260 | # Backup & report files from converting an old project file 261 | # to a newer Visual Studio version. Backup files are not needed, 262 | # because we have git ;-) 263 | _UpgradeReport_Files/ 264 | Backup*/ 265 | UpgradeLog*.XML 266 | UpgradeLog*.htm 267 | ServiceFabricBackup/ 268 | *.rptproj.bak 269 | 270 | # SQL Server files 271 | *.mdf 272 | *.ldf 273 | *.ndf 274 | 275 | # Business Intelligence projects 276 | *.rdl.data 277 | *.bim.layout 278 | *.bim_*.settings 279 | *.rptproj.rsuser 280 | *- [Bb]ackup.rdl 281 | *- [Bb]ackup ([0-9]).rdl 282 | *- [Bb]ackup ([0-9][0-9]).rdl 283 | 284 | # Microsoft Fakes 285 | FakesAssemblies/ 286 | 287 | # GhostDoc plugin setting file 288 | *.GhostDoc.xml 289 | 290 | # Node.js Tools for Visual Studio 291 | .ntvs_analysis.dat 292 | node_modules/ 293 | 294 | # Visual Studio 6 build log 295 | *.plg 296 | 297 | # Visual Studio 6 workspace options file 298 | *.opt 299 | 300 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 301 | *.vbw 302 | 303 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 304 | *.vbp 305 | 306 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 307 | *.dsw 308 | *.dsp 309 | 310 | # Visual Studio 6 technical files 311 | *.ncb 312 | *.aps 313 | 314 | # Visual Studio LightSwitch build output 315 | **/*.HTMLClient/GeneratedArtifacts 316 | **/*.DesktopClient/GeneratedArtifacts 317 | **/*.DesktopClient/ModelManifest.xml 318 | **/*.Server/GeneratedArtifacts 319 | **/*.Server/ModelManifest.xml 320 | _Pvt_Extensions 321 | 322 | # Paket dependency manager 323 | .paket/paket.exe 324 | paket-files/ 325 | 326 | # FAKE - F# Make 327 | .fake/ 328 | 329 | # CodeRush personal settings 330 | .cr/personal 331 | 332 | # Python Tools for Visual Studio (PTVS) 333 | __pycache__/ 334 | *.pyc 335 | 336 | # Cake - Uncomment if you are using it 337 | # tools/** 338 | # !tools/packages.config 339 | 340 | # Tabs Studio 341 | *.tss 342 | 343 | # Telerik's JustMock configuration file 344 | *.jmconfig 345 | 346 | # BizTalk build output 347 | *.btp.cs 348 | *.btm.cs 349 | *.odx.cs 350 | *.xsd.cs 351 | 352 | # OpenCover UI analysis results 353 | OpenCover/ 354 | 355 | # Azure Stream Analytics local run output 356 | ASALocalRun/ 357 | 358 | # MSBuild Binary and Structured Log 359 | *.binlog 360 | 361 | # NVidia Nsight GPU debugger configuration file 362 | *.nvuser 363 | 364 | # MFractors (Xamarin productivity tool) working folder 365 | .mfractor/ 366 | 367 | # Local History for Visual Studio 368 | .localhistory/ 369 | 370 | # Visual Studio History (VSHistory) files 371 | .vshistory/ 372 | 373 | # BeatPulse healthcheck temp database 374 | healthchecksdb 375 | 376 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 377 | MigrationBackup/ 378 | 379 | # Ionide (cross platform F# VS Code tools) working folder 380 | .ionide/ 381 | 382 | # Fody - auto-generated XML schema 383 | FodyWeavers.xsd 384 | 385 | # VS Code files for those working on multiple tools 386 | .vscode/* 387 | !.vscode/settings.json 388 | !.vscode/tasks.json 389 | !.vscode/launch.json 390 | !.vscode/extensions.json 391 | *.code-workspace 392 | 393 | # Local History for Visual Studio Code 394 | .history/ 395 | 396 | # Windows Installer files from build outputs 397 | *.cab 398 | *.msi 399 | *.msix 400 | *.msm 401 | *.msp 402 | 403 | # JetBrains Rider 404 | *.sln.iml 405 | .idea 406 | 407 | ## 408 | ## Visual studio for Mac 409 | ## 410 | 411 | 412 | # globs 413 | Makefile.in 414 | *.userprefs 415 | *.usertasks 416 | config.make 417 | config.status 418 | aclocal.m4 419 | install-sh 420 | autom4te.cache/ 421 | *.tar.gz 422 | tarballs/ 423 | test-results/ 424 | 425 | # Mac bundle stuff 426 | *.dmg 427 | *.app 428 | 429 | # content below from: https://github.com/github/gitignore/blob/master/Global/macOS.gitignore 430 | # General 431 | .DS_Store 432 | .AppleDouble 433 | .LSOverride 434 | 435 | # Icon must end with two \r 436 | Icon 437 | 438 | 439 | # Thumbnails 440 | ._* 441 | 442 | # Files that might appear in the root of a volume 443 | .DocumentRevisions-V100 444 | .fseventsd 445 | .Spotlight-V100 446 | .TemporaryItems 447 | .Trashes 448 | .VolumeIcon.icns 449 | .com.apple.timemachine.donotpresent 450 | 451 | # Directories potentially created on remote AFP share 452 | .AppleDB 453 | .AppleDesktop 454 | Network Trash Folder 455 | Temporary Items 456 | .apdisk 457 | 458 | # content below from: https://github.com/github/gitignore/blob/master/Global/Windows.gitignore 459 | # Windows thumbnail cache files 460 | Thumbs.db 461 | ehthumbs.db 462 | ehthumbs_vista.db 463 | 464 | # Dump file 465 | *.stackdump 466 | 467 | # Folder config file 468 | [Dd]esktop.ini 469 | 470 | # Recycle Bin used on file shares 471 | $RECYCLE.BIN/ 472 | 473 | # Windows Installer files 474 | *.cab 475 | *.msi 476 | *.msix 477 | *.msm 478 | *.msp 479 | 480 | # Windows shortcuts 481 | *.lnk 482 | 483 | # Vim temporary swap files 484 | *.swp 485 | 486 | # Managed by git-multi (sub repo git@github.com:elgatopanzon/godotegp-framework.git) (commit 969623f) 487 | /libs/godotegp-framework 488 | # Managed by git-multi (sub repo git@github.com:elgatopanzon/godotegp-framework-nongame.git) (commit efb12b9) 489 | /libs/godotegp-framework-nongame 490 | 491 | # data directory when using docker compose 492 | /GatoGPT_Data 493 | -------------------------------------------------------------------------------- /Config/GatoGPT.Config.TextGenerationPresetsConfig/DefaultModelProfiles.json: -------------------------------------------------------------------------------- 1 | { 2 | "DefaultModelProfiles": { 3 | "alpaca": { 4 | "Name": "Alpaca", 5 | "InferenceParams": { 6 | "InputPrefix": "### Instruction:\n", 7 | "InputSuffix": "\n### Response:\n", 8 | "Antiprompts": [ 9 | "### Instruction:" 10 | ], 11 | "PrePrompt": "Below is an instruction that describes a task. Write a response that appropriately completes the request.", 12 | "PrePromptSuffix": "\n", 13 | "PrePromptPrefix": "" 14 | } 15 | }, 16 | "chatml": { 17 | "Name": "ChatML", 18 | "InferenceParams": { 19 | "InputPrefix": "<|im_start|>user\n", 20 | "InputSuffix": "\n<|im_end|>\n<|im_start|>assistant\n", 21 | "Antiprompts": [ 22 | "<|im_start|>", 23 | "<|im_end|>" 24 | ], 25 | "PrePromptPrefix": "<|im_start|>system\n", 26 | "PrePromptSuffix": "<|im_end|>\n", 27 | "ChatMessageTemplate": "<|im_start|>{{ Role }} {{ Name }}:\n{{ Message }}\n<|im_end|>", 28 | "ChatTemplate": "{{ PrePromptPrefix }}{{ PrePrompt }}{{ PrePromptSuffix }}{{ Input }}", 29 | "ChatMessageGenerationTemplate": "<|im_start|>assistant {{ AssistantName }}:\n", 30 | "PrePrompt": "Perform the task to the best of your ability." 31 | } 32 | }, 33 | "codellama_completion": { 34 | "Name": "CodeLlama Completion", 35 | "LoadParams": { 36 | "RopeFreqBase": 0, 37 | "RopeFreqScale": 0 38 | }, 39 | "InferenceParams": { 40 | "TopP": 0.9, 41 | "Temp": 0.2, 42 | "InputPrefix": "", 43 | "InputSuffix": "", 44 | "PrePrompt": "", 45 | "PrePromptPrefix": "", 46 | "PrePromptSuffix": "", 47 | "Antiprompts": [ 48 | "" 49 | ] 50 | } 51 | }, 52 | "codellama_instruct": { 53 | "Name": "CodeLlama Instruct", 54 | "LoadParams": { 55 | "RopeFreqBase": 0, 56 | "RopeFreqScale": 0 57 | }, 58 | "InferenceParams": { 59 | "TopP": 0.95, 60 | "Temp": 0.2, 61 | "InputPrefix": "[INST]", 62 | "InputSuffix": "[/INST]", 63 | "PrePrompt": "You are a helpful coding AI assistant.", 64 | "PrePromptPrefix": "[INST]<>", 65 | "PrePromptSuffix": "<>[/INST]", 66 | "Antiprompts": [ 67 | "[INST]" 68 | ] 69 | } 70 | }, 71 | "codellama_wizardcoder": { 72 | "Name": "CodeLlama WizardCoder" 73 | }, 74 | "deepseek_coder": { 75 | "Name": "Deepseek Coder", 76 | "LoadParams": { 77 | "RopeFreqBase": 0, 78 | "RopeFreqScale": 0 79 | }, 80 | "InferenceParams": { 81 | "InputPrefix": "### Instruction:\n", 82 | "InputSuffix": "\n### Response:\n", 83 | "Antiprompts": [ 84 | "### Instruction:" 85 | ], 86 | "PrePrompt": "You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science.", 87 | "PrePromptSuffix": "\n", 88 | "PrePromptPrefix": "" 89 | } 90 | }, 91 | "metaai_llama_2_chat": { 92 | "Name": "MetaAI Llama 2 Chat", 93 | "LoadParams": { 94 | "RopeFreqBase": 0, 95 | "RopeFreqScale": 0 96 | }, 97 | "InferenceParams": { 98 | "InputPrefix": "[INST]", 99 | "InputSuffix": "[/INST]\n", 100 | "PrePrompt": "You are a helpful coding AI assistant.", 101 | "PrePromptPrefix": "[INST]<>\n", 102 | "PrePromptSuffix": "<>[/INST]\n", 103 | "Antiprompts": [ 104 | "[INST]" 105 | ] 106 | } 107 | }, 108 | "mistral_instruct": { 109 | "Name": "Mistral Instruct", 110 | "LoadParams": { 111 | "RopeFreqBase": 0, 112 | "RopeFreqScale": 0 113 | }, 114 | "InferenceParams": { 115 | "InputPrefix": "[INST]", 116 | "InputSuffix": "[/INST]", 117 | "PrePromptPrefix": "", 118 | "PrePromptSuffix": "", 119 | "Antiprompts": [ 120 | "[INST]" 121 | ] 122 | } 123 | }, 124 | "mistral_instruct_chat": { 125 | "Name": "Mistral Instruct (for chat only)", 126 | "LoadParams": { 127 | "RopeFreqBase": 0, 128 | "RopeFreqScale": 0 129 | }, 130 | "InferenceParams": { 131 | "InputPrefix": "", 132 | "InputSuffix": "", 133 | "PrePromptPrefix": "", 134 | "PrePromptSuffix": "", 135 | "Antiprompts": [ 136 | "[INST]" 137 | ], 138 | "ChatTemplate": "{{ PrePromptPrefix }}{{ PrePrompt }}{{ PrePromptSuffix }}\nBelow is a chat log:\n{{ Input }}", 139 | "ChatMessageTemplate": "<{{ Name }}> {{ Message }}", 140 | "ChatMessageGenerationTemplate": "<{{ AssistantName }}>" 141 | } 142 | }, 143 | "obsidian_vision": { 144 | "Name": "Obsidian Vision", 145 | "LoadParams": { 146 | "NCtx": 2048, 147 | "RopeFreqBase": 10000, 148 | "RopeFreqScale": 1 149 | }, 150 | "InferenceParams": { 151 | "InputPrefix": "<|im_start|>user\n", 152 | "InputSuffix": "\n###\n<|im_start|>assistant:", 153 | "Antiprompts": [ 154 | "<|im_start|>", 155 | "<|im_end|>", 156 | "###" 157 | ], 158 | "PrePrompt": "", 159 | "PrePromptSuffix": "", 160 | "PrePromptPrefix": "" 161 | } 162 | }, 163 | "phi_2": { 164 | "Name": "Phi 2", 165 | "LoadParams": { 166 | "RopeFreqBase": 0, 167 | "RopeFreqScale": 0 168 | }, 169 | "InferenceParams": { 170 | "InputPrefix": "Instruct: ", 171 | "InputSuffix": "\nOutput:", 172 | "PrePromptPrefix": "", 173 | "PrePromptSuffix": "", 174 | "Antiprompts": [ 175 | "Instruct:", 176 | "Output:" 177 | ] 178 | } 179 | }, 180 | "phind_codellama": { 181 | "Name": "Phind CodeLlama", 182 | "LoadParams": { 183 | "RopeFreqBase": 0, 184 | "RopeFreqScale": 0 185 | }, 186 | "InferenceParams": { 187 | "InputPrefix": "### User Message\n", 188 | "InputSuffix": "### Assistant\n", 189 | "PrePromptPrefix": "### System Prompt\n", 190 | "PrePromptSuffix": "\n", 191 | "Antiprompts": [ 192 | "[INST]" 193 | ] 194 | } 195 | }, 196 | "vicuna_v1_1": { 197 | "Name": "Vicuna v1.1", 198 | "LoadParams": { 199 | "RopeFreqScale": 0 200 | }, 201 | "InferenceParams": { 202 | "InputPrefix": "USER:", 203 | "InputSuffix": "ASSISTANT:", 204 | "PrePromptPrefix": "", 205 | "PrePromptSuffix": "\n\n", 206 | "PrePrompt": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.", 207 | "Antiprompts": [ 208 | "USER:" 209 | ] 210 | } 211 | }, 212 | "vicuna_v1_5_16k": { 213 | "Name": "Vicuna v1.5 16K", 214 | "LoadParams": { 215 | "RopeFreqScale": 0.25 216 | }, 217 | "InferenceParams": { 218 | "InputPrefix": "USER:", 219 | "InputSuffix": "ASSISTANT:", 220 | "PrePromptPrefix": "", 221 | "PrePromptSuffix": "\n\n", 222 | "PrePrompt": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.", 223 | "Antiprompts": [ 224 | "USER:" 225 | ] 226 | } 227 | }, 228 | "zephyr": { 229 | "Name": "Zephyr", 230 | "InferenceParams": { 231 | "PrePromptPrefix": "<|system|>\n", 232 | "PrePromptSuffix": "\n", 233 | "InputPrefix": "\n<|user|>\n", 234 | "InputSuffix": "\n<|assistant|>\n", 235 | "Antiprompts": [ 236 | "<|system|>", 237 | "<|user|>", 238 | "<|assistant|>" 239 | ] 240 | }, 241 | "LoadParams": { 242 | "RopeFreqScale": 0, 243 | "RopeFreqBase": 0 244 | } 245 | }, 246 | "synthia": { 247 | "Name": "Synthia", 248 | "InferenceParams": { 249 | "InputPrefix": "", 250 | "InputSuffix": "", 251 | "Antiprompts": [ 252 | "SYSTEM:", 253 | "USER:" 254 | ], 255 | "PrePrompt": "Always answer without hesitation.", 256 | "PrePromptSuffix": "\n\n", 257 | "PrePromptPrefix": "SYSTEM: ", 258 | "ChatTemplate": "{{ PrePromptPrefix }}{{ PrePrompt }}{{ PrePromptSuffix }}\n{{ Input }}", 259 | "ChatMessageTemplate": "{{ Name }}: {{ Message }}", 260 | "ChatMessageGenerationTemplate": "ASSISTANT: " 261 | } 262 | } 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /Config/GatoGPT.Config.TextGenerationPresetsConfig/FilenamePresetMap.json: -------------------------------------------------------------------------------- 1 | { 2 | "FilenamePresetMap": { 3 | "*/Llama-2-*B-*hat-*/*.*": "metaai_llama_2_chat", 4 | "*/vicuna-*B-v*-16K-*/*.*": "vicuna_v1_5_16k", 5 | "*/CodeLlama-*B-Python-*/*.*": "codellama_instruct", 6 | "*/CodeLlama-*B-Instruct-*/*.*": "codellama_instruct", 7 | "*/Phind-CodeLlama-34B-v*-GGUF/*/*.*": "phind_codellama", 8 | "*/*istral-*B-Instruct-v*-GGUF/*.gguf": "mistral_instruct", 9 | "*/OpenHermes-2*-Mistral-7B-GGUF/*.gguf": "chatml", 10 | "*/dolphin-*-mistral-*B-GGUF/*.gguf": "chatml", 11 | "*/*istral-*B-OpenOrca-GGUF/*.gguf": "chatml", 12 | "*/zephyr-*-GGUF/*.gguf": "zephyr", 13 | "*/stablelm-zephyr-*-GGUF/*.gguf": "zephyr", 14 | "*/deepseek-*-instruct-GGUF/*.gguf": "deepseek_coder", 15 | "*/Mixtral-8x7B-Instruct-v*-GGUF/*.gguf": "mistral_instruct", 16 | "*/phi-2-GGUF/*.gguf": "phi_2", 17 | "*/Synthia-*/*.gguf": "synthia" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Config/GodotEGP.Config.EngineConfig/LoggerManager.json: -------------------------------------------------------------------------------- 1 | { 2 | "LoggerManager": { 3 | "LogLevel": 1, 4 | "LogLevelOverrides": null 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Config/GodotEGP.Config.ResourceDefinitionConfig/Resources.json: -------------------------------------------------------------------------------- 1 | { 2 | "Resources": { 3 | "LlamaGrammars": { 4 | "json": { 5 | "Path": "res://Grammars/json.gbnf", 6 | "Class": "GatoGPT.Resource.LlamaGrammar" 7 | } 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env 2 | 3 | # Copy code into /code 4 | COPY . ./code 5 | 6 | RUN git clone https://github.com/elgatopanzon/godotegp-framework.git /code/libs/godotegp-framework 7 | RUN git clone https://github.com/elgatopanzon/godotegp-framework-nongame.git /code/libs/godotegp-framework-nongame 8 | 9 | WORKDIR /code 10 | # Restore as distinct layers 11 | RUN dotnet restore 12 | # Build and publish a release 13 | RUN dotnet publish -c Debug -o out 14 | 15 | # Build runtime image 16 | FROM mcr.microsoft.com/dotnet/aspnet:8.0 17 | 18 | # Install dependencies 19 | RUN apt update -y 20 | RUN apt install -y build-essential cmake gcc software-properties-common wget 21 | RUN apt install -y git openssh-client 22 | 23 | # Download nvidia cuda toolkit 24 | RUN wget --quiet https://developer.download.nvidia.com/compute/cuda/12.3.2/local_installers/cuda-repo-debian12-12-3-local_12.3.2-545.23.08-1_amd64.deb 25 | 26 | # Copy everything and clone required project libraries 27 | RUN mkdir /code 28 | 29 | # Install nvidia cuda toolkit 30 | RUN dpkg -i cuda-repo-debian12-12-3-local_12.3.2-545.23.08-1_amd64.deb \ 31 | && cp /var/cuda-repo-debian12-12-3-local/cuda-*-keyring.gpg /usr/share/keyrings/ \ 32 | && add-apt-repository contrib \ 33 | && apt-get update \ 34 | && apt-get -y install cuda-toolkit-12-3 35 | 36 | # Compile llama.cpp 37 | RUN git clone https://github.com/ggerganov/llama.cpp \ 38 | && cd llama.cpp \ 39 | && mkdir build \ 40 | && cd build \ 41 | && CUDACXX=/usr/local/cuda-12/bin/nvcc cmake .. -DLLAMA_CUBLAS=ON -DCMAKE_CUDA_ARCHITECTURES=all \ 42 | && CUDACXX=/usr/local/cuda-12/bin/nvcc cmake --build . --config Release 43 | 44 | # Move required binaries 45 | RUN mv llama.cpp/build/bin/main /code/llama.cpp 46 | RUN mv llama.cpp/build/bin/llava-cli /code/llama.cpp-llava-cli 47 | RUN mv llama.cpp/build/bin/server /code/llama.cpp-server 48 | RUN mv llama.cpp/build/bin/tokenize /code/llama.cpp-tokenize 49 | 50 | # Create running user 51 | ARG UNAME=user 52 | ARG UID=1000 53 | ARG GID=1000 54 | RUN groupadd -g $GID -o $UNAME 55 | RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME 56 | 57 | # Copy deployment 58 | COPY --from=build-env /code/out /code 59 | COPY --from=build-env /code/out /code/bin/Debug/net8.0 60 | WORKDIR /code 61 | 62 | RUN chown -R $UNAME:$UNAME /code 63 | 64 | # Switch to running user 65 | USER $UNAME 66 | 67 | # Release build not working, for now issue debug run command 68 | ENTRYPOINT ["dotnet", "/code/bin/Debug/net8.0/GatoGPT.dll"] 69 | -------------------------------------------------------------------------------- /GatoGPT.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | true 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /GatoGPT.http: -------------------------------------------------------------------------------- 1 | @gatogpt_HostAddress = http://localhost:5036 2 | 3 | GET {{gatogpt_HostAddress}}/weatherforecast/ 4 | Accept: application/json 5 | 6 | ### 7 | -------------------------------------------------------------------------------- /Grammars/json.gbnf: -------------------------------------------------------------------------------- 1 | root ::= object 2 | 3 | object ::= "{" ws ( string ":" ws value ("," ws string ":" ws value)* )? "}" 4 | 5 | value ::= object | array | string | number | ("true" | "false" | "null") ws 6 | 7 | array ::= "[" ws ( value ("," ws value)* )? "]" ws 8 | 9 | string ::= "\"" ( [a-zA-Z0-9 ] )* "\"" ws 10 | 11 | number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws 12 | 13 | 14 | ws ::= ([ \t\n] ws)? 15 | -------------------------------------------------------------------------------- /Program.cs: -------------------------------------------------------------------------------- 1 | namespace GatoGPT; 2 | 3 | using GodotEGPNonGame.ServiceWorkers; 4 | 5 | using GatoGPT.CLI; 6 | 7 | using GatoGPT.Service; 8 | using GatoGPT.Handler; 9 | using GatoGPT.Config; 10 | using GatoGPT.Resource; 11 | using GatoGPT.AI.TextGeneration; 12 | using GatoGPT.Event; 13 | 14 | using GodotEGP; 15 | using GodotEGP.Logging; 16 | using GodotEGP.Service; 17 | using GodotEGP.Random; 18 | using GodotEGP.Objects.Extensions; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Event.Filters; 21 | using Godot; 22 | 23 | class Program 24 | { 25 | public static GodotEGP.Main GodotEGP; 26 | 27 | static async Task Main(string[] args) 28 | { 29 | // init GodotEGP 30 | GodotEGP = new GodotEGP.Main(); 31 | SceneTree.Instance.Root.AddChild(GodotEGP); 32 | 33 | // init LLMConfigHandler 34 | SceneTree.Instance.Root.AddChild(new LlamaConfigHandler()); 35 | SceneTree.Instance.Root.AddChild(new EmbeddingConfigHandler()); 36 | SceneTree.Instance.Root.AddChild(new DownloadConfigHandler()); 37 | 38 | // wait for services to be ready 39 | if (!ServiceRegistry.WaitForServices( 40 | typeof(ConfigManager), 41 | typeof(ResourceManager), 42 | typeof(ScriptService), 43 | typeof(TextGenerationModelManager), 44 | typeof(EmbeddingModelManager) 45 | )) 46 | { 47 | LoggerManager.LogCritical("Required services never became ready"); 48 | 49 | return 0; 50 | } 51 | 52 | LoggerManager.LogInfo("Services ready"); 53 | 54 | // create SceneTree service worker instance 55 | var serviceWorker = new SceneTreeServiceWorker(); 56 | await serviceWorker.StartAsync(new CancellationToken()); 57 | 58 | LoggerManager.LogInfo("GodotEGP ready!"); 59 | 60 | // init LlamaCacheService 61 | ServiceRegistry.Get(); 62 | 63 | // CLI application 64 | var cli = new GatoGPTCLI(args); 65 | 66 | // execute the CLI parser 67 | return await cli.Run(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/launchsettings.json", 3 | "iisSettings": { 4 | "windowsAuthentication": false, 5 | "anonymousAuthentication": true, 6 | "iisExpress": { 7 | "applicationUrl": "http://localhost:64216", 8 | "sslPort": 44326 9 | } 10 | }, 11 | "profiles": { 12 | "http": { 13 | "commandName": "Project", 14 | "dotnetRunMessages": true, 15 | "launchBrowser": true, 16 | "launchUrl": "swagger", 17 | "applicationUrl": "http://localhost:5036", 18 | "environmentVariables": { 19 | "ASPNETCORE_ENVIRONMENT": "Development" 20 | } 21 | }, 22 | "https": { 23 | "commandName": "Project", 24 | "dotnetRunMessages": true, 25 | "launchBrowser": true, 26 | "launchUrl": "swagger", 27 | "applicationUrl": "https://localhost:7086;http://localhost:5036", 28 | "environmentVariables": { 29 | "ASPNETCORE_ENVIRONMENT": "Development" 30 | } 31 | }, 32 | "IIS Express": { 33 | "commandName": "IISExpress", 34 | "launchBrowser": true, 35 | "launchUrl": "swagger", 36 | "environmentVariables": { 37 | "ASPNETCORE_ENVIRONMENT": "Development" 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | }, 8 | "AllowedHosts": "*" 9 | } 10 | -------------------------------------------------------------------------------- /classes/AI/Embedding/Backends/BuiltinSentenceTransformerBackend.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : BuiltinSentenceTransformer 4 | * @created : Friday Jan 12, 2024 21:52:50 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.Embedding.Backends; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using AllMiniLmL6V2Sharp; 17 | using AllMiniLmL6V2Sharp.Tokenizer; 18 | 19 | public partial class BuiltinSentenceTransformerBackend : ModelBackend, IEmbeddingBackend 20 | { 21 | public new EmbeddingModelDefinition ModelDefinition { get; set; } 22 | private BertTokenizer _tokenizer; 23 | private AllMiniLmL6V2Embedder _onnxEmbedder; 24 | 25 | public BuiltinSentenceTransformerBackend(EmbeddingModelDefinition modelDefinition, bool isStateful = false) : base(modelDefinition, isStateful) 26 | { 27 | ModelDefinition = modelDefinition; 28 | _state.Enter(); 29 | } 30 | 31 | public float[] GenerateEmbedding(string input) 32 | { 33 | LoadModel(); 34 | 35 | float[] embedding = _onnxEmbedder.GenerateEmbedding(input).ToArray(); 36 | 37 | UnloadModel(); 38 | 39 | return embedding; 40 | } 41 | 42 | public override void LoadModel() 43 | { 44 | // extract the paths and expected vocab.txt in same model path 45 | string modelPath = ProjectSettings.GlobalizePath(ModelDefinition.ModelResource.Definition.Path); 46 | string modelVocabPath = ProjectSettings.GlobalizePath(modelPath.Replace("/"+modelPath.GetFile(), "")+"/vocab.txt"); 47 | 48 | LoggerManager.LogDebug("Embeddings model path", "", "modelPath", modelPath); 49 | LoggerManager.LogDebug("Embeddings model vocab", "", "modelVocabPath", modelVocabPath); 50 | 51 | _tokenizer = new BertTokenizer(modelVocabPath); 52 | _onnxEmbedder = new AllMiniLmL6V2Embedder(modelPath: modelPath, tokenizer: _tokenizer); 53 | } 54 | 55 | public override void UnloadModel() 56 | { 57 | _tokenizer = null; 58 | _onnxEmbedder = null; 59 | 60 | GC.Collect(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /classes/AI/Embedding/Backends/EmbeddingBackend.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingBackend 4 | * @created : Friday Jan 12, 2024 22:01:47 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.Embedding.Backends; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class EmbeddingBackend : AI.ModelBackend, IEmbeddingBackend 17 | { 18 | public EmbeddingBackend(EmbeddingModelDefinition modelDefinition, bool isStateful = false) : base(modelDefinition, isStateful) 19 | { 20 | 21 | } 22 | 23 | public float[] GenerateEmbedding(string input) 24 | { 25 | return null; 26 | } 27 | 28 | public static IEmbeddingBackend CreateBackend(EmbeddingModelDefinition modelDefinition) 29 | { 30 | string fqClassName = typeof(IEmbeddingBackend).FullName; 31 | fqClassName = fqClassName.Replace("."+nameof(IEmbeddingBackend), ""); 32 | fqClassName = fqClassName+"."+modelDefinition.Backend+"Backend"; 33 | 34 | LoggerManager.LogDebug("Creating model backend instance", "", "backend", fqClassName); 35 | 36 | Type t = Type.GetType(fqClassName); 37 | 38 | if (t == null) 39 | { 40 | throw new Exception($"Invalid model backend: '{modelDefinition.Backend}'"); 41 | } 42 | return (IEmbeddingBackend) Activator.CreateInstance(t, modelDefinition); 43 | } 44 | } 45 | 46 | -------------------------------------------------------------------------------- /classes/AI/Embedding/Backends/IEmbeddingBackend.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : IEmbeddingBackend 4 | * @created : Friday Jan 12, 2024 21:48:58 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.Embedding.Backends; 8 | 9 | using GatoGPT.AI; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial interface IEmbeddingBackend : AI.IModelBackend 19 | { 20 | public float[] GenerateEmbedding(string input); 21 | } 22 | -------------------------------------------------------------------------------- /classes/AI/Embedding/EmbeddingModelDefinition.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingModelDefinition 4 | * @created : Friday Jan 05, 2024 23:22:38 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.Embedding; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | using GatoGPT.Resource; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | public partial class EmbeddingModelDefinition : ModelDefinition 20 | { 21 | public EmbeddingModelDefinition(string modelResourceId, string profilePreset = "") : base(modelResourceId, profilePreset) 22 | { 23 | Backend = "BuiltinSentenceTransformer"; 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /classes/AI/IModelBackend.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : IModelInstance 4 | * @created : Friday Jan 12, 2024 18:39:12 CST 5 | */ 6 | 7 | namespace GatoGPT.AI; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial interface IModelBackend 17 | { 18 | public string InstanceId { get; set; } 19 | public bool Stateful { get; set; } 20 | public bool IsFirstRun { get; set; } 21 | public bool Running { get; set; } 22 | public AI.ModelDefinition ModelDefinition { get; set; } 23 | 24 | public void LoadModel(); 25 | public void UnloadModel(); 26 | public bool SafeToUnloadModel(); 27 | public void DeleteInstanceState(bool keepCache); 28 | public void SetInstanceId(string id, bool keepState = true); 29 | } 30 | -------------------------------------------------------------------------------- /classes/AI/ModelBackend.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelInstance 4 | * @created : Friday Jan 12, 2024 18:55:55 CST 5 | */ 6 | 7 | namespace GatoGPT.AI; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | using GodotEGP.Threading; 16 | using GodotEGP.State; 17 | 18 | public partial class ModelBackend : BackgroundJob, AI.IModelBackend 19 | { 20 | public string InstanceId { get; set; } 21 | public bool Stateful { get; set; } = false; 22 | public bool IsFirstRun { get; set; } = true; 23 | public bool Running { get; set; } 24 | public bool Persistent { get; set; } 25 | public AI.ModelDefinition ModelDefinition { get; set; } 26 | 27 | protected const int SETUP_STATE = 0; 28 | protected const int LOAD_MODEL_STATE = 1; 29 | protected const int UNLOAD_MODEL_STATE = 2; 30 | protected const int INFERENCE_RUNNING_STATE = 3; 31 | protected const int INFERENCE_FINISHED_STATE = 4; 32 | 33 | protected class ModelInstanceState : HStateMachine {}; 34 | protected class SetupState : HStateMachine {}; 35 | protected class LoadModelState : HStateMachine {}; 36 | protected class UnloadModelState : HStateMachine {}; 37 | protected class InferenceRunningState : HStateMachine {}; 38 | protected class InferenceFinishedState : HStateMachine {}; 39 | 40 | protected ModelInstanceState _state {get; set;} 41 | protected SetupState _setupState { get; set; } 42 | protected LoadModelState _loadModelState { get; set; } 43 | protected UnloadModelState _unloadModelState { get; set; } 44 | protected InferenceRunningState _inferenceRunningState { get; set; } 45 | protected InferenceFinishedState _inferenceFinishedState { get; set; } 46 | 47 | public ModelBackend(AI.ModelDefinition modelDefinition, bool isStateful = false) 48 | { 49 | ModelDefinition = modelDefinition; 50 | 51 | SetInstanceId(); 52 | 53 | Stateful = isStateful; 54 | 55 | // setup states 56 | _state = new(); 57 | _setupState = new(); 58 | _loadModelState = new(); 59 | _unloadModelState = new(); 60 | _inferenceRunningState = new(); 61 | _inferenceFinishedState = new(); 62 | 63 | // add states to root state 64 | _state.AddState(_setupState); 65 | _state.AddState(_loadModelState); 66 | _state.AddState(_unloadModelState); 67 | _state.AddState(_inferenceRunningState); 68 | _state.AddState(_inferenceFinishedState); 69 | 70 | // add state change callbacks 71 | _setupState.OnEnter = _State_Setup_OnEnter; 72 | _loadModelState.OnEnter = _State_LoadModel_OnEnter; 73 | _loadModelState.OnUpdate = _State_LoadModel_OnUpdate; 74 | _unloadModelState.OnEnter = _State_UnloadModel_OnEnter; 75 | _unloadModelState.OnUpdate = _State_UnloadModel_OnUpdate; 76 | _inferenceRunningState.OnEnter = _State_InferenceRunning_OnEnter; 77 | _inferenceRunningState.OnUpdate = _State_InferenceRunning_OnUpdate; 78 | _inferenceFinishedState.OnEnter = _State_InferenceFinished_OnEnter; 79 | 80 | // configure state transitions 81 | // after setup is finished, we can load the model 82 | _state.AddTransition(_setupState, _loadModelState, LOAD_MODEL_STATE); 83 | 84 | // with the model loaded, we can begin running the inference loop 85 | _state.AddTransition(_loadModelState, _inferenceRunningState, INFERENCE_RUNNING_STATE); 86 | 87 | // from the running state we can change to the unload state 88 | _state.AddTransition(_inferenceRunningState, _unloadModelState, UNLOAD_MODEL_STATE); 89 | 90 | // once model is unloaded inference is considered finished 91 | _state.AddTransition(_unloadModelState, _inferenceFinishedState, INFERENCE_FINISHED_STATE); 92 | 93 | // if we want to re-run inferrence, we need to restart from loadModel state 94 | _state.AddTransition(_inferenceFinishedState, _loadModelState, LOAD_MODEL_STATE); 95 | 96 | LoggerManager.LogDebug("Created instance", "", "modelDefinition", modelDefinition); 97 | } 98 | 99 | public static T CreateBackend(AI.ModelDefinition modelDefinition, bool isStateful = false) where T : IModelBackend 100 | { 101 | string fqClassName = typeof(T).FullName; 102 | fqClassName = fqClassName.Replace("."+typeof(T).Name, ""); 103 | fqClassName = fqClassName+"."+modelDefinition.Backend+"Backend"; 104 | 105 | LoggerManager.LogDebug("Creating model backend instance", "", "backend", fqClassName); 106 | 107 | Type t = Type.GetType(fqClassName); 108 | 109 | if (t == null) 110 | { 111 | throw new Exception($"Invalid model backend: '{modelDefinition.Backend}'"); 112 | } 113 | return (T) Activator.CreateInstance(t, modelDefinition, isStateful); 114 | } 115 | 116 | /******************************** 117 | * Model instance manageement * 118 | ********************************/ 119 | public void SetInstanceId(string id = "", bool keepState = true) 120 | { 121 | if (id == "") 122 | { 123 | id = $"{ModelDefinition.Id}-{GetHashCode()}"; 124 | } 125 | 126 | InstanceId = id; 127 | } 128 | 129 | public virtual void LoadModel() 130 | { 131 | 132 | } 133 | public virtual void UnloadModel() 134 | { 135 | 136 | } 137 | public virtual bool SafeToUnloadModel() 138 | { 139 | return true; 140 | } 141 | public virtual void DeleteInstanceState(bool keepCache = true) 142 | { 143 | } 144 | 145 | /******************* 146 | * State methods * 147 | *******************/ 148 | 149 | public virtual void _State_Setup_OnEnter() 150 | { 151 | } 152 | public virtual void _State_LoadModel_OnEnter() 153 | { 154 | } 155 | public virtual void _State_LoadModel_OnUpdate() 156 | { 157 | } 158 | public virtual void _State_UnloadModel_OnEnter() 159 | { 160 | } 161 | public virtual void _State_UnloadModel_OnUpdate() 162 | { 163 | } 164 | public virtual void _State_InferenceRunning_OnEnter() 165 | { 166 | } 167 | public virtual void _State_InferenceRunning_OnUpdate() 168 | { 169 | } 170 | public virtual void _State_InferenceFinished_OnEnter() 171 | { 172 | } 173 | public virtual void _State_InferenceFinished_OnUpdate() 174 | { 175 | } 176 | } 177 | 178 | -------------------------------------------------------------------------------- /classes/AI/ModelDefinition.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelDefinition 4 | * @created : Monday Jan 01, 2024 23:58:19 CST 5 | */ 6 | 7 | namespace GatoGPT.AI; 8 | 9 | using GatoGPT.Resource; 10 | using GatoGPT.AI.TextGeneration; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | using GodotEGP.Resource; 19 | 20 | using GodotEGP.Objects.Validated; 21 | 22 | public partial class ModelDefinition : VObject 23 | { 24 | // friendly ID of the model definition 25 | private string _id; 26 | public string Id 27 | { 28 | get { return _id; } 29 | set { _id = value; } 30 | } 31 | 32 | internal readonly VValue _persistent; 33 | 34 | public int Persistent 35 | { 36 | get { return _persistent.Value; } 37 | set { _persistent.Value = value; } 38 | } 39 | 40 | // model profile preset code (used to override the filename one) 41 | internal readonly VValue _profilePreset; 42 | 43 | public string ProfilePreset 44 | { 45 | get { return _profilePreset.Value; } 46 | set { _profilePreset.Value = value; } 47 | } 48 | 49 | internal readonly VValue _modelResourceId; 50 | 51 | public string ModelResourceId 52 | { 53 | get { return _modelResourceId.Value; } 54 | set { _modelResourceId.Value = value; } 55 | } 56 | 57 | internal readonly VValue> _modelResource; 58 | public ResourceObject ModelResource 59 | { 60 | get { return _modelResource.Value; } 61 | set { _modelResource.Value = value; } 62 | } 63 | 64 | internal readonly VValue _ownedBy; 65 | 66 | public string OwnedBy 67 | { 68 | get { return _ownedBy.Value; } 69 | set { _ownedBy.Value = value; } 70 | } 71 | 72 | internal readonly VValue _backend; 73 | 74 | public string Backend 75 | { 76 | get { return _backend.Value; } 77 | set { _backend.Value = value; } 78 | } 79 | 80 | public ModelDefinition(string modelResourceId, string profilePreset = "") 81 | { 82 | _persistent = AddValidatedValue(this) 83 | .Default(0) 84 | .ChangeEventsEnabled(); 85 | 86 | _profilePreset = AddValidatedValue(this) 87 | .Default("") 88 | .ChangeEventsEnabled(); 89 | 90 | _modelResourceId = AddValidatedValue(this) 91 | .Default("") 92 | .ChangeEventsEnabled(); 93 | 94 | _ownedBy = AddValidatedValue(this) 95 | .Default("local") 96 | .ChangeEventsEnabled(); 97 | 98 | _backend = AddValidatedValue(this) 99 | .Default("builtin") 100 | .ChangeEventsEnabled(); 101 | 102 | ModelResourceId = modelResourceId; 103 | ProfilePreset = profilePreset; 104 | } 105 | } 106 | 107 | // WIP: generic class for model definition 108 | public partial class ModelDefinition : ModelDefinition where TModelResource : Resource 109 | { 110 | // instance of the model resource to load for this model definition 111 | internal readonly new VValue> _modelResource; 112 | 113 | public new ResourceObject ModelResource 114 | { 115 | get { return _modelResource.Value; } 116 | set { _modelResource.Value = value; } 117 | } 118 | 119 | public ModelDefinition(string modelResourceId, string profilePreset = "") : base(modelResourceId, profilePreset) 120 | { 121 | _modelResource = AddValidatedValue>(this) 122 | .ChangeEventsEnabled(); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /classes/AI/ModelProfile.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelProfile 4 | * @created : Friday Jan 12, 2024 17:17:25 CST 5 | */ 6 | 7 | namespace GatoGPT.AI; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Objects.Validated; 12 | using GodotEGP.Logging; 13 | using GodotEGP.Service; 14 | using GodotEGP.Event.Events; 15 | using GodotEGP.Config; 16 | 17 | public partial class ModelProfileBase : VObject 18 | { 19 | internal readonly VValue _name; 20 | 21 | public string Name 22 | { 23 | get { return _name.Value; } 24 | set { _name.Value = value; } 25 | } 26 | 27 | public ModelProfileBase() 28 | { 29 | _name = AddValidatedValue(this) 30 | .Default("Default Profile"); 31 | 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/Backends/ITextGenerationBackend.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : IModelInstance 4 | * @created : Friday Jan 12, 2024 18:40:09 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration.Backends; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial interface ITextGenerationBackend : AI.IModelBackend 17 | { 18 | public void StartInference(string promptText, AI.TextGeneration.LoadParams loadParams = null, AI.TextGeneration.InferenceParams inferenceParams = null); 19 | 20 | public new AI.TextGeneration.ModelDefinition ModelDefinition { get; set; } 21 | public AI.TextGeneration.LoadParams LoadParams { get; set; } 22 | public AI.TextGeneration.InferenceParams InferenceParams { get; set; } 23 | public string Prompt { get; set; } 24 | public string CurrentInferenceLine { get; set; } 25 | public InferenceResult InferenceResult { get; set; } 26 | public Dictionary Metadata { get; set; } 27 | 28 | public bool Persistent { get; set; } 29 | 30 | public bool Finished 31 | { 32 | get { 33 | if (InferenceResult != null) 34 | { 35 | return InferenceResult.Finished; 36 | } 37 | else 38 | { 39 | return false; 40 | } 41 | } 42 | } 43 | public abstract List TokenizeString(string content, bool skipBos = true); 44 | public abstract string FormatPrompt(string prompt); 45 | } 46 | 47 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/InferenceResult.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : InferenceResult 4 | * @created : Tuesday Jan 02, 2024 17:56:14 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class InferenceResult 17 | { 18 | public int GenerationTokenCount { 19 | get { 20 | return Tokens.Count; 21 | } 22 | } 23 | 24 | public int PromptTokenCount { get; set; } 25 | 26 | public int TotalTokenCount { 27 | get { 28 | return GenerationTokenCount + PromptTokenCount; 29 | } 30 | } 31 | 32 | internal List Tokens { get; set; } 33 | 34 | public TimeSpan TimeToFirstToken { 35 | get { 36 | return FirstTokenTime - StartTime; 37 | } 38 | } 39 | 40 | public TimeSpan GenerationTime { 41 | get { 42 | return PrevTokenTime - StartTime; 43 | } 44 | } 45 | 46 | public double TokensPerSec { 47 | get { 48 | return GenerationTokenCount / (GenerationTime.TotalSeconds - TimeToFirstToken.TotalSeconds); 49 | } 50 | } 51 | 52 | public DateTime StartTime { get; set; } 53 | public DateTime EndTime { 54 | get { 55 | return PrevTokenTime; 56 | } 57 | } 58 | public DateTime FirstTokenTime { get; set; } 59 | public DateTime PrevTokenTime { get; set; } 60 | 61 | public string Output { 62 | get { 63 | return String.Join("", Tokens); 64 | } 65 | } 66 | 67 | public string OutputStripped { get; set; } 68 | 69 | internal bool Finished { get; set; } 70 | 71 | public InferenceError Error { get; set; } 72 | public bool Success { 73 | get { 74 | return (Error == null); 75 | } 76 | } 77 | 78 | public InferenceResult() 79 | { 80 | Tokens = new(); 81 | 82 | StartTime = DateTime.Now; 83 | } 84 | 85 | public void AddToken(string token) 86 | { 87 | Tokens.Add(token); 88 | } 89 | } 90 | 91 | public partial class InferenceError 92 | { 93 | public string Type { get; set; } 94 | public string Code { get; set; } 95 | public string Message { get; set; } 96 | public Exception Exception { get; set; } 97 | } 98 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/LlamaCacheManager.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaCacheManager 4 | * @created : Tuesday Jan 09, 2024 22:55:11 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using LLama; 17 | using LLama.Common; 18 | 19 | using System.Security.Cryptography; 20 | using System.Text.RegularExpressions; 21 | 22 | public partial class LlamaCacheManager 23 | { 24 | private string _stateId { get; set; } 25 | 26 | private string _cacheDataDir { get; set; } = Path.Combine(OS.GetUserDataDir(), "Cache"); 27 | 28 | public LlamaCacheManager(uint? modelContextSize, float? modelRopeBase, float? modelRopeScale, string modelId, string modelHash) 29 | { 30 | _stateId = $"{modelId}-{modelHash}-{modelContextSize}-{modelRopeBase}-{modelRopeScale}"; 31 | 32 | LoggerManager.LogDebug("Creating cache manager", "", "stateId", _stateId); 33 | 34 | CreateCacheDir(); 35 | } 36 | 37 | public void CreateCacheDir() 38 | { 39 | Directory.CreateDirectory(_cacheDataDir); 40 | 41 | Directory.CreateDirectory(GetCacheBaseDir()); 42 | } 43 | 44 | public string GetCacheBaseDir() 45 | { 46 | return Path.Combine(_cacheDataDir, _stateId); 47 | } 48 | 49 | public string GetCacheSaveDir(string cacheId, string subName = "") 50 | { 51 | return Path.Combine(GetCacheBaseDir(), cacheId, subName); 52 | } 53 | 54 | public async Task SavePromptCache(string prompt, LLamaContext context, InstructExecutor executor) 55 | { 56 | string cacheId = GetPromptContentHash(prompt); 57 | string promptSavePath = GetCacheSaveDir(cacheId, "prompt"); 58 | 59 | LoggerManager.LogDebug("Saving prompt cache", "", "stateId", _stateId); 60 | LoggerManager.LogDebug("", "", "prompt", prompt); 61 | LoggerManager.LogDebug("", "", "cacheId", cacheId); 62 | LoggerManager.LogDebug("", "", "saveDir", promptSavePath); 63 | 64 | // create directory to hold the state files 65 | Directory.CreateDirectory(GetCacheSaveDir(cacheId)); 66 | 67 | // save the full prompt content 68 | using (StreamWriter writer = new StreamWriter(promptSavePath)) 69 | { 70 | writer.Write(prompt); 71 | } 72 | 73 | // save the context and executor states 74 | context.SaveState(GetCacheSaveDir(cacheId, "context")); 75 | await executor.SaveState(GetCacheSaveDir(cacheId, "executor")); 76 | 77 | return true; 78 | } 79 | 80 | public string GetPromptContentHash(string content) 81 | { 82 | string hash = ""; 83 | 84 | byte[] bytes; 85 | 86 | using (HashAlgorithm algorithm = SHA256.Create()) 87 | bytes = algorithm.ComputeHash(System.Text.Encoding.UTF8.GetBytes(content)); 88 | 89 | hash = ""; 90 | foreach (byte b in bytes) 91 | { 92 | hash += b.ToString("x2"); 93 | } 94 | 95 | return hash; 96 | } 97 | 98 | public async Task GetCachedPrompt(string prompt, AI.TextGeneration.InferenceParams inferenceParams, LLamaContext context, InstructExecutor executor) 99 | { 100 | // search the cache directory for existing states, then search for 101 | // prompt files matching the given prompt 102 | DirectoryInfo dir = new DirectoryInfo (GetCacheBaseDir()); 103 | 104 | DirectoryInfo[] dirs = dir.GetDirectories().OrderByDescending(p => p.CreationTime).ToArray(); 105 | 106 | foreach (DirectoryInfo cacheDir in dirs) 107 | { 108 | // LoggerManager.LogDebug("Found cache dir", "", "cacheDir", cacheDir.ToString()); 109 | 110 | // strip the input prefix and suffix when comparing the prompts 111 | string promptCache = File.ReadAllText(Path.Combine(cacheDir.ToString(), "prompt")); 112 | string currentPrompt = prompt; 113 | if (inferenceParams.InputPrefix.Length > 0) 114 | { 115 | promptCache = promptCache.Replace(inferenceParams.InputPrefix, ""); 116 | currentPrompt = currentPrompt.Replace(inferenceParams.InputPrefix, ""); 117 | } 118 | if (inferenceParams.InputSuffix.Length > 0) 119 | { 120 | promptCache = promptCache.Replace(inferenceParams.InputSuffix, ""); 121 | currentPrompt = currentPrompt.Replace(inferenceParams.InputSuffix, ""); 122 | } 123 | 124 | // LoggerManager.LogDebug("", "", "cachePrompt", promptCache); 125 | // LoggerManager.LogDebug("", "", "currentPrompt", currentPrompt); 126 | 127 | if (currentPrompt.StartsWith(promptCache)) 128 | { 129 | LoggerManager.LogDebug("Prompt cache hit!"); 130 | 131 | string strippedPrompt = ExtractPromptAdditionalText(currentPrompt, promptCache); 132 | 133 | LoggerManager.LogDebug("Stripped prompt after cache", "", "strippedPrompt", strippedPrompt); 134 | 135 | // load the state 136 | string cacheId = cacheDir.ToString().GetFile(); 137 | context.LoadState(GetCacheSaveDir(cacheId, "context")); 138 | await executor.LoadState(GetCacheSaveDir(cacheId, "executor")); 139 | 140 | // return stripped prompt including the suffix we removed 141 | // earlier 142 | return strippedPrompt+inferenceParams.InputSuffix; 143 | } 144 | 145 | } 146 | 147 | LoggerManager.LogDebug("Cache miss!"); 148 | 149 | return prompt; 150 | } 151 | 152 | public string ExtractPromptAdditionalText(string currentPrompt, string promptCache) 153 | { 154 | var regex = new Regex(Regex.Escape(promptCache)); 155 | var newText = regex.Replace(currentPrompt, "", 1); 156 | 157 | return newText; 158 | } 159 | 160 | public void DeleteCache() 161 | { 162 | LoggerManager.LogDebug("Deleting all cache", "", "stateId", _stateId); 163 | 164 | Directory.Delete(GetCacheBaseDir(), true); 165 | } 166 | } 167 | 168 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/LlamaModelDefinition.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaModelDefinition 4 | * @created : Friday Jan 05, 2024 23:05:17 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using GatoGPT.Resource; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Objects.Validated; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | // WIP: new class using generic modeldefinition class 20 | public partial class LlamaModelDefinition : ModelDefinition 21 | { 22 | // the profile instance used for the model definition 23 | internal readonly VNative _modelProfile; 24 | 25 | public ModelProfile ModelProfile 26 | { 27 | get { return _modelProfile.Value; } 28 | set { _modelProfile.Value = value; } 29 | } 30 | 31 | internal readonly VNative _modelProfileOverride; 32 | 33 | public ModelProfile ModelProfileOverride 34 | { 35 | get { return _modelProfileOverride.Value; } 36 | set { _modelProfileOverride.Value = value; } 37 | } 38 | 39 | internal readonly VValue _promptCache; 40 | 41 | public bool PromptCache 42 | { 43 | get { return _promptCache.Value; } 44 | set { _promptCache.Value = value; } 45 | } 46 | 47 | internal readonly VValue _vision; 48 | 49 | public bool Vision 50 | { 51 | get { return _vision.Value; } 52 | set { _vision.Value = value; } 53 | } 54 | 55 | internal readonly VValue> _dynamicCtxConfigs; 56 | 57 | public List DynamicCtxConfigs 58 | { 59 | get { return _dynamicCtxConfigs.Value; } 60 | set { _dynamicCtxConfigs.Value = value; } 61 | } 62 | 63 | public LlamaModelDefinition(string modelResourceId, string profilePreset = "", ModelProfile modelProfile = null) : base(modelResourceId, profilePreset) 64 | { 65 | _modelProfile = AddValidatedNative(this) 66 | .ChangeEventsEnabled(); 67 | 68 | _modelProfileOverride = AddValidatedNative(this) 69 | .ChangeEventsEnabled(); 70 | 71 | _promptCache = AddValidatedValue(this) 72 | .Default(false) 73 | .ChangeEventsEnabled(); 74 | 75 | _vision = AddValidatedValue(this) 76 | .Default(true) 77 | .ChangeEventsEnabled(); 78 | 79 | _dynamicCtxConfigs = AddValidatedValue>(this) 80 | .Default(new List()) 81 | .ChangeEventsEnabled(); 82 | 83 | if (modelProfile != null) 84 | { 85 | ModelProfileOverride = modelProfile; 86 | } 87 | } 88 | } 89 | 90 | public class DynamicCtxConfig { 91 | public int NCtx { get; set; } 92 | public int NGpuLayers { get; set; } 93 | public int NThreads { get; set; } 94 | } 95 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/LoadParams.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaLoadParams 4 | * @created : Friday Jan 12, 2024 17:21:03 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Objects.Validated; 12 | using GodotEGP.Logging; 13 | using GodotEGP.Service; 14 | using GodotEGP.Event.Events; 15 | using GodotEGP.Config; 16 | 17 | public partial class LoadParams : VObject 18 | { 19 | // token context length 20 | internal readonly VValue _nCtx; 21 | 22 | public int NCtx 23 | { 24 | get { return _nCtx.Value; } 25 | set { _nCtx.Value = value; } 26 | } 27 | 28 | // size of tokens per batch 29 | internal readonly VValue _nBatch; 30 | 31 | public int NBatch 32 | { 33 | get { return _nBatch.Value; } 34 | set { _nBatch.Value = value; } 35 | } 36 | 37 | // rope_freq_base 38 | internal readonly VValue _ropeFreqBase; 39 | 40 | public double RopeFreqBase 41 | { 42 | get { return _ropeFreqBase.Value; } 43 | set { _ropeFreqBase.Value = value; } 44 | } 45 | 46 | // rope_freq_scale 47 | internal readonly VValue _ropeFreqScale; 48 | 49 | public double RopeFreqScale 50 | { 51 | get { return _ropeFreqScale.Value; } 52 | set { _ropeFreqScale.Value = value; } 53 | } 54 | 55 | // number of layers to offload to the GPU 56 | internal readonly VValue _nGpuLayers; 57 | 58 | public int NGpuLayers 59 | { 60 | get { return _nGpuLayers.Value; } 61 | set { _nGpuLayers.Value = value; } 62 | } 63 | 64 | // use mlock when loading the model to memory 65 | internal readonly VValue _useMlock; 66 | 67 | public bool UseMlock 68 | { 69 | get { return _useMlock.Value; } 70 | set { _useMlock.Value = value; } 71 | } 72 | 73 | // GPU id to use for the main model 74 | internal readonly VValue _mainGpu; 75 | 76 | public int MainGpu 77 | { 78 | get { return _mainGpu.Value; } 79 | set { _mainGpu.Value = value; } 80 | } 81 | 82 | // seed to use for random generation 83 | internal readonly VValue _seed; 84 | 85 | public int Seed 86 | { 87 | get { return _seed.Value; } 88 | set { _seed.Value = value; } 89 | } 90 | 91 | // whether the model should use half-precision for the key/value cache 92 | internal readonly VValue _f16Kv; 93 | 94 | public bool F16KV 95 | { 96 | get { return _f16Kv.Value; } 97 | set { _f16Kv.Value = value; } 98 | } 99 | 100 | // whether to load using mmap 101 | internal readonly VValue _useMMap; 102 | 103 | public bool UseMMap 104 | { 105 | get { return _useMMap.Value; } 106 | set { _useMMap.Value = value; } 107 | } 108 | 109 | internal readonly VValue _kvOffload; 110 | 111 | public bool KVOffload 112 | { 113 | get { return _kvOffload.Value; } 114 | set { _kvOffload.Value = value; } 115 | } 116 | 117 | // multimodal --mmproj 118 | internal readonly VValue _mmProjPath; 119 | 120 | public string MMProjPath 121 | { 122 | get { return _mmProjPath.Value; } 123 | set { _mmProjPath.Value = value; } 124 | } 125 | 126 | public LoadParams() 127 | { 128 | _nCtx = AddValidatedValue(this) 129 | .Default(2048) 130 | .ChangeEventsEnabled(); 131 | 132 | _nBatch = AddValidatedValue(this) 133 | .Default(512) 134 | .ChangeEventsEnabled(); 135 | 136 | _ropeFreqBase = AddValidatedValue(this) 137 | .Default(0) 138 | .ChangeEventsEnabled(); 139 | 140 | _ropeFreqScale = AddValidatedValue(this) 141 | .Default(0) 142 | .ChangeEventsEnabled(); 143 | 144 | _nGpuLayers = AddValidatedValue(this) 145 | .Default(0) 146 | .ChangeEventsEnabled(); 147 | 148 | _useMlock = AddValidatedValue(this) 149 | .Default(false) 150 | .ChangeEventsEnabled(); 151 | 152 | _mainGpu = AddValidatedValue(this) 153 | .Default(0) 154 | .ChangeEventsEnabled(); 155 | 156 | _seed = AddValidatedValue(this) 157 | .Default(-1) 158 | .ChangeEventsEnabled(); 159 | 160 | _f16Kv = AddValidatedValue(this) 161 | .Default(true) 162 | .ChangeEventsEnabled(); 163 | 164 | _useMMap = AddValidatedValue(this) 165 | .Default(true) 166 | .ChangeEventsEnabled(); 167 | 168 | _kvOffload = AddValidatedValue(this) 169 | .Default(true) 170 | .ChangeEventsEnabled(); 171 | 172 | _mmProjPath = AddValidatedValue(this) 173 | .Default("") 174 | .ChangeEventsEnabled(); 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/ModelDefinition.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : TextGenerationModelDefinition 4 | * @created : Friday Jan 12, 2024 17:44:45 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | // TODO: stop depending on LlamaModelDefintion class 17 | public partial class ModelDefinition : LlamaModelDefinition 18 | { 19 | public ModelDefinition(string modelResourceId, string profilePreset = "", ModelProfile modelProfile = null) : base(modelResourceId, profilePreset, modelProfile) 20 | { 21 | Backend = "LlamaCpp"; 22 | } 23 | } 24 | 25 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/ModelProfile.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelProfile 4 | * @created : Monday Jan 01, 2024 21:04:31 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | using GodotEGP.Objects.Validated; 16 | 17 | public partial class ModelProfile : AI.ModelProfileBase 18 | { 19 | internal readonly VNative _loadParams; 20 | 21 | public LoadParams LoadParams 22 | { 23 | get { return _loadParams.Value; } 24 | set { _loadParams.Value = value; } 25 | } 26 | 27 | internal readonly VNative _inferenceParams; 28 | 29 | public InferenceParams InferenceParams 30 | { 31 | get { return _inferenceParams.Value; } 32 | set { _inferenceParams.Value = value; } 33 | } 34 | 35 | public ModelProfile() 36 | { 37 | _loadParams = AddValidatedNative(this) 38 | .Default(new LoadParams()) 39 | .ChangeEventsEnabled(); 40 | 41 | _inferenceParams = AddValidatedNative(this) 42 | .Default(new InferenceParams()) 43 | .ChangeEventsEnabled(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/TokenFilter/CaptureMarkdownOutput.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CaptureMarkdownOutput 4 | * @created : Tuesday Jan 30, 2024 23:16:15 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration.TokenFilter; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using System.Text.RegularExpressions; 17 | 18 | public partial class CaptureMarkdownOutput : ITokenFilter 19 | { 20 | string _codeBlockStartPattern = @"[`]+[\w]*(\n*.)*\n*[`]*"; 21 | string _codeBlockFullPattern = @"```([a-z]*)\n([\s\S]*?)\n```"; 22 | 23 | public bool Match(string[] tokens, string[] allTokens) 24 | { 25 | LoggerManager.LogDebug("Matching with tokens", "", "tokens", tokens); 26 | 27 | bool match = false; 28 | string tokensString = String.Join("", tokens); 29 | 30 | 31 | Match blockStart = Regex.Match(tokensString, _codeBlockStartPattern, RegexOptions.Multiline); 32 | Match blockFull = Regex.Match(tokensString, _codeBlockFullPattern, RegexOptions.Multiline); 33 | 34 | if (blockStart.Success) 35 | { 36 | match = true; 37 | } 38 | if (blockFull.Success) 39 | { 40 | LoggerManager.LogDebug("Matched full code block", "", "tokens", tokensString); 41 | 42 | match = false; 43 | } 44 | 45 | return match; 46 | } 47 | 48 | public string[] Filter(string[] tokens, string[] allTokens) 49 | { 50 | string tokensString = String.Join("", tokens); 51 | Match blockFull = Regex.Match(tokensString, _codeBlockFullPattern, RegexOptions.Multiline); 52 | 53 | string code = blockFull.Groups[2].Value; 54 | string codeType = blockFull.Groups[1].Value; 55 | 56 | LoggerManager.LogDebug("Parsed code block from tokens", "", codeType, code); 57 | 58 | return new string[] { code }; 59 | } 60 | } 61 | 62 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/TokenFilter/ITokenFilter.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ITokenFilter 4 | * @created : Tuesday Jan 30, 2024 19:55:48 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial interface ITokenFilter 17 | { 18 | public bool Match(string[] tokens, string[] allTokens); 19 | public string[] Filter(string[] tokens, string[] allTokens); 20 | } 21 | 22 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/TokenFilter/StripAntiprompt.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : StripAntiprompt 4 | * @created : Tuesday Jan 30, 2024 21:48:52 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration.TokenFilter; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using System.Text.RegularExpressions; 17 | 18 | public partial class StripAntiprompt : ITokenFilter 19 | { 20 | public List Antiprompts { get; set; } 21 | 22 | public StripAntiprompt(List antiprompts) 23 | { 24 | Antiprompts = antiprompts; 25 | } 26 | 27 | public bool Match(string[] tokens, string[] allTokens) 28 | { 29 | LoggerManager.LogDebug("Matching with tokens", "", "tokens", tokens); 30 | 31 | bool match = false; 32 | string tokensString = String.Join("", tokens); 33 | 34 | foreach (string antiprompt in Antiprompts) 35 | { 36 | if (tokensString.Trim().Length > 0 && antiprompt.StartsWith(tokensString)) 37 | { 38 | LoggerManager.LogDebug("Token partial match anti-prompt", "", tokensString, antiprompt); 39 | 40 | match = true; 41 | break; 42 | } 43 | } 44 | 45 | return match; 46 | } 47 | 48 | public string[] Filter(string[] tokens, string[] allTokens) 49 | { 50 | int tokensCount = tokens.Count(); 51 | string tokensString = String.Join("", tokens); 52 | 53 | foreach (var antiprompt in Antiprompts) 54 | { 55 | tokensString = tokensString.Replace(antiprompt, String.Empty); 56 | } 57 | 58 | // make fake array of tokens simply because after stripping some away 59 | // it's not possible to return them to token form 60 | string[] fakeArray = new string[] { tokensString }; 61 | 62 | return fakeArray; 63 | } 64 | } 65 | 66 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/TokenFilter/StripLeadingSpace.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : StripLeadingSpace 4 | * @created : Tuesday Jan 30, 2024 20:19:18 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration.TokenFilter; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using System.Text.RegularExpressions; 17 | 18 | public partial class StripLeadingSpace : ITokenFilter 19 | { 20 | public bool Match(string[] tokens, string[] allTokens) 21 | { 22 | bool match = false; 23 | string tokensString = String.Join("", tokens); 24 | 25 | // check if we're dealing with a token which starts with a space and all 26 | // current tokens once stripped don't equal a length 27 | match = Regex.IsMatch(tokensString, @"^[ ]+[\w]+$") && String.Join("", allTokens).Trim().Length == 0; 28 | 29 | return match; 30 | } 31 | 32 | public string[] Filter(string[] tokens, string[] allTokens) 33 | { 34 | string str = String.Join("", tokens); 35 | str = str.Trim(); 36 | 37 | LoggerManager.LogDebug("After filter", "", "after", str); 38 | 39 | return new string[] { str }; 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/TokenFilter/TokenFilterProcessor.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : TokenFilterProcessor 4 | * @created : Tuesday Jan 30, 2024 19:54:16 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration.TokenFilter; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class StreamingTokenFilter 17 | { 18 | public List FilteredTokens = new(); 19 | public List ReleasedTokens = new(); 20 | public List Filters { get; set; } = new(); 21 | 22 | public void AddFilter(ITokenFilter filter) 23 | { 24 | Filters.Add(filter); 25 | } 26 | 27 | public bool FilterToken(string token, string[] allTokens) 28 | { 29 | bool filtered = false; 30 | 31 | foreach (ITokenFilter filter in Filters) 32 | { 33 | LoggerManager.LogDebug("Running filter", "", filter.GetType().Name, String.Join("", FilteredTokens)+token); 34 | 35 | filtered = filter.Match(FilteredTokens.Concat(new string[] { token }).ToArray(), allTokens); 36 | 37 | LoggerManager.LogDebug("Filter result", "", filter.GetType().Name, filtered); 38 | 39 | if (filtered) 40 | { 41 | FilteredTokens.Add(token); 42 | 43 | LoggerManager.LogDebug("Filtering matched token", "", "token", token); 44 | LoggerManager.LogDebug("Current filtered tokens", "", "filteredTokens", FilteredTokens); 45 | 46 | return filtered; 47 | } 48 | } 49 | 50 | if (!filtered) 51 | { 52 | if (FilteredTokens.Count > 0 && ReleasedTokens.Count == 0) 53 | { 54 | LoggerManager.LogDebug("Release filtered tokens", "", "filteredTokens", FilteredTokens); 55 | 56 | var t = FilteredTokens.Concat(new string[] { token }).ToArray(); 57 | 58 | foreach (ITokenFilter filter in Filters) 59 | { 60 | if (filter.Match(t, allTokens)) 61 | { 62 | continue; 63 | } 64 | t = filter.Filter(t, allTokens); 65 | } 66 | 67 | ReleasedTokens = t.ToList(); 68 | 69 | FilteredTokens = new(); 70 | } 71 | } 72 | 73 | return filtered; 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /classes/AI/TextGeneration/TokenizedString.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : TokenizedString 4 | * @created : Wednesday Jan 31, 2024 17:55:49 CST 5 | */ 6 | 7 | namespace GatoGPT.AI.TextGeneration; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class TokenizedString 17 | { 18 | public int Id { get; set; } 19 | public string Token { get; set; } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /classes/CLI/ProcessRunner.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ProcessRunner 4 | * @created : Friday Jan 12, 2024 23:37:39 CST 5 | */ 6 | 7 | namespace GatoGPT.CLI; 8 | 9 | using GatoGPT.Event; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | using GodotEGP.Threading; 18 | using System.ComponentModel; 19 | 20 | using System.Diagnostics; 21 | using System.Text; 22 | 23 | public partial class ProcessRunner : BackgroundJob 24 | { 25 | public string Command { get; set; } 26 | public string[] Args { get; set; } 27 | public int ReturnCode { get; set; } = -1; 28 | 29 | public bool Success { 30 | get { 31 | return ReturnCode == 0; 32 | } 33 | } 34 | 35 | 36 | private ProcessStartInfo _processStartInfo; 37 | private Process _process; 38 | private TaskCompletionSource _task; 39 | 40 | public string Output { 41 | get { 42 | return String.Join("", OutputStrings); 43 | } 44 | } 45 | public string OutputStripped { 46 | get { 47 | return Output.Trim(); 48 | } 49 | } 50 | public string Error { 51 | get { 52 | return String.Join("\n", ErrorLines); 53 | } 54 | } 55 | public string ErrorStripped { 56 | get { 57 | return Error.Trim(); 58 | } 59 | } 60 | public List OutputStrings { get; set; } 61 | public List ErrorLines { get; set; } 62 | 63 | public List OutputFilters { get; set; } = new(); 64 | 65 | public DateTime StartTime { get; set; } = DateTime.Now; 66 | public DateTime EndTime { get; set; } = DateTime.Now; 67 | public TimeSpan ExecutionTime { 68 | get { 69 | return EndTime - StartTime; 70 | } 71 | } 72 | 73 | public ConsoleAutomator _automator { get; set; } 74 | 75 | public ProcessRunner(string command, params string[] args) 76 | { 77 | Command = command; 78 | Args = args; 79 | _task = new(); 80 | 81 | OutputStrings = new(); 82 | ErrorLines = new(); 83 | } 84 | 85 | public void AddArguments(params string[] args) 86 | { 87 | Args = Args.Concat(args).ToArray(); 88 | } 89 | 90 | public void AddOutputFilter(Func func) 91 | { 92 | OutputFilters.Add(new ProcessOutputFilter(func)); 93 | } 94 | 95 | public bool GetOutputFilterMatch(string output) 96 | { 97 | bool match = false; 98 | 99 | foreach (var filter in OutputFilters) 100 | { 101 | if (filter.Run(output)) 102 | { 103 | match = true; 104 | } 105 | } 106 | 107 | return match; 108 | } 109 | 110 | public async Task Execute() 111 | { 112 | _processStartInfo = new ProcessStartInfo() { 113 | FileName = Command, 114 | Arguments = String.Join(" ", Args), 115 | RedirectStandardError = true, 116 | RedirectStandardInput = true, 117 | RedirectStandardOutput = true, 118 | }; 119 | 120 | _process = new Process() { 121 | StartInfo = _processStartInfo, 122 | EnableRaisingEvents = true, 123 | }; 124 | 125 | _process.OutputDataReceived += _On_Process_OutputData; 126 | _process.ErrorDataReceived += _On_Process_ErrorData; 127 | 128 | Run(); 129 | 130 | return await _task.Task; 131 | } 132 | 133 | public void Kill() 134 | { 135 | _process.Kill(true); 136 | } 137 | 138 | public void ProcessExitSuccess() 139 | { 140 | this.Emit((e) => e.SetData(ReturnCode)); 141 | } 142 | public void ProcessExitError() 143 | { 144 | this.Emit((e) => e.SetData(ReturnCode)); 145 | } 146 | 147 | public void AutomatorStandardInputRead(object sender, ConsoleInputReadEventArgs e) 148 | { 149 | ProcessConsoleOutput(e.Input); 150 | } 151 | 152 | public override void DoWork(object sender, DoWorkEventArgs e) 153 | { 154 | LoggerManager.LogDebug("Executing process", "", "process", $"{Command} {String.Join(" ", Args)}"); 155 | 156 | this.Emit(); 157 | 158 | _process.Start(); 159 | 160 | _automator = new ConsoleAutomator(_process.StandardInput, _process.StandardOutput); 161 | _automator.StandardInputRead += AutomatorStandardInputRead; 162 | _automator.StartAutomate(); 163 | 164 | _process.BeginErrorReadLine(); 165 | // _process.BeginOutputReadLine(); 166 | 167 | _process.WaitForExit(); 168 | _automator.StandardInputRead -= AutomatorStandardInputRead; 169 | 170 | ReturnCode = _process.ExitCode; 171 | 172 | LoggerManager.LogDebug($"Process exited with code {ReturnCode}", "", "process", $"{Command} {String.Join(" ", Args)}"); 173 | _process.Dispose(); 174 | 175 | EndTime = DateTime.Now; 176 | 177 | e.Result = ReturnCode; 178 | } 179 | 180 | public override void ProgressChanged(object sender, ProgressChangedEventArgs e) 181 | { 182 | } 183 | 184 | public void ProcessConsoleOutput(string output) 185 | { 186 | if (!String.IsNullOrEmpty(output)) 187 | { 188 | if (GetOutputFilterMatch(output)) 189 | { 190 | LoggerManager.LogDebug("Output filter match, excluding", "", "excludedOutput", output); 191 | return; 192 | } 193 | 194 | LoggerManager.LogDebug("Process output", "", "output", output); 195 | 196 | OutputStrings.Add(output); 197 | 198 | this.Emit((e) => e.Line = output); 199 | } 200 | } 201 | 202 | public void _On_Process_OutputData(object sender, DataReceivedEventArgs args) 203 | { 204 | ProcessConsoleOutput(args.Data+"\n"); 205 | } 206 | public void _On_Process_ErrorData(object sender, DataReceivedEventArgs args) 207 | { 208 | string output = args.Data; 209 | if (!String.IsNullOrEmpty(args.Data)) 210 | { 211 | if (GetOutputFilterMatch(output)) 212 | { 213 | LoggerManager.LogDebug("Error output filter match, excluding", "", "excludedErrorOutput", output); 214 | return; 215 | } 216 | 217 | LoggerManager.LogDebug("Process output error", "", "output", output); 218 | 219 | ErrorLines.Add(output); 220 | 221 | this.Emit((e) => e.Line = output); 222 | } 223 | } 224 | 225 | 226 | public override void RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e) 227 | { 228 | if (ReturnCode == 0) 229 | { 230 | ProcessExitSuccess(); 231 | } 232 | else { 233 | ProcessExitError(); 234 | } 235 | 236 | _task.SetResult(ReturnCode); 237 | } 238 | 239 | public override void RunWorkerError(object sender, RunWorkerCompletedEventArgs e) 240 | { 241 | ProcessExitError(); 242 | 243 | _task.SetResult(ReturnCode); 244 | } 245 | } 246 | 247 | public partial class ProcessOutputFilter 248 | { 249 | public Func Func { get; set; } 250 | 251 | public ProcessOutputFilter(Func func) 252 | { 253 | Func = func; 254 | } 255 | 256 | public bool Run(string output) 257 | { 258 | return Func(output); 259 | } 260 | } 261 | 262 | public class ConsoleInputReadEventArgs : EventArgs 263 | { 264 | public ConsoleInputReadEventArgs(string input) 265 | { 266 | this.Input = input; 267 | } 268 | 269 | public string Input { get; private set; } 270 | } 271 | 272 | public interface IConsoleAutomator 273 | { 274 | StreamWriter StandardInput { get; } 275 | 276 | event EventHandler StandardInputRead; 277 | } 278 | 279 | public abstract class ConsoleAutomatorBase : IConsoleAutomator 280 | { 281 | protected readonly StringBuilder inputAccumulator = new StringBuilder(); 282 | 283 | protected readonly byte[] buffer = new byte[256]; 284 | 285 | protected volatile bool stopAutomation; 286 | 287 | public StreamWriter StandardInput { get; protected set; } 288 | 289 | protected StreamReader StandardOutput { get; set; } 290 | 291 | protected StreamReader StandardError { get; set; } 292 | 293 | public event EventHandler StandardInputRead; 294 | 295 | protected void BeginReadAsync() 296 | { 297 | if (!this.stopAutomation) { 298 | this.StandardOutput.BaseStream.BeginRead(this.buffer, 0, this.buffer.Length, this.ReadHappened, null); 299 | } 300 | } 301 | 302 | protected virtual void OnAutomationStopped() 303 | { 304 | this.stopAutomation = true; 305 | this.StandardOutput.DiscardBufferedData(); 306 | } 307 | 308 | private void ReadHappened(IAsyncResult asyncResult) 309 | { 310 | var bytesRead = this.StandardOutput.BaseStream.EndRead(asyncResult); 311 | if (bytesRead == 0) { 312 | this.OnAutomationStopped(); 313 | return; 314 | } 315 | 316 | var input = this.StandardOutput.CurrentEncoding.GetString(this.buffer, 0, bytesRead); 317 | this.inputAccumulator.Append(input); 318 | 319 | if (bytesRead < this.buffer.Length) { 320 | this.OnInputRead(this.inputAccumulator.ToString()); 321 | } 322 | 323 | this.BeginReadAsync(); 324 | } 325 | 326 | private void OnInputRead(string input) 327 | { 328 | var handler = this.StandardInputRead; 329 | if (handler == null) { 330 | return; 331 | } 332 | 333 | handler(this, new ConsoleInputReadEventArgs(input)); 334 | this.inputAccumulator.Clear(); 335 | } 336 | } 337 | 338 | public class ConsoleAutomator : ConsoleAutomatorBase, IConsoleAutomator 339 | { 340 | public ConsoleAutomator(StreamWriter standardInput, StreamReader standardOutput) 341 | { 342 | this.StandardInput = standardInput; 343 | this.StandardOutput = standardOutput; 344 | } 345 | 346 | public void StartAutomate() 347 | { 348 | this.stopAutomation = false; 349 | this.BeginReadAsync(); 350 | } 351 | 352 | public void StopAutomation() 353 | { 354 | this.OnAutomationStopped(); 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /classes/Config/EmbeddingModelManagerConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingModelManagerConfig 4 | * @created : Friday Jan 05, 2024 23:27:14 CST 5 | */ 6 | 7 | namespace GatoGPT.Config; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using GodotEGP.Objects.Validated; 17 | 18 | public partial class EmbeddingModelManagerConfig : VConfig 19 | { 20 | public EmbeddingModelManagerConfig() 21 | { 22 | } 23 | } 24 | 25 | -------------------------------------------------------------------------------- /classes/Config/GlobalConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : GlobalConfig 4 | * @created : Sunday Jan 14, 2024 18:02:07 CST 5 | */ 6 | 7 | namespace GodotEGP.Config; 8 | 9 | using GatoGPT.Config; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Objects.Validated; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | public partial class GlobalConfig : VConfig 20 | { 21 | internal VNative _openAiConfig = new(); 22 | 23 | public OpenAIConfig OpenAIConfig 24 | { 25 | get { return _openAiConfig.Value; } 26 | set { _openAiConfig.Value = value; } 27 | } 28 | 29 | partial void InitConfigParams() 30 | { 31 | _openAiConfig = AddValidatedNative(this) 32 | .Default(new OpenAIConfig()) 33 | .ChangeEventsEnabled(); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /classes/Config/LlamaCacheManagerConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaCacheManagerConfig 4 | * @created : Wednesday Jan 10, 2024 17:46:44 CST 5 | */ 6 | 7 | namespace GatoGPT.Config; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using GodotEGP.Objects.Validated; 17 | 18 | public partial class LlamaCacheManagerConfig : VConfig 19 | { 20 | internal readonly VValue _maxCacheSizeMb; 21 | 22 | public long MaxCacheSizeMb 23 | { 24 | get { return _maxCacheSizeMb.Value; } 25 | set { _maxCacheSizeMb.Value = value; } 26 | } 27 | 28 | internal readonly VValue _maxCacheAgeMin; 29 | 30 | public int MaxCacheAgeMin 31 | { 32 | get { return _maxCacheAgeMin.Value; } 33 | set { _maxCacheAgeMin.Value = value; } 34 | } 35 | 36 | internal readonly VValue _cacheTimoutSec; 37 | 38 | public int CacheTimeoutSec 39 | { 40 | get { return _cacheTimoutSec.Value; } 41 | set { _cacheTimoutSec.Value = value; } 42 | } 43 | 44 | 45 | public LlamaCacheManagerConfig() 46 | { 47 | _maxCacheSizeMb = AddValidatedValue(this) 48 | .Default(1000) 49 | .ChangeEventsEnabled(); 50 | 51 | _maxCacheAgeMin = AddValidatedValue(this) 52 | .Default(72 * 60) 53 | .ChangeEventsEnabled(); 54 | 55 | _cacheTimoutSec = AddValidatedValue(this) 56 | .Default(600) 57 | .ChangeEventsEnabled(); 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /classes/Config/LlamaModelManagerConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : TextGenerationModelManagerConfig 4 | * @created : Tuesday Jan 02, 2024 00:48:03 CST 5 | */ 6 | 7 | namespace GatoGPT.Config; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using GodotEGP.Objects.Validated; 17 | 18 | public partial class TextGenerationModelManagerConfig : VConfig 19 | { 20 | internal readonly VValue _maxThreads; 21 | 22 | public int MaxThreads 23 | { 24 | get { return _maxThreads.Value; } 25 | set { _maxThreads.Value = value; } 26 | } 27 | 28 | internal readonly VValue _maxLoadedModels; 29 | 30 | public int MaxLoadedModels 31 | { 32 | get { return _maxLoadedModels.Value; } 33 | set { _maxLoadedModels.Value = value; } 34 | } 35 | 36 | internal readonly VValue _modelIdleUnloadTimeout; 37 | 38 | public int ModelIdleUnloadTimeout 39 | { 40 | get { return _modelIdleUnloadTimeout.Value; } 41 | set { _modelIdleUnloadTimeout.Value = value; } 42 | } 43 | 44 | public TextGenerationModelManagerConfig() 45 | { 46 | _maxThreads = AddValidatedValue(this) 47 | .Default(-1) 48 | .ChangeEventsEnabled(); 49 | 50 | _maxLoadedModels = AddValidatedValue(this) 51 | .Default(1) 52 | .ChangeEventsEnabled(); 53 | 54 | _modelIdleUnloadTimeout = AddValidatedValue(this) 55 | .Default(-1) 56 | .ChangeEventsEnabled(); 57 | } 58 | } 59 | 60 | -------------------------------------------------------------------------------- /classes/Config/ModelDefinitionsConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelDefinitionsConfig 4 | * @created : Friday Jan 12, 2024 17:27:36 CST 5 | */ 6 | 7 | namespace GatoGPT.Config; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | using GatoGPT.AI.Embedding; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Objects.Validated; 15 | using GodotEGP.Logging; 16 | using GodotEGP.Service; 17 | using GodotEGP.Event.Events; 18 | using GodotEGP.Config; 19 | 20 | public partial class ModelDefinitionsConfig : VConfig 21 | { 22 | // holds llama model definitions 23 | internal readonly VValue> _textGenerationModelDefinitions; 24 | 25 | public Dictionary TextGeneration 26 | { 27 | get { return _textGenerationModelDefinitions.Value; } 28 | set { _textGenerationModelDefinitions.Value = value; } 29 | } 30 | 31 | // holds embedding model definitions 32 | internal readonly VValue> _embeddingModelDefinitions; 33 | 34 | public Dictionary Embedding 35 | { 36 | get { return _embeddingModelDefinitions.Value; } 37 | set { _embeddingModelDefinitions.Value = value; } 38 | } 39 | 40 | public ModelDefinitionsConfig() 41 | { 42 | _textGenerationModelDefinitions = AddValidatedValue>(this) 43 | .Default(new Dictionary()) 44 | .ChangeEventsEnabled(); 45 | 46 | _textGenerationModelDefinitions.MergeCollections = true; 47 | 48 | _embeddingModelDefinitions = AddValidatedValue>(this) 49 | .Default(new Dictionary()) 50 | .ChangeEventsEnabled(); 51 | 52 | _embeddingModelDefinitions.MergeCollections = true; 53 | } 54 | } 55 | 56 | -------------------------------------------------------------------------------- /classes/Config/ModelDownloadConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelDownloadConfig 4 | * @created : Thursday Feb 01, 2024 23:59:05 CST 5 | */ 6 | 7 | namespace GatoGPT.Config; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Objects.Validated; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | public partial class ModelDownloadConfig : VConfig 20 | { 21 | internal readonly VValue> _urlDownloads; 22 | 23 | public List UrlDownloads 24 | { 25 | get { return _urlDownloads.Value; } 26 | set { _urlDownloads.Value = value; } 27 | } 28 | 29 | internal readonly VValue _downloadBasePath; 30 | 31 | public string DownloadBasePath 32 | { 33 | get { return _downloadBasePath.Value; } 34 | set { _downloadBasePath.Value = value; } 35 | } 36 | 37 | internal readonly VValue _downloadProcessSec; 38 | 39 | public int DownloadProcessSec 40 | { 41 | get { return _downloadProcessSec.Value; } 42 | set { _downloadProcessSec.Value = value; } 43 | } 44 | 45 | internal readonly VValue _maxConcurrentDownloads; 46 | 47 | public int MaxConcurrentDownloads 48 | { 49 | get { return _maxConcurrentDownloads.Value; } 50 | set { _maxConcurrentDownloads.Value = value; } 51 | } 52 | 53 | internal readonly VValue _downloadBandwidthLimit; 54 | 55 | public long DownloadBandwidthLimit 56 | { 57 | get { return _downloadBandwidthLimit.Value; } 58 | set { _downloadBandwidthLimit.Value = value; } 59 | } 60 | 61 | 62 | public ModelDownloadConfig() 63 | { 64 | _urlDownloads = AddValidatedValue>(this) 65 | .Default(new List()) 66 | .ChangeEventsEnabled(); 67 | 68 | _downloadBasePath = AddValidatedValue(this) 69 | .Default("user://Models") 70 | .ChangeEventsEnabled(); 71 | 72 | _downloadProcessSec = AddValidatedValue(this) 73 | .Default(60) 74 | .ChangeEventsEnabled(); 75 | 76 | _maxConcurrentDownloads = AddValidatedValue(this) 77 | .Default(1) 78 | .ChangeEventsEnabled(); 79 | 80 | _downloadBandwidthLimit = AddValidatedValue(this) 81 | .Default(0) 82 | .ChangeEventsEnabled(); 83 | } 84 | } 85 | 86 | public partial class DownloadConfig : VConfig 87 | { 88 | public string Id { get; set; } 89 | public string Type { get; set; } 90 | public string Quantization { get; set; } 91 | public ModelDefinition ModelDefinition { get; set; } 92 | public bool CreateResourceDefinition { get; set; } = true; 93 | } 94 | 95 | public partial class UrlDownloadConfig : DownloadConfig 96 | { 97 | public string Url { get; set; } 98 | } 99 | -------------------------------------------------------------------------------- /classes/Config/SaveDataManagerConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : SaveDataManagerConfig 4 | * @created : Monday Jan 15, 2024 18:58:03 CST 5 | */ 6 | 7 | namespace GodotEGP.Config; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Objects.Validated; 12 | using GodotEGP.Logging; 13 | using GodotEGP.Service; 14 | using GodotEGP.Event.Events; 15 | using GodotEGP.Config; 16 | 17 | public partial class SaveDataManagerConfig : VObject 18 | { 19 | partial void InitConfigParams() 20 | { 21 | // disable creation of System data 22 | AutocreateSystemData = false; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /classes/Config/TextGenerationPresetsConfig.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LLMConfig 4 | * @created : Monday Jan 01, 2024 22:20:59 CST 5 | */ 6 | 7 | namespace GatoGPT.Config; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | using GodotEGP.Objects.Validated; 18 | 19 | using System.Collections.Generic; 20 | using System.Text.RegularExpressions; 21 | 22 | public partial class TextGenerationPresetsConfig : VConfig 23 | { 24 | // holds definitions for default model profiles 25 | internal readonly VValue> _defaultModelProfiles; 26 | 27 | public Dictionary DefaultModelProfiles 28 | { 29 | get { return _defaultModelProfiles.Value; } 30 | set { _defaultModelProfiles.Value = value; } 31 | } 32 | 33 | // map of filename matches to preset IDs 34 | internal readonly VValue> _filenamePresetMap; 35 | 36 | public Dictionary FilenamePresetMap 37 | { 38 | get { return _filenamePresetMap.Value; } 39 | set { _filenamePresetMap.Value = value; } 40 | } 41 | 42 | 43 | public TextGenerationPresetsConfig() 44 | { 45 | _defaultModelProfiles = AddValidatedValue>(this) 46 | .Default(new Dictionary() { }) 47 | .ChangeEventsEnabled(); 48 | 49 | _filenamePresetMap = AddValidatedValue>(this) 50 | .Default(new Dictionary()) 51 | .ChangeEventsEnabled(); 52 | } 53 | 54 | public ModelProfile GetPresetForFilename(string filename) 55 | { 56 | foreach (var obj in FilenamePresetMap) 57 | { 58 | if (Regex.IsMatch(filename, WildCardToRegular(obj.Key))) 59 | { 60 | LoggerManager.LogDebug("Found profile matching filename", "", "match", $"{filename} = {obj.Value}"); 61 | 62 | return GetDefaultProfile(obj.Value); 63 | } 64 | } 65 | 66 | return new ModelProfile(); 67 | } 68 | 69 | public ModelProfile GetDefaultProfile(string profileKey) 70 | { 71 | if (DefaultModelProfiles.ContainsKey(profileKey)) 72 | { 73 | return DefaultModelProfiles[profileKey]; 74 | } 75 | 76 | throw new InvalidModelPreset($"The model preset {profileKey} does not exist in DefaultModelProfiles!"); 77 | } 78 | 79 | public bool PresetExists(string profileKey) 80 | { 81 | return DefaultModelProfiles.ContainsKey(profileKey); 82 | } 83 | 84 | private String WildCardToRegular(String value) { 85 | return "^" + Regex.Escape(value).Replace("\\*", ".*") + "$"; 86 | } 87 | 88 | /**************** 89 | * Exceptions * 90 | ****************/ 91 | 92 | public class InvalidModelPreset : Exception 93 | { 94 | public InvalidModelPreset() { } 95 | public InvalidModelPreset(string message) : base(message) { } 96 | public InvalidModelPreset(string message, Exception inner) : base(message, inner) { } 97 | protected InvalidModelPreset( 98 | System.Runtime.Serialization.SerializationInfo info, 99 | System.Runtime.Serialization.StreamingContext context) 100 | : base(info, context) { } 101 | } 102 | } 103 | 104 | -------------------------------------------------------------------------------- /classes/Event/Events.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : Events 4 | * @created : Tuesday Jan 02, 2024 14:13:40 CST 5 | */ 6 | 7 | namespace GatoGPT.Event; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial class LlamaModelInstanceEvent : Event 19 | { 20 | public string Id; 21 | } 22 | static public partial class LlamaModelInstanceEventExtensions 23 | { 24 | static public T SetInstanceId(this T o, string id) where T : LlamaModelInstanceEvent 25 | { 26 | o.Id = id; 27 | return o; 28 | } 29 | } 30 | 31 | public partial class LlamaModelLoadStart : LlamaModelInstanceEvent {}; 32 | public partial class LlamaModelLoadFinished : LlamaModelInstanceEvent {}; 33 | public partial class LlamaModelUnloadStart : LlamaModelInstanceEvent {}; 34 | public partial class LlamaModelUnloadFinished : LlamaModelInstanceEvent {}; 35 | 36 | public partial class TextGenerationInferenceStart : LlamaModelInstanceEvent {}; 37 | public partial class TextGenerationInferenceToken : LlamaModelInstanceEvent { 38 | public string Token; 39 | }; 40 | public partial class TextGenerationInferenceLine : LlamaModelInstanceEvent { 41 | public string Line; 42 | }; 43 | public partial class TextGenerationInferenceFinished : LlamaModelInstanceEvent { 44 | public InferenceResult Result; 45 | }; 46 | 47 | static public partial class LlamaModelInstanceEventExtensions 48 | { 49 | static public T SetToken(this T o, string token) where T : TextGenerationInferenceToken 50 | { 51 | o.Token = token; 52 | return o; 53 | } 54 | } 55 | static public partial class LlamaModelInstanceEventExtensions 56 | { 57 | static public T SetLine(this T o, string line) where T : TextGenerationInferenceLine 58 | { 59 | o.Line = line; 60 | return o; 61 | } 62 | } 63 | static public partial class LlamaModelInstanceEventExtensions 64 | { 65 | static public T SetResult(this T o, InferenceResult result) where T : TextGenerationInferenceFinished 66 | { 67 | o.Result = result; 68 | return o; 69 | } 70 | } 71 | 72 | 73 | public partial class ProcessRunnerEvent : Event {} 74 | 75 | public partial class ProcessStarted : ProcessRunnerEvent {} 76 | public partial class ProcessOutputLine : ProcessRunnerEvent { 77 | public string Line { get; set; } 78 | } 79 | public partial class ProcessOutputErrorLine : ProcessOutputLine {} 80 | public partial class ProcessFinishedSuccess : ProcessRunnerEvent {} 81 | public partial class ProcessFinishedError : ProcessRunnerEvent {} 82 | -------------------------------------------------------------------------------- /classes/Handler/DownloadConfigHandler.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : DownloadConfigHandler 4 | * @created : Friday Feb 02, 2024 00:16:24 CST 5 | */ 6 | 7 | namespace GatoGPT.Handler; 8 | 9 | using GatoGPT.Config; 10 | using GatoGPT.AI.TextGeneration; 11 | using GatoGPT.Service; 12 | using GatoGPT.Resource; 13 | 14 | using Godot; 15 | using GodotEGP; 16 | using GodotEGP.Objects.Extensions; 17 | using GodotEGP.Logging; 18 | using GodotEGP.Service; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Event.Filters; 21 | using GodotEGP.Config; 22 | using GodotEGP.Handler; 23 | 24 | public partial class DownloadConfigHandler : Handler 25 | { 26 | private ModelDownloadManager _modelDownloadManager; 27 | 28 | public DownloadConfigHandler() 29 | { 30 | ServiceRegistry.Get().Subscribe(_On_ConfigManager_Ready).Filters(new OwnerObjectTypeFilter(typeof(ConfigManager))); 31 | 32 | // run config update when ResourceManager is ready to populate model 33 | // resources 34 | ServiceRegistry.Get().Subscribe(_On_ConfigManager_Ready).Filters(new OwnerObjectTypeFilter(typeof(ResourceManager))); 35 | 36 | _modelDownloadManager = ServiceRegistry.Get(); 37 | } 38 | 39 | public void _On_ConfigManager_Ready(IEvent e) 40 | { 41 | // subscribe to changes on model download config 42 | var mdc = ServiceRegistry.Get().Get(); 43 | mdc.SubscribeOwner(_On_ModelDownloadConfig_ValueChanged, isHighPriority: true); 44 | 45 | // trigger changed event 46 | _On_ModelDownloadConfig_Changed(mdc); 47 | } 48 | 49 | public void _On_ModelDownloadConfig_ValueChanged(IEvent e) 50 | { 51 | var mdc = ServiceRegistry.Get().Get(); 52 | 53 | _On_ModelDownloadConfig_Changed(mdc); 54 | } 55 | 56 | public void _On_ModelDownloadConfig_Changed(ModelDownloadConfig mdc) 57 | { 58 | _modelDownloadManager.SetConfig(mdc); 59 | 60 | _modelDownloadManager.SetResourceDefinitions(ServiceRegistry.Get().Get()); 61 | } 62 | } 63 | 64 | -------------------------------------------------------------------------------- /classes/Handler/EmbeddingConfigHandler.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingConfigHandler 4 | * @created : Friday Jan 05, 2024 23:39:39 CST 5 | */ 6 | 7 | namespace GatoGPT.Handler; 8 | 9 | using GatoGPT.Config; 10 | using GatoGPT.AI.TextGeneration; 11 | using GatoGPT.Service; 12 | using GatoGPT.Resource; 13 | 14 | using Godot; 15 | using GodotEGP; 16 | using GodotEGP.Objects.Extensions; 17 | using GodotEGP.Logging; 18 | using GodotEGP.Service; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Event.Filters; 21 | using GodotEGP.Config; 22 | using GodotEGP.Handler; 23 | 24 | public partial class EmbeddingConfigHandler : Handler 25 | { 26 | private EmbeddingModelManager _embeddingManager; 27 | 28 | public EmbeddingConfigHandler() 29 | { 30 | ServiceRegistry.Get().Subscribe(_On_ConfigManager_Ready).Filters(new OwnerObjectTypeFilter(typeof(ConfigManager))); 31 | 32 | // run config update when ResourceManager is ready to populate model 33 | // resources 34 | ServiceRegistry.Get().Subscribe(_On_ConfigManager_Ready).Filters(new OwnerObjectTypeFilter(typeof(ResourceManager))); 35 | 36 | _embeddingManager = ServiceRegistry.Get(); 37 | } 38 | 39 | public void _On_ConfigManager_Ready(IEvent e) 40 | { 41 | // subscribe to changes on model preset and definitions config 42 | var sc = ServiceRegistry.Get().Get(); 43 | sc.SubscribeOwner(_On_ModelsConfig_ValueChanged, isHighPriority: true); 44 | 45 | // var pc = ServiceRegistry.Get().Get(); 46 | // pc.SubscribeOwner(_On_ModelsConfig_ValueChanged, isHighPriority: true); 47 | 48 | var dc = ServiceRegistry.Get().Get(); 49 | dc.SubscribeOwner(_On_ModelsConfig_ValueChanged, isHighPriority: true); 50 | 51 | // trigger changed event 52 | _On_ModelsConfig_Changed(sc, dc); 53 | } 54 | 55 | public void _On_ModelsConfig_ValueChanged(IEvent e) 56 | { 57 | var sc = ServiceRegistry.Get().Get(); 58 | // var pc = ServiceRegistry.Get().Get(); 59 | var dc = ServiceRegistry.Get().Get(); 60 | 61 | _On_ModelsConfig_Changed(sc, dc); 62 | } 63 | 64 | public void _On_ModelsConfig_Changed(EmbeddingModelManagerConfig managerConfig, ModelDefinitionsConfig definitionsConfig) 65 | { 66 | _embeddingManager.SetConfig(managerConfig, definitionsConfig); 67 | 68 | _embeddingManager.SetModelResources(ServiceRegistry.Get().GetResources()); 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /classes/Handler/LlamaConfigHandler.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LLMConfigHandler 4 | * @created : Tuesday Jan 02, 2024 00:28:09 CST 5 | */ 6 | 7 | namespace GatoGPT.Handler; 8 | 9 | using GatoGPT.Config; 10 | using GatoGPT.AI.TextGeneration; 11 | using GatoGPT.Service; 12 | using GatoGPT.Resource; 13 | 14 | using Godot; 15 | using GodotEGP; 16 | using GodotEGP.Objects.Extensions; 17 | using GodotEGP.Logging; 18 | using GodotEGP.Service; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Event.Filters; 21 | using GodotEGP.Config; 22 | using GodotEGP.Handler; 23 | 24 | public partial class LlamaConfigHandler : Handler 25 | { 26 | private TextGenerationModelManager _LLMModelManager; 27 | private LlamaCacheService _LlamaCacheService; 28 | 29 | public LlamaConfigHandler() 30 | { 31 | ServiceRegistry.Get().Subscribe(_On_ConfigManager_Ready).Filters(new OwnerObjectTypeFilter(typeof(ConfigManager))); 32 | 33 | // run config update when ResourceManager is ready to populate model 34 | // resources 35 | ServiceRegistry.Get().Subscribe(_On_ConfigManager_Ready).Filters(new OwnerObjectTypeFilter(typeof(ResourceManager))); 36 | 37 | _LLMModelManager = ServiceRegistry.Get(); 38 | _LlamaCacheService = ServiceRegistry.Get(); 39 | } 40 | 41 | public void _On_ConfigManager_Ready(IEvent e) 42 | { 43 | // subscribe to changes on model preset and definitions config 44 | var sc = ServiceRegistry.Get().Get(); 45 | sc.SubscribeOwner(_On_ModelsConfig_ValueChanged, isHighPriority: true); 46 | 47 | var pc = ServiceRegistry.Get().Get(); 48 | pc.SubscribeOwner(_On_ModelsConfig_ValueChanged, isHighPriority: true); 49 | 50 | var dc = ServiceRegistry.Get().Get(); 51 | dc.SubscribeOwner(_On_ModelsConfig_ValueChanged, isHighPriority: true); 52 | 53 | // trigger changed event 54 | _On_ModelsConfig_Changed(sc, pc, dc); 55 | 56 | // subscribe to LlamaCacheManagerConfig 57 | var cm = ServiceRegistry.Get().Get(); 58 | cm.SubscribeOwner(_On_CacheConfig_ValueChanged, isHighPriority: true); 59 | 60 | _On_CacheConfig_Changed(cm); 61 | } 62 | 63 | public void _On_ModelsConfig_ValueChanged(IEvent e) 64 | { 65 | var sc = ServiceRegistry.Get().Get(); 66 | var pc = ServiceRegistry.Get().Get(); 67 | var dc = ServiceRegistry.Get().Get(); 68 | 69 | _On_ModelsConfig_Changed(sc, pc, dc); 70 | } 71 | 72 | public void _On_ModelsConfig_Changed(TextGenerationModelManagerConfig managerConfig, TextGenerationPresetsConfig presetsConfig, ModelDefinitionsConfig definitionsConfig) 73 | { 74 | _LLMModelManager.SetConfig(managerConfig, presetsConfig, definitionsConfig); 75 | 76 | _LLMModelManager.SetModelResources(ServiceRegistry.Get().GetResources()); 77 | } 78 | 79 | public void _On_CacheConfig_ValueChanged(IEvent e) 80 | { 81 | var cm = ServiceRegistry.Get().Get(); 82 | 83 | _On_CacheConfig_Changed(cm); 84 | } 85 | 86 | public void _On_CacheConfig_Changed(LlamaCacheManagerConfig cm) 87 | { 88 | _LlamaCacheService.SetConfig(cm); 89 | } 90 | } 91 | 92 | -------------------------------------------------------------------------------- /classes/Resource/EmbeddingModel.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingModel 4 | * @created : Friday Jan 05, 2024 23:18:59 CST 5 | */ 6 | 7 | namespace GatoGPT.Resource; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class EmbeddingModel : Resource 17 | { 18 | public EmbeddingModel() 19 | { 20 | 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /classes/Resource/LlamaGrammar.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaGrammar 4 | * @created : Friday Jan 19, 2024 20:52:01 CST 5 | */ 6 | 7 | namespace GatoGPT.Resource; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class LlamaGrammar : Resource 17 | { 18 | public LlamaGrammar() 19 | { 20 | 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /classes/Resource/LlamaModel.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaModel 4 | * @created : Tuesday Jan 02, 2024 01:02:29 CST 5 | */ 6 | 7 | namespace GatoGPT.Resource; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class LlamaModel : Resource 17 | { 18 | public LlamaModel() 19 | { 20 | 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /classes/Resource/SentenceTransformerModel.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : SentenceTransformerModel 4 | * @created : Friday Jan 05, 2024 19:15:03 CST 5 | */ 6 | 7 | namespace GatoGPT.Resource; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class SentenceTransformerModel : Resource 17 | { 18 | public SentenceTransformerModel() 19 | { 20 | 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /classes/Service/EmbeddingModelManager.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingModelManager 4 | * @created : Friday Jan 05, 2024 22:47:23 CST 5 | */ 6 | 7 | namespace GatoGPT.Service; 8 | 9 | using GatoGPT.AI.Embedding; 10 | using GatoGPT.Config; 11 | using GatoGPT.Resource; 12 | using GatoGPT.Event; 13 | 14 | using Godot; 15 | using GodotEGP.Objects.Extensions; 16 | using GodotEGP.Logging; 17 | using GodotEGP.Service; 18 | using GodotEGP.Event.Events; 19 | using GodotEGP.Config; 20 | using GodotEGP.Resource; 21 | 22 | public partial class EmbeddingModelManager : Service 23 | { 24 | private EmbeddingModelManagerConfig _config = new EmbeddingModelManagerConfig(); 25 | // private EmbeddingModelPresetsConfig _presetsConfig = new EmbeddingModelPresetsConfig(); 26 | private ModelDefinitionsConfig _definitionsConfig = new ModelDefinitionsConfig(); 27 | 28 | public Dictionary ModelDefinitions { 29 | get { 30 | return _definitionsConfig.Embedding; 31 | } 32 | } 33 | 34 | 35 | private Dictionary> _modelResources; 36 | 37 | public Dictionary> ModelResources { 38 | get { 39 | return _modelResources; 40 | } 41 | } 42 | 43 | public void SetConfig(EmbeddingModelManagerConfig config, ModelDefinitionsConfig definitionsConfig) 44 | { 45 | LoggerManager.LogDebug("Setting config", "", "config", config); 46 | // LoggerManager.LogDebug("Setting model presets config", "", "modelPresets", presetsConfig); 47 | LoggerManager.LogDebug("Setting model definitions config", "", "modelDefinitions", definitionsConfig); 48 | 49 | _config = config; 50 | // _presetsConfig = presetsConfig; 51 | _definitionsConfig = definitionsConfig; 52 | 53 | PrepareDefinitionConfigs(); 54 | 55 | if (!GetReady()) 56 | { 57 | _SetServiceReady(true); 58 | } 59 | } 60 | 61 | public void PrepareDefinitionConfigs() 62 | { 63 | // check there's resources and model definitions before processing 64 | if (_definitionsConfig.Embedding.Count == 0 || _modelResources == null || _modelResources.Count == 0) 65 | { 66 | return; 67 | } 68 | 69 | foreach (var def in _definitionsConfig.Embedding) 70 | { 71 | if (def.Value.ModelResourceId != null && def.Value.ModelResourceId.Length > 0) 72 | { 73 | def.Value.Id = def.Key; 74 | 75 | // fetch the resource object from resources 76 | def.Value.ModelResource = GetModelResource(def.Value.ModelResourceId); 77 | 78 | // find matching preset for filename 79 | // def.Value.ModelProfile = _presetsConfig.GetPresetForFilename(def.Value.ModelResource.Definition.Path); 80 | 81 | // merge profile with profile overrides, if set 82 | // if (def.Value.ModelProfileOverride != null) 83 | // { 84 | // LoggerManager.LogDebug("Applying model profile overrides", "", "overrides", def.Value.ModelProfileOverride); 85 | // 86 | // def.Value.ModelProfile.MergeFrom(def.Value.ModelProfileOverride); 87 | // } 88 | } 89 | } 90 | } 91 | 92 | public ResourceObject GetModelResource(string resourceId) 93 | { 94 | return _modelResources[resourceId]; 95 | } 96 | 97 | public void SetModelResources(Dictionary> modelResources) 98 | { 99 | LoggerManager.LogDebug("Setting model resources config", "", "modelResources", modelResources); 100 | 101 | _modelResources = modelResources; 102 | 103 | PrepareDefinitionConfigs(); 104 | } 105 | 106 | // Called when the node enters the scene tree for the first time. 107 | public override void _Ready() 108 | { 109 | } 110 | 111 | // Called every frame. 'delta' is the elapsed time since the previous frame. 112 | public override void _Process(double delta) 113 | { 114 | } 115 | 116 | // Called when service is registered in manager 117 | public override void _OnServiceRegistered() 118 | { 119 | } 120 | 121 | // Called when service is deregistered from manager 122 | public override void _OnServiceDeregistered() 123 | { 124 | // LoggerManager.LogDebug($"Service deregistered!", "", "service", this.GetType().Name); 125 | } 126 | 127 | // Called when service is considered ready 128 | public override void _OnServiceReady() 129 | { 130 | LoggerManager.LogDebug("Model resources", "", "modelResources", _modelResources); 131 | LoggerManager.LogDebug("Model definitions", "", "modelDefinitions", _definitionsConfig); 132 | } 133 | 134 | /****************************** 135 | * Model management methods * 136 | ******************************/ 137 | 138 | public bool ModelDefinitionIsValid(string id) 139 | { 140 | return _definitionsConfig.Embedding.ContainsKey(id); 141 | } 142 | 143 | public EmbeddingModelDefinition GetModelDefinition(string id) 144 | { 145 | return _definitionsConfig.Embedding[id]; 146 | } 147 | } 148 | 149 | -------------------------------------------------------------------------------- /classes/Service/EmbeddingService.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingService 4 | * @created : Friday Jan 05, 2024 22:44:51 CST 5 | */ 6 | 7 | namespace GatoGPT.Service; 8 | 9 | using GatoGPT.AI.Embedding; 10 | using GatoGPT.AI.Embedding.Backends; 11 | using GatoGPT.Config; 12 | using GatoGPT.Event; 13 | 14 | using Godot; 15 | using GodotEGP; 16 | using GodotEGP.Objects.Extensions; 17 | using GodotEGP.Logging; 18 | using GodotEGP.Service; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Config; 21 | 22 | using System.Collections.Generic; 23 | 24 | public partial class EmbeddingService : Service 25 | { 26 | private EmbeddingModelManager _modelManager; 27 | 28 | public EmbeddingService() 29 | { 30 | _modelManager = ServiceRegistry.Get(); 31 | } 32 | 33 | public float[] GenerateEmbedding(string modelDefinitionId, string input) 34 | { 35 | var modelDefinition = _modelManager.GetModelDefinition(modelDefinitionId); 36 | 37 | IEmbeddingBackend backend = AI.ModelBackend.CreateBackend(modelDefinition); 38 | 39 | return backend.GenerateEmbedding(input).ToArray(); 40 | } 41 | 42 | public List GenerateEmbeddings(string modelDefinitionId, IEnumerable inputs) 43 | { 44 | List results = new(); 45 | 46 | foreach (var input in inputs) 47 | { 48 | results.Add(GenerateEmbedding(modelDefinitionId, input)); 49 | } 50 | 51 | return results; 52 | } 53 | } 54 | 55 | -------------------------------------------------------------------------------- /classes/Service/LlamaCacheService.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LlamaCacheService 4 | * @created : Wednesday Jan 10, 2024 15:15:44 CST 5 | */ 6 | 7 | namespace GatoGPT.Service; 8 | 9 | using GatoGPT.Config; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | using GodotEGP.Misc; 18 | 19 | public partial class LlamaCacheService : Service 20 | { 21 | private string _cacheBaseDir { get; set; } = Path.Combine(OS.GetUserDataDir(), "Cache"); 22 | 23 | private LlamaCacheManagerConfig _config { get; set; } 24 | 25 | private Timer _cleanupTimer; 26 | 27 | public LlamaCacheService() 28 | { 29 | _cleanupTimer = new Timer(); 30 | } 31 | 32 | public void SetConfig(LlamaCacheManagerConfig config) 33 | { 34 | LoggerManager.LogDebug("Setting cache config", "", "config", config); 35 | 36 | _config = config; 37 | 38 | _cleanupTimer.WaitTime = _config.CacheTimeoutSec; 39 | 40 | if (!GetReady()) 41 | { 42 | _SetServiceReady(true); 43 | } 44 | 45 | _cleanupTimer.Stop(); 46 | _cleanupTimer.Start(_config.CacheTimeoutSec); 47 | 48 | Cleanup(); 49 | } 50 | 51 | // Called when the node enters the scene tree for the first time. 52 | public override void _Ready() 53 | { 54 | } 55 | 56 | // Called every frame. 'delta' is the elapsed time since the previous frame. 57 | public override void _Process(double delta) 58 | { 59 | } 60 | 61 | // Called when service is registered in manager 62 | public override void _OnServiceRegistered() 63 | { 64 | } 65 | 66 | // Called when service is deregistered from manager 67 | public override void _OnServiceDeregistered() 68 | { 69 | // LoggerManager.LogDebug($"Service deregistered!", "", "service", this.GetType().Name); 70 | } 71 | 72 | // Called when service is considered ready 73 | public override void _OnServiceReady() 74 | { 75 | // setup a Timer node to run the cleanup process 76 | LoggerManager.LogDebug("Setting up cleanup timer"); 77 | 78 | _cleanupTimer.WaitTime = _config.CacheTimeoutSec; 79 | _cleanupTimer.Autostart = true; 80 | _cleanupTimer.OneShot = false; 81 | _cleanupTimer.SubscribeSignal(StringNames.Instance["timeout"], false, _On_CleanupTimer_Timeout); 82 | 83 | AddChild(_cleanupTimer); 84 | } 85 | 86 | public void Cleanup() 87 | { 88 | if (!Directory.Exists(_cacheBaseDir)) 89 | { 90 | return; 91 | } 92 | 93 | DirectoryInfo cacheIdDir = new DirectoryInfo (_cacheBaseDir); 94 | 95 | DirectoryInfo[] cacheIdDirs = cacheIdDir.GetDirectories().OrderBy(p => p.CreationTime).ToArray(); 96 | 97 | long totalCacheSizeMb = 0; 98 | 99 | List allCaches = new(); 100 | 101 | foreach (DirectoryInfo cacheDir in cacheIdDirs) 102 | { 103 | DirectoryInfo[] caches = cacheDir.GetDirectories().OrderBy(p => p.CreationTime).ToArray(); 104 | 105 | foreach (DirectoryInfo cache in caches) 106 | { 107 | long cacheSize = GetDirectoryTotalSize(cache); 108 | totalCacheSizeMb += cacheSize; 109 | 110 | if ((DateTime.Now - cache.CreationTime).TotalMinutes > _config.MaxCacheAgeMin) 111 | { 112 | allCaches.Add(cache); 113 | } 114 | } 115 | } 116 | 117 | if (allCaches.Count > 0) 118 | { 119 | LoggerManager.LogDebug("Caches eligable for cleaning", "", "total", allCaches.Count); 120 | LoggerManager.LogDebug("Total cache size", "", "totalCacheSize", $"{totalCacheSizeMb}, maxCacheSize:{_config.MaxCacheSizeMb}"); 121 | 122 | // cleanup caches until cache size is below the max 123 | if (totalCacheSizeMb > _config.MaxCacheSizeMb) 124 | { 125 | foreach(var c in allCaches.OrderBy(p => p.CreationTime).ToArray()) 126 | { 127 | LoggerManager.LogDebug("Purging cache dir", "", "cacheDir", $"{c.ToString()}, CreationTime:{c.CreationTime}"); 128 | 129 | totalCacheSizeMb -= GetDirectoryTotalSize(c); 130 | Directory.Delete(c.ToString(), true); 131 | 132 | if (totalCacheSizeMb < _config.MaxCacheSizeMb) 133 | { 134 | break; 135 | } 136 | } 137 | } 138 | } 139 | } 140 | 141 | public void _On_CleanupTimer_Timeout(IEvent e) 142 | { 143 | Cleanup(); 144 | } 145 | 146 | public long BytesToMb(long bytes) 147 | { 148 | return bytes / 1000000; 149 | } 150 | 151 | public long GetDirectoryTotalSize(DirectoryInfo dir) 152 | { 153 | return BytesToMb(dir.EnumerateFiles().Sum(file => file.Length)); 154 | } 155 | } 156 | 157 | -------------------------------------------------------------------------------- /classes/Service/ModelManager.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelManager 4 | * @created : Friday Jan 05, 2024 22:55:41 CST 5 | */ 6 | 7 | namespace GatoGPT.Service; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | using GatoGPT.Config; 11 | using GatoGPT.Resource; 12 | using GatoGPT.Event; 13 | 14 | using Godot; 15 | using GodotEGP; 16 | using GodotEGP.Objects.Extensions; 17 | using GodotEGP.Logging; 18 | using GodotEGP.Service; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Config; 21 | using GodotEGP.Resource; 22 | 23 | using LLama; 24 | using LLama.Common; 25 | 26 | // public partial class ModelManager : Service where TModelResource : Resource 27 | // { 28 | // private TManagerConfig _config = new TManagerConfig(); 29 | // private TPresetsConfig _presetsConfig = new TPresetsConfig(); 30 | // private TDefinitionsConfig _definitionsConfig = new TDefinitionsConfig(); 31 | // 32 | // public Dictionary> ModelDefinitions { 33 | // get { 34 | // return _definitionsConfig.ModelDefinitions; 35 | // } 36 | // } 37 | // 38 | // 39 | // private Dictionary> _modelResources; 40 | // 41 | // public Dictionary> ModelResources { 42 | // get { 43 | // return _modelResources; 44 | // } 45 | // } 46 | // 47 | // public void SetConfig(TManagerConfig config, TPresetsConfig presetsConfig, TDefinitionsConfig definitionsConfig) 48 | // { 49 | // LoggerManager.LogDebug("Setting config", "", "config", config); 50 | // LoggerManager.LogDebug("Setting model presets config", "", "modelPresets", presetsConfig); 51 | // LoggerManager.LogDebug("Setting model definitions config", "", "modelDefinitions", definitionsConfig); 52 | // 53 | // _config = config; 54 | // _presetsConfig = presetsConfig; 55 | // _definitionsConfig = definitionsConfig; 56 | // 57 | // PrepareDefinitionConfigs(); 58 | // 59 | // if (!GetReady()) 60 | // { 61 | // _SetServiceReady(true); 62 | // } 63 | // } 64 | // 65 | // public void PrepareDefinitionConfigs() 66 | // { 67 | // // check there's resources and model definitions before processing 68 | // if (_definitionsConfig.ModelDefinitions.Count == 0 || _modelResources == null || _modelResources.Count == 0) 69 | // { 70 | // return; 71 | // } 72 | // 73 | // foreach (var def in _definitionsConfig.ModelDefinitions) 74 | // { 75 | // if (def.Value.ModelResourceId.Length > 0) 76 | // { 77 | // def.Value.Id = def.Key; 78 | // 79 | // // fetch the resource object from resources 80 | // def.Value.ModelResource = GetModelResource(def.Value.ModelResourceId); 81 | // 82 | // // find matching preset for filename 83 | // def.Value.ModelProfile = _presetsConfig.GetPresetForFilename(def.Value.ModelResource.Definition.Path); 84 | // 85 | // // merge profile with profile overrides, if set 86 | // if (def.Value.ModelProfileOverride != null) 87 | // { 88 | // LoggerManager.LogDebug("Applying model profile overrides", "", "overrides", def.Value.ModelProfileOverride); 89 | // 90 | // def.Value.ModelProfile.MergeFrom(def.Value.ModelProfileOverride); 91 | // } 92 | // } 93 | // } 94 | // } 95 | // 96 | // public Resource GetModelResource(string resourceId) 97 | // { 98 | // return _modelResources[resourceId]; 99 | // } 100 | // 101 | // public void SetModelResources(Dictionary> modelResources) 102 | // { 103 | // LoggerManager.LogDebug("Setting model resources config", "", "modelResources", modelResources); 104 | // 105 | // _modelResources = modelResources; 106 | // 107 | // PrepareDefinitionConfigs(); 108 | // } 109 | // 110 | // // Called when the node enters the scene tree for the first time. 111 | // public override void _Ready() 112 | // { 113 | // } 114 | // 115 | // // Called every frame. 'delta' is the elapsed time since the previous frame. 116 | // public override void _Process(double delta) 117 | // { 118 | // } 119 | // 120 | // // Called when service is registered in manager 121 | // public override void _OnServiceRegistered() 122 | // { 123 | // } 124 | // 125 | // // Called when service is deregistered from manager 126 | // public override void _OnServiceDeregistered() 127 | // { 128 | // // LoggerManager.LogDebug($"Service deregistered!", "", "service", this.GetType().Name); 129 | // } 130 | // 131 | // // Called when service is considered ready 132 | // public override void _OnServiceReady() 133 | // { 134 | // LoggerManager.LogDebug("Model resources", "", "modelResources", _modelResources); 135 | // LoggerManager.LogDebug("Model definitions", "", "modelDefinitions", _definitionsConfig); 136 | // } 137 | // 138 | // /****************************** 139 | // * Model management methods * 140 | // ******************************/ 141 | // 142 | // public bool ModelDefinitionIsValid(string id) 143 | // { 144 | // return _definitionsConfig.ModelDefinitions.ContainsKey(id); 145 | // } 146 | // 147 | // public ModelDefinition GetModelDefinition(string id) 148 | // { 149 | // return _definitionsConfig.ModelDefinitions[id]; 150 | // } 151 | // } 152 | // 153 | -------------------------------------------------------------------------------- /classes/Service/TextGenerationModelManager.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : LLMModelManager 4 | * @created : Tuesday Jan 02, 2024 00:19:47 CST 5 | */ 6 | 7 | namespace GatoGPT.Service; 8 | 9 | using GatoGPT.AI; 10 | using GatoGPT.AI.TextGeneration; 11 | using GatoGPT.Config; 12 | using GatoGPT.Resource; 13 | using GatoGPT.Event; 14 | 15 | using Godot; 16 | using GodotEGP; 17 | using GodotEGP.Objects.Extensions; 18 | using GodotEGP.Logging; 19 | using GodotEGP.Service; 20 | using GodotEGP.Event.Events; 21 | using GodotEGP.Config; 22 | using GodotEGP.Resource; 23 | 24 | using LLama; 25 | using LLama.Common; 26 | 27 | public partial class TextGenerationModelManager : Service 28 | { 29 | private TextGenerationModelManagerConfig _config = new TextGenerationModelManagerConfig(); 30 | private TextGenerationPresetsConfig _presetsConfig = new TextGenerationPresetsConfig(); 31 | private ModelDefinitionsConfig _definitionsConfig = new ModelDefinitionsConfig(); 32 | 33 | public Dictionary ModelDefinitions { 34 | get { 35 | return _definitionsConfig.TextGeneration; 36 | } 37 | } 38 | 39 | 40 | private Dictionary> _modelResources; 41 | 42 | public Dictionary> ModelResources { 43 | get { 44 | return _modelResources; 45 | } 46 | } 47 | 48 | public TextGenerationModelManager() 49 | { 50 | 51 | } 52 | 53 | public void SetConfig(TextGenerationModelManagerConfig config, TextGenerationPresetsConfig presetsConfig, ModelDefinitionsConfig definitionsConfig) 54 | { 55 | LoggerManager.LogDebug("Setting config", "", "config", config); 56 | LoggerManager.LogDebug("Setting model presets config", "", "modelPresets", presetsConfig); 57 | LoggerManager.LogDebug("Setting model definitions config", "", "modelDefinitions", definitionsConfig); 58 | 59 | _config = config; 60 | _presetsConfig = presetsConfig; 61 | _definitionsConfig = definitionsConfig; 62 | 63 | PrepareDefinitionConfigs(); 64 | 65 | if (!GetReady()) 66 | { 67 | _SetServiceReady(true); 68 | } 69 | } 70 | 71 | public void PrepareDefinitionConfigs() 72 | { 73 | // check there's resources and model definitions before processing 74 | if (_definitionsConfig.TextGeneration.Count == 0 || _modelResources == null || _modelResources.Count == 0) 75 | { 76 | return; 77 | } 78 | 79 | foreach (var def in _definitionsConfig.TextGeneration) 80 | { 81 | if (def.Value.ModelResourceId != null && def.Value.ModelResourceId.Length > 0) 82 | { 83 | def.Value.Id = def.Key; 84 | 85 | LoggerManager.LogDebug("Preparing model definition profile", "", "modelDefinition", def.Key); 86 | 87 | // fetch the resource object from resources 88 | def.Value.ModelResource = GetModelResource(def.Value.ModelResourceId); 89 | 90 | if (def.Value.ModelResource == null) 91 | { 92 | continue; 93 | } 94 | 95 | // find matching preset for filename 96 | if (def.Value.ProfilePreset != null && def.Value.ProfilePreset.Length > 0 && _presetsConfig.PresetExists(def.Value.ProfilePreset)) 97 | { 98 | LoggerManager.LogDebug("Overriding model with preset", "", "preset", $"{def.Key}={def.Value.ProfilePreset}"); 99 | 100 | def.Value.ModelProfile = _presetsConfig.GetDefaultProfile(def.Value.ProfilePreset).DeepCopy(); 101 | 102 | } 103 | else 104 | { 105 | def.Value.ModelProfile = _presetsConfig.GetPresetForFilename(def.Value.ModelResource.Definition.Path).DeepCopy(); 106 | } 107 | 108 | // merge profile with profile overrides, if set 109 | if (def.Value.ModelProfileOverride != null) 110 | { 111 | LoggerManager.LogDebug("Applying model profile overrides", "", "overrides", def.Value.ModelProfileOverride); 112 | 113 | def.Value.ModelProfile.MergeFrom(def.Value.ModelProfileOverride); 114 | } 115 | 116 | LoggerManager.LogDebug("Final model profile", "", def.Key, def.Value.ModelProfile); 117 | } 118 | } 119 | } 120 | 121 | public ResourceObject GetModelResource(string resourceId) 122 | { 123 | if (_modelResources.TryGetValue(resourceId, out var resource)) 124 | { 125 | return resource; 126 | } 127 | return null; 128 | } 129 | 130 | public void SetModelResources(Dictionary> modelResources) 131 | { 132 | LoggerManager.LogDebug("Setting model resources config", "", "modelResources", modelResources); 133 | 134 | _modelResources = modelResources; 135 | 136 | PrepareDefinitionConfigs(); 137 | } 138 | 139 | // Called when the node enters the scene tree for the first time. 140 | public override void _Ready() 141 | { 142 | } 143 | 144 | // Called every frame. 'delta' is the elapsed time since the previous frame. 145 | public override void _Process(double delta) 146 | { 147 | } 148 | 149 | // Called when service is registered in manager 150 | public override void _OnServiceRegistered() 151 | { 152 | } 153 | 154 | // Called when service is deregistered from manager 155 | public override void _OnServiceDeregistered() 156 | { 157 | // LoggerManager.LogDebug($"Service deregistered!", "", "service", this.GetType().Name); 158 | } 159 | 160 | // Called when service is considered ready 161 | public override void _OnServiceReady() 162 | { 163 | LoggerManager.LogDebug("Model resources", "", "modelResources", _modelResources); 164 | LoggerManager.LogDebug("Model definitions", "", "modelDefinitions", _definitionsConfig); 165 | } 166 | 167 | /****************************** 168 | * Model management methods * 169 | ******************************/ 170 | 171 | public bool ModelDefinitionIsValid(string id) 172 | { 173 | return _definitionsConfig.TextGeneration.ContainsKey(id); 174 | } 175 | 176 | public AI.TextGeneration.ModelDefinition GetModelDefinition(string id) 177 | { 178 | if (ModelDefinitionIsValid(id)) 179 | { 180 | return _definitionsConfig.TextGeneration[id]; 181 | } 182 | 183 | return null; 184 | } 185 | } 186 | 187 | -------------------------------------------------------------------------------- /classes/Service/TextGenerationService.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : TextGenerationService 4 | * @created : Tuesday Jan 02, 2024 12:28:50 CST 5 | */ 6 | 7 | namespace GatoGPT.Service; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | using GatoGPT.AI.TextGeneration.Backends; 11 | using GatoGPT.Config; 12 | using GatoGPT.Event; 13 | using GatoGPT.Resource; 14 | 15 | using Godot; 16 | using GodotEGP; 17 | using GodotEGP.Objects.Extensions; 18 | using GodotEGP.Logging; 19 | using GodotEGP.Service; 20 | using GodotEGP.Event.Events; 21 | using GodotEGP.Config; 22 | 23 | using System.Collections.Generic; 24 | 25 | public partial class TextGenerationService : Service 26 | { 27 | private TextGenerationModelManager _modelManager; 28 | private ResourceManager _resourceManager; 29 | 30 | private Dictionary _modelInstances = new(); 31 | 32 | private Queue _inferenceQueue = new(); 33 | 34 | public TextGenerationService() 35 | { 36 | // assign the model manager instance 37 | _modelManager = ServiceRegistry.Get(); 38 | _resourceManager = ServiceRegistry.Get(); 39 | } 40 | 41 | /*********************** 42 | * Inference methods * 43 | ***********************/ 44 | 45 | public AI.TextGeneration.Backends.ITextGenerationBackend Infer(string modelDefinitionId, string prompt, bool stateful = false, string existingInstanceId = "", AI.TextGeneration.LoadParams loadParams = null, AI.TextGeneration.InferenceParams inferenceParams = null) 46 | { 47 | var modelInstance = QueueInferenceRequest(modelDefinitionId, prompt, stateful, existingInstanceId, loadParams, inferenceParams); 48 | 49 | return modelInstance; 50 | } 51 | 52 | public async Task InferAsync(string modelDefinitionId, string prompt, bool stateful = false, string existingInstanceId = "", AI.TextGeneration.LoadParams loadParams = null, AI.TextGeneration.InferenceParams inferenceParams = null) 53 | { 54 | // call the normal infer which queues request 55 | var modelInstance = Infer(modelDefinitionId, prompt, stateful, existingInstanceId, loadParams, inferenceParams); 56 | 57 | // wait for the model 58 | while (!modelInstance.Finished) 59 | { 60 | // LoggerManager.LogDebug("Waiting for inference", "", "inferenceRequest", $"prompt:{prompt}, model:{modelDefinitionId}, instanceId:{modelInstance.InstanceId}, queueSize:{_inferenceQueue.Count}"); 61 | await Task.Delay(100); 62 | } 63 | 64 | return modelInstance.InferenceResult; 65 | } 66 | 67 | /***************************** 68 | * Inference queue methods * 69 | *****************************/ 70 | 71 | public AI.TextGeneration.Backends.ITextGenerationBackend QueueInferenceRequest(string modelDefinitionId, string prompt, bool stateful = false, string existingInstanceId = "", AI.TextGeneration.LoadParams loadParams = null, AI.TextGeneration.InferenceParams inferenceParams = null) 72 | { 73 | var modelInstance = CreateModelInstance(modelDefinitionId, stateful, existingInstanceId); 74 | 75 | LoggerManager.LogDebug("Queuing inference request", "", "inferenceRequest", $"prompt:{prompt}, model:{modelDefinitionId}"); 76 | 77 | _inferenceQueue.Enqueue(new InferenceRequest(modelInstance, prompt, loadParams, inferenceParams)); 78 | 79 | return modelInstance; 80 | } 81 | 82 | public override void _Process(double delta) 83 | { 84 | // if there's a queued request and there's no running instances, then 85 | // dequeue a request and start inference 86 | if (_inferenceQueue.TryPeek(out var request) && !IsRunningInstances()) 87 | { 88 | _inferenceQueue.Dequeue(); 89 | 90 | LoggerManager.LogDebug("Running queued inference", "", "request", request); 91 | 92 | // set GrammarResource 93 | // TODO: put this in a better location? 94 | if (request.ModelInstance.ModelDefinition.ModelProfileOverride != null) 95 | { 96 | string grammarResourceId = request.ModelInstance.ModelDefinition.ModelProfileOverride.InferenceParams.GrammarResourceId; 97 | if (request.InferenceParams.GrammarResourceId.Length > 0) 98 | { 99 | grammarResourceId = request.InferenceParams.GrammarResourceId; 100 | } 101 | if (grammarResourceId.Length > 0) 102 | { 103 | var grammarResources = _resourceManager.GetResources(); 104 | if (grammarResources.TryGetValue(grammarResourceId, out var grammarResource)) 105 | { 106 | LoggerManager.LogDebug("Setting grammar resource", "", "grammarResource", grammarResource); 107 | request.InferenceParams.GrammarResource = grammarResource; 108 | } 109 | } 110 | } 111 | 112 | request.ModelInstance.StartInference(request.Prompt, request.LoadParams, request.InferenceParams); 113 | } 114 | } 115 | 116 | /**************************** 117 | * Model instance methods * 118 | ****************************/ 119 | 120 | public AI.TextGeneration.Backends.ITextGenerationBackend CreateModelInstance(string modelDefinitionId, bool stateful = false, string existingInstanceId = "") 121 | { 122 | // check if the requested definition is valid 123 | if (!_modelManager.ModelDefinitionIsValid(modelDefinitionId)) 124 | { 125 | throw new InvalidModelDefinitionException($"'{modelDefinitionId}' is an invalid model definition!"); 126 | } 127 | 128 | // obtain the definition and create an instance 129 | var modelDefinition = _modelManager.GetModelDefinition(modelDefinitionId); 130 | 131 | var modelInstance = AI.ModelBackend.CreateBackend(modelDefinition, stateful); 132 | 133 | if (existingInstanceId.Length == 0) 134 | { 135 | // add model instance to dictionary using InstanceId 136 | AddModelInstance(modelInstance); 137 | } 138 | else { 139 | LoggerManager.LogDebug("Using existing instance", "", "instanceId", existingInstanceId); 140 | 141 | modelInstance = (ITextGenerationBackend) _modelInstances[existingInstanceId]; 142 | modelInstance.InferenceResult = null; 143 | } 144 | 145 | // return the instance for external management 146 | return modelInstance; 147 | } 148 | 149 | public void AddModelInstance(AI.TextGeneration.Backends.ITextGenerationBackend instance) 150 | { 151 | _modelInstances.Add(instance.InstanceId, instance); 152 | } 153 | 154 | public void SetModelInstanceId(string instanceId, string newInstanceId) 155 | { 156 | _modelInstances[newInstanceId] = _modelInstances[instanceId]; 157 | _modelInstances[newInstanceId].SetInstanceId(newInstanceId); 158 | } 159 | 160 | public void UnloadExistingModels(string excludeInstanceId = "") 161 | { 162 | foreach (var modelObj in _modelInstances) 163 | { 164 | if (modelObj.Value.SafeToUnloadModel() && modelObj.Value.InstanceId != excludeInstanceId) 165 | { 166 | LoggerManager.LogDebug("Unloading model instance", "", "instanceId", modelObj.Value.InstanceId); 167 | 168 | modelObj.Value.UnloadModel(); 169 | } 170 | } 171 | } 172 | 173 | public bool IsRunningInstances() 174 | { 175 | foreach (var instance in _modelInstances) 176 | { 177 | if (instance.Value.Running) 178 | { 179 | return true; 180 | } 181 | } 182 | 183 | return false; 184 | } 185 | 186 | public ITextGenerationBackend GetPersistentInstance(string modelDefinitionId, bool stateful = false) 187 | { 188 | foreach (var instance in _modelInstances) 189 | { 190 | if (instance.Value.Persistent && instance.Value.ModelDefinition.Id == modelDefinitionId) 191 | { 192 | LoggerManager.LogDebug("Found persistent instance", "", "modelId", modelDefinitionId); 193 | LoggerManager.LogDebug("", "", "instance", instance.Value); 194 | 195 | return instance.Value; 196 | } 197 | } 198 | 199 | return null; 200 | } 201 | 202 | public void DestroyExistingInstances(bool keepStateFiles = true) 203 | { 204 | LoggerManager.LogDebug("Destroying all instances", "", "keepStateFiles", keepStateFiles); 205 | 206 | foreach (var modelObj in _modelInstances) 207 | { 208 | if (modelObj.Value.SafeToUnloadModel()) 209 | { 210 | modelObj.Value.DeleteInstanceState(keepStateFiles); 211 | } 212 | } 213 | } 214 | 215 | /****************** 216 | * Tokenization * 217 | ******************/ 218 | public List TokenizeString(string modelDefinitionId, string content) 219 | { 220 | // create model instance 221 | var modelInstance = CreateModelInstance(modelDefinitionId); 222 | 223 | // return tokenized string 224 | return modelInstance.TokenizeString(content); 225 | } 226 | 227 | /**************** 228 | * Exceptions * 229 | ****************/ 230 | 231 | public class InvalidModelDefinitionException : Exception 232 | { 233 | public InvalidModelDefinitionException() { } 234 | public InvalidModelDefinitionException(string message) : base(message) { } 235 | public InvalidModelDefinitionException(string message, Exception inner) : base(message, inner) { } 236 | protected InvalidModelDefinitionException( 237 | System.Runtime.Serialization.SerializationInfo info, 238 | System.Runtime.Serialization.StreamingContext context) 239 | : base(info, context) { } 240 | } 241 | } 242 | 243 | public partial class InferenceRequest 244 | { 245 | public AI.TextGeneration.Backends.ITextGenerationBackend ModelInstance { get; set; } 246 | public string Prompt { get; set; } 247 | public AI.TextGeneration.LoadParams LoadParams { get; set; } 248 | public AI.TextGeneration.InferenceParams InferenceParams { get; set; } 249 | 250 | public InferenceRequest(AI.TextGeneration.Backends.ITextGenerationBackend modelInstance, string prompt, AI.TextGeneration.LoadParams loadParams = null, AI.TextGeneration.InferenceParams inferenceParams = null) 251 | { 252 | ModelInstance = modelInstance; 253 | Prompt = prompt; 254 | LoadParams = loadParams; 255 | InferenceParams = inferenceParams; 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /classes/WebAPI/Application.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : Application 4 | * @created : Thursday Jan 04, 2024 20:16:27 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI; 8 | 9 | using GatoGPT.WebAPI.Extensions; 10 | using GatoGPT.WebAPI.MappingProfiles; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | using Microsoft.AspNetCore.Mvc; 20 | using Microsoft.AspNetCore.Mvc.ApiExplorer; 21 | using Microsoft.AspNetCore.Mvc.Infrastructure; 22 | using Microsoft.AspNetCore.Mvc.Routing; 23 | using Microsoft.Extensions.Options; 24 | using Newtonsoft.Json.Serialization; 25 | using Swashbuckle.AspNetCore.SwaggerGen; 26 | 27 | 28 | public partial class Application 29 | { 30 | private WebApplicationBuilder builder; 31 | private WebApplication app; 32 | 33 | public Application(string[] args, string host = "", int port = 0) 34 | { 35 | builder = WebApplication.CreateBuilder(args); 36 | 37 | // add services to the app 38 | builder.Services.AddControllers() 39 | .AddNewtonsoftJson(options => options.SerializerSettings.ContractResolver = new DefaultContractResolver() { NamingStrategy = new SnakeCaseNamingStrategy() }); 40 | 41 | builder.Services.Configure(options => 42 | { 43 | options.SerializerOptions.PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseLower; 44 | options.SerializerOptions.WriteIndented = true; 45 | }); 46 | 47 | builder.Services.AddEndpointsApiExplorer(); 48 | builder.Services.AddSwaggerGen(); 49 | builder.Services.AddSwaggerGenNewtonsoftSupport(); 50 | 51 | builder.Services.AddCustomCors("AllowAllOrigins"); 52 | 53 | // configure the Swagger UI 54 | builder.Services.AddTransient, ConfigureSwaggerOptions>(); 55 | 56 | builder.Services.AddRouting(options => options.LowercaseUrls = true); 57 | 58 | // configure API versions 59 | builder.Services.AddVersioning(); 60 | 61 | // override the host and port 62 | if (port > 0 && host.Length > 0) 63 | { 64 | builder.WebHost.UseUrls($"http://{host}:{port}"); 65 | } 66 | 67 | // add automatter services 68 | builder.Services.AddAutoMapper(typeof(ModelMappings)); 69 | 70 | // build app 71 | app = builder.Build(); 72 | 73 | // setup swagger UI for development version 74 | var apiVersionDescriptionProvider = app.Services.GetRequiredService(); 75 | var loggerFactory = app.Services.GetRequiredService(); 76 | 77 | // Configure the HTTP request pipeline. 78 | if (app.Environment.IsDevelopment()) 79 | { 80 | app.UseSwagger(); 81 | app.UseSwaggerUI( 82 | options => 83 | { 84 | foreach (var description in apiVersionDescriptionProvider.ApiVersionDescriptions) 85 | { 86 | options.SwaggerEndpoint( 87 | $"/swagger/{description.GroupName}/swagger.json", 88 | description.GroupName.ToUpperInvariant()); 89 | } 90 | }); 91 | } 92 | else 93 | { 94 | app.AddProductionExceptionHandling(loggerFactory); 95 | } 96 | 97 | app.UseCors("AllowAllOrigins"); 98 | // app.UseHttpsRedirection(); 99 | 100 | // app.UseAuthorization(); 101 | 102 | // create routes for defined API controllers 103 | app.MapControllers(); 104 | 105 | // launch the web app! 106 | app.Run(); 107 | } 108 | } 109 | 110 | -------------------------------------------------------------------------------- /classes/WebAPI/ConfigureSwaggerOptions.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ConfigureSwaggerOptions 4 | * @created : Thursday Jan 04, 2024 20:30:00 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI; 8 | 9 | using Microsoft.AspNetCore.Mvc.ApiExplorer; 10 | using Microsoft.Extensions.Options; 11 | using Microsoft.OpenApi.Models; 12 | using Swashbuckle.AspNetCore.SwaggerGen; 13 | 14 | public class ConfigureSwaggerOptions : IConfigureOptions 15 | { 16 | readonly IApiVersionDescriptionProvider provider; 17 | 18 | public ConfigureSwaggerOptions(IApiVersionDescriptionProvider provider) => this.provider = provider; 19 | 20 | public void Configure(SwaggerGenOptions options) 21 | { 22 | foreach (var description in provider.ApiVersionDescriptions) 23 | { 24 | options.SwaggerDoc(description.GroupName, CreateInfoForApiVersion(description)); 25 | } 26 | } 27 | 28 | static OpenApiInfo CreateInfoForApiVersion(ApiVersionDescription description) 29 | { 30 | var info = new OpenApiInfo() 31 | { 32 | Title = System.Reflection.Assembly.GetEntryAssembly().GetName().Name +" API", 33 | Version = description.ApiVersion.ToString(), 34 | Description = "OpenAI compatible API for text generation.", 35 | }; 36 | 37 | if (description.IsDeprecated) 38 | { 39 | info.Description += " This API version has been deprecated."; 40 | } 41 | 42 | return info; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /classes/WebAPI/Controllers/v1/CompletionsController.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionsController 4 | * @created : Friday Jan 05, 2024 00:03:44 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.v1.Controllers; 8 | 9 | using GatoGPT.Service; 10 | using GatoGPT.Config; 11 | using GatoGPT.AI.TextGeneration; 12 | using GatoGPT.WebAPI.Dtos; 13 | using GatoGPT.WebAPI.Entities; 14 | 15 | using Godot; 16 | using GodotEGP; 17 | using GodotEGP.Objects.Extensions; 18 | using GodotEGP.Logging; 19 | using GodotEGP.Service; 20 | using GodotEGP.Event.Events; 21 | using GodotEGP.Config; 22 | 23 | using AutoMapper; 24 | // using Microsoft.AspNetCore.JsonPatch; 25 | using Microsoft.AspNetCore.Mvc; 26 | // using GatoGPT.WebAPI.Dtos; 27 | // using GatoGPT.WebAPI.Entities; 28 | using System.Text.Json; 29 | 30 | using GodotEGP.AI.OpenAI; 31 | 32 | [ApiController] 33 | [ApiVersion("1.0")] 34 | [Route("v{version:apiVersion}/[controller]")] 35 | public partial class CompletionsController : ControllerBase 36 | { 37 | private readonly IMapper _mapper; 38 | private readonly TextGenerationModelManager _modelManager; 39 | private readonly TextGenerationService _inferenceService; 40 | 41 | public CompletionsController(IMapper mapper) 42 | { 43 | _mapper = mapper; 44 | _modelManager = ServiceRegistry.Get(); 45 | _inferenceService = ServiceRegistry.Get(); 46 | } 47 | 48 | [HttpPost(Name = nameof(CreateCompletion))] 49 | public async Task> CreateCompletion(ApiVersion version, [FromBody] CompletionCreateDto completionCreateDto) 50 | { 51 | LoggerManager.LogDebug("Recieved completionCreateDto", "", "create", completionCreateDto); 52 | 53 | // validate required params 54 | if (completionCreateDto.Model.Length == 0) 55 | { 56 | return BadRequest(new InvalidRequestErrorDto( 57 | message: "You must provide a model parameter", 58 | code: null, 59 | param:null 60 | )); 61 | } 62 | 63 | // check model is valid 64 | if (!_modelManager.ModelDefinitions.ContainsKey(completionCreateDto.Model)) 65 | { 66 | return NotFound(new InvalidRequestErrorDto(message:$"The model '{completionCreateDto.Model}' does not exist", code:"model_not_found", param:"model")); 67 | } 68 | 69 | // openai backend passthrough 70 | if (_modelManager.GetModelDefinition(completionCreateDto.Model).Backend == "openai") 71 | { 72 | var openAi = new OpenAI(ServiceRegistry.Get().Get().OpenAIConfig); 73 | 74 | var openaiSse = new ServerSentEventManager(HttpContext); 75 | 76 | openaiSse.Start(); 77 | 78 | // stream responses when there's no tool calls 79 | openAi.SubscribeOwner(async (e) => { 80 | await openaiSse.SendEvent(e.Event); 81 | }, isHighPriority: true); 82 | 83 | var openaiResult = await openAi 84 | .Completions(_mapper.Map(new CompletionCreateOpenAIDto(completionCreateDto))); 85 | 86 | if (completionCreateDto.Stream) 87 | { 88 | return new EmptyResult(); 89 | } 90 | 91 | if (openaiResult == null) 92 | { 93 | return BadRequest(openAi.Error); 94 | } 95 | 96 | return Ok(openaiResult); 97 | } 98 | 99 | // extract prompts 100 | List prompts = new(); 101 | if (completionCreateDto.Prompt is System.String promptString) 102 | { 103 | prompts.Add(promptString); 104 | } 105 | else if (completionCreateDto.Prompt is Newtonsoft.Json.Linq.JArray promptList) 106 | { 107 | prompts = promptList.ToArray().Select(x => x.ToString()).ToList(); 108 | } 109 | 110 | // extract stops 111 | List stops = new(); 112 | if (completionCreateDto.Stop is System.String stopString) 113 | { 114 | stops.Add(stopString); 115 | } 116 | else if (completionCreateDto.Stop is Newtonsoft.Json.Linq.JArray stopsList) 117 | { 118 | stops = stopsList.ToArray().Select(x => x.ToString()).ToList(); 119 | } 120 | 121 | LoggerManager.LogDebug("Completion dto extracted prompts", "", "prompts", prompts); 122 | LoggerManager.LogDebug("Completion dto extracted stops", "", "stops", stops); 123 | 124 | // queue and generate responses 125 | List inferenceInstances = new(); 126 | 127 | // create LoadParams and InferenceParams objects from dto 128 | AI.TextGeneration.LoadParams loadParams = _modelManager.GetModelDefinition(completionCreateDto.Model).ModelProfile.LoadParams.DeepCopy(); 129 | AI.TextGeneration.InferenceParams inferenceParams = _modelManager.GetModelDefinition(completionCreateDto.Model).ModelProfile.InferenceParams.DeepCopy(); 130 | 131 | var completionCreateDtoDefault = new CompletionCreateDto(); 132 | 133 | if (completionCreateDto.Temperature != completionCreateDtoDefault.Temperature) 134 | inferenceParams.Temp = completionCreateDto.Temperature; 135 | if (completionCreateDto.FrequencyPenalty != completionCreateDtoDefault.FrequencyPenalty) 136 | inferenceParams.FrequencyPenalty = completionCreateDto.FrequencyPenalty; 137 | if (completionCreateDto.PresencePenalty != completionCreateDtoDefault.PresencePenalty) 138 | inferenceParams.PresencePenalty = completionCreateDto.PresencePenalty; 139 | if (completionCreateDto.RepeatPenalty != completionCreateDtoDefault.RepeatPenalty) 140 | inferenceParams.RepeatPenalty = completionCreateDto.RepeatPenalty; 141 | if (completionCreateDto.MaxTokens != completionCreateDtoDefault.MaxTokens) 142 | inferenceParams.NPredict = completionCreateDto.MaxTokens; 143 | if (completionCreateDto.Seed != completionCreateDtoDefault.Seed) 144 | loadParams.Seed = completionCreateDto.Seed; 145 | 146 | inferenceParams.Antiprompts = inferenceParams.Antiprompts.Concat(stops).ToList(); 147 | 148 | if (completionCreateDto.MinP != completionCreateDtoDefault.MinP) 149 | inferenceParams.MinP = completionCreateDto.MinP; 150 | if (completionCreateDto.TopP != completionCreateDtoDefault.TopP) 151 | inferenceParams.TopP = completionCreateDto.TopP; 152 | if (completionCreateDto.TopK != completionCreateDtoDefault.TopK) 153 | inferenceParams.TopK = completionCreateDto.TopK; 154 | 155 | // create N * prompts count instances 156 | foreach (string prompt in prompts) 157 | { 158 | for (int i = 0; i < completionCreateDto.N; i++) 159 | { 160 | var modelInstance = _inferenceService.Infer(completionCreateDto.Model, prompt, stateful:false, loadParams:loadParams, inferenceParams:inferenceParams); 161 | 162 | inferenceInstances.Add(modelInstance); 163 | } 164 | } 165 | 166 | // wait for them all to be finished 167 | while (true) 168 | { 169 | int finishedCount = 0; 170 | 171 | foreach (var instance in inferenceInstances) 172 | { 173 | if (instance.Finished) 174 | { 175 | finishedCount++; 176 | } 177 | } 178 | 179 | if (finishedCount == inferenceInstances.Count) 180 | { 181 | break; 182 | } 183 | 184 | await Task.Delay(100); 185 | } 186 | 187 | // create CompletionDto object and populate choice results 188 | CompletionDto completionDto = new(); 189 | 190 | int currentIndex = 0; 191 | foreach (var instance in inferenceInstances) 192 | { 193 | completionDto.Usage.PromptTokens += instance.InferenceResult.PromptTokenCount; 194 | completionDto.Usage.CompletionTokens += instance.InferenceResult.GenerationTokenCount; 195 | 196 | completionDto.Choices.Add(new CompletionChoiceDto() { 197 | FinishReason = ((instance.InferenceResult.Tokens.Count >= completionCreateDto.MaxTokens && completionCreateDto.MaxTokens > -1) ? "length" : "stop"), 198 | Index = currentIndex, 199 | Text = instance.InferenceResult.OutputStripped, 200 | InferenceResult = instance.InferenceResult, 201 | }); 202 | 203 | currentIndex++; 204 | 205 | _inferenceService.DestroyExistingInstances(); 206 | } 207 | 208 | completionDto.Id = $"cmpl-{GetHashCode()}-{completionDto.GetHashCode()}-{completionCreateDto.GetHashCode()}"; 209 | completionDto.Created = ((DateTimeOffset) DateTime.UtcNow).ToUnixTimeSeconds(); 210 | completionDto.Model = completionCreateDto.Model; 211 | completionDto.SystemFingerprint = GetHashCode().ToString(); 212 | 213 | LoggerManager.LogDebug("Returning completionDto", "", "completionDto", completionDto); 214 | 215 | return Ok(completionDto); 216 | } 217 | } 218 | 219 | -------------------------------------------------------------------------------- /classes/WebAPI/Controllers/v1/EmbeddingsController.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingsController 4 | * @created : Friday Jan 05, 2024 18:47:39 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.v1.Controllers; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using GatoGPT.Service; 12 | using GatoGPT.Config; 13 | using GatoGPT.Resource; 14 | using GatoGPT.WebAPI.Dtos; 15 | using GatoGPT.WebAPI.Entities; 16 | 17 | using Godot; 18 | using GodotEGP; 19 | using GodotEGP.Objects.Extensions; 20 | using GodotEGP.Logging; 21 | using GodotEGP.Service; 22 | using GodotEGP.Event.Events; 23 | using GodotEGP.Config; 24 | 25 | using LLama; 26 | using LLama.Common; 27 | 28 | using AutoMapper; 29 | // using Microsoft.AspNetCore.JsonPatch; 30 | using Microsoft.AspNetCore.Mvc; 31 | // using GatoGPT.WebAPI.Dtos; 32 | // using GatoGPT.WebAPI.Entities; 33 | using System.Text.Json; 34 | 35 | using GodotEGP.AI.OpenAI; 36 | 37 | [ApiController] 38 | [ApiVersion("1.0")] 39 | [Route("v{version:apiVersion}/[controller]")] 40 | public partial class EmbeddingsController : ControllerBase 41 | { 42 | private readonly IMapper _mapper; 43 | private readonly ResourceManager _resourceManager; 44 | private readonly EmbeddingModelManager _modelManager; 45 | private readonly EmbeddingService _embeddingService; 46 | 47 | public EmbeddingsController(IMapper mapper) 48 | { 49 | _mapper = mapper; 50 | _resourceManager = ServiceRegistry.Get(); 51 | _modelManager = ServiceRegistry.Get(); 52 | _embeddingService = ServiceRegistry.Get(); 53 | } 54 | 55 | [HttpPost(Name = nameof(CreateEmbedding))] 56 | public async Task> CreateEmbedding(ApiVersion version, [FromBody] EmbeddingCreateDto embeddingCreateDto) 57 | { 58 | LoggerManager.LogDebug("Recieved embeddingCreateDto", "", "create", embeddingCreateDto); 59 | 60 | // // validate required params 61 | if (embeddingCreateDto.Model.Length == 0) 62 | { 63 | return BadRequest(new InvalidRequestErrorDto( 64 | message: "You must provide a model parameter", 65 | code: null, 66 | param:null 67 | )); 68 | } 69 | 70 | // check model is valid 71 | if (!_modelManager.ModelDefinitions.ContainsKey(embeddingCreateDto.Model)) 72 | { 73 | return NotFound(new InvalidRequestErrorDto(message:$"The model '{embeddingCreateDto.Model}' does not exist", code:"model_not_found", param:"model")); 74 | } 75 | 76 | // openai backend passthrough 77 | if (_modelManager.GetModelDefinition(embeddingCreateDto.Model).Backend == "openai") 78 | { 79 | var openAi = new OpenAI(ServiceRegistry.Get().Get().OpenAIConfig); 80 | var openaiResult = await openAi 81 | .Embeddings(_mapper.Map(embeddingCreateDto)); 82 | 83 | if (openaiResult == null) 84 | { 85 | return BadRequest(openAi.Error); 86 | } 87 | 88 | return Ok(openaiResult); 89 | } 90 | 91 | // create the EmbeddingsDto object 92 | EmbeddingsDto embeddingsDto = new EmbeddingsDto() { 93 | Model = embeddingCreateDto.Model 94 | }; 95 | 96 | // create embeddings for each passed input 97 | for (int i = 0; i < embeddingCreateDto.GetInputs().Count; i++) 98 | { 99 | string input = embeddingCreateDto.GetInputs()[i]; 100 | 101 | EmbeddingDto embedding = new EmbeddingDto() { 102 | Index = i, 103 | Embedding = _embeddingService.GenerateEmbedding(embeddingCreateDto.Model, input) 104 | }; 105 | 106 | embeddingsDto.Data.Add(embedding); 107 | } 108 | 109 | return Ok(embeddingsDto); 110 | } 111 | } 112 | 113 | -------------------------------------------------------------------------------- /classes/WebAPI/Controllers/v1/ExtendedTokenizeController.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ExtendedTokenizeController 4 | * @created : Wednesday Jan 31, 2024 17:29:07 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.v1.Controllers; 8 | 9 | using GatoGPT.Service; 10 | using GatoGPT.Config; 11 | using GatoGPT.WebAPI.Dtos; 12 | using GatoGPT.WebAPI.Entities; 13 | using GatoGPT.AI.TextGeneration; 14 | 15 | using Godot; 16 | using GodotEGP; 17 | using GodotEGP.Objects.Extensions; 18 | using GodotEGP.Logging; 19 | using GodotEGP.Service; 20 | using GodotEGP.Event.Events; 21 | using GodotEGP.Config; 22 | using GodotEGP.AI.OpenAI; 23 | 24 | using AutoMapper; 25 | using Microsoft.AspNetCore.Mvc; 26 | using System.ComponentModel; 27 | 28 | [ApiController] 29 | [ApiVersion("1.0")] 30 | [Tags("Tokenize (Extended API)")] 31 | [Route("v{version:apiVersion}/extended/tokenize")] 32 | public partial class ExtendedTokenizeController : ControllerBase 33 | { 34 | private readonly IMapper _mapper; 35 | private readonly TextGenerationModelManager _modelManager; 36 | private readonly TextGenerationService _textGenerationService; 37 | 38 | public ExtendedTokenizeController(IMapper mapper) 39 | { 40 | _mapper = mapper; 41 | _modelManager = ServiceRegistry.Get(); 42 | _textGenerationService = ServiceRegistry.Get(); 43 | } 44 | 45 | [HttpPost(Name = nameof(Tokenize))] 46 | public async Task> Tokenize(ApiVersion version, [FromBody] ExtendedTokenizeCreateDto tokenizeCreateDto) 47 | { 48 | LoggerManager.LogDebug("Tokenize tokenizeCreateDto", "", "tokenizeCreateDto", tokenizeCreateDto); 49 | 50 | // validate required params 51 | if (tokenizeCreateDto.Model.Length == 0) 52 | { 53 | return BadRequest(new InvalidRequestErrorDto( 54 | message: "You must provide a model parameter", 55 | code: null, 56 | param:null 57 | )); 58 | } 59 | 60 | // check model is valid 61 | if (!_modelManager.ModelDefinitions.ContainsKey(tokenizeCreateDto.Model)) 62 | { 63 | return NotFound(new InvalidRequestErrorDto(message:$"The model '{tokenizeCreateDto.Model}' does not exist", code:"model_not_found", param:"model")); 64 | } 65 | 66 | var tokenizedStringDto = new ExtendedTokenizeDto(); 67 | 68 | try 69 | { 70 | var tokenized = _textGenerationService.TokenizeString(tokenizeCreateDto.Model, tokenizeCreateDto.Content); 71 | 72 | tokenizedStringDto.Tokens = tokenized; 73 | } 74 | catch (System.Exception e) 75 | { 76 | ErrorResultExtended err = new() { 77 | Error = new() { 78 | Code = "tokenize_error", 79 | Type = e.GetType().Name, 80 | Message = e.Message, 81 | Exception = e, 82 | } 83 | }; 84 | 85 | LoggerManager.LogDebug("Tokenize content exception", "", "error", err); 86 | 87 | return BadRequest(err); 88 | } 89 | 90 | LoggerManager.LogDebug("Tokenized string count", "", tokenizeCreateDto.Content, tokenizedStringDto.Tokens.Count); 91 | 92 | LoggerManager.LogDebug("Returning tokenizedStringDto", "", "tokenizedStringDto", tokenizedStringDto); 93 | 94 | return Ok(tokenizedStringDto); 95 | } 96 | 97 | [HttpPost("chat", Name = nameof(TokenizeChatCompletion))] 98 | public async Task> TokenizeChatCompletion(ApiVersion version, [FromBody] ChatCompletionCreateDto chatCompletionCreateDto) 99 | { 100 | LoggerManager.LogDebug("Tokenize chatCompletionCreateDto", "", "chatCompletionCreateDto", chatCompletionCreateDto); 101 | 102 | // validate required params 103 | if (chatCompletionCreateDto.Model.Length == 0) 104 | { 105 | return BadRequest(new InvalidRequestErrorDto( 106 | message: "You must provide a model parameter", 107 | code: null, 108 | param:null 109 | )); 110 | } 111 | 112 | // check model is valid 113 | if (!_modelManager.ModelDefinitions.ContainsKey(chatCompletionCreateDto.Model)) 114 | { 115 | return NotFound(new InvalidRequestErrorDto(message:$"The model '{chatCompletionCreateDto.Model}' does not exist", code:"model_not_found", param:"model")); 116 | } 117 | 118 | ModelDefinition modelDefinition = _modelManager.GetModelDefinition(chatCompletionCreateDto.Model); 119 | 120 | // create instance of model and get full prompt 121 | var modelInstance = _textGenerationService.CreateModelInstance(modelDefinition.Id); 122 | 123 | var inferenceParams = modelDefinition.ModelProfile.InferenceParams.DeepCopy(); 124 | var loadParams = modelDefinition.ModelProfile.LoadParams.DeepCopy(); 125 | modelInstance.InferenceParams = inferenceParams; 126 | modelInstance.LoadParams = loadParams; 127 | 128 | // apply extended parameters which can affect token length 129 | if (chatCompletionCreateDto.Extended != null) 130 | { 131 | if (chatCompletionCreateDto.Extended.Model != null) 132 | { 133 | if (chatCompletionCreateDto.Extended.Model.Backend != null) 134 | modelDefinition.Backend = (string) chatCompletionCreateDto.Extended.Model.Backend; 135 | } 136 | if (chatCompletionCreateDto.Extended.Inference != null) 137 | { 138 | if (chatCompletionCreateDto.Extended.Inference.ChatMessageTemplate != null) 139 | inferenceParams.ChatMessageTemplate = (string) chatCompletionCreateDto.Extended.Inference.ChatMessageTemplate; 140 | if (chatCompletionCreateDto.Extended.Inference.ChatMessageGenerationTemplate != null) 141 | inferenceParams.ChatMessageGenerationTemplate = (string) chatCompletionCreateDto.Extended.Inference.ChatMessageGenerationTemplate; 142 | if (chatCompletionCreateDto.Extended.Inference.PrePrompt != null) 143 | inferenceParams.PrePrompt = (string) chatCompletionCreateDto.Extended.Inference.PrePrompt; 144 | if (chatCompletionCreateDto.Extended.Inference.CfgNegativePrompt != null) 145 | inferenceParams.NegativeCfgPrompt = (string) chatCompletionCreateDto.Extended.Inference.CfgNegativePrompt; 146 | } 147 | } 148 | 149 | // create a StatefulChat instance and parse the prompt 150 | StatefulChat chatInstance = new(false, loadParams, inferenceParams); 151 | 152 | List chatMessages = new(); 153 | foreach (var message in chatCompletionCreateDto.Messages) 154 | { 155 | chatMessages.Add(new() { 156 | Role = message.Role, 157 | Name = message.Name, 158 | Content = message.GetContent(), 159 | }); 160 | } 161 | 162 | chatInstance.SetChatMessages(chatMessages); 163 | chatInstance.UpdateStatefulInstanceId(); 164 | 165 | string chatPrompt = chatInstance.GetPrompt(); 166 | 167 | chatPrompt = modelInstance.FormatPrompt(chatPrompt); 168 | 169 | // create tokenizeDto 170 | ExtendedTokenizeCreateDto tokenizeCreateDto = new() { 171 | Model = chatCompletionCreateDto.Model, 172 | Content = chatPrompt+inferenceParams.NegativeCfgPrompt, 173 | }; 174 | 175 | var res = await Tokenize(HttpContext.GetRequestedApiVersion(), tokenizeCreateDto); 176 | 177 | if (res.Result is OkObjectResult okRes) 178 | { 179 | LoggerManager.LogDebug("Tokenize result", "", "res", okRes.Value); 180 | 181 | return Ok(okRes.Value); 182 | } 183 | else if (res.Result is BadRequestResult badRes) 184 | { 185 | return BadRequest(badRes); 186 | } 187 | else { 188 | return BadRequest(new InvalidRequestErrorDto(message:$"Unknown error", code:"tokenize_unknown_error")); 189 | } 190 | 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /classes/WebAPI/Controllers/v1/ModelsController.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelsController 4 | * @created : Thursday Jan 04, 2024 21:41:23 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.v1.Controllers; 8 | 9 | using GatoGPT.Service; 10 | using GatoGPT.Config; 11 | using GatoGPT.WebAPI.Dtos; 12 | using GatoGPT.WebAPI.Entities; 13 | 14 | using Godot; 15 | using GodotEGP; 16 | using GodotEGP.Objects.Extensions; 17 | using GodotEGP.Logging; 18 | using GodotEGP.Service; 19 | using GodotEGP.Event.Events; 20 | using GodotEGP.Config; 21 | 22 | using AutoMapper; 23 | // using Microsoft.AspNetCore.JsonPatch; 24 | using Microsoft.AspNetCore.Mvc; 25 | // using GatoGPT.WebAPI.Dtos; 26 | // using GatoGPT.WebAPI.Entities; 27 | using System.Text.Json; 28 | 29 | [ApiController] 30 | [ApiVersion("1.0")] 31 | [Route("v{version:apiVersion}/[controller]")] 32 | public partial class ModelsController : ControllerBase 33 | { 34 | private readonly IMapper _mapper; 35 | private readonly TextGenerationModelManager _modelManager; 36 | 37 | public ModelsController(IMapper mapper) 38 | { 39 | _mapper = mapper; 40 | _modelManager = ServiceRegistry.Get(); 41 | } 42 | 43 | [HttpGet(Name = nameof(ListModels))] 44 | public ActionResult ListModels(ApiVersion version) 45 | { 46 | ModelListDto modelsDto = new(); 47 | 48 | 49 | // get list of mapped ModelDto objects 50 | IEnumerable dtos = GetModelEntities().Select(x => _mapper.Map(x)); 51 | 52 | // assign model Dto list to modelsListDto object 53 | modelsDto.Data = dtos.ToList(); 54 | 55 | return Ok(modelsDto); 56 | } 57 | 58 | [HttpGet("{model}", Name = nameof(GetModel))] 59 | public ActionResult GetModel(ApiVersion version, string model) 60 | { 61 | if (_modelManager.ModelDefinitions.ContainsKey(model)) 62 | { 63 | ModelFullDto modelDto = GetModelEntities().Where(x => x.Id == model).Select(x => _mapper.Map(x)).FirstOrDefault(); 64 | 65 | modelDto.Definition = _modelManager.GetModelDefinition(model); 66 | 67 | return Ok(modelDto); 68 | } 69 | 70 | return NotFound(new InvalidRequestErrorDto(message:$"The model '{model}' does not exist", code:"model_not_found", param:"model")); 71 | } 72 | 73 | [ApiExplorerSettings(IgnoreApi = true)] 74 | public List GetModelEntities() 75 | { 76 | // convert model definitions into ModelEntity objects 77 | List modelEntities = new(); 78 | 79 | foreach (var def in _modelManager.ModelDefinitions) 80 | { 81 | var entity = new ModelEntity() { 82 | Model = def.Value, 83 | }; 84 | 85 | modelEntities.Add(entity); 86 | } 87 | 88 | return modelEntities; 89 | } 90 | } 91 | 92 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/BaseDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : BaseDto 4 | * @created : Thursday Jan 04, 2024 22:23:27 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class BaseDto 17 | { 18 | public string Object { get; set; } 19 | } 20 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionChoiceDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionChoiceDto 4 | * @created : Friday Jan 05, 2024 12:44:22 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial class ChatCompletionChoiceDto : CompletionChoiceBaseDto 19 | { 20 | public ChatCompletionMessageDto Message { get; set; } 21 | } 22 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionChunkDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionChunkDto 4 | * @created : Sunday Jan 07, 2024 19:53:04 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial class ChatCompletionChunkDto : BaseDto 19 | { 20 | public string Id { get; set; } 21 | public List Choices { get; set; } 22 | public long Created { get; set; } 23 | public string Model { get; set; } 24 | public string SystemFingerprint { get; set; } 25 | public InferenceResult InferenceResult { get; set; } 26 | 27 | public ChatCompletionChunkDto() 28 | { 29 | Object = "chat.completion.chunk"; 30 | Created = ((DateTimeOffset) DateTime.Now).ToUnixTimeSeconds(); 31 | Choices = new(); 32 | } 33 | } 34 | 35 | public partial class ChatCompletionChunkChoicesDto 36 | { 37 | public int Index { get; set; } 38 | public ChatCompletionMessageDto Delta { get; set; } 39 | 40 | public ChatCompletionChunkChoicesDto() 41 | { 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionCreateDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionCreateDto 4 | * @created : Friday Jan 05, 2024 13:07:04 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json; 17 | 18 | public partial class ChatCompletionCreateDto : CompletionCreateBaseDto 19 | { 20 | public ChatCompletionCreateResponseFormatDto ResponseFormat { get; set; } 21 | public List Messages { get; set; } 22 | public List Tools { get; set; } 23 | public object ToolChoice { get; set; } 24 | 25 | // extended property implements optional extended parameters 26 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 27 | public ChatCompletionCreateExtendedDto? Extended { get; set; } 28 | 29 | public ChatCompletionCreateDto() 30 | { 31 | ResponseFormat = new(); 32 | Tools = new(); 33 | ToolChoice = (string) "auto"; 34 | 35 | // the Completion API has 16 as default, while the Chat Completion API 36 | // has it set to null (presumably unlimited/max context length) 37 | MaxTokens = -1; 38 | } 39 | 40 | public string GetToolChoice() 41 | { 42 | if (ToolChoice is string ts) 43 | return ts; 44 | else 45 | return ""; 46 | } 47 | 48 | public ChatCompletionToolChoiceDto GetToolChoiceObject() 49 | { 50 | if (ToolChoice is ChatCompletionToolChoiceDto dto) 51 | { 52 | return dto; 53 | } 54 | 55 | return new(); 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionCreateExtendedDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionCreateExtendedDto 4 | * @created : Sunday Jan 28, 2024 20:43:57 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json; 17 | 18 | public partial class ChatCompletionCreateExtendedDto 19 | { 20 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 21 | public ChatCompletionCreateExtendedModelDto? Model { get; set; } 22 | 23 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 24 | public ChatCompletionCreateExtendedInferenceDto? Inference { get; set; } 25 | } 26 | 27 | public partial class ChatCompletionCreateExtendedModelDto 28 | { 29 | public int? NCtx { get; set; } 30 | public int? NBatch { get; set; } 31 | public int? NGpuLayers { get; set; } 32 | public string? Backend { get; set; } 33 | public bool? PromptCache { get; set; } 34 | public double? RopeFreqBase { get; set; } 35 | public double? RopeFreqScale { get; set; } 36 | } 37 | public partial class ChatCompletionCreateExtendedInferenceDto 38 | { 39 | public int? NThreads { get; set; } 40 | public int? NKeep { get; set; } 41 | public int? TopK { get; set; } 42 | public double? Tfs { get; set; } 43 | public double? Typical { get; set; } 44 | public double? Mirostat { get; set; } 45 | public double? MirostatLearningRate { get; set; } 46 | public double? MirostatEntropy { get; set; } 47 | public double? RepeatPenalty { get; set; } 48 | public int? RepeatLastN { get; set; } 49 | public bool? Vision { get; set; } 50 | public string? GrammarResourceId { get; set; } 51 | 52 | public string? ChatMessageTemplate { get; set; } 53 | public string? ChatMessageGenerationTemplate { get; set; } 54 | public string? PrePrompt { get; set; } 55 | 56 | public string? CfgNegativePrompt { get; set; } 57 | public double? CfgScale { get; set; } 58 | 59 | public string? PromptCacheId { get; set; } 60 | 61 | public List? Samplers { get; set; } 62 | } 63 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionCreateFunctionDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionCreateFunctionDto 4 | * @created : Saturday Jan 06, 2024 00:52:47 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json; 17 | 18 | public partial class ChatCompletionCreateFunctionDto 19 | { 20 | public string Description { get; set; } 21 | public string Name { get; set; } 22 | public object Parameters { get; set; } 23 | 24 | public ChatCompletionCreateFunctionDto() 25 | { 26 | Description = ""; 27 | Parameters = new(); 28 | } 29 | 30 | public ChatCompletionCreateFunctionParametersDto GetParametersDto() 31 | { 32 | return JsonConvert.DeserializeObject(Parameters.ToString()); 33 | } 34 | } 35 | 36 | public partial class ChatCompletionCreateFunctionParametersDto 37 | { 38 | public string Type { get; set; } 39 | public Dictionary Properties { get; set; } 40 | public List Required { get; set; } 41 | 42 | public ChatCompletionCreateFunctionParametersDto() 43 | { 44 | Properties = new(); 45 | Required = new(); 46 | } 47 | } 48 | 49 | public partial class ChatCompletionCreateFunctionPropertyDto 50 | { 51 | public string Type { get; set; } 52 | public string Description { get; set; } 53 | public List Enum { get; set; } 54 | 55 | public ChatCompletionCreateFunctionPropertyDto() 56 | { 57 | Type = "string"; 58 | Description = ""; 59 | Enum = new(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionCreateOpenAIDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionCreateOpenAIDto 4 | * @created : Sunday Jan 14, 2024 21:26:08 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json; 17 | 18 | public partial class ChatCompletionCreateOpenAIDto : ChatCompletionCreateOpenAIBaseDto 19 | { 20 | public ChatCompletionCreateResponseFormatDto ResponseFormat { 21 | get { 22 | return BaseDto.ResponseFormat; 23 | } 24 | } 25 | public List Messages { 26 | get { 27 | List messages = new(); 28 | 29 | foreach (var mes in BaseDto.Messages) 30 | { 31 | messages.Add(new() { 32 | Role = mes.Role, 33 | Name = (mes.Name.Length > 0 ? mes.Name : mes.Role), 34 | Content = mes.Content, 35 | }); 36 | } 37 | 38 | return messages; 39 | } 40 | } 41 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 42 | public List? Tools { 43 | get { 44 | return (BaseDto.Tools.Count > 0 ? BaseDto.Tools : null); 45 | } 46 | } 47 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 48 | public object? ToolChoice { 49 | get { 50 | return (BaseDto.Tools.Count > 0 ? BaseDto.ToolChoice : null); 51 | } 52 | } 53 | 54 | public ChatCompletionCreateOpenAIDto(ChatCompletionCreateDto baseDto) : base(baseDto) 55 | { 56 | 57 | } 58 | } 59 | 60 | public partial class ChatCompletionMessageCreateOpenAIDto 61 | { 62 | public string Role { get; set; } = "system"; 63 | public string Name { get; set; } = "User"; 64 | public object Content { get; set; } 65 | } 66 | 67 | public partial class ChatCompletionCreateOpenAIBaseDto 68 | { 69 | internal ChatCompletionCreateDto BaseDto { get; set; } 70 | 71 | public string Model { 72 | get { 73 | return BaseDto.Model; 74 | } 75 | } 76 | public double FrequencyPenalty { 77 | get { 78 | return BaseDto.FrequencyPenalty; 79 | } 80 | } 81 | 82 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 83 | public int? MaxTokens { 84 | get { 85 | return (BaseDto.MaxTokens < 1 ? null : BaseDto.MaxTokens); 86 | } 87 | } 88 | public int N { 89 | get { 90 | return BaseDto.N; 91 | } 92 | } 93 | public double PresencePenalty { 94 | get { 95 | return BaseDto.PresencePenalty; 96 | } 97 | } 98 | public int Seed { 99 | get { 100 | return BaseDto.Seed; 101 | } 102 | } 103 | public object Stop { 104 | get { 105 | return BaseDto.Stop; 106 | } 107 | } 108 | public bool Stream { 109 | get { 110 | return BaseDto.Stream; 111 | } 112 | } 113 | public double Temperature { 114 | get { 115 | return BaseDto.Temperature; 116 | } 117 | } 118 | public double TopP { 119 | get { 120 | return BaseDto.TopP; 121 | } 122 | } 123 | public string User { 124 | get { 125 | return BaseDto.User; 126 | } 127 | } 128 | 129 | public ChatCompletionCreateOpenAIBaseDto(ChatCompletionCreateDto baseDto) 130 | { 131 | BaseDto = baseDto; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionCreateResponseFormatDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionCreateResponseFormatDto 4 | * @created : Friday Jan 05, 2024 13:08:30 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ChatCompletionCreateResponseFormatDto 17 | { 18 | public string Type { get; set; } 19 | 20 | public ChatCompletionCreateResponseFormatDto() 21 | { 22 | Type = "text"; 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionCreateToolDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionCreateToolDto 4 | * @created : Saturday Jan 06, 2024 00:51:41 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ChatCompletionCreateToolDto 17 | { 18 | public string Type { get; set; } 19 | public ChatCompletionCreateFunctionDto Function { get; set; } 20 | 21 | public ChatCompletionCreateToolDto() 22 | { 23 | Function = new(); 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionDto 4 | * @created : Friday Jan 05, 2024 12:51:26 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ChatCompletionDto : CompletionBaseDto 17 | { 18 | public ChatCompletionDto() 19 | { 20 | Object = "chat.completion"; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionMessageCreateDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionMessageCreateDto 4 | * @created : Friday Jan 05, 2024 17:06:13 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json; 17 | 18 | public partial class ChatCompletionMessageCreateDto : ChatCompletionMessageDto 19 | { 20 | [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] 21 | public string Name { get; set; } 22 | 23 | public ChatCompletionMessageCreateDto() 24 | { 25 | Name = ""; 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionMessageDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionMessageDto 4 | * @created : Friday Jan 05, 2024 12:47:45 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json.Linq; 17 | 18 | public partial class ChatCompletionMessageDto 19 | { 20 | public object Content { get; set; } 21 | public string Role { get; set; } 22 | public List ToolCalls { get; set; } 23 | 24 | public ChatCompletionMessageDto() 25 | { 26 | ToolCalls = new(); 27 | } 28 | 29 | public string GetContent() 30 | { 31 | if (Content is Newtonsoft.Json.Linq.JArray) 32 | { 33 | LoggerManager.LogDebug("Contents object", "", "contents", GetContents()); 34 | 35 | return String.Join(" ", GetContents().Where(x => x.Type == "text").Select(x => x.Text).ToArray()); 36 | } 37 | else if (Content is List dto) 38 | { 39 | return String.Join(" ", GetContents().Where(x => x.Type == "text").Select(x => x.Text).ToArray()); 40 | } 41 | else 42 | { 43 | return (string) Content; 44 | } 45 | } 46 | public List GetContents() 47 | { 48 | List contentDtos = new(); 49 | 50 | if (Content is Newtonsoft.Json.Linq.JArray c) 51 | { 52 | 53 | foreach (Newtonsoft.Json.Linq.JToken content in c) 54 | { 55 | IDictionary dict = content.ToObject>(); 56 | 57 | ChatCompletionMessageContentDto contentDto = new(); 58 | 59 | if (dict.TryGetValue("type", out var type)) 60 | { 61 | contentDto.Type = (string) type; 62 | } 63 | if (dict.TryGetValue("text", out var text)) 64 | { 65 | contentDto.Text = (string) text; 66 | } 67 | if (dict.TryGetValue("image_url", out var imageUrl)) 68 | { 69 | var values = JObject.FromObject(imageUrl).ToObject>(); 70 | 71 | if (values.TryGetValue("url", out var url)) 72 | { 73 | contentDto.ImageUrl = (string) url; 74 | } 75 | } 76 | 77 | contentDtos.Add(contentDto); 78 | } 79 | } 80 | else if (Content is List dto) 81 | { 82 | contentDtos = dto; 83 | } 84 | 85 | LoggerManager.LogDebug("Content dtos", "", "contentDtos", contentDtos); 86 | 87 | return contentDtos; 88 | } 89 | } 90 | 91 | public partial class ChatCompletionMessageContentDto 92 | { 93 | public string Type { get; set; } = "text"; 94 | public string Text { get; set; } = ""; 95 | public string ImageUrl { get; set; } = ""; 96 | } 97 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionToolCallsDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionToolCallsDto 4 | * @created : Saturday Jan 06, 2024 01:15:14 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ChatCompletionToolCallDto 17 | { 18 | public string Id { get; set; } 19 | public string Type { get; set; } 20 | public ChatCompletionToolFunctionDto Function { get; set; } 21 | 22 | public ChatCompletionToolCallDto() 23 | { 24 | Function = new(); 25 | } 26 | } 27 | 28 | public partial class ChatCompletionToolCallRawDto 29 | { 30 | public string Function { get; set; } 31 | public Dictionary Arguments { get; set; } 32 | 33 | public ChatCompletionToolCallRawDto() 34 | { 35 | Arguments = new(); 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionToolChoiceDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionToolChoiceDto 4 | * @created : Saturday Jan 06, 2024 00:59:47 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ChatCompletionToolChoiceDto 17 | { 18 | public string Type { get; set; } 19 | public ChatCompletionToolChoiceFunctionDto Function { get; set; } 20 | 21 | public ChatCompletionToolChoiceDto() 22 | { 23 | Function = new(); 24 | } 25 | } 26 | 27 | public partial class ChatCompletionToolChoiceFunctionDto 28 | { 29 | public string Name { get; set; } 30 | } 31 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ChatCompletionToolFunctionDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionToolFunctionDto 4 | * @created : Saturday Jan 06, 2024 01:18:28 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ChatCompletionToolFunctionDto 17 | { 18 | public string Name { get; set; } 19 | public string Arguments { get; set; } 20 | 21 | public ChatCompletionToolFunctionDto() 22 | { 23 | 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionBaseDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionBaseDto 4 | * @created : Friday Jan 05, 2024 12:54:07 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class CompletionBaseDto : BaseDto 17 | { 18 | public string Id { get; set; } 19 | public List Choices { get; set; } 20 | public long Created { get; set; } 21 | public string Model { get; set; } 22 | public string SystemFingerprint { get; set; } 23 | public CompletionUsageDto Usage { get; set; } 24 | 25 | public CompletionBaseDto() 26 | { 27 | Choices = new(); 28 | Usage = new(); 29 | } 30 | } 31 | 32 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionChoiceBaseDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionChoiceBaseDto 4 | * @created : Friday Jan 05, 2024 12:45:37 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial class CompletionChoiceBaseDto 19 | { 20 | public string FinishReason { get; set; } 21 | public int Index { get; set; } 22 | public object Logprobs { get; set; } // TODO 23 | public InferenceResult InferenceResult { get; set; } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionChoiceDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionChoiceDto 4 | * @created : Friday Jan 05, 2024 00:18:23 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial class CompletionChoiceDto : CompletionChoiceBaseDto 19 | { 20 | public string Text { get; set; } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionCreateBaseDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionCreateBaseDto 4 | * @created : Friday Jan 05, 2024 13:03:26 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class CompletionCreateBaseDto 17 | { 18 | public string Model { get; set; } 19 | public bool Echo { get; set; } 20 | public double FrequencyPenalty { get; set; } 21 | public int MaxTokens { get; set; } 22 | public int N { get; set; } 23 | public double PresencePenalty { get; set; } 24 | public double RepeatPenalty { get; set; } 25 | public int Seed { get; set; } 26 | public object Stop { get; set; } 27 | public bool Stream { get; set; } 28 | public double Temperature { get; set; } 29 | public double MinP { get; set; } 30 | public double TopP { get; set; } 31 | public int TopK { get; set; } 32 | public string User { get; set; } 33 | 34 | public CompletionCreateBaseDto() 35 | { 36 | Echo = false; 37 | FrequencyPenalty = 0; 38 | MaxTokens = 16; 39 | N = 1; 40 | PresencePenalty = 0; 41 | RepeatPenalty = 1.1; 42 | Seed = -1; 43 | Stop = new(); 44 | Stream = false; 45 | Temperature = 1; 46 | MinP = 0.05; 47 | TopP = 0.95; 48 | TopK = 40; 49 | User = ""; 50 | } 51 | } 52 | 53 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionCreateDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionCreateDto 4 | * @created : Friday Jan 05, 2024 00:26:00 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class CompletionCreateDto : CompletionCreateBaseDto 17 | { 18 | public object Prompt { get; set; } 19 | public int BestOf { get; set; } 20 | public string Suffix { get; set; } 21 | 22 | public CompletionCreateDto() 23 | { 24 | Prompt = new(); 25 | BestOf = 1; 26 | Suffix = ""; 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionCreateOpenAIDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionCreateOpenAIDto 4 | * @created : Sunday Jan 14, 2024 21:07:22 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class CompletionCreateOpenAIDto : CompletionCreateOpenAIBaseDto 17 | { 18 | public object Prompt { 19 | get { 20 | return BaseDto.Prompt; 21 | } 22 | } 23 | public int BestOf { 24 | get { 25 | return BaseDto.BestOf; 26 | } 27 | } 28 | public string Suffix { 29 | get { 30 | return BaseDto.Suffix; 31 | } 32 | } 33 | 34 | public CompletionCreateOpenAIDto(CompletionCreateDto baseDto) : base(baseDto) 35 | { 36 | 37 | } 38 | } 39 | 40 | public partial class CompletionCreateOpenAIBaseDto 41 | { 42 | internal CompletionCreateDto BaseDto { get; set; } 43 | 44 | public string Model { 45 | get { 46 | return BaseDto.Model; 47 | } 48 | } 49 | public bool Echo { 50 | get { 51 | return BaseDto.Echo; 52 | } 53 | } 54 | public double FrequencyPenalty { 55 | get { 56 | return BaseDto.FrequencyPenalty; 57 | } 58 | } 59 | public int MaxTokens { 60 | get { 61 | return BaseDto.MaxTokens; 62 | } 63 | } 64 | public int N { 65 | get { 66 | return BaseDto.N; 67 | } 68 | } 69 | public double PresencePenalty { 70 | get { 71 | return BaseDto.PresencePenalty; 72 | } 73 | } 74 | public int Seed { 75 | get { 76 | return BaseDto.Seed; 77 | } 78 | } 79 | public object Stop { 80 | get { 81 | return (Stop is string) ? new List() { (string) Stop } : Stop; 82 | } 83 | } 84 | public bool Stream { 85 | get { 86 | return BaseDto.Stream; 87 | } 88 | } 89 | public double Temperature { 90 | get { 91 | return BaseDto.Temperature; 92 | } 93 | } 94 | public double TopP { 95 | get { 96 | return BaseDto.TopP; 97 | } 98 | } 99 | public string User { 100 | get { 101 | return BaseDto.User; 102 | } 103 | } 104 | 105 | public CompletionCreateOpenAIBaseDto(CompletionCreateDto baseDto) 106 | { 107 | BaseDto = baseDto; 108 | } 109 | } 110 | 111 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionDto 4 | * @created : Friday Jan 05, 2024 00:14:54 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class CompletionDto : CompletionBaseDto 17 | { 18 | public CompletionDto() 19 | { 20 | Object = "text_completion"; 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/CompletionUsageDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionUsageDto 4 | * @created : Friday Jan 05, 2024 00:17:04 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class CompletionUsageDto 17 | { 18 | public int PromptTokens { get; set; } 19 | public int CompletionTokens { get; set; } 20 | public int TotalTokens { 21 | get { 22 | return PromptTokens + CompletionTokens; 23 | } 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/EmbeddingCreateDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingCreateDto 4 | * @created : Friday Jan 05, 2024 18:56:19 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class EmbeddingCreateDto 17 | { 18 | public object Input { get; set; } 19 | public string Model { get; set; } 20 | public string EncodingFormat { get; set; } 21 | 22 | public EmbeddingCreateDto() 23 | { 24 | EncodingFormat = "float"; 25 | } 26 | 27 | public List GetInputs() 28 | { 29 | if (Input is string s) 30 | { 31 | return new List() {s}; 32 | } 33 | 34 | return (List) Input; 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/EmbeddingDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingDto 4 | * @created : Friday Jan 05, 2024 18:49:46 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class EmbeddingsDto : ListBaseDto 17 | { 18 | public string Model { get; set; } 19 | public EmbeddingUsageDto Usage { get; set; } 20 | 21 | public EmbeddingsDto() 22 | { 23 | Object = "list"; 24 | Data = new(); 25 | } 26 | } 27 | 28 | public partial class EmbeddingDto : BaseDto 29 | { 30 | public float[] Embedding { get; set; } 31 | public int Index { get; set; } 32 | 33 | public EmbeddingDto() 34 | { 35 | Object = "embedding"; 36 | Index = 0; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/EmbeddingUsageDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingUsageDto 4 | * @created : Friday Jan 05, 2024 18:54:28 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class EmbeddingUsageDto 17 | { 18 | public int PromptTokens { get; set; } 19 | public int TotalTokens { 20 | get { 21 | return PromptTokens; 22 | } 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ErrorDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ErrorDto 4 | * @created : Thursday Jan 04, 2024 23:48:22 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ErrorDto 17 | { 18 | public string Message { get; set; } 19 | public string Type { get; set; } 20 | public string Param { get; set; } 21 | public string Code { get; set; } 22 | 23 | public ErrorDto(string message = "", string type = "", string param = "", string code = "") 24 | { 25 | Message = message; 26 | Type = type; 27 | Param = param; 28 | Code = code; 29 | } 30 | } 31 | 32 | public partial class ErrorObjectDto 33 | { 34 | public ErrorDto Error { get; set; } 35 | 36 | public ErrorObjectDto(string message = "", string type = "", string param = "", string code = "") 37 | { 38 | Error = new(); 39 | Error.Message = message; 40 | Error.Type = type; 41 | Error.Param = param; 42 | Error.Code = code; 43 | } 44 | } 45 | 46 | public partial class InvalidRequestErrorDto : ErrorObjectDto 47 | { 48 | public InvalidRequestErrorDto(string message = "", string type = "", string param = "", string code = "") : base(message:message, type:"invalid_request_error", param:param, code:code) 49 | { 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ExtendedTokenizeCreateDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ExtendedTokenizeCreateDto 4 | * @created : Wednesday Jan 31, 2024 17:39:33 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ExtendedTokenizeCreateDto 17 | { 18 | public string Model { get; set; } 19 | public string Content { get; set; } 20 | } 21 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ExtendedTokenizeDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ExtendedTokenizeDto 4 | * @created : Wednesday Jan 31, 2024 17:39:56 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | 11 | using Godot; 12 | using GodotEGP.Objects.Extensions; 13 | using GodotEGP.Logging; 14 | using GodotEGP.Service; 15 | using GodotEGP.Event.Events; 16 | using GodotEGP.Config; 17 | 18 | public partial class ExtendedTokenizeDto 19 | { 20 | public List Tokens { get; set; } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ListBaseDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ListBaseDto 4 | * @created : Thursday Jan 04, 2024 22:18:49 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class ListBaseDto : BaseDto where T : BaseDto 17 | { 18 | public List Data { get; set; } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /classes/WebAPI/Dtos/ModelDto.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelDto 4 | * @created : Thursday Jan 04, 2024 22:08:31 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Dtos; 8 | 9 | using GatoGPT.AI; 10 | using GatoGPT.AI.TextGeneration; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | public partial class ModelDto : BaseDto 20 | { 21 | public string Id { get; set; } 22 | public long Created { get; set; } 23 | public string OwnedBy { get; set; } 24 | 25 | public ModelDto() 26 | { 27 | Created = ((DateTimeOffset) DateTime.UtcNow).ToUnixTimeSeconds(); 28 | Object = "model"; 29 | OwnedBy = "local"; 30 | } 31 | } 32 | 33 | public partial class ModelFullDto : ModelDto 34 | { 35 | public AI.ModelDefinition Definition { get; set; } 36 | } 37 | 38 | public partial class ModelListDto : ListBaseDto 39 | { 40 | public ModelListDto() 41 | { 42 | Data = new(); 43 | Object = "list"; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /classes/WebAPI/Entities/ModelEntity.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelEntity 4 | * @created : Thursday Jan 04, 2024 22:06:35 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Entities; 8 | 9 | using GatoGPT.AI.TextGeneration; 10 | using GatoGPT.AI; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | public partial class ModelEntity 20 | { 21 | public AI.ModelDefinition Model; 22 | 23 | public string Id 24 | { 25 | get { return Model.Id; } 26 | } 27 | public string OwnedBy 28 | { 29 | get { return Model.OwnedBy; } 30 | } 31 | } 32 | 33 | -------------------------------------------------------------------------------- /classes/WebAPI/Extensions/CustomCorsExtension.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CustomCorsExtension 4 | * @created : Thursday Jan 04, 2024 20:27:09 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Extensions; 8 | 9 | public static class CorsExtension 10 | { 11 | public static void AddCustomCors(this IServiceCollection services, string policyName) 12 | { 13 | services.AddCors(options => 14 | { 15 | options.AddPolicy(policyName, 16 | builder => 17 | { 18 | builder 19 | .AllowAnyOrigin() 20 | .AllowAnyHeader() 21 | .AllowAnyMethod(); 22 | }); 23 | }); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /classes/WebAPI/Extensions/ExceptionExtension.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ExceptionExtension 4 | * @created : Thursday Jan 04, 2024 20:46:05 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Extensions; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Microsoft.AspNetCore.Diagnostics; 17 | 18 | public static class ExceptionExtension 19 | { 20 | public static void AddProductionExceptionHandling(this IApplicationBuilder app, ILoggerFactory loggerFactory) 21 | { 22 | app.UseExceptionHandler(errorApp => 23 | { 24 | errorApp.Run(async context => 25 | { 26 | context.Response.StatusCode = 500; 27 | context.Response.ContentType = "text/plain"; 28 | var errorFeature = context.Features.Get(); 29 | if (errorFeature != null) 30 | { 31 | var logger = loggerFactory.CreateLogger("Global exception logger"); 32 | logger.LogError(500, errorFeature.Error, errorFeature.Error.Message); 33 | } 34 | 35 | await context.Response.WriteAsync("There was an error"); 36 | }); 37 | }); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /classes/WebAPI/Extensions/VersioningExtension.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : VersioningExtension 4 | * @created : Thursday Jan 04, 2024 20:40:10 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.Extensions; 8 | 9 | using Microsoft.AspNetCore.Mvc; 10 | using Microsoft.AspNetCore.Mvc.Versioning; 11 | using Microsoft.Extensions.DependencyInjection; 12 | 13 | public static class VersioningExtension 14 | { 15 | public static void AddVersioning(this IServiceCollection services) 16 | { 17 | services.AddApiVersioning( 18 | config => 19 | { 20 | config.ReportApiVersions = true; 21 | config.AssumeDefaultVersionWhenUnspecified = true; 22 | config.DefaultApiVersion = new ApiVersion(1, 0); 23 | config.ApiVersionReader = ApiVersionReader.Combine(new UrlSegmentApiVersionReader(), 24 | new HeaderApiVersionReader("x-api-version"), 25 | new MediaTypeApiVersionReader("x-api-version")); 26 | }); 27 | services.AddVersionedApiExplorer( 28 | options => 29 | { 30 | options.GroupNameFormat = "'v'VVV"; 31 | 32 | // note: this option is only necessary when versioning by url segment. the SubstitutionFormat 33 | // can also be used to control the format of the API version in route templates 34 | options.SubstituteApiVersionInUrl = true; 35 | }); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /classes/WebAPI/MappingProfiles/ChatCompletionMappings.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ChatCompletionMappings 4 | * @created : Friday Jan 26, 2024 22:22:29 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.MappingProfiles; 8 | 9 | using GatoGPT.WebAPI.Entities; 10 | using GatoGPT.WebAPI.Dtos; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | using GodotEGP.AI.OpenAI; 19 | 20 | using AutoMapper; 21 | // using GatoGPT.WebAPI.Dtos; 22 | // using GatoGPT.WebAPI.Entities; 23 | 24 | public class ChatCompletionMappings : Profile 25 | { 26 | public ChatCompletionMappings() 27 | { 28 | CreateMap().ReverseMap(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /classes/WebAPI/MappingProfiles/CompletionMappings.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : CompletionMappings 4 | * @created : Friday Jan 26, 2024 22:24:13 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.MappingProfiles; 8 | 9 | using GatoGPT.WebAPI.Entities; 10 | using GatoGPT.WebAPI.Dtos; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | using GodotEGP.AI.OpenAI; 19 | 20 | using AutoMapper; 21 | // using GatoGPT.WebAPI.Dtos; 22 | // using GatoGPT.WebAPI.Entities; 23 | 24 | public class CompletionMappings : Profile 25 | { 26 | public CompletionMappings() 27 | { 28 | CreateMap().ReverseMap(); 29 | CreateMap().ReverseMap(); 30 | CreateMap().ReverseMap(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /classes/WebAPI/MappingProfiles/EmbeddingsMappings.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : EmbeddingsMappings 4 | * @created : Friday Jan 26, 2024 22:25:17 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.MappingProfiles; 8 | 9 | using GatoGPT.WebAPI.Entities; 10 | using GatoGPT.WebAPI.Dtos; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | using GodotEGP.AI.OpenAI; 19 | 20 | using AutoMapper; 21 | // using GatoGPT.WebAPI.Dtos; 22 | // using GatoGPT.WebAPI.Entities; 23 | 24 | public class EmbeddingsMappings : Profile 25 | { 26 | public EmbeddingsMappings() 27 | { 28 | CreateMap().ReverseMap(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /classes/WebAPI/MappingProfiles/ModelMappings.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ModelMappings 4 | * @created : Thursday Jan 04, 2024 21:47:18 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI.MappingProfiles; 8 | 9 | using GatoGPT.WebAPI.Entities; 10 | using GatoGPT.WebAPI.Dtos; 11 | 12 | using Godot; 13 | using GodotEGP.Objects.Extensions; 14 | using GodotEGP.Logging; 15 | using GodotEGP.Service; 16 | using GodotEGP.Event.Events; 17 | using GodotEGP.Config; 18 | 19 | using AutoMapper; 20 | // using GatoGPT.WebAPI.Dtos; 21 | // using GatoGPT.WebAPI.Entities; 22 | 23 | public class ModelMappings : Profile 24 | { 25 | public ModelMappings() 26 | { 27 | CreateMap().ReverseMap(); 28 | CreateMap().ReverseMap(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /classes/WebAPI/ServerSentEventManager.cs: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : ElGatoPanzon (contact@elgatopanzon.io) Copyright (c) ElGatoPanzon 3 | * @file : ServerSentEventManager 4 | * @created : Sunday Jan 07, 2024 19:22:08 CST 5 | */ 6 | 7 | namespace GatoGPT.WebAPI; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | using Newtonsoft.Json; 17 | using Newtonsoft.Json.Serialization; 18 | 19 | public partial class ServerSentEventManager 20 | { 21 | private HttpContext _httpContext { get; set; } 22 | private bool _done = false; 23 | 24 | public ServerSentEventManager(HttpContext httpContext) 25 | { 26 | _httpContext = httpContext; 27 | } 28 | 29 | public void Start() 30 | { 31 | _httpContext.Response.Headers.Add("Content-Type", "text/event-stream"); 32 | } 33 | 34 | public async Task SendEvent(object eventObject, string eventPrefix = "data: ") 35 | { 36 | if (eventPrefix.Length > 0) 37 | { 38 | await _httpContext.Response.WriteAsync(eventPrefix); 39 | } 40 | if (eventObject is string stringObj) 41 | { 42 | await _httpContext.Response.WriteAsync(stringObj); 43 | } 44 | else 45 | await _httpContext.Response.WriteAsync(JsonConvert.SerializeObject(eventObject, 46 | new JsonSerializerSettings 47 | { 48 | ContractResolver = new DefaultContractResolver() { NamingStrategy = new SnakeCaseNamingStrategy() }, 49 | })); 50 | await _httpContext.Response.WriteAsync($"\n\n"); 51 | await _httpContext.Response.Body.FlushAsync(); 52 | 53 | return true; 54 | } 55 | 56 | public async Task Done() 57 | { 58 | _done = true; 59 | return await SendEvent("[DONE]"); 60 | } 61 | 62 | public async Task WaitDone() 63 | { 64 | while (!_done) 65 | { 66 | await Task.Delay(100); 67 | } 68 | 69 | return true; 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.1' 3 | name: gatogpt 4 | services: 5 | gatogpt: 6 | build: . 7 | image: elgatopanzon/gatogpt:latest 8 | ports: 9 | - 8181:8181 10 | volumes: 11 | # - .:/code 12 | # main config files e.g. model resources/definitions 13 | - ./GatoGPT_Data/Config:/code/GatoGPT_Data/Config 14 | # model files 15 | - ./GatoGPT_Data/Models:/code/GatoGPT_Data/Models 16 | # location to store downloaded files the API creates (e.g. images) 17 | - ./GatoGPT_Data/Downloads:/code/GatoGPT_Data/Downloads 18 | # holds prompt caches and states, optional 19 | # - ./GatoGPT_Data/Cache:/code/GatoGPT_Data/Cache 20 | 21 | # start the API 22 | command: ["api", "--host", "0.0.0.0", "--port", "8181"] 23 | 24 | logging: 25 | driver: json-file 26 | options: 27 | max-file: "3" # number of files or file count 28 | max-size: '10m' 29 | 30 | deploy: 31 | resources: 32 | reservations: 33 | devices: 34 | - driver: nvidia 35 | device_ids: ['0'] 36 | capabilities: [gpu] 37 | -------------------------------------------------------------------------------- /templates/cs.template: -------------------------------------------------------------------------------- 1 | /** 2 | * @author : {{NAME}} ({{EMAIL}}) {{COPYRIGHT}} 3 | * @file : {{FILE}} 4 | * @created : {{TIMESTAMP}} 5 | */ 6 | 7 | namespace GatoGPT.{{FILEP}}; 8 | 9 | using Godot; 10 | using GodotEGP.Objects.Extensions; 11 | using GodotEGP.Logging; 12 | using GodotEGP.Service; 13 | using GodotEGP.Event.Events; 14 | using GodotEGP.Config; 15 | 16 | public partial class {{CAMEL_CLASS}} 17 | { 18 | public {{CAMEL_CLASS}}() 19 | { 20 | {{CURSOR}} 21 | } 22 | } 23 | --------------------------------------------------------------------------------