├── .clang-format ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ └── config.yml └── workflows │ └── builds.yml ├── .gitignore ├── .gitmodules ├── .vscode └── extensions.json ├── LICENSE.md ├── README.md ├── binding_generator.py ├── build.zig ├── godot ├── .gitattributes ├── .gitignore ├── addons │ └── godot-llama-cpp │ │ ├── assets │ │ ├── godot-llama-cpp-1024x1024.svg │ │ ├── godot-llama-cpp-1024x1024.svg.import │ │ ├── godot-llama-cpp-16x16.svg │ │ └── godot-llama-cpp-16x16.svg.import │ │ ├── chat │ │ └── chat_formatter.gd │ │ ├── plugin.cfg │ │ ├── plugin.gd │ │ └── plugin.gdextension ├── examples │ └── simple │ │ ├── TextEdit.gd │ │ ├── form.gd │ │ ├── message.gd │ │ ├── message.tscn │ │ ├── simple.gd │ │ ├── simple.tscn │ │ ├── system.svg │ │ ├── system.svg.import │ │ ├── user.svg │ │ └── user.svg.import ├── icon.svg ├── icon.svg.import ├── models │ └── .gitkeep └── project.godot ├── src ├── llama_context.cpp ├── llama_context.h ├── llama_model.cpp ├── llama_model.h ├── llama_model_loader.cpp ├── llama_model_loader.h ├── register_types.cpp └── register_types.h └── tools ├── concat_files.zig └── expand_metal.zig /.clang-format: -------------------------------------------------------------------------------- 1 | # Commented out parameters are those with the same value as base LLVM style. 2 | # We can uncomment them if we want to change their value, or enforce the 3 | # chosen value in case the base style changes (last sync: Clang 14.0). 4 | --- 5 | ### General config, applies to all languages ### 6 | BasedOnStyle: LLVM 7 | AccessModifierOffset: -4 8 | AlignAfterOpenBracket: DontAlign 9 | # AlignArrayOfStructures: None 10 | # AlignConsecutiveMacros: None 11 | # AlignConsecutiveAssignments: None 12 | # AlignConsecutiveBitFields: None 13 | # AlignConsecutiveDeclarations: None 14 | # AlignEscapedNewlines: Right 15 | AlignOperands: DontAlign 16 | AlignTrailingComments: false 17 | # AllowAllArgumentsOnNextLine: true 18 | AllowAllParametersOfDeclarationOnNextLine: false 19 | # AllowShortEnumsOnASingleLine: true 20 | # AllowShortBlocksOnASingleLine: Never 21 | # AllowShortCaseLabelsOnASingleLine: false 22 | # AllowShortFunctionsOnASingleLine: All 23 | # AllowShortLambdasOnASingleLine: All 24 | # AllowShortIfStatementsOnASingleLine: Never 25 | # AllowShortLoopsOnASingleLine: false 26 | # AlwaysBreakAfterDefinitionReturnType: None 27 | # AlwaysBreakAfterReturnType: None 28 | # AlwaysBreakBeforeMultilineStrings: false 29 | # AlwaysBreakTemplateDeclarations: MultiLine 30 | # AttributeMacros: 31 | # - __capability 32 | # BinPackArguments: true 33 | # BinPackParameters: true 34 | # BraceWrapping: 35 | # AfterCaseLabel: false 36 | # AfterClass: false 37 | # AfterControlStatement: Never 38 | # AfterEnum: false 39 | # AfterFunction: false 40 | # AfterNamespace: false 41 | # AfterObjCDeclaration: false 42 | # AfterStruct: false 43 | # AfterUnion: false 44 | # AfterExternBlock: false 45 | # BeforeCatch: false 46 | # BeforeElse: false 47 | # BeforeLambdaBody: false 48 | # BeforeWhile: false 49 | # IndentBraces: false 50 | # SplitEmptyFunction: true 51 | # SplitEmptyRecord: true 52 | # SplitEmptyNamespace: true 53 | # BreakBeforeBinaryOperators: None 54 | # BreakBeforeConceptDeclarations: true 55 | # BreakBeforeBraces: Attach 56 | # BreakBeforeInheritanceComma: false 57 | # BreakInheritanceList: BeforeColon 58 | # BreakBeforeTernaryOperators: true 59 | # BreakConstructorInitializersBeforeComma: false 60 | BreakConstructorInitializers: AfterColon 61 | # BreakStringLiterals: true 62 | ColumnLimit: 0 63 | # CommentPragmas: '^ IWYU pragma:' 64 | # QualifierAlignment: Leave 65 | # CompactNamespaces: false 66 | ConstructorInitializerIndentWidth: 8 67 | ContinuationIndentWidth: 8 68 | Cpp11BracedListStyle: false 69 | # DeriveLineEnding: true 70 | # DerivePointerAlignment: false 71 | # DisableFormat: false 72 | # EmptyLineAfterAccessModifier: Never 73 | # EmptyLineBeforeAccessModifier: LogicalBlock 74 | # ExperimentalAutoDetectBinPacking: false 75 | # PackConstructorInitializers: BinPack 76 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 77 | # AllowAllConstructorInitializersOnNextLine: true 78 | # FixNamespaceComments: true 79 | # ForEachMacros: 80 | # - foreach 81 | # - Q_FOREACH 82 | # - BOOST_FOREACH 83 | # IfMacros: 84 | # - KJ_IF_MAYBE 85 | # IncludeBlocks: Preserve 86 | IncludeCategories: 87 | - Regex: '".*"' 88 | Priority: 1 89 | - Regex: '^<.*\.h>' 90 | Priority: 2 91 | - Regex: '^<.*' 92 | Priority: 3 93 | # IncludeIsMainRegex: '(Test)?$' 94 | # IncludeIsMainSourceRegex: '' 95 | # IndentAccessModifiers: false 96 | IndentCaseLabels: true 97 | # IndentCaseBlocks: false 98 | # IndentGotoLabels: true 99 | # IndentPPDirectives: None 100 | # IndentExternBlock: AfterExternBlock 101 | # IndentRequires: false 102 | IndentWidth: 4 103 | # IndentWrappedFunctionNames: false 104 | # InsertTrailingCommas: None 105 | # JavaScriptQuotes: Leave 106 | # JavaScriptWrapImports: true 107 | KeepEmptyLinesAtTheStartOfBlocks: false 108 | # LambdaBodyIndentation: Signature 109 | # MacroBlockBegin: '' 110 | # MacroBlockEnd: '' 111 | # MaxEmptyLinesToKeep: 1 112 | # NamespaceIndentation: None 113 | # PenaltyBreakAssignment: 2 114 | # PenaltyBreakBeforeFirstCallParameter: 19 115 | # PenaltyBreakComment: 300 116 | # PenaltyBreakFirstLessLess: 120 117 | # PenaltyBreakOpenParenthesis: 0 118 | # PenaltyBreakString: 1000 119 | # PenaltyBreakTemplateDeclaration: 10 120 | # PenaltyExcessCharacter: 1000000 121 | # PenaltyReturnTypeOnItsOwnLine: 60 122 | # PenaltyIndentedWhitespace: 0 123 | # PointerAlignment: Right 124 | # PPIndentWidth: -1 125 | # ReferenceAlignment: Pointer 126 | # ReflowComments: true 127 | # RemoveBracesLLVM: false 128 | # SeparateDefinitionBlocks: Leave 129 | # ShortNamespaceLines: 1 130 | # SortIncludes: CaseSensitive 131 | # SortJavaStaticImport: Before 132 | # SortUsingDeclarations: true 133 | # SpaceAfterCStyleCast: false 134 | # SpaceAfterLogicalNot: false 135 | # SpaceAfterTemplateKeyword: true 136 | # SpaceBeforeAssignmentOperators: true 137 | # SpaceBeforeCaseColon: false 138 | # SpaceBeforeCpp11BracedList: false 139 | # SpaceBeforeCtorInitializerColon: true 140 | # SpaceBeforeInheritanceColon: true 141 | # SpaceBeforeParens: ControlStatements 142 | # SpaceBeforeParensOptions: 143 | # AfterControlStatements: true 144 | # AfterForeachMacros: true 145 | # AfterFunctionDefinitionName: false 146 | # AfterFunctionDeclarationName: false 147 | # AfterIfMacros: true 148 | # AfterOverloadedOperator: false 149 | # BeforeNonEmptyParentheses: false 150 | # SpaceAroundPointerQualifiers: Default 151 | # SpaceBeforeRangeBasedForLoopColon: true 152 | # SpaceInEmptyBlock: false 153 | # SpaceInEmptyParentheses: false 154 | # SpacesBeforeTrailingComments: 1 155 | # SpacesInAngles: Never 156 | # SpacesInConditionalStatement: false 157 | # SpacesInContainerLiterals: true 158 | # SpacesInCStyleCastParentheses: false 159 | ## Godot TODO: We'll want to use a min of 1, but we need to see how to fix 160 | ## our comment capitalization at the same time. 161 | SpacesInLineCommentPrefix: 162 | Minimum: 0 163 | Maximum: -1 164 | # SpacesInParentheses: false 165 | # SpacesInSquareBrackets: false 166 | # SpaceBeforeSquareBrackets: false 167 | # BitFieldColonSpacing: Both 168 | # StatementAttributeLikeMacros: 169 | # - Q_EMIT 170 | # StatementMacros: 171 | # - Q_UNUSED 172 | # - QT_REQUIRE_VERSION 173 | TabWidth: 4 174 | # UseCRLF: false 175 | UseTab: Always 176 | # WhitespaceSensitiveMacros: 177 | # - STRINGIZE 178 | # - PP_STRINGIZE 179 | # - BOOST_PP_STRINGIZE 180 | # - NS_SWIFT_NAME 181 | # - CF_SWIFT_NAME 182 | --- 183 | ### C++ specific config ### 184 | Language: Cpp 185 | Standard: c++17 186 | --- 187 | ### ObjC specific config ### 188 | Language: ObjC 189 | # ObjCBinPackProtocolList: Auto 190 | ObjCBlockIndentWidth: 4 191 | # ObjCBreakBeforeNestedBlockParam: true 192 | # ObjCSpaceAfterProperty: false 193 | # ObjCSpaceBeforeProtocolList: true 194 | --- 195 | ### Java specific config ### 196 | Language: Java 197 | # BreakAfterJavaFieldAnnotations: false 198 | JavaImportGroups: ['org.godotengine', 'android', 'androidx', 'com.android', 'com.google', 'java', 'javax'] 199 | ... 200 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Normalize EOL for all files that Git considers text files. 2 | * text=auto eol=lf 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug report 2 | description: Report a bug with the godot-cpp template 3 | body: 4 | 5 | - type: markdown 6 | attributes: 7 | value: | 8 | - Write a descriptive issue title above. 9 | - Search [open](https://github.com/godotengine/godot-cpp-template/issues) and [closed](https://github.com/godotengine/godot-cpp-template/issues?q=is%3Aissue+is%3Aclosed) issues to ensure it has not already been reported. 10 | - type: input 11 | attributes: 12 | label: Godot version 13 | description: > 14 | Specify the Git commit hash of your Godot build. 15 | placeholder: v4.0.stable.official [92bee43ad] 16 | validations: 17 | required: true 18 | 19 | - type: input 20 | attributes: 21 | label: godot-cpp version 22 | description: > 23 | Specify the Git commit hash of the godot-cpp submodule in your project. You can run `git status` inside the folder to check it. 24 | placeholder: v4.0.stable.official [9d1c396c5] 25 | validations: 26 | required: true 27 | 28 | - type: input 29 | attributes: 30 | label: System information 31 | description: | 32 | Specify the OS version. 33 | placeholder: Windows 10 34 | validations: 35 | required: true 36 | 37 | - type: textarea 38 | attributes: 39 | label: Issue description 40 | description: | 41 | Describe your issue briefly. What doesn't work, and how do you expect it to work instead? 42 | You can include images or videos with drag and drop, and format code blocks or logs with ``` tags. 43 | validations: 44 | required: true -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | 3 | contact_links: 4 | - name: Godot proposals 5 | url: https://github.com/godotengine/godot-proposals 6 | about: Please submit feature proposals on the Godot proposals repository, not here. 7 | 8 | - name: Godot documentation repository 9 | url: https://github.com/godotengine/godot-docs 10 | about: Please report issues with documentation on the Godot documentation repository, not here. 11 | 12 | - name: Godot community channels 13 | url: https://godotengine.org/community 14 | about: Please ask for technical support on one of the other community channels, not here. 15 | -------------------------------------------------------------------------------- /.github/workflows/builds.yml: -------------------------------------------------------------------------------- 1 | name: Builds 2 | 3 | on: push 4 | 5 | env: 6 | LIBNAME: example 7 | 8 | concurrency: 9 | group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-macos 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | build: 14 | runs-on: ${{matrix.os}} 15 | # temporarily disable 16 | if: false 17 | name: ${{matrix.name}} 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | include: 22 | - identifier: windows-debug 23 | os: windows-latest 24 | name: 🏁 Windows Debug 25 | target: template_debug 26 | platform: windows 27 | arch: x86_64 28 | - identifier: windows-release 29 | os: windows-latest 30 | name: 🏁 Windows Release 31 | target: template_release 32 | platform: windows 33 | arch: x86_64 34 | - identifier: macos-debug 35 | os: macos-latest 36 | name: 🍎 macOS (universal) Debug 37 | target: template_debug 38 | platform: macos 39 | arch: universal 40 | - identifier: macos-release 41 | os: macos-latest 42 | name: 🍎 macOS (universal) Release 43 | target: template_release 44 | platform: macos 45 | arch: universal 46 | - identifier: linux-debug 47 | os: ubuntu-latest 48 | name: 🐧 Linux Debug 49 | runner: ubuntu-20.04 50 | target: template_debug 51 | platform: linux 52 | arch: x86_64 53 | - identifier: linux-release 54 | os: ubuntu-latest 55 | name: 🐧 Linux Release 56 | runner: ubuntu-20.04 57 | target: template_release 58 | platform: linux 59 | arch: x86_64 60 | 61 | steps: 62 | - name: Checkout project 63 | uses: actions/checkout@v3 64 | with: 65 | submodules: recursive 66 | 67 | - name: Set up Python 68 | uses: actions/setup-python@v4 69 | with: 70 | python-version: '3.x' 71 | 72 | - name: Set up SCons 73 | shell: bash 74 | run: | 75 | python -c "import sys; print(sys.version)" 76 | python -m pip install scons 77 | scons --version 78 | - name: Linux dependencies 79 | if: ${{ matrix.platform == 'linux' }} 80 | run: | 81 | sudo apt-get update -qq 82 | sudo apt-get install -qqq build-essential pkg-config 83 | - name: Setup MinGW for Windows/MinGW build 84 | if: ${{ matrix.platform == 'windows' }} 85 | uses: egor-tensin/setup-mingw@v2 86 | with: 87 | version: 12.2.0 88 | 89 | - name: Compile godot-cpp 90 | shell: sh 91 | run: | 92 | scons target='${{ matrix.target }}' platform='${{ matrix.platform }}' arch='${{ matrix.arch }}' 93 | working-directory: godot-cpp 94 | 95 | - name: Compile Extension 96 | shell: sh 97 | run: | 98 | scons target='${{ matrix.target }}' platform='${{ matrix.platform }}' arch='${{ matrix.arch }}' 99 | - name: Delete compilation files 100 | if: ${{ matrix.platform == 'windows' }} 101 | run: | 102 | Remove-Item bin/* -Include *.exp,*.lib,*.pdb -Force 103 | - name: Upload artifact 104 | uses: actions/upload-artifact@v3 105 | with: 106 | name: ${{ github.event.repository.name }} 107 | path: | 108 | ${{ github.workspace }}/bin/* 109 | - name: Archive Release 110 | uses: thedoctor0/zip-release@0.7.1 111 | with: 112 | type: 'zip' 113 | filename: '${{ env.LIBNAME }}.${{ matrix.platform }}.${{ matrix.arch }}.zip' 114 | path: '${{ github.workspace }}/bin/' 115 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 116 | 117 | - name: Create and upload asset 118 | if: success() && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 119 | uses: ncipollo/release-action@v1 120 | with: 121 | allowUpdates: true 122 | artifacts: "${{ env.LIBNAME }}.${{ matrix.platform }}.${{ matrix.arch }}.zip" 123 | omitNameDuringUpdate: true 124 | omitBodyDuringUpdate: true 125 | token: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Godot 4+ specific ignores 2 | .godot/ 3 | 4 | # Ignore library files but not the gdextension file 5 | demo/bin/* 6 | !demo/bin/android 7 | demo/bin/android/* 8 | !demo/bin/android/.gitkeep 9 | !demo/bin/linux 10 | demo/bin/linux/* 11 | !demo/bin/linux/.gitkeep 12 | !demo/bin/macos 13 | demo/bin/macos/* 14 | !demo/bin/macos/.gitkeep 15 | !demo/bin/windows 16 | demo/bin/windows/* 17 | !demo/bin/windows/.gitkeep 18 | !demo/bin/*.gdextension 19 | .sconsign*.dblite 20 | 21 | # Ignore custom.py 22 | custom.py 23 | 24 | # Ignore generated compile_commands.json 25 | compile_commands.json 26 | 27 | # Binaries 28 | *.o 29 | *.os 30 | *.so 31 | *.obj 32 | *.bc 33 | *.pyc 34 | *.dblite 35 | *.pdb 36 | *.lib 37 | *.config 38 | *.creator 39 | *.creator.user 40 | *.files 41 | *.includes 42 | *.idb 43 | *.exp 44 | 45 | # Other stuff 46 | *.log 47 | 48 | # VSCode 49 | .vscode/* 50 | !.vscode/extensions.json 51 | 52 | .zig-cache 53 | zig-out 54 | *.gguf 55 | 56 | godot/addons/godot-llama-cpp/lib/* -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "llama.cpp"] 2 | path = llama.cpp 3 | url = https://github.com/ggerganov/llama.cpp.git 4 | [submodule "godot-cpp"] 5 | path = godot_cpp 6 | url = git@github.com:godotengine/godot-cpp.git 7 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-vscode.cpptools-extension-pack", 4 | "ms-python.python" 5 | ] 6 | } -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright © 2024 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 |

godot-llama-cpp

6 | 7 | Run large language models in [Godot](https://godotengine.org). Powered by [llama.cpp](https://github.com/ggerganov/llama.cpp). 8 | 9 |
10 |
11 | 12 | ![Godot v4.2](https://img.shields.io/badge/Godot-v4.2-%23478cbf?logo=godot-engine&logoColor=white) 13 | ![GitHub last commit](https://img.shields.io/github/last-commit/hazelnutcloud/godot-llama-cpp) 14 | ![GitHub License](https://img.shields.io/github/license/hazelnutcloud/godot-llama-cpp) 15 | 16 |
17 | 18 | ## Overview 19 | 20 | This library aims to provide a high-level interface to run large language models in Godot, following Godot's node-based design principles. 21 | 22 | ```gdscript 23 | @onready var llama_context = %LlamaContext 24 | 25 | var messages = [ 26 | { "sender": "system", "text": "You are a pirate chatbot who always responds in pirate speak!" }, 27 | { "sender": "user", "text": "Who are you?" } 28 | ] 29 | var prompt = ChatFormatter.apply("llama3", messages) 30 | var completion_id = llama_context.request_completion(prompt) 31 | 32 | while (true): 33 | var response = await llama_context.completion_generated 34 | print(response["text"]) 35 | 36 | if response["done"]: break 37 | ``` 38 | 39 | ## Features 40 | - Platform and compute backend support: 41 | | Platform | CPU | Metal | Vulkan | CUDA | 42 | |----------|-----|-------|--------|------| 43 | | macOS | ✅ | ✅ | ❌ | ❌ | 44 | | Linux | ✅ | ❌ | ✅ | 🚧 | 45 | | Windows | ✅ | ❌ | 🚧 | 🚧 | 46 | - Asynchronous completion generation 47 | - Support any language model that llama.cpp supports in GGUF format 48 | - GGUF files are Godot resources 49 | 50 | ## Roadmap 51 | - [ ] Chat completions support via dedicated library for jinja2 templating in zig 52 | - [ ] Grammar support 53 | - [ ] Multimodal models support 54 | - [ ] Embeddings 55 | - [ ] Vector database using LibSQL 56 | 57 | ## Building & Installation 58 | 59 | 1. Download zig v0.13.0 from https://ziglang.org/download/ 60 | 2. Clone the repository: 61 | ```bash 62 | git clone --recurse-submodules https://github.com/hazelnutcloud/godot-llama-cpp.git 63 | ``` 64 | 3. Copy the `godot-llama-cpp` addon folder in `godot/addons` to your Godot project's `addons` folder. 65 | ```bash 66 | cp -r godot-llama-cpp/godot/addons/godot-llama-cpp /addons 67 | ``` 68 | 4. Build the extension and install it in your Godot project: 69 | ```bash 70 | cd godot-llama-cpp 71 | zig build --prefix /addons/godot-llama-cpp 72 | ``` 73 | 5. Enable the plugin in your Godot project settings. 74 | 6. Add the `LlamaContext` node to your scene. 75 | 7. Run your Godot project. 76 | 8. Enjoy! 77 | 78 | ## License 79 | 80 | This project is licensed under the MIT License - see the [LICENSE](LICENSE.md) file for details. 81 | -------------------------------------------------------------------------------- /binding_generator.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from godot_cpp import binding_generator 3 | from pathlib import Path 4 | from os.path import abspath 5 | 6 | if __name__ == "__main__": 7 | if len(sys.argv) < 3: 8 | print("Please provide the path to the godot extension_api.json file and the output directory") 9 | exit(1) 10 | 11 | if sys.argv[1] is None: 12 | print("Please provide the path to the godot extension_api.json file") 13 | exit(1) 14 | 15 | if sys.argv[2] is None: 16 | print("Please provide the path to the output directory") 17 | exit(1) 18 | 19 | api_filepath = Path(sys.argv[1]).resolve() 20 | output_dir = abspath(Path(sys.argv[2]).resolve()) 21 | 22 | binding_generator.generate_bindings( 23 | api_filepath=api_filepath, use_template_get_node=True, output_dir=output_dir 24 | ) 25 | -------------------------------------------------------------------------------- /build.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const builtin = @import("builtin"); 3 | 4 | const cfiles_exts = [_][]const u8{ ".c", ".cpp", ".cxx", ".c++", ".cc" }; 5 | const extension_name = "godot-llama-cpp"; 6 | 7 | const ComputeBackend = enum { 8 | metal, 9 | vulkan, 10 | cuda, 11 | cpu, 12 | }; 13 | 14 | const Extension = enum { 15 | @".c", 16 | @".cpp", 17 | @".m", 18 | }; 19 | 20 | const Source = struct { 21 | name: []const u8, 22 | source_file: []const u8, 23 | root: ?std.Build.LazyPath = null, 24 | dependencies: ?[]const *std.Build.Step = null, 25 | }; 26 | 27 | pub fn build(b: *std.Build) !void { 28 | const target = b.standardTargetOptions(.{}); 29 | const optimize = b.standardOptimizeOption(.{}); 30 | const triple = try target.result.linuxTriple(b.allocator); 31 | const compute_backend = b.option( 32 | ComputeBackend, 33 | "compute-backend", 34 | "The compute backend to use.", 35 | ) orelse ComputeBackend.cpu; 36 | 37 | const gen_run = b.addSystemCommand(&.{ "python", "binding_generator.py" }); 38 | gen_run.addFileArg(b.path("godot_cpp/gdextension/extension_api.json")); 39 | const gen_out = gen_run.addOutputDirectoryArg("godot-cpp-gen"); 40 | 41 | // godot-llama-cpp 42 | const plugin = b.addSharedLibrary(.{ 43 | .name = b.fmt("{s}-{s}-{s}", .{ extension_name, triple, @tagName(optimize) }), 44 | .target = target, 45 | .optimize = optimize, 46 | }); 47 | b.installArtifact(plugin); 48 | 49 | plugin.addCSourceFiles(.{ .files = try findFilesRecursive(b, "src", &cfiles_exts) }); 50 | plugin.addIncludePath(b.path("src")); 51 | plugin.addIncludePath(b.path("godot_cpp/gdextension")); 52 | plugin.addIncludePath(b.path("godot_cpp/include")); 53 | plugin.addIncludePath(gen_out.path(b, "gen/include")); 54 | plugin.addIncludePath(b.path("llama.cpp/src")); 55 | plugin.addIncludePath(b.path("llama.cpp/include")); 56 | plugin.addIncludePath(b.path("llama.cpp/common")); 57 | plugin.addIncludePath(b.path("llama.cpp/ggml/include")); 58 | plugin.addIncludePath(b.path("llama.cpp/ggml/src")); 59 | 60 | // godot-cpp 61 | const lib_godot = b.addStaticLibrary(.{ 62 | .name = "godot-cpp", 63 | .target = target, 64 | .optimize = optimize, 65 | }); 66 | plugin.linkLibrary(lib_godot); 67 | lib_godot.linkLibCpp(); 68 | lib_godot.step.dependOn(&gen_run.step); 69 | 70 | lib_godot.addIncludePath(b.path("godot_cpp/gdextension")); 71 | lib_godot.addIncludePath(b.path("godot_cpp/include")); 72 | lib_godot.addIncludePath(gen_out.path(b, "gen/include")); 73 | 74 | const concat_gen_exe = b.addExecutable(.{ 75 | .name = "concat_gen", 76 | .target = target, 77 | .optimize = optimize, 78 | .root_source_file = b.path("tools/concat_files.zig"), 79 | }); 80 | var concat_gen_run = b.addRunArtifact(concat_gen_exe); 81 | concat_gen_run.addDirectoryArg(gen_out.path(b, "gen/src")); 82 | const concat_gen_out = concat_gen_run.addOutputFileArg("gen_concat.cpp"); 83 | const lib_godot_sources = try findFilesRecursive(b, "godot_cpp/src", &cfiles_exts); 84 | 85 | lib_godot.addCSourceFile(.{ .file = concat_gen_out, .flags = &.{ "-std=c++17", "-fno-exceptions" } }); 86 | lib_godot.addCSourceFiles(.{ .files = lib_godot_sources, .flags = &.{ "-std=c++17", "-fno-exceptions" } }); 87 | 88 | // llama.cpp 89 | const lib_llama_cpp = b.addStaticLibrary(.{ 90 | .name = "llama.cpp", 91 | .target = target, 92 | .optimize = optimize, 93 | }); 94 | plugin.linkLibrary(lib_llama_cpp); 95 | 96 | var base_flags = std.ArrayList([]const u8).init(b.allocator); 97 | var c_flags = std.ArrayList([]const u8).init(b.allocator); 98 | var cpp_flags = std.ArrayList([]const u8).init(b.allocator); 99 | var include_paths = std.ArrayList(std.Build.LazyPath).init(b.allocator); 100 | var system_libs = std.ArrayList([]const u8).init(b.allocator); 101 | var library_paths = std.ArrayList(std.Build.LazyPath).init(b.allocator); 102 | 103 | var sources = std.ArrayList(Source).init(b.allocator); 104 | 105 | try c_flags.append("-std=c11"); 106 | try cpp_flags.append("-std=c++17"); 107 | try include_paths.appendSlice(&.{ 108 | b.path("llama.cpp/src"), 109 | b.path("llama.cpp/include"), 110 | b.path("llama.cpp/common"), 111 | b.path("llama.cpp/ggml/include"), 112 | b.path("llama.cpp/ggml/src"), 113 | }); 114 | 115 | switch (target.result.os.tag) { 116 | .linux => { 117 | try base_flags.append("-D_GNU_SOURCE"); 118 | }, 119 | .macos => { 120 | try base_flags.append("-D_DARWIN_C_SOURCE"); 121 | }, 122 | else => {}, 123 | } 124 | 125 | switch (compute_backend) { 126 | .metal => { 127 | try base_flags.append("-DGGML_USE_METAL"); 128 | try sources.append(.{ .name = "ggml_metal", .source_file = "llama.cpp/ggml/src/ggml-metal.m" }); 129 | 130 | lib_llama_cpp.linkFramework("Foundation"); 131 | lib_llama_cpp.linkFramework("Metal"); 132 | lib_llama_cpp.linkFramework("MetalKit"); 133 | 134 | const expand_metal = b.addExecutable(.{ 135 | .name = "expand_metal", 136 | .target = target, 137 | .root_source_file = b.path("tools/expand_metal.zig"), 138 | }); 139 | var run_expand_metal = b.addRunArtifact(expand_metal); 140 | run_expand_metal.addArg("--metal-file"); 141 | run_expand_metal.addFileArg(b.path("llama.cpp/ggml/src/ggml-metal.metal")); 142 | run_expand_metal.addArg("--common-file"); 143 | run_expand_metal.addFileArg(b.path("llama.cpp/ggml/src/ggml-common.h")); 144 | run_expand_metal.addArg("--output-file"); 145 | const metal_expanded = run_expand_metal.addOutputFileArg("ggml-metal.metal"); 146 | const install_metal = b.addInstallFileWithDir(metal_expanded, .lib, "ggml-metal.metal"); 147 | lib_llama_cpp.step.dependOn(&install_metal.step); 148 | }, 149 | .vulkan => { 150 | try base_flags.append("-DGGML_USE_VULKAN"); 151 | try sources.append(.{ .name = "ggml_vulkan", .source_file = "llama.cpp/ggml/src/ggml-vulkan.cpp" }); 152 | 153 | const env_map = try std.process.getEnvMap(b.allocator); 154 | const vulkan_sdk = env_map.get("VULKAN_SDK") orelse return error.MissingVulkanSDK; 155 | 156 | const vk_library_path = b.pathJoin(&.{ vulkan_sdk, "lib" }); 157 | const vk_include_path = b.pathJoin(&.{ vulkan_sdk, "include" }); 158 | try include_paths.append(std.Build.LazyPath{ .cwd_relative = vk_include_path }); 159 | try library_paths.append(.{ .cwd_relative = vk_library_path }); 160 | try system_libs.append("vulkan"); 161 | 162 | lib_llama_cpp.addLibraryPath(.{ .cwd_relative = b.pathJoin(&.{ vulkan_sdk, "lib" }) }); 163 | plugin.addLibraryPath(.{ .cwd_relative = b.pathJoin(&.{ vulkan_sdk, "lib" }) }); 164 | }, 165 | else => {}, 166 | } 167 | 168 | const zig_triple = try target.result.zigTriple(b.allocator); 169 | const build_info_run = b.addSystemCommand(&.{ 170 | "echo", 171 | "-e", 172 | b.fmt( 173 | "int LLAMA_BUILD_NUMBER = {d};\\nchar const *LLAMA_COMMIT = \"$(git rev-parse HEAD)\";\\nchar const *LLAMA_COMPILER = \"Zig {s}\";\\nchar const *LLAMA_BUILD_TARGET = \"{s}\";\\n", 174 | .{ 0, builtin.zig_version_string, zig_triple }, 175 | ), 176 | }); 177 | var build_info_wf = b.addWriteFiles(); 178 | _ = build_info_wf.addCopyFile(build_info_run.captureStdOut(), "build-info.cpp"); 179 | 180 | try sources.appendSlice(&.{ 181 | .{ .name = "build_info", .source_file = "build-info.cpp", .root = build_info_wf.getDirectory(), .dependencies = &.{&build_info_wf.step} }, 182 | .{ .name = "ggml", .source_file = "llama.cpp/ggml/src/ggml.c" }, 183 | .{ .name = "sgemm", .source_file = "llama.cpp/ggml/src/sgemm.cpp" }, 184 | .{ .name = "ggml_alloc", .source_file = "llama.cpp/ggml/src/ggml-alloc.c" }, 185 | .{ .name = "ggml_backend", .source_file = "llama.cpp/ggml/src/ggml-backend.c" }, 186 | .{ .name = "ggml_quants", .source_file = "llama.cpp/ggml/src/ggml-quants.c" }, 187 | .{ .name = "llama", .source_file = "llama.cpp/src/llama.cpp" }, 188 | .{ .name = "unicode", .source_file = "llama.cpp/src/unicode.cpp" }, 189 | .{ .name = "unicode_data", .source_file = "llama.cpp/src/unicode-data.cpp" }, 190 | .{ .name = "common", .source_file = "llama.cpp/common/common.cpp" }, 191 | .{ .name = "console", .source_file = "llama.cpp/common/console.cpp" }, 192 | .{ .name = "sampling", .source_file = "llama.cpp/common/sampling.cpp" }, 193 | .{ .name = "grammar_parser", .source_file = "llama.cpp/common/grammar-parser.cpp" }, 194 | .{ .name = "json_schema_to_grammar", .source_file = "llama.cpp/common/json-schema-to-grammar.cpp" }, 195 | }); 196 | 197 | try c_flags.appendSlice(base_flags.items); 198 | try cpp_flags.appendSlice(base_flags.items); 199 | 200 | for (sources.items) |source| { 201 | const obj = b.addObject(.{ 202 | .name = source.name, 203 | .target = target, 204 | .optimize = optimize, 205 | }); 206 | lib_llama_cpp.addObject(obj); 207 | if (source.dependencies) |deps| { 208 | for (deps) |dep| { 209 | obj.step.dependOn(dep); 210 | } 211 | } 212 | const file = if (source.root) |root| 213 | root.path(b, source.source_file) 214 | else 215 | b.path(source.source_file); 216 | const extension = std.meta.stringToEnum( 217 | Extension, 218 | std.fs.path.extension(source.source_file), 219 | ) orelse return error.UnknownExtension; 220 | const flags = switch (extension) { 221 | .@".c", .@".m" => c_flags.items, 222 | .@".cpp" => cpp_flags.items, 223 | }; 224 | obj.addCSourceFile(.{ 225 | .file = file, 226 | .flags = flags, 227 | }); 228 | for (include_paths.items) |path| { 229 | obj.addIncludePath(path); 230 | } 231 | for (system_libs.items) |lib| { 232 | obj.linkSystemLibrary(lib); 233 | } 234 | for (library_paths.items) |path| { 235 | obj.addLibraryPath(path); 236 | } 237 | switch (extension) { 238 | .@".c", .@".m" => obj.linkLibC(), 239 | .@".cpp" => obj.linkLibCpp(), 240 | } 241 | } 242 | 243 | // check step 244 | const check = b.step("check", "Check if plugin compiles"); 245 | check.dependOn(&plugin.step); 246 | } 247 | 248 | fn findFilesRecursive(b: *std.Build, dir_name: []const u8, exts: []const []const u8) ![][]const u8 { 249 | var sources = std.ArrayList([]const u8).init(b.allocator); 250 | 251 | var dir = try b.build_root.handle.openDir(dir_name, .{ .iterate = true }); 252 | var walker = try dir.walk(b.allocator); 253 | defer walker.deinit(); 254 | while (try walker.next()) |entry| { 255 | const ext = std.fs.path.extension(entry.basename); 256 | const include_file = for (exts) |e| { 257 | if (std.mem.eql(u8, ext, e)) { 258 | break true; 259 | } 260 | } else false; 261 | if (include_file) { 262 | try sources.append(b.fmt("{s}/{s}", .{ dir_name, entry.path })); 263 | } 264 | } 265 | 266 | return sources.items; 267 | } 268 | -------------------------------------------------------------------------------- /godot/.gitattributes: -------------------------------------------------------------------------------- 1 | # Normalize EOL for all files that Git considers text files. 2 | * text=auto eol=lf 3 | -------------------------------------------------------------------------------- /godot/.gitignore: -------------------------------------------------------------------------------- 1 | # Godot 4+ specific ignores 2 | .godot/ 3 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 33 | 36 | 39 | 43 | 48 | 53 | 58 | 63 | 66 | 71 | 76 | 77 | 80 | 85 | 90 | 91 | 95 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg.import: -------------------------------------------------------------------------------- 1 | [remap] 2 | 3 | importer="texture" 4 | type="CompressedTexture2D" 5 | uid="uid://dplw232htshgc" 6 | path="res://.godot/imported/godot-llama-cpp-1024x1024.svg-0f412fe282a705cdf65256e51c97ddd2.ctex" 7 | metadata={ 8 | "vram_texture": false 9 | } 10 | 11 | [deps] 12 | 13 | source_file="res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg" 14 | dest_files=["res://.godot/imported/godot-llama-cpp-1024x1024.svg-0f412fe282a705cdf65256e51c97ddd2.ctex"] 15 | 16 | [params] 17 | 18 | compress/mode=0 19 | compress/high_quality=false 20 | compress/lossy_quality=0.7 21 | compress/hdr_compression=1 22 | compress/normal_map=0 23 | compress/channel_pack=0 24 | mipmaps/generate=false 25 | mipmaps/limit=-1 26 | roughness/mode=0 27 | roughness/src_normal="" 28 | process/fix_alpha_border=true 29 | process/premult_alpha=false 30 | process/normal_map_invert_y=false 31 | process/hdr_as_srgb=false 32 | process/hdr_clamp_exposure=false 33 | process/size_limit=0 34 | detect_3d/compress_to=1 35 | svg/scale=1.0 36 | editor/scale_with_editor_scale=false 37 | editor/convert_colors_with_editor_theme=false 38 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/assets/godot-llama-cpp-16x16.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 75 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/assets/godot-llama-cpp-16x16.svg.import: -------------------------------------------------------------------------------- 1 | [remap] 2 | 3 | importer="texture" 4 | type="CompressedTexture2D" 5 | uid="uid://du1ewals7kll1" 6 | path="res://.godot/imported/godot-llama-cpp-16x16.svg-65f33723bf99167c9acd957fe1b937f8.ctex" 7 | metadata={ 8 | "vram_texture": false 9 | } 10 | 11 | [deps] 12 | 13 | source_file="res://addons/godot-llama-cpp/assets/godot-llama-cpp-16x16.svg" 14 | dest_files=["res://.godot/imported/godot-llama-cpp-16x16.svg-65f33723bf99167c9acd957fe1b937f8.ctex"] 15 | 16 | [params] 17 | 18 | compress/mode=0 19 | compress/high_quality=false 20 | compress/lossy_quality=0.7 21 | compress/hdr_compression=1 22 | compress/normal_map=0 23 | compress/channel_pack=0 24 | mipmaps/generate=false 25 | mipmaps/limit=-1 26 | roughness/mode=0 27 | roughness/src_normal="" 28 | process/fix_alpha_border=true 29 | process/premult_alpha=false 30 | process/normal_map_invert_y=false 31 | process/hdr_as_srgb=false 32 | process/hdr_clamp_exposure=false 33 | process/size_limit=0 34 | detect_3d/compress_to=1 35 | svg/scale=1.0 36 | editor/scale_with_editor_scale=false 37 | editor/convert_colors_with_editor_theme=false 38 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/chat/chat_formatter.gd: -------------------------------------------------------------------------------- 1 | class_name ChatFormatter 2 | 3 | static func apply(format: String, messages: Array) -> String: 4 | match format: 5 | "llama3": 6 | return format_llama3(messages) 7 | "phi3": 8 | return format_phi3(messages) 9 | "mistral": 10 | return format_mistral(messages) 11 | _: 12 | printerr("Unknown chat format: ", format) 13 | return "" 14 | 15 | static func format_llama3(messages: Array) -> String: 16 | var res = "" 17 | 18 | for i in range(messages.size()): 19 | match messages[i]: 20 | {"text": var text, "sender": var sender}: 21 | res += """<|start_header_id|>%s<|end_header_id|> 22 | 23 | %s<|eot_id|> 24 | """ % [sender, text] 25 | _: 26 | printerr("Invalid message at index ", i) 27 | 28 | res += "<|start_header_id|>assistant<|end_header_id|>\n\n" 29 | return res 30 | 31 | static func format_phi3(messages: Array) -> String: 32 | var res = "" 33 | 34 | for i in range(messages.size()): 35 | match messages[i]: 36 | {"text": var text, "sender": var sender}: 37 | res +="<|%s|>\n%s<|end|>\n" % [sender, text] 38 | _: 39 | printerr("Invalid message at index ", i) 40 | res += "<|assistant|>\n" 41 | return res 42 | 43 | static func format_mistral(messages: Array) -> String: 44 | var res = "" 45 | 46 | for i in range(messages.size()): 47 | match messages[i]: 48 | {"text": var text, "sender": var sender}: 49 | if sender == "user": 50 | res += "[INST] %s [/INST]" % text 51 | else: 52 | res += "%s" 53 | _: 54 | printerr("Invalid message at index ", i) 55 | 56 | return res 57 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/plugin.cfg: -------------------------------------------------------------------------------- 1 | [plugin] 2 | 3 | name="godot-llama-cpp" 4 | description="Run large language models in Godot. Powered by llama.cpp." 5 | author="hazelnutcloud" 6 | version="0.0.1" 7 | script="plugin.gd" 8 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/plugin.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends EditorPlugin 3 | 4 | 5 | func _enter_tree(): 6 | # Initialization of the plugin goes here. 7 | pass 8 | 9 | 10 | func _exit_tree(): 11 | # Clean-up of the plugin goes here. 12 | pass 13 | -------------------------------------------------------------------------------- /godot/addons/godot-llama-cpp/plugin.gdextension: -------------------------------------------------------------------------------- 1 | [configuration] 2 | 3 | entry_symbol = "init_library" 4 | compatibility_minimum = "4.2" 5 | 6 | [libraries] 7 | 8 | macos.debug = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-aarch64-macos-none-Debug.dylib" 9 | macos.release = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-aarch64-macos-none-ReleaseSafe.dylib" 10 | windows.debug.x86_64 = "res://addons/godot-llama-cpp/lib/godot-llama-cpp-x86_64-windows-gnu-Debug.dll" 11 | windows.release.x86_64 = "res://addons/godot-llama-cpp/lib/godot-llama-cpp-x86_64-windows-gnu-ReleaseSafe.dll" 12 | linux.debug.x86_64 = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-x86_64-linux-gnu-Debug.so" 13 | linux.release.x86_64 = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-x86_64-linux-gnu-ReleaseSafe.so" 14 | 15 | [icons] 16 | 17 | LlamaModel = "res://addons/godot-llama-cpp/assets/godot-llama-cpp-16x16.svg" 18 | LlamaContext = "res://addons/godot-llama-cpp/assets/godot-llama-cpp-16x16.svg" -------------------------------------------------------------------------------- /godot/examples/simple/TextEdit.gd: -------------------------------------------------------------------------------- 1 | extends TextEdit 2 | 3 | signal submit(input: String) 4 | 5 | func _gui_input(event: InputEvent) -> void: 6 | if event is InputEventKey: 7 | var keycode = event.get_keycode_with_modifiers() 8 | if keycode == KEY_ENTER and event.is_pressed(): 9 | handle_submit() 10 | accept_event() 11 | if keycode == KEY_ENTER | KEY_MASK_SHIFT and event.is_pressed(): 12 | insert_text_at_caret("\n") 13 | accept_event() 14 | 15 | func _on_button_pressed() -> void: 16 | handle_submit() 17 | 18 | func handle_submit() -> void: 19 | submit.emit(text) 20 | text = "" 21 | -------------------------------------------------------------------------------- /godot/examples/simple/form.gd: -------------------------------------------------------------------------------- 1 | extends HBoxContainer 2 | 3 | @onready var text_edit = %TextEdit 4 | 5 | func _on_button_pressed() -> void: 6 | text_edit.handle_submit() 7 | -------------------------------------------------------------------------------- /godot/examples/simple/message.gd: -------------------------------------------------------------------------------- 1 | class_name Message 2 | extends Node 3 | 4 | var ai_avatar = preload ("res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg") 5 | var user_avatar = preload ("res://examples/simple/user.svg") 6 | var system_avatar = preload("res://examples/simple/system.svg") 7 | var stylebox: StyleBoxTexture = StyleBoxTexture.new() 8 | 9 | @onready var text_container = %Text 10 | @onready var icon = %Panel 11 | @export_enum("user", "assistant", "system") var sender: String: 12 | get: 13 | return sender 14 | set(value): 15 | sender = value 16 | if icon == null: return 17 | if value == "user": 18 | stylebox.texture = user_avatar 19 | elif value == "assistant": 20 | stylebox.texture = ai_avatar 21 | else: 22 | stylebox.texture = system_avatar 23 | icon.add_theme_stylebox_override("panel", stylebox) 24 | @export var include_in_prompt: bool = true 25 | @export var text: String: 26 | get: 27 | return text 28 | set(value): 29 | text = value 30 | if text_container == null: 31 | return 32 | text_container.text = value 33 | 34 | var completion_id: int = -1 35 | var pending: bool = false 36 | var errored: bool = false 37 | 38 | func _ready(): 39 | text_container.text = text 40 | sender = sender 41 | 42 | 43 | -------------------------------------------------------------------------------- /godot/examples/simple/message.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=4 format=3 uid="uid://t862t0v8ht2q"] 2 | 3 | [ext_resource type="Script" path="res://examples/simple/message.gd" id="1_pko33"] 4 | [ext_resource type="Texture2D" uid="uid://dplw232htshgc" path="res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg" id="2_xf8it"] 5 | 6 | [sub_resource type="StyleBoxTexture" id="StyleBoxTexture_ki268"] 7 | texture = ExtResource("2_xf8it") 8 | 9 | [node name="RichTextLabel" type="HBoxContainer"] 10 | anchors_preset = 15 11 | anchor_right = 1.0 12 | anchor_bottom = 1.0 13 | grow_horizontal = 2 14 | grow_vertical = 2 15 | size_flags_horizontal = 3 16 | theme_override_constants/separation = 20 17 | script = ExtResource("1_pko33") 18 | 19 | [node name="Panel" type="Panel" parent="."] 20 | unique_name_in_owner = true 21 | custom_minimum_size = Vector2(80, 80) 22 | layout_mode = 2 23 | size_flags_vertical = 0 24 | theme_override_styles/panel = SubResource("StyleBoxTexture_ki268") 25 | 26 | [node name="Text" type="RichTextLabel" parent="."] 27 | unique_name_in_owner = true 28 | layout_mode = 2 29 | size_flags_horizontal = 3 30 | focus_mode = 2 31 | text = "..." 32 | fit_content = true 33 | selection_enabled = true 34 | -------------------------------------------------------------------------------- /godot/examples/simple/simple.gd: -------------------------------------------------------------------------------- 1 | extends Node 2 | 3 | const message = preload("res://examples/simple/message.tscn") 4 | 5 | @onready var messages_container = %MessagesContainer 6 | @onready var llama_context = %LlamaContext 7 | 8 | func _on_text_edit_submit(input: String) -> void: 9 | handle_input(input) 10 | 11 | func handle_input(input: String) -> void: 12 | var messages = [] 13 | messages.append_array(messages_container.get_children().filter(func(msg: Message): return msg.include_in_prompt).map( 14 | func(msg: Message) -> Dictionary: 15 | return { "text": msg.text, "sender": msg.sender } 16 | )) 17 | messages.append({"text": input, "sender": "user"}) 18 | var prompt = ChatFormatter.apply("llama3", messages) 19 | print("prompt: ", prompt) 20 | 21 | var completion_id = llama_context.request_completion(prompt) 22 | 23 | var user_message: Message = message.instantiate() 24 | messages_container.add_child(user_message) 25 | user_message.text = input 26 | user_message.sender = "user" 27 | user_message.completion_id = completion_id 28 | 29 | var ai_message: Message = message.instantiate() 30 | messages_container.add_child(ai_message) 31 | ai_message.sender = "assistant" 32 | ai_message.completion_id = completion_id 33 | ai_message.pending = true 34 | 35 | func _on_llama_context_completion_generated(chunk: Dictionary) -> void: 36 | var completion_id = chunk.id 37 | for msg: Message in messages_container.get_children(): 38 | if msg.completion_id != completion_id or msg.sender != "assistant": 39 | continue 40 | if chunk.has("error"): 41 | msg.errored = true 42 | elif chunk.has("text"): 43 | if msg.pending: 44 | msg.pending = false 45 | msg.text = chunk["text"] 46 | else: 47 | msg.text += chunk["text"] 48 | -------------------------------------------------------------------------------- /godot/examples/simple/simple.tscn: -------------------------------------------------------------------------------- 1 | [gd_scene load_steps=6 format=3 uid="uid://c55kb4qvg6geq"] 2 | 3 | [ext_resource type="Texture2D" uid="uid://dplw232htshgc" path="res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg" id="1_gjsev"] 4 | [ext_resource type="Script" path="res://examples/simple/simple.gd" id="1_sruc3"] 5 | [ext_resource type="PackedScene" uid="uid://t862t0v8ht2q" path="res://examples/simple/message.tscn" id="2_7iip7"] 6 | [ext_resource type="Script" path="res://examples/simple/TextEdit.gd" id="2_7usqw"] 7 | [ext_resource type="LlamaModel" path="res://models/Meta-Llama-3-8B-Instruct.Q4_K_M.gguf" id="5_yssjj"] 8 | 9 | [node name="Node" type="Node"] 10 | script = ExtResource("1_sruc3") 11 | 12 | [node name="Panel" type="Panel" parent="."] 13 | anchors_preset = 15 14 | anchor_right = 1.0 15 | anchor_bottom = 1.0 16 | grow_horizontal = 2 17 | grow_vertical = 2 18 | 19 | [node name="MarginContainer" type="MarginContainer" parent="Panel"] 20 | layout_mode = 1 21 | anchors_preset = 15 22 | anchor_right = 1.0 23 | anchor_bottom = 1.0 24 | grow_horizontal = 2 25 | grow_vertical = 2 26 | theme_override_constants/margin_left = 10 27 | theme_override_constants/margin_top = 10 28 | theme_override_constants/margin_right = 10 29 | theme_override_constants/margin_bottom = 10 30 | 31 | [node name="VBoxContainer" type="VBoxContainer" parent="Panel/MarginContainer"] 32 | layout_mode = 2 33 | 34 | [node name="ScrollContainer" type="ScrollContainer" parent="Panel/MarginContainer/VBoxContainer"] 35 | layout_mode = 2 36 | size_flags_vertical = 3 37 | follow_focus = true 38 | 39 | [node name="MessagesContainer" type="VBoxContainer" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer"] 40 | unique_name_in_owner = true 41 | layout_mode = 2 42 | size_flags_horizontal = 3 43 | size_flags_vertical = 3 44 | theme_override_constants/separation = 30 45 | 46 | [node name="RichTextLabel2" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer/MessagesContainer" instance=ExtResource("2_7iip7")] 47 | layout_mode = 2 48 | sender = "system" 49 | text = "You are a pirate chatbot who always responds in pirate speak!" 50 | 51 | [node name="HBoxContainer" type="HBoxContainer" parent="Panel/MarginContainer/VBoxContainer"] 52 | layout_mode = 2 53 | 54 | [node name="TextEdit" type="TextEdit" parent="Panel/MarginContainer/VBoxContainer/HBoxContainer"] 55 | custom_minimum_size = Vector2(2.08165e-12, 100) 56 | layout_mode = 2 57 | size_flags_horizontal = 3 58 | placeholder_text = "Ask me anything..." 59 | wrap_mode = 1 60 | script = ExtResource("2_7usqw") 61 | 62 | [node name="Button" type="Button" parent="Panel/MarginContainer/VBoxContainer/HBoxContainer"] 63 | custom_minimum_size = Vector2(100, 2.08165e-12) 64 | layout_mode = 2 65 | icon = ExtResource("1_gjsev") 66 | expand_icon = true 67 | 68 | [node name="LlamaContext" type="LlamaContext" parent="."] 69 | model = ExtResource("5_yssjj") 70 | unique_name_in_owner = true 71 | 72 | [connection signal="submit" from="Panel/MarginContainer/VBoxContainer/HBoxContainer/TextEdit" to="." method="_on_text_edit_submit"] 73 | [connection signal="pressed" from="Panel/MarginContainer/VBoxContainer/HBoxContainer/Button" to="Panel/MarginContainer/VBoxContainer/HBoxContainer/TextEdit" method="_on_button_pressed"] 74 | [connection signal="completion_generated" from="LlamaContext" to="." method="_on_llama_context_completion_generated"] 75 | -------------------------------------------------------------------------------- /godot/examples/simple/system.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 8 | 9 | -------------------------------------------------------------------------------- /godot/examples/simple/system.svg.import: -------------------------------------------------------------------------------- 1 | [remap] 2 | 3 | importer="texture" 4 | type="CompressedTexture2D" 5 | uid="uid://ckurx0p0r4v1t" 6 | path="res://.godot/imported/system.svg-d1b27c65464663409463ff5f8dfc9953.ctex" 7 | metadata={ 8 | "vram_texture": false 9 | } 10 | 11 | [deps] 12 | 13 | source_file="res://examples/simple/system.svg" 14 | dest_files=["res://.godot/imported/system.svg-d1b27c65464663409463ff5f8dfc9953.ctex"] 15 | 16 | [params] 17 | 18 | compress/mode=0 19 | compress/high_quality=false 20 | compress/lossy_quality=0.7 21 | compress/hdr_compression=1 22 | compress/normal_map=0 23 | compress/channel_pack=0 24 | mipmaps/generate=false 25 | mipmaps/limit=-1 26 | roughness/mode=0 27 | roughness/src_normal="" 28 | process/fix_alpha_border=true 29 | process/premult_alpha=false 30 | process/normal_map_invert_y=false 31 | process/hdr_as_srgb=false 32 | process/hdr_clamp_exposure=false 33 | process/size_limit=0 34 | detect_3d/compress_to=1 35 | svg/scale=1.0 36 | editor/scale_with_editor_scale=false 37 | editor/convert_colors_with_editor_theme=false 38 | -------------------------------------------------------------------------------- /godot/examples/simple/user.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /godot/examples/simple/user.svg.import: -------------------------------------------------------------------------------- 1 | [remap] 2 | 3 | importer="texture" 4 | type="CompressedTexture2D" 5 | uid="uid://l8xuk1t4hnof" 6 | path="res://.godot/imported/user.svg-fbe84d041cfa09d8365142161c3b195f.ctex" 7 | metadata={ 8 | "vram_texture": false 9 | } 10 | 11 | [deps] 12 | 13 | source_file="res://examples/simple/user.svg" 14 | dest_files=["res://.godot/imported/user.svg-fbe84d041cfa09d8365142161c3b195f.ctex"] 15 | 16 | [params] 17 | 18 | compress/mode=0 19 | compress/high_quality=false 20 | compress/lossy_quality=0.7 21 | compress/hdr_compression=1 22 | compress/normal_map=0 23 | compress/channel_pack=0 24 | mipmaps/generate=false 25 | mipmaps/limit=-1 26 | roughness/mode=0 27 | roughness/src_normal="" 28 | process/fix_alpha_border=true 29 | process/premult_alpha=false 30 | process/normal_map_invert_y=false 31 | process/hdr_as_srgb=false 32 | process/hdr_clamp_exposure=false 33 | process/size_limit=0 34 | detect_3d/compress_to=1 35 | svg/scale=1.0 36 | editor/scale_with_editor_scale=false 37 | editor/convert_colors_with_editor_theme=false 38 | -------------------------------------------------------------------------------- /godot/icon.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /godot/icon.svg.import: -------------------------------------------------------------------------------- 1 | [remap] 2 | 3 | importer="texture" 4 | type="CompressedTexture2D" 5 | uid="uid://beeg0oqle7bnk" 6 | path="res://.godot/imported/icon.svg-218a8f2b3041327d8a5756f3a245f83b.ctex" 7 | metadata={ 8 | "vram_texture": false 9 | } 10 | 11 | [deps] 12 | 13 | source_file="res://icon.svg" 14 | dest_files=["res://.godot/imported/icon.svg-218a8f2b3041327d8a5756f3a245f83b.ctex"] 15 | 16 | [params] 17 | 18 | compress/mode=0 19 | compress/high_quality=false 20 | compress/lossy_quality=0.7 21 | compress/hdr_compression=1 22 | compress/normal_map=0 23 | compress/channel_pack=0 24 | mipmaps/generate=false 25 | mipmaps/limit=-1 26 | roughness/mode=0 27 | roughness/src_normal="" 28 | process/fix_alpha_border=true 29 | process/premult_alpha=false 30 | process/normal_map_invert_y=false 31 | process/hdr_as_srgb=false 32 | process/hdr_clamp_exposure=false 33 | process/size_limit=0 34 | detect_3d/compress_to=1 35 | svg/scale=1.0 36 | editor/scale_with_editor_scale=false 37 | editor/convert_colors_with_editor_theme=false 38 | -------------------------------------------------------------------------------- /godot/models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hazelnutcloud/godot-llama-cpp/17c224035f509b1a4ccd0c5b8b2162d376bd2a22/godot/models/.gitkeep -------------------------------------------------------------------------------- /godot/project.godot: -------------------------------------------------------------------------------- 1 | ; Engine configuration file. 2 | ; It's best edited using the editor UI and not directly, 3 | ; since the parameters that go here are not all obvious. 4 | ; 5 | ; Format: 6 | ; [section] ; section goes between [] 7 | ; param=value ; assign values to parameters 8 | 9 | config_version=5 10 | 11 | [application] 12 | 13 | config/name="godot-llama-cpp" 14 | run/main_scene="res://examples/simple/simple.tscn" 15 | config/features=PackedStringArray("4.2", "Forward Plus") 16 | config/icon="res://icon.svg" 17 | 18 | [editor_plugins] 19 | 20 | enabled=PackedStringArray("res://addons/godot-llama-cpp/plugin.cfg") 21 | -------------------------------------------------------------------------------- /src/llama_context.cpp: -------------------------------------------------------------------------------- 1 | #include "llama_context.h" 2 | #include "common.h" 3 | #include "llama.h" 4 | #include "llama_model.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | using namespace godot; 14 | 15 | void LlamaContext::_bind_methods() { 16 | ClassDB::bind_method(D_METHOD("set_model", "model"), &LlamaContext::set_model); 17 | ClassDB::bind_method(D_METHOD("get_model"), &LlamaContext::get_model); 18 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::OBJECT, "model", PROPERTY_HINT_RESOURCE_TYPE, "LlamaModel"), "set_model", "get_model"); 19 | 20 | ClassDB::bind_method(D_METHOD("get_seed"), &LlamaContext::get_seed); 21 | ClassDB::bind_method(D_METHOD("set_seed", "seed"), &LlamaContext::set_seed); 22 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::INT, "seed"), "set_seed", "get_seed"); 23 | 24 | ClassDB::bind_method(D_METHOD("get_temperature"), &LlamaContext::get_temperature); 25 | ClassDB::bind_method(D_METHOD("set_temperature", "temperature"), &LlamaContext::set_temperature); 26 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::FLOAT, "temperature"), "set_temperature", "get_temperature"); 27 | 28 | ClassDB::bind_method(D_METHOD("get_top_p"), &LlamaContext::get_top_p); 29 | ClassDB::bind_method(D_METHOD("set_top_p", "top_p"), &LlamaContext::set_top_p); 30 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::FLOAT, "top_p"), "set_top_p", "get_top_p"); 31 | 32 | ClassDB::bind_method(D_METHOD("get_frequency_penalty"), &LlamaContext::get_frequency_penalty); 33 | ClassDB::bind_method(D_METHOD("set_frequency_penalty", "frequency_penalty"), &LlamaContext::set_frequency_penalty); 34 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::FLOAT, "frequency_penalty"), "set_frequency_penalty", "get_frequency_penalty"); 35 | 36 | ClassDB::bind_method(D_METHOD("get_presence_penalty"), &LlamaContext::get_presence_penalty); 37 | ClassDB::bind_method(D_METHOD("set_presence_penalty", "presence_penalty"), &LlamaContext::set_presence_penalty); 38 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::FLOAT, "presence_penalty"), "set_presence_penalty", "get_presence_penalty"); 39 | 40 | ClassDB::bind_method(D_METHOD("get_n_ctx"), &LlamaContext::get_n_ctx); 41 | ClassDB::bind_method(D_METHOD("set_n_ctx", "n_ctx"), &LlamaContext::set_n_ctx); 42 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::INT, "n_ctx"), "set_n_ctx", "get_n_ctx"); 43 | 44 | ClassDB::bind_method(D_METHOD("get_n_len"), &LlamaContext::get_n_len); 45 | ClassDB::bind_method(D_METHOD("set_n_len", "n_len"), &LlamaContext::set_n_len); 46 | ClassDB::add_property("LlamaContext", PropertyInfo(Variant::INT, "n_len"), "set_n_len", "get_n_len"); 47 | 48 | ClassDB::bind_method(D_METHOD("request_completion", "prompt"), &LlamaContext::request_completion); 49 | ClassDB::bind_method(D_METHOD("__thread_loop"), &LlamaContext::__thread_loop); 50 | 51 | ADD_SIGNAL(MethodInfo("completion_generated", PropertyInfo(Variant::DICTIONARY, "chunk"))); 52 | } 53 | 54 | LlamaContext::LlamaContext() { 55 | ctx_params = llama_context_default_params(); 56 | ctx_params.seed = -1; 57 | ctx_params.n_ctx = 4096; 58 | 59 | int32_t n_threads = OS::get_singleton()->get_processor_count(); 60 | ctx_params.n_threads = n_threads; 61 | ctx_params.n_threads_batch = n_threads; 62 | } 63 | 64 | void LlamaContext::_enter_tree() { 65 | // TODO: remove this and use runtime classes once godot 4.3 lands, see https://github.com/godotengine/godot/pull/82554 66 | if (Engine::get_singleton()->is_editor_hint()) { 67 | return; 68 | } 69 | 70 | if (model->model == NULL) { 71 | UtilityFunctions::printerr(vformat("%s: Failed to initialize llama context, model property not defined", __func__)); 72 | return; 73 | } 74 | 75 | mutex.instantiate(); 76 | semaphore.instantiate(); 77 | thread.instantiate(); 78 | 79 | llama_backend_init(); 80 | llama_numa_init(ggml_numa_strategy::GGML_NUMA_STRATEGY_DISABLED); 81 | 82 | ctx = llama_new_context_with_model(model->model, ctx_params); 83 | if (ctx == NULL) { 84 | UtilityFunctions::printerr(vformat("%s: Failed to initialize llama context, null ctx", __func__)); 85 | return; 86 | } 87 | 88 | sampling_ctx = llama_sampling_init(sampling_params); 89 | 90 | UtilityFunctions::print(vformat("%s: Context initialized", __func__)); 91 | 92 | thread->start(callable_mp(this, &LlamaContext::__thread_loop)); 93 | } 94 | 95 | void LlamaContext::__thread_loop() { 96 | while (true) { 97 | semaphore->wait(); 98 | 99 | mutex->lock(); 100 | if (exit_thread) { 101 | mutex->unlock(); 102 | break; 103 | } 104 | if (completion_requests.size() == 0) { 105 | mutex->unlock(); 106 | continue; 107 | } 108 | completion_request req = completion_requests.get(0); 109 | completion_requests.remove_at(0); 110 | mutex->unlock(); 111 | 112 | UtilityFunctions::print(vformat("%s: Running completion for prompt id: %d", __func__, req.id)); 113 | 114 | std::vector request_tokens; 115 | request_tokens = ::llama_tokenize(ctx, req.prompt.utf8().get_data(), true, true); 116 | 117 | size_t shared_prefix_idx = 0; 118 | auto diff = std::mismatch(context_tokens.begin(), context_tokens.end(), request_tokens.begin(), request_tokens.end()); 119 | if (diff.first != context_tokens.end()) { 120 | shared_prefix_idx = std::distance(context_tokens.begin(), diff.first); 121 | } else { 122 | shared_prefix_idx = std::min(context_tokens.size(), request_tokens.size()); 123 | } 124 | 125 | bool rm_success = llama_kv_cache_seq_rm(ctx, -1, shared_prefix_idx, -1); 126 | if (!rm_success) { 127 | UtilityFunctions::printerr(vformat("%s: Failed to remove tokens from kv cache", __func__)); 128 | Dictionary response; 129 | response["id"] = req.id; 130 | response["error"] = "Failed to remove tokens from kv cache"; 131 | call_thread_safe("emit_signal", "completion_generated", response); 132 | continue; 133 | } 134 | context_tokens.erase(context_tokens.begin() + shared_prefix_idx, context_tokens.end()); 135 | request_tokens.erase(request_tokens.begin(), request_tokens.begin() + shared_prefix_idx); 136 | 137 | int32_t batch_size = std::min(ctx_params.n_batch, (uint32_t)request_tokens.size()); 138 | 139 | llama_batch batch = llama_batch_init(batch_size, 0, 1); 140 | 141 | // chunk request_tokens into sequences of size batch_size 142 | std::vector> sequences; 143 | for (size_t i = 0; i < request_tokens.size(); i += batch_size) { 144 | sequences.push_back(std::vector(request_tokens.begin() + i, request_tokens.begin() + std::min(i + batch_size, request_tokens.size()))); 145 | } 146 | 147 | printf("Request tokens: \n"); 148 | for (auto sequence : sequences) { 149 | for (auto token : sequence) { 150 | printf("%s", llama_token_to_piece(ctx, token).c_str()); 151 | } 152 | } 153 | printf("\n"); 154 | 155 | int curr_token_pos = context_tokens.size(); 156 | bool decode_failed = false; 157 | 158 | for (size_t i = 0; i < sequences.size(); i++) { 159 | llama_batch_clear(batch); 160 | 161 | std::vector sequence = sequences[i]; 162 | 163 | for (size_t j = 0; j < sequence.size(); j++) { 164 | llama_batch_add(batch, sequence[j], j + curr_token_pos, { 0 }, false); 165 | } 166 | 167 | curr_token_pos += sequence.size(); 168 | 169 | if (i == sequences.size() - 1) { 170 | batch.logits[batch.n_tokens - 1] = true; 171 | } 172 | 173 | if (llama_decode(ctx, batch) != 0) { 174 | decode_failed = true; 175 | break; 176 | } 177 | } 178 | 179 | printf("Request tokens: %d\n", (int32_t)request_tokens.size()); 180 | printf("Batch tokens: %d\n", batch.n_tokens); 181 | printf("Current token pos: %d\n", curr_token_pos); 182 | 183 | if (decode_failed) { 184 | Dictionary response; 185 | response["id"] = req.id; 186 | response["error"] = "llama_decode() failed"; 187 | call_thread_safe("emit_signal", "completion_generated", response); 188 | continue; 189 | } 190 | 191 | context_tokens.insert(context_tokens.end(), request_tokens.begin(), request_tokens.end()); 192 | 193 | while (true) { 194 | if (exit_thread) { 195 | return; 196 | } 197 | llama_token new_token_id = llama_sampling_sample(sampling_ctx, ctx, NULL, batch.n_tokens - 1); 198 | llama_sampling_accept(sampling_ctx, ctx, new_token_id, false); 199 | 200 | Dictionary response; 201 | response["id"] = req.id; 202 | 203 | context_tokens.push_back(new_token_id); 204 | 205 | bool eog = llama_token_is_eog(model->model, new_token_id); 206 | bool curr_eq_n_len = curr_token_pos == n_len; 207 | 208 | if (eog || curr_eq_n_len) { 209 | response["done"] = true; 210 | call_thread_safe("emit_signal", "completion_generated", response); 211 | break; 212 | } 213 | 214 | response["text"] = llama_token_to_piece(ctx, new_token_id).c_str(); 215 | response["done"] = false; 216 | call_thread_safe("emit_signal", "completion_generated", response); 217 | 218 | llama_batch_clear(batch); 219 | 220 | llama_batch_add(batch, new_token_id, curr_token_pos, { 0 }, true); 221 | 222 | curr_token_pos++; 223 | 224 | if (llama_decode(ctx, batch) != 0) { 225 | decode_failed = true; 226 | break; 227 | } 228 | } 229 | 230 | llama_sampling_reset(sampling_ctx); 231 | 232 | if (decode_failed) { 233 | Dictionary response; 234 | response["id"] = req.id; 235 | response["error"] = "llama_decode() failed"; 236 | call_thread_safe("emit_signal", "completion_generated", response); 237 | continue; 238 | } 239 | } 240 | } 241 | 242 | PackedStringArray LlamaContext::_get_configuration_warnings() const { 243 | PackedStringArray warnings; 244 | if (model == NULL) { 245 | warnings.push_back("Model resource property not defined"); 246 | } 247 | return warnings; 248 | } 249 | 250 | int LlamaContext::request_completion(const String &prompt) { 251 | int id = request_id++; 252 | 253 | UtilityFunctions::print(vformat("%s: Requesting completion for prompt id: %d", __func__, id)); 254 | 255 | mutex->lock(); 256 | completion_request req = { id, prompt }; 257 | completion_requests.append(req); 258 | mutex->unlock(); 259 | 260 | semaphore->post(); 261 | 262 | return id; 263 | } 264 | 265 | void LlamaContext::set_model(const Ref p_model) { 266 | model = p_model; 267 | } 268 | Ref LlamaContext::get_model() { 269 | return model; 270 | } 271 | 272 | uint32_t LlamaContext::get_seed() { 273 | return ctx_params.seed; 274 | } 275 | void LlamaContext::set_seed(uint32_t seed) { 276 | ctx_params.seed = seed; 277 | } 278 | 279 | uint32_t LlamaContext::get_n_ctx() { 280 | return ctx_params.n_ctx; 281 | } 282 | void LlamaContext::set_n_ctx(uint32_t n_ctx) { 283 | ctx_params.n_ctx = n_ctx; 284 | } 285 | 286 | int32_t LlamaContext::get_n_len() { 287 | return n_len; 288 | } 289 | void LlamaContext::set_n_len(int32_t n_len) { 290 | this->n_len = n_len; 291 | } 292 | 293 | float LlamaContext::get_temperature() { 294 | return sampling_params.temp; 295 | } 296 | void LlamaContext::set_temperature(float temperature) { 297 | sampling_params.temp = temperature; 298 | } 299 | 300 | float LlamaContext::get_top_p() { 301 | return sampling_params.top_p; 302 | } 303 | void LlamaContext::set_top_p(float top_p) { 304 | sampling_params.top_p = top_p; 305 | } 306 | 307 | float LlamaContext::get_frequency_penalty() { 308 | return sampling_params.penalty_freq; 309 | } 310 | void LlamaContext::set_frequency_penalty(float frequency_penalty) { 311 | sampling_params.penalty_freq = frequency_penalty; 312 | } 313 | 314 | float LlamaContext::get_presence_penalty() { 315 | return sampling_params.penalty_present; 316 | } 317 | void LlamaContext::set_presence_penalty(float presence_penalty) { 318 | sampling_params.penalty_present = presence_penalty; 319 | } 320 | 321 | void LlamaContext::_exit_tree() { 322 | if (Engine::get_singleton()->is_editor_hint()) { 323 | return; 324 | } 325 | 326 | mutex->lock(); 327 | exit_thread = true; 328 | mutex->unlock(); 329 | 330 | semaphore->post(); 331 | 332 | thread->wait_to_finish(); 333 | 334 | if (ctx) { 335 | llama_free(ctx); 336 | } 337 | if (sampling_ctx) { 338 | llama_sampling_free(sampling_ctx); 339 | } 340 | llama_backend_free(); 341 | } -------------------------------------------------------------------------------- /src/llama_context.h: -------------------------------------------------------------------------------- 1 | #ifndef LLAMA_CONTEXT_H 2 | #define LLAMA_CONTEXT_H 3 | 4 | #include "llama.h" 5 | #include "common.h" 6 | #include "llama_model.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | namespace godot { 13 | 14 | struct completion_request { 15 | int id; 16 | String prompt; 17 | }; 18 | 19 | class LlamaContext : public Node { 20 | GDCLASS(LlamaContext, Node) 21 | 22 | private: 23 | Ref model; 24 | llama_context *ctx = nullptr; 25 | llama_sampling_context *sampling_ctx = nullptr; 26 | llama_context_params ctx_params; 27 | llama_sampling_params sampling_params; 28 | int32_t n_len = 1024; 29 | int request_id = 0; 30 | Vector completion_requests; 31 | 32 | Ref thread; 33 | Ref semaphore; 34 | Ref mutex; 35 | std::vector context_tokens; 36 | bool exit_thread = false; 37 | 38 | protected: 39 | static void _bind_methods(); 40 | 41 | public: 42 | void set_model(const Ref model); 43 | Ref get_model(); 44 | 45 | int request_completion(const String &prompt); 46 | void __thread_loop(); 47 | 48 | uint32_t get_seed(); 49 | void set_seed(uint32_t seed); 50 | uint32_t get_n_ctx(); 51 | void set_n_ctx(uint32_t n_ctx); 52 | int32_t get_n_len(); 53 | void set_n_len(int32_t n_len); 54 | float get_temperature(); 55 | void set_temperature(float temperature); 56 | float get_top_p(); 57 | void set_top_p(float top_p); 58 | float get_frequency_penalty(); 59 | void set_frequency_penalty(float frequency_penalty); 60 | float get_presence_penalty(); 61 | void set_presence_penalty(float presence_penalty); 62 | 63 | virtual PackedStringArray _get_configuration_warnings() const override; 64 | virtual void _enter_tree() override; 65 | virtual void _exit_tree() override; 66 | LlamaContext(); 67 | }; 68 | } //namespace godot 69 | 70 | #endif -------------------------------------------------------------------------------- /src/llama_model.cpp: -------------------------------------------------------------------------------- 1 | #include "llama_model.h" 2 | #include "llama.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace godot; 9 | 10 | void LlamaModel::_bind_methods() { 11 | ClassDB::bind_method(D_METHOD("load_model"), &LlamaModel::load_model); 12 | 13 | ClassDB::bind_method(D_METHOD("get_n_gpu_layers"), &LlamaModel::get_n_gpu_layers); 14 | ClassDB::bind_method(D_METHOD("set_n_gpu_layers", "n"), &LlamaModel::set_n_gpu_layers); 15 | ClassDB::add_property("LlamaModel", PropertyInfo(Variant::INT, "n_gpu_layers"), "set_n_gpu_layers", "get_n_gpu_layers"); 16 | } 17 | 18 | LlamaModel::LlamaModel() { 19 | model_params = llama_model_default_params(); 20 | } 21 | 22 | void LlamaModel::load_model() { 23 | if (model) { 24 | return; 25 | } 26 | 27 | if (Engine::get_singleton()->is_editor_hint()) { 28 | return; 29 | } 30 | 31 | String absPath = ProjectSettings::get_singleton()->globalize_path(get_path()); 32 | 33 | model = llama_load_model_from_file(absPath.utf8().get_data(), model_params); 34 | 35 | if (model == NULL) { 36 | UtilityFunctions::printerr(vformat("%s: Unable to load model from %s", __func__, absPath)); 37 | return; 38 | } 39 | 40 | UtilityFunctions::print(vformat("%s: Model loaded from %s", __func__, absPath)); 41 | } 42 | 43 | int32_t LlamaModel::get_n_gpu_layers() { 44 | return model_params.n_gpu_layers; 45 | } 46 | 47 | void LlamaModel::set_n_gpu_layers(int32_t n) { 48 | model_params.n_gpu_layers = n; 49 | } 50 | 51 | LlamaModel::~LlamaModel() { 52 | if (model) { 53 | llama_free_model(model); 54 | } 55 | } -------------------------------------------------------------------------------- /src/llama_model.h: -------------------------------------------------------------------------------- 1 | #ifndef LLAMA_MODEL_H 2 | #define LLAMA_MODEL_H 3 | 4 | #include 5 | #include 6 | 7 | namespace godot { 8 | 9 | class LlamaModel : public Resource { 10 | GDCLASS(LlamaModel, Resource) 11 | 12 | private: 13 | llama_model_params model_params; 14 | 15 | protected: 16 | static void _bind_methods(); 17 | 18 | public: 19 | llama_model *model = nullptr; 20 | void load_model(); 21 | 22 | int32_t get_n_gpu_layers(); 23 | void set_n_gpu_layers(int32_t n); 24 | 25 | LlamaModel(); 26 | ~LlamaModel(); 27 | }; 28 | 29 | } //namespace godot 30 | 31 | #endif -------------------------------------------------------------------------------- /src/llama_model_loader.cpp: -------------------------------------------------------------------------------- 1 | #include "llama_model_loader.h" 2 | #include "llama_model.h" 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace godot; 8 | 9 | PackedStringArray LlamaModelLoader::_get_recognized_extensions() const { 10 | PackedStringArray arr; 11 | arr.push_back("gguf"); 12 | return arr; 13 | } 14 | 15 | Variant godot::LlamaModelLoader::_load(const String &path, const String &original_path, bool use_sub_threads, int32_t cache_mode) const { 16 | Ref model = memnew(LlamaModel); 17 | 18 | if (!FileAccess::file_exists(path)) { 19 | return ERR_FILE_NOT_FOUND; 20 | } 21 | 22 | model->set_path(path); 23 | model->load_model(); 24 | 25 | return { model }; 26 | } 27 | 28 | bool LlamaModelLoader::_handles_type(const StringName &type) const { 29 | return ClassDB::is_parent_class(type, "LlamaModel"); 30 | } 31 | 32 | String LlamaModelLoader::_get_resource_type(const String &p_path) const { 33 | String el = p_path.get_extension().to_lower(); 34 | 35 | if (el == "gguf") { 36 | return "LlamaModel"; 37 | } 38 | 39 | return ""; 40 | } -------------------------------------------------------------------------------- /src/llama_model_loader.h: -------------------------------------------------------------------------------- 1 | #ifndef LLAMA_MODEL_LOADER_H 2 | #define LLAMA_MODEL_LOADER_H 3 | 4 | #include 5 | 6 | namespace godot { 7 | 8 | class LlamaModelLoader : public ResourceFormatLoader { 9 | GDCLASS(LlamaModelLoader, ResourceFormatLoader) 10 | 11 | protected: 12 | static void _bind_methods(){}; 13 | 14 | public: 15 | PackedStringArray _get_recognized_extensions() const override; 16 | Variant _load(const String &path, const String &original_path, bool use_sub_threads, int32_t cache_mode) const override; 17 | virtual bool _handles_type(const StringName &type) const override; 18 | virtual String _get_resource_type(const String &p_path) const override; 19 | }; 20 | 21 | } //namespace godot 22 | 23 | #endif -------------------------------------------------------------------------------- /src/register_types.cpp: -------------------------------------------------------------------------------- 1 | #include "register_types.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "llama_model.h" 8 | #include "llama_model_loader.h" 9 | #include "llama_context.h" 10 | 11 | using namespace godot; 12 | 13 | static Ref llamaModelLoader; 14 | 15 | void initialize_types(ModuleInitializationLevel p_level) 16 | { 17 | if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) { 18 | return; 19 | } 20 | ClassDB::register_class(); 21 | llamaModelLoader.instantiate(); 22 | ResourceLoader::get_singleton()->add_resource_format_loader(llamaModelLoader); 23 | 24 | ClassDB::register_class(); 25 | ClassDB::register_class(); 26 | } 27 | 28 | void uninitialize_types(ModuleInitializationLevel p_level) { 29 | if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) { 30 | return; 31 | } 32 | ResourceLoader::get_singleton()->remove_resource_format_loader(llamaModelLoader); 33 | llamaModelLoader.unref(); 34 | } 35 | 36 | extern "C" 37 | { 38 | // Initialization 39 | GDExtensionBool GDE_EXPORT init_library(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization) 40 | { 41 | GDExtensionBinding::InitObject init_obj(p_get_proc_address, p_library, r_initialization); 42 | init_obj.register_initializer(initialize_types); 43 | init_obj.register_terminator(uninitialize_types); 44 | init_obj.set_minimum_library_initialization_level(MODULE_INITIALIZATION_LEVEL_SCENE); 45 | 46 | return init_obj.init(); 47 | } 48 | } -------------------------------------------------------------------------------- /src/register_types.h: -------------------------------------------------------------------------------- 1 | #ifndef REGISTER_TYPES_H 2 | #define REGISTER_TYPES_H 3 | #include 4 | 5 | using namespace godot; 6 | 7 | void initialize_types(ModuleInitializationLevel p_level); 8 | void uninitialize_types(ModuleInitializationLevel p_level); 9 | 10 | #endif // REGISTER_TYPES_H -------------------------------------------------------------------------------- /tools/concat_files.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | const usage = "Usage: ./concat_files ... "; 4 | 5 | pub fn main() !void { 6 | var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator); 7 | defer arena_state.deinit(); 8 | const arena = arena_state.allocator(); 9 | 10 | const args = try std.process.argsAlloc(arena); 11 | 12 | if (args.len < 3) { 13 | std.debug.panic("expected at least 2 arguments", .{}); 14 | } 15 | 16 | var input_dirs = std.ArrayList([]const u8).init(arena); 17 | var output_file_path: []const u8 = undefined; 18 | 19 | { 20 | var i: usize = 1; 21 | while (i < args.len) : (i += 1) { 22 | const arg = args[i]; 23 | if (std.mem.eql(u8, "-h", arg) or std.mem.eql(u8, "--help", arg)) { 24 | try std.io.getStdOut().writeAll(usage); 25 | return std.process.cleanExit(); 26 | } else if (i == args.len - 1) { 27 | output_file_path = arg; 28 | } else { 29 | try input_dirs.append(arg); 30 | } 31 | } 32 | } 33 | 34 | const cwd = std.fs.cwd(); 35 | const output = try cwd.createFile(output_file_path, .{}); 36 | var pos: u64 = 0; 37 | 38 | for (input_dirs.items) |dir_path| { 39 | var dir = try cwd.openDir(dir_path, .{ .iterate = true }); 40 | defer dir.close(); 41 | 42 | var walker = try dir.walk(arena); 43 | defer walker.deinit(); 44 | 45 | while (try walker.next()) |entry| { 46 | if (entry.kind != .file) continue; 47 | const file = try entry.dir.openFile(entry.basename, .{}); 48 | defer file.close(); 49 | 50 | const len = (try file.stat()).size; 51 | const bytes_copied = try file.copyRangeAll(0, output, pos, len); 52 | pos += bytes_copied; 53 | 54 | try output.pwriteAll("\n", pos); 55 | pos += 1; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /tools/expand_metal.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | const usage = 4 | \\Usage: ./embed_metal [options] 5 | \\ 6 | \\Options: 7 | \\ --metal-file ggml-metal.metal 8 | \\ --common-file ggml-common.h 9 | \\ --output-file ggml-metal-embed.metal 10 | \\ 11 | ; 12 | 13 | pub fn main() !void { 14 | var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator); 15 | defer arena_state.deinit(); 16 | const arena = arena_state.allocator(); 17 | 18 | const args = try std.process.argsAlloc(arena); 19 | 20 | var opt_metal_file_path: ?[]const u8 = null; 21 | var opt_common_file_path: ?[]const u8 = null; 22 | var opt_output_file_path: ?[]const u8 = null; 23 | 24 | { 25 | var i: usize = 1; 26 | while (i < args.len) : (i += 1) { 27 | const arg = args[i]; 28 | if (std.mem.eql(u8, "-h", arg) or std.mem.eql(u8, "--help", arg)) { 29 | try std.io.getStdOut().writeAll(usage); 30 | return std.process.cleanExit(); 31 | } else if (std.mem.eql(u8, "--metal-file", arg)) { 32 | i += 1; 33 | if (i > args.len) std.debug.panic("expected arg after '{s}'", .{arg}); 34 | if (opt_metal_file_path != null) std.debug.panic("duplicated {s} argument", .{arg}); 35 | opt_metal_file_path = args[i]; 36 | } else if (std.mem.eql(u8, "--common-file", arg)) { 37 | i += 1; 38 | if (i > args.len) std.debug.panic("expected arg after '{s}'", .{arg}); 39 | if (opt_common_file_path != null) std.debug.panic("duplicated {s} argument", .{arg}); 40 | opt_common_file_path = args[i]; 41 | } else if (std.mem.eql(u8, "--output-file", arg)) { 42 | i += 1; 43 | if (i > args.len) std.debug.panic("expected arg after '{s}'", .{arg}); 44 | if (opt_output_file_path != null) std.debug.panic("duplicated {s} argument", .{arg}); 45 | opt_output_file_path = args[i]; 46 | } else { 47 | std.debug.panic("unrecognized arg: '{s}'", .{arg}); 48 | } 49 | } 50 | } 51 | 52 | const metal_file_path = opt_metal_file_path orelse std.debug.panic("missing --input-file", .{}); 53 | const common_file_path = opt_common_file_path orelse std.debug.panic("missing --output-file", .{}); 54 | const output_file_path = opt_output_file_path orelse std.debug.panic("missing --lang", .{}); 55 | 56 | const cwd = std.fs.cwd(); 57 | 58 | var metal_file = try cwd.openFile(metal_file_path, .{}); 59 | defer metal_file.close(); 60 | 61 | var common_file = try cwd.openFile(common_file_path, .{}); 62 | defer common_file.close(); 63 | 64 | const metal_size = (try metal_file.stat()).size; 65 | const metal_contents = try arena.alloc(u8, metal_size); 66 | defer arena.free(metal_contents); 67 | _ = try metal_file.readAll(metal_contents); 68 | 69 | const common_size = (try common_file.stat()).size; 70 | const common_contents = try arena.alloc(u8, common_size); 71 | defer arena.free(common_contents); 72 | _ = try common_file.readAll(common_contents); 73 | 74 | const output = try std.mem.replaceOwned(u8, arena, metal_contents, "#include \"ggml-common.h\"", common_contents); 75 | defer arena.free(output); 76 | 77 | const output_file = try cwd.createFile(output_file_path, .{}); 78 | try output_file.writeAll(output); 79 | } 80 | --------------------------------------------------------------------------------