├── .circleci ├── config.yml └── continue_config.yml ├── .codespellrc ├── .github ├── ISSUE_TEMPLATE │ ├── bindings-bug.md │ ├── chat-bug.md │ ├── config.yml │ ├── documentation.md │ ├── feature-request.md │ └── other-bug.md ├── pull_request_template.md └── workflows │ ├── close_issues.yml │ └── codespell.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── gpt4all-backend ├── CMakeLists.txt ├── README.md ├── dlhandle.cpp ├── dlhandle.h ├── gptj.cpp ├── gptj_impl.h ├── llama.cpp.cmake ├── llamamodel.cpp ├── llamamodel_impl.h ├── llmodel.cpp ├── llmodel.h ├── llmodel_c.cpp ├── llmodel_c.h ├── llmodel_shared.cpp ├── llmodel_shared.h ├── scripts │ ├── convert_bert_hf_to_gguf.py │ └── convert_gptj_to_gguf.py ├── sysinfo.h ├── utils.cpp └── utils.h ├── gpt4all-bindings ├── README.md ├── cli │ ├── README.md │ ├── app.py │ └── developer_notes.md ├── csharp │ ├── .editorconfig │ ├── .gitignore │ ├── Directory.Build.props │ ├── Gpt4All.Samples │ │ ├── Gpt4All.Samples.csproj │ │ └── Program.cs │ ├── Gpt4All.Tests │ │ ├── Constants.cs │ │ ├── Gpt4All.Tests.csproj │ │ ├── ModelFactoryTests.cs │ │ ├── NativeLibraryLoaderTests.cs │ │ ├── PlatformSpecificFactAttribute.cs │ │ └── Traits.cs │ ├── Gpt4All.sln │ ├── Gpt4All │ │ ├── Bindings │ │ │ ├── ILLModel.cs │ │ │ ├── LLModel.cs │ │ │ ├── LLPromptContext.cs │ │ │ ├── NativeMethods.cs │ │ │ └── NativeTypeNameAttribute.cs │ │ ├── Extensions │ │ │ ├── LLPromptContextExtensions.cs │ │ │ └── PredictRequestOptionsExtensions.cs │ │ ├── GenLLModelBindings.rsp │ │ ├── Gpt4All.cs │ │ ├── Gpt4All.csproj │ │ ├── LibraryLoader │ │ │ ├── ILibraryLoader.cs │ │ │ ├── LinuxLibraryLoader.cs │ │ │ ├── LoadResult.cs │ │ │ ├── MacOsLibraryLoader.cs │ │ │ ├── NativeLibraryLoader.cs │ │ │ └── WindowsLibraryLoader.cs │ │ ├── Model │ │ │ ├── DefaultPromptFormatter.cs │ │ │ ├── Gpt4AllModelFactory.cs │ │ │ ├── IGpt4AllModel.cs │ │ │ ├── IGpt4AllModelFactory.cs │ │ │ ├── IPromptFormatter.cs │ │ │ └── ModelOptions.cs │ │ ├── Prediction │ │ │ ├── ITextPrediction.cs │ │ │ ├── ITextPredictionResult.cs │ │ │ ├── ITextPredictionStreamingResult.cs │ │ │ ├── PredictRequestOptions.cs │ │ │ ├── TextPredictionResult.cs │ │ │ └── TextPredictionStreamingResult.cs │ │ └── gen_bindings.ps1 │ ├── README.md │ ├── build_linux.sh │ ├── build_win-mingw.ps1 │ ├── build_win-msvc.ps1 │ └── docs │ │ └── gpt4all_csharp.md ├── golang │ ├── Makefile │ ├── README.md │ ├── binding.cpp │ ├── binding.h │ ├── example │ │ └── main.go │ ├── go.mod │ ├── go.sum │ ├── gpt4all.go │ ├── gpt4all_suite_test.go │ ├── gpt4all_test.go │ └── options.go ├── java │ ├── .gitignore │ ├── Developer_docs.md │ ├── README.md │ ├── TODO.md │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── hexadevlabs │ │ │ └── gpt4all │ │ │ ├── LLModel.java │ │ │ ├── LLModelLibrary.java │ │ │ ├── PromptIsTooLongException.java │ │ │ └── Util.java │ │ └── test │ │ └── java │ │ └── com │ │ └── hexadevlabs │ │ └── gpt4all │ │ ├── BasicTests.java │ │ ├── Example1.java │ │ ├── Example2.java │ │ ├── Example3.java │ │ ├── Example4.java │ │ └── Example5.java ├── python │ ├── .gitignore │ ├── .isort.cfg │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── README.md │ ├── docs │ │ ├── assets │ │ │ ├── favicon.ico │ │ │ └── nomic.png │ │ ├── css │ │ │ └── custom.css │ │ ├── gpt4all_chat.md │ │ ├── gpt4all_cli.md │ │ ├── gpt4all_faq.md │ │ ├── gpt4all_monitoring.md │ │ ├── gpt4all_nodejs.md │ │ ├── gpt4all_python.md │ │ ├── gpt4all_python_embedding.md │ │ └── index.md │ ├── gpt4all │ │ ├── __init__.py │ │ ├── _pyllmodel.py │ │ ├── gpt4all.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── test_embed_timings.py │ │ │ └── test_gpt4all.py │ ├── makefile │ ├── mkdocs.yml │ └── setup.py └── typescript │ ├── .clang-format │ ├── .gitignore │ ├── .npmignore │ ├── .yarnrc.yml │ ├── README.md │ ├── binding.ci.gyp │ ├── binding.gyp │ ├── index.cc │ ├── index.h │ ├── package.json │ ├── prompt.cc │ ├── prompt.h │ ├── scripts │ ├── build.js │ ├── build_mingw.ps1 │ ├── build_msvc.bat │ ├── build_unix.sh │ ├── docs.js │ ├── mkclangd.js │ └── prebuild.js │ ├── spec │ ├── callbacks.mjs │ ├── chat-memory.mjs │ ├── chat-minimal.mjs │ ├── concurrency.mjs │ ├── embed-jsonl.mjs │ ├── embed.mjs │ ├── llmodel.mjs │ ├── long-context.mjs │ ├── model-switching.mjs │ ├── stateless.mjs │ ├── streaming.mjs │ └── system.mjs │ ├── src │ ├── chat-session.js │ ├── config.js │ ├── gpt4all.d.ts │ ├── gpt4all.js │ ├── models.js │ └── util.js │ ├── test │ ├── gpt4all.test.js │ └── models.json │ └── yarn.lock ├── gpt4all-chat ├── CMakeLists.txt ├── LICENSE ├── README.md ├── build_and_run.md ├── chat.cpp ├── chat.h ├── chatapi.cpp ├── chatapi.h ├── chatlistmodel.cpp ├── chatlistmodel.h ├── chatllm.cpp ├── chatllm.h ├── chatmodel.h ├── cmake │ ├── config.h.in │ ├── deploy-qt-linux.cmake.in │ ├── deploy-qt-mac.cmake.in │ ├── deploy-qt-windows.cmake.in │ ├── installerscript.qs │ └── sign_dmg.py ├── database.cpp ├── database.h ├── download.cpp ├── download.h ├── embeddings.cpp ├── embeddings.h ├── embllm.cpp ├── embllm.h ├── flatpak-manifest │ ├── io.gpt4all.gpt4all.appdata.xml │ ├── io.gpt4all.gpt4all.desktop │ └── io.gpt4all.gpt4all.yml ├── hnswlib │ ├── bruteforce.h │ ├── hnswalg.h │ ├── hnswlib.h │ ├── space_ip.h │ ├── space_l2.h │ └── visited_list_pool.h ├── icons │ ├── close.svg │ ├── copy.svg │ ├── db.svg │ ├── download.svg │ ├── edit.svg │ ├── eject.svg │ ├── image.svg │ ├── left_panel_closed.svg │ ├── left_panel_open.svg │ ├── logo-1024.png │ ├── logo-128.png │ ├── logo-16.png │ ├── logo-256.png │ ├── logo-32.png │ ├── logo-48.png │ ├── logo-512.png │ ├── logo-64.png │ ├── logo.svg │ ├── network.svg │ ├── question.svg │ ├── regenerate.svg │ ├── send_message.svg │ ├── settings.svg │ ├── stop_generating.svg │ ├── thumbs_down.svg │ ├── thumbs_up.svg │ └── trash.svg ├── llm.cpp ├── llm.h ├── localdocs.cpp ├── localdocs.h ├── localdocsmodel.cpp ├── localdocsmodel.h ├── logger.cpp ├── logger.h ├── main.cpp ├── main.qml ├── metadata │ ├── models.json │ ├── models2.json │ ├── models3.json │ └── release.json ├── modellist.cpp ├── modellist.h ├── mysettings.cpp ├── mysettings.h ├── network.cpp ├── network.h ├── qml │ ├── AboutDialog.qml │ ├── ApplicationSettings.qml │ ├── ChatDrawer.qml │ ├── ChatView.qml │ ├── CollectionsDialog.qml │ ├── LocalDocsSettings.qml │ ├── ModelDownloaderDialog.qml │ ├── ModelSettings.qml │ ├── MyBusyIndicator.qml │ ├── MyButton.qml │ ├── MyCheckBox.qml │ ├── MyComboBox.qml │ ├── MyDialog.qml │ ├── MyDirectoryField.qml │ ├── MyMiniButton.qml │ ├── MySettingsButton.qml │ ├── MySettingsDestructiveButton.qml │ ├── MySettingsLabel.qml │ ├── MySettingsStack.qml │ ├── MySettingsTab.qml │ ├── MySlug.qml │ ├── MyTextArea.qml │ ├── MyTextField.qml │ ├── MyToolButton.qml │ ├── NetworkDialog.qml │ ├── NewVersionDialog.qml │ ├── PopupDialog.qml │ ├── SettingsDialog.qml │ ├── StartupDialog.qml │ ├── SwitchModelDialog.qml │ ├── Theme.qml │ └── ThumbsDownDialog.qml ├── resources │ ├── gpt4all.icns │ ├── gpt4all.ico │ └── gpt4all.rc ├── responsetext.cpp ├── responsetext.h ├── server.cpp └── server.h ├── gpt4all-lora-demo.gif └── gpt4all-training ├── GPT-J_MAP.md ├── README.md ├── TRAINING_LOG.md ├── build_map.py ├── clean.py ├── configs ├── deepspeed │ ├── ds_config.json │ ├── ds_config_gptj.json │ ├── ds_config_gptj_lora.json │ ├── ds_config_mpt.json │ └── ds_config_pythia.json ├── eval │ ├── generate_baseline.yaml │ ├── generate_gpt4all_gptj.yaml │ ├── generate_gpt4all_gptj_lora.yaml │ └── generate_gpt4all_llama_lora.yaml ├── generate │ ├── generate.yaml │ ├── generate_gptj.yaml │ ├── generate_gptj_lora.yaml │ └── generate_llama.yaml ├── inference │ └── gptj.yaml └── train │ ├── finetune.yaml │ ├── finetune_falcon.yaml │ ├── finetune_gptj.yaml │ ├── finetune_gptj_lora.yaml │ ├── finetune_lora.yaml │ ├── finetune_mpt.yaml │ └── finetune_openllama.yaml ├── create_hostname.sh ├── data.py ├── env.yaml ├── eval_figures.py ├── eval_self_instruct.py ├── figs ├── clustering_overfit.png ├── duplicate_loss.png ├── first_lora.png ├── overfit-gpt-j.png ├── perplexity_hist.png └── single_epoch.png ├── generate.py ├── inference.py ├── launcher.sh ├── old-README.md ├── read.py ├── requirements.txt └── train.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | setup: true 3 | orbs: 4 | path-filtering: circleci/path-filtering@0.0.1 5 | 6 | workflows: 7 | version: 2.1 8 | generate-config: 9 | jobs: 10 | - path-filtering/filter: 11 | base-revision: main 12 | config-path: .circleci/continue_config.yml 13 | mapping: | 14 | .circleci/.* run-all-workflows true 15 | gpt4all-backend/.* run-all-workflows true 16 | gpt4all-bindings/python/.* run-python-workflow true 17 | gpt4all-bindings/typescript/.* run-ts-workflow true 18 | gpt4all-bindings/csharp/.* run-csharp-workflow true 19 | gpt4all-chat/.* run-chat-workflow true 20 | .* run-default-workflow true 21 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | ignore-words-list = blong, afterall, som, assistent, crasher 3 | skip = .git,*.pdf,*.svg,*.lock 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bindings-bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F6E0 Bindings Bug Report" 3 | about: A bug report for the GPT4All Bindings 4 | labels: ["bindings", "bug-unconfirmed"] 5 | --- 6 | 7 | 8 | 9 | ### Bug Report 10 | 11 | 12 | 13 | ### Example Code 14 | 15 | 16 | 17 | ### Steps to Reproduce 18 | 19 | 20 | 21 | 1. 22 | 2. 23 | 3. 24 | 25 | ### Expected Behavior 26 | 27 | 28 | 29 | ### Your Environment 30 | 31 | - Bindings version (e.g. "Version" from `pip show gpt4all`): 32 | - Operating System: 33 | - Chat model used (if applicable): 34 | 35 | 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/chat-bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F4AC GPT4All Bug Report" 3 | about: A bug report for GPT4All Chat 4 | labels: ["chat", "bug-unconfirmed"] 5 | --- 6 | 7 | 8 | 9 | ### Bug Report 10 | 11 | 12 | 13 | ### Steps to Reproduce 14 | 15 | 16 | 17 | 1. 18 | 2. 19 | 3. 20 | 21 | ### Expected Behavior 22 | 23 | 24 | 25 | ### Your Environment 26 | 27 | - GPT4All version: 28 | - Operating System: 29 | - Chat model used (if applicable): 30 | 31 | 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F4C4 Documentation" 3 | about: An issue related to the GPT4All documentation 4 | labels: ["documentation"] 5 | --- 6 | 7 | ### Documentation 8 | 9 | 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature Request" 3 | about: Submit a proposal/request for a new GPT4All feature 4 | title: "[Feature] Feature request title..." 5 | labels: ["enhancement"] 6 | --- 7 | 8 | ### Feature Request 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/other-bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Other Bug Report" 3 | about: A bug in another component of GPT4All 4 | labels: ["bug-unconfirmed"] 5 | --- 6 | 7 | 8 | 9 | ### Bug Report 10 | 11 | 12 | 13 | ### Steps to Reproduce 14 | 15 | 16 | 17 | 1. 18 | 2. 19 | 3. 20 | 21 | ### Expected Behavior 22 | 23 | 24 | 25 | ### Your Environment 26 | 27 | - GPT4All version (if applicable): 28 | - Operating System: 29 | - Chat model used (if applicable): 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Describe your changes 2 | 3 | ## Issue ticket number and link 4 | 5 | ## Checklist before requesting a review 6 | - [ ] I have performed a self-review of my code. 7 | - [ ] If it is a core feature, I have added thorough tests. 8 | - [ ] I have added thorough documentation for my code. 9 | - [ ] I have tagged PR with relevant project labels. I acknowledge that a PR without labels may be dismissed. 10 | - [ ] If this PR addresses a bug, I have provided both a screenshot/video of the original bug and the working solution. 11 | 12 | ## Demo 13 | 14 | 15 | ### Steps to Reproduce 16 | 17 | 18 | ## Notes 19 | 20 | -------------------------------------------------------------------------------- /.github/workflows/close_issues.yml: -------------------------------------------------------------------------------- 1 | # This workflow will close issues that do not have labels or additional comments. 2 | # Trigger manually. 3 | 4 | name: "Close Issues" 5 | on: 6 | workflow_dispatch: 7 | 8 | jobs: 9 | close_issues: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Close issues without label or comment 13 | uses: actions/github-script@v3 14 | with: 15 | github-token: ${{secrets.GITHUB_TOKEN}} 16 | script: | 17 | const repo = context.repo; 18 | let page = 1; 19 | let issues = []; 20 | while (true) { 21 | const result = await github.issues.listForRepo({...repo, per_page: 100, page: page}); 22 | if (result.data.length === 0) break; 23 | issues = issues.concat(result.data); 24 | page += 1; 25 | } 26 | for (let { number } of issues) { 27 | const issueData = await github.issues.get({...repo, issue_number: number}); 28 | const comments = await github.issues.listComments({...repo, issue_number: number}); 29 | if (issueData.data.labels.length === 0 && comments.data.length < 1) { 30 | await github.issues.update({...repo, issue_number: number, state: 'closed'}); 31 | await github.issues.createComment({...repo, issue_number: number, body: 'Issue closed as it does not have any labels or comments.'}); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Codespell 3 | 4 | on: 5 | push: 6 | branches: [main] 7 | pull_request: 8 | branches: [main] 9 | 10 | jobs: 11 | codespell: 12 | name: Check for spelling errors 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | - name: Codespell 19 | uses: codespell-project/actions-codespell@v2 20 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "llama.cpp-mainline"] 2 | path = gpt4all-backend/llama.cpp-mainline 3 | url = https://github.com/nomic-ai/llama.cpp.git 4 | branch = master 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Nomic, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /gpt4all-backend/dlhandle.cpp: -------------------------------------------------------------------------------- 1 | #include "dlhandle.h" 2 | 3 | #include 4 | 5 | #ifndef _WIN32 6 | # include 7 | #else 8 | # include 9 | # include 10 | # define WIN32_LEAN_AND_MEAN 11 | # ifndef NOMINMAX 12 | # define NOMINMAX 13 | # endif 14 | # include 15 | #endif 16 | 17 | using namespace std::string_literals; 18 | namespace fs = std::filesystem; 19 | 20 | 21 | #ifndef _WIN32 22 | 23 | Dlhandle::Dlhandle(const fs::path &fpath) { 24 | chandle = dlopen(fpath.c_str(), RTLD_LAZY | RTLD_LOCAL); 25 | if (!chandle) { 26 | throw Exception("dlopen: "s + dlerror()); 27 | } 28 | } 29 | 30 | Dlhandle::~Dlhandle() { 31 | if (chandle) dlclose(chandle); 32 | } 33 | 34 | void *Dlhandle::get_internal(const char *symbol) const { 35 | return dlsym(chandle, symbol); 36 | } 37 | 38 | #else // defined(_WIN32) 39 | 40 | Dlhandle::Dlhandle(const fs::path &fpath) { 41 | fs::path afpath = fs::absolute(fpath); 42 | 43 | // Suppress the "Entry Point Not Found" dialog, caused by outdated nvcuda.dll from the GPU driver 44 | UINT lastErrorMode = GetErrorMode(); 45 | UINT success = SetErrorMode(lastErrorMode | SEM_FAILCRITICALERRORS); 46 | assert(success); 47 | 48 | chandle = LoadLibraryExW(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR); 49 | 50 | success = SetErrorMode(lastErrorMode); 51 | assert(success); 52 | 53 | if (!chandle) { 54 | DWORD err = GetLastError(); 55 | std::ostringstream ss; 56 | ss << "LoadLibraryExW failed with error 0x" << std::hex << err; 57 | throw Exception(ss.str()); 58 | } 59 | } 60 | 61 | Dlhandle::~Dlhandle() { 62 | if (chandle) FreeLibrary(HMODULE(chandle)); 63 | } 64 | 65 | void *Dlhandle::get_internal(const char *symbol) const { 66 | return GetProcAddress(HMODULE(chandle), symbol); 67 | } 68 | 69 | #endif // defined(_WIN32) 70 | -------------------------------------------------------------------------------- /gpt4all-backend/dlhandle.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace fs = std::filesystem; 9 | 10 | 11 | class Dlhandle { 12 | void *chandle = nullptr; 13 | 14 | public: 15 | class Exception : public std::runtime_error { 16 | public: 17 | using std::runtime_error::runtime_error; 18 | }; 19 | 20 | Dlhandle() = default; 21 | Dlhandle(const fs::path &fpath); 22 | Dlhandle(const Dlhandle &o) = delete; 23 | Dlhandle(Dlhandle &&o) 24 | : chandle(o.chandle) 25 | { 26 | o.chandle = nullptr; 27 | } 28 | 29 | ~Dlhandle(); 30 | 31 | Dlhandle &operator=(Dlhandle &&o) { 32 | chandle = std::exchange(o.chandle, nullptr); 33 | return *this; 34 | } 35 | 36 | template 37 | T *get(const std::string &symbol) const { 38 | return reinterpret_cast(get_internal(symbol.c_str())); 39 | } 40 | 41 | auto get_fnc(const std::string &symbol) const { 42 | return get(symbol); 43 | } 44 | 45 | private: 46 | void *get_internal(const char *symbol) const; 47 | }; 48 | -------------------------------------------------------------------------------- /gpt4all-backend/gptj_impl.h: -------------------------------------------------------------------------------- 1 | #ifndef GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE 2 | #error This file is NOT meant to be included outside of gptj.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE 3 | #endif 4 | #ifndef GPTJ_H 5 | #define GPTJ_H 6 | 7 | #include "llmodel.h" 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | struct GPTJPrivate; 14 | class GPTJ : public LLModel { 15 | public: 16 | GPTJ(); 17 | ~GPTJ(); 18 | 19 | bool supportsEmbedding() const override { return false; } 20 | bool supportsCompletion() const override { return true; } 21 | bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override; 22 | bool isModelLoaded() const override; 23 | size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override; 24 | size_t stateSize() const override; 25 | size_t saveState(uint8_t *dest) const override; 26 | size_t restoreState(const uint8_t *src) override; 27 | void setThreadCount(int32_t n_threads) override; 28 | int32_t threadCount() const override; 29 | 30 | private: 31 | GPTJPrivate *d_ptr; 32 | 33 | protected: 34 | std::vector tokenize(PromptContext &ctx, const std::string &str, bool special) const override; 35 | Token sampleToken(PromptContext &ctx) const override; 36 | std::string tokenToString(Token id) const override; 37 | bool evalTokens(PromptContext &ctx, const std::vector &tokens) const override; 38 | int32_t contextLength() const override; 39 | const std::vector &endTokens() const override; 40 | bool shouldAddBOS() const override { return false; } 41 | }; 42 | 43 | #endif // GPTJ_H 44 | -------------------------------------------------------------------------------- /gpt4all-backend/llmodel_shared.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | struct llm_buffer { 10 | uint8_t * addr = NULL; 11 | size_t size = 0; 12 | 13 | void resize(size_t size) { 14 | delete[] addr; 15 | addr = new uint8_t[size]; 16 | this->size = size; 17 | } 18 | 19 | ~llm_buffer() { 20 | delete[] addr; 21 | } 22 | }; 23 | 24 | struct llm_kv_cache { 25 | struct ggml_tensor * k; 26 | struct ggml_tensor * v; 27 | 28 | struct ggml_context * ctx = NULL; 29 | 30 | llm_buffer buf; 31 | 32 | int n; // number of tokens currently in the cache 33 | 34 | ~llm_kv_cache() { 35 | if (ctx) { 36 | ggml_free(ctx); 37 | } 38 | } 39 | }; 40 | 41 | inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads) { 42 | struct ggml_cplan plan = ggml_graph_plan(graph, n_threads); 43 | if (plan.work_size > 0) { 44 | buf.resize(plan.work_size); 45 | plan.work_data = buf.addr; 46 | } 47 | ggml_graph_compute(graph, &plan); 48 | } 49 | -------------------------------------------------------------------------------- /gpt4all-backend/sysinfo.h: -------------------------------------------------------------------------------- 1 | #ifndef SYSINFO_H 2 | #define SYSINFO_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #if defined(__linux__) 10 | # include 11 | #elif defined(__APPLE__) 12 | # include 13 | # include 14 | #elif defined(_WIN32) 15 | # define WIN32_LEAN_AND_MEAN 16 | # ifndef NOMINMAX 17 | # define NOMINMAX 18 | # endif 19 | # include 20 | #endif 21 | 22 | static long long getSystemTotalRAMInBytes() 23 | { 24 | long long totalRAM = 0; 25 | 26 | #if defined(__linux__) 27 | std::ifstream file("/proc/meminfo"); 28 | std::string line; 29 | while (std::getline(file, line)) { 30 | if (line.find("MemTotal") != std::string::npos) { 31 | std::string memTotalStr = line.substr(line.find(":") + 1); 32 | memTotalStr.erase(0, memTotalStr.find_first_not_of(" ")); 33 | memTotalStr = memTotalStr.substr(0, memTotalStr.find(" ")); 34 | totalRAM = std::stoll(memTotalStr) * 1024; // Convert from KB to bytes 35 | break; 36 | } 37 | } 38 | file.close(); 39 | #elif defined(__APPLE__) 40 | int mib[2] = {CTL_HW, HW_MEMSIZE}; 41 | size_t length = sizeof(totalRAM); 42 | sysctl(mib, 2, &totalRAM, &length, NULL, 0); 43 | #elif defined(_WIN32) 44 | MEMORYSTATUSEX memoryStatus; 45 | memoryStatus.dwLength = sizeof(memoryStatus); 46 | GlobalMemoryStatusEx(&memoryStatus); 47 | totalRAM = memoryStatus.ullTotalPhys; 48 | #endif 49 | 50 | return totalRAM; 51 | } 52 | 53 | static double getSystemTotalRAMInGB() 54 | { 55 | return static_cast(getSystemTotalRAMInBytes()) / (1024 * 1024 * 1024); 56 | } 57 | 58 | static std::string getSystemTotalRAMInGBString() 59 | { 60 | std::stringstream ss; 61 | ss << std::fixed << std::setprecision(2) << getSystemTotalRAMInGB() << " GB"; 62 | return ss.str(); 63 | } 64 | 65 | #endif // SYSINFO_H 66 | -------------------------------------------------------------------------------- /gpt4all-bindings/README.md: -------------------------------------------------------------------------------- 1 | # GPT4All Bindings 2 | This directory will contain language specific bindings on top of the C/C++ model backends. 3 | We will have one directory per language binding (e.g. Python, Typescript, Golang, etc.). -------------------------------------------------------------------------------- /gpt4all-bindings/cli/README.md: -------------------------------------------------------------------------------- 1 | # GPT4All Command-Line Interface (CLI) 2 | 3 | GPT4All on the command-line. 4 | 5 | ## Documentation 6 | 7 | 8 | ## Quickstart 9 | 10 | The CLI is based on the `gpt4all` Python bindings and the `typer` package. 11 | 12 | The following shows one way to get started with the CLI, the documentation has more information. 13 | Typically, you will want to replace `python` with `python3` on _Unix-like_ systems and `py -3` on 14 | _Windows_. Also, it's assumed you have all the necessary Python components already installed. 15 | 16 | The CLI is a self-contained Python script named [app.py] ([download][app.py-download]). As long as 17 | its package dependencies are present, you can download and run it from wherever you like. 18 | 19 | [app.py]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/cli/app.py 20 | [app.py-download]: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-bindings/cli/app.py 21 | 22 | ```shell 23 | # optional but recommended: create and use a virtual environment 24 | python -m venv gpt4all-cli 25 | ``` 26 | _Windows_ and _Unix-like_ systems differ slightly in how you activate a _virtual environment_: 27 | - _Unix-like_, typically: `. gpt4all-cli/bin/activate` 28 | - _Windows_: `gpt4all-cli\Scripts\activate` 29 | 30 | Then: 31 | ```shell 32 | # pip-install the necessary packages; omit '--user' if using a virtual environment 33 | python -m pip install --user --upgrade gpt4all typer 34 | # run the CLI 35 | python app.py repl 36 | ``` 37 | By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user 38 | directory, if necessary. 39 | 40 | If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument, 41 | for example: 42 | ```shell 43 | python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf 44 | ``` 45 | -------------------------------------------------------------------------------- /gpt4all-bindings/cli/developer_notes.md: -------------------------------------------------------------------------------- 1 | # Developing the CLI 2 | ## Documentation 3 | Documentation can be found in three places: 4 | - `app.py` docstrings & comments 5 | - a Readme: `gpt4all-bindings/cli/README.md` 6 | - the actual CLI documentation: `gpt4all-bindings/python/docs/gpt4all_cli.md` 7 | 8 | The _docstrings_ are meant for programmatic use. Since the CLI is primarily geared towards users and 9 | not to build on top, they're kept terse. 10 | 11 | The _Readme_ is mostly meant for users and includes: 12 | - a link to the _CLI documentation_ (on the [website]) 13 | - a Quickstart section with some guidance on how to get started with a sane setup 14 | 15 | The _CLI documentation_ and other documentation are located in the above mentioned `docs/` folder. 16 | They're in Markdown format and built for the [website]. Of the three, they should be the most 17 | detailed. 18 | 19 | [website]: https://docs.gpt4all.io/gpt4all_cli.html 20 | 21 | 22 | ## Versioning 23 | The version number should now follow the `gpt4all` PyPI package, so compatibility is more clear. 24 | 25 | The one place to change it is the `namedtuple` called `VERSION_INFO`. 26 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | en-US 8 | 0.6.4-alpha 9 | $(VersionSuffix) 10 | $(Version)$(VersionSuffix) 11 | true 12 | 13 | git 14 | true 15 | true 16 | latest-minimum 17 | true 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | preview 26 | strict 27 | 28 | 29 | 30 | 31 | all 32 | runtime; build; native; contentfiles; analyzers 33 | 34 | 35 | all 36 | runtime; build; native; contentfiles; analyzers 37 | 38 | 39 | all 40 | runtime; build; native; contentfiles; analyzers 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | true 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Samples/Program.cs: -------------------------------------------------------------------------------- 1 | using Gpt4All; 2 | 3 | var modelFactory = new Gpt4AllModelFactory(); 4 | if (args.Length < 2) 5 | { 6 | Console.WriteLine($"Usage: Gpt4All.Samples "); 7 | return; 8 | } 9 | 10 | var modelPath = args[0]; 11 | var prompt = args[1]; 12 | 13 | using var model = modelFactory.LoadModel(modelPath); 14 | 15 | var result = await model.GetStreamingPredictionAsync( 16 | prompt, 17 | PredictRequestOptions.Defaults); 18 | 19 | await foreach (var token in result.GetPredictionStreamingAsync()) 20 | { 21 | Console.Write(token); 22 | } 23 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Tests/Constants.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All.Tests; 2 | 3 | public static class Constants 4 | { 5 | public const string MODELS_BASE_DIR = "../../../models"; 6 | public const string LLAMA_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-l13b-snoozy.bin"; 7 | public const string GPTJ_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-gpt4all-j-v1.3-groovy.bin"; 8 | public const string MPT_MODEL_PATH = $"{MODELS_BASE_DIR}/ggml-mpt-7b-chat.bin"; 9 | } 10 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Tests/ModelFactoryTests.cs: -------------------------------------------------------------------------------- 1 | using Xunit; 2 | 3 | namespace Gpt4All.Tests; 4 | 5 | public class ModelFactoryTests 6 | { 7 | private readonly Gpt4AllModelFactory _modelFactory; 8 | 9 | public ModelFactoryTests() 10 | { 11 | _modelFactory = new Gpt4AllModelFactory(); 12 | } 13 | 14 | [Fact] 15 | [Trait(Traits.SkipOnCI, "True")] 16 | public void CanLoadLlamaModel() 17 | { 18 | using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH); 19 | } 20 | 21 | [Fact] 22 | [Trait(Traits.SkipOnCI, "True")] 23 | public void CanLoadGptjModel() 24 | { 25 | using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH); 26 | } 27 | 28 | [Fact] 29 | [Trait(Traits.SkipOnCI, "True")] 30 | public void CanLoadMptModel() 31 | { 32 | using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Tests/NativeLibraryLoaderTests.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using Gpt4All.LibraryLoader; 3 | using Xunit; 4 | 5 | namespace Gpt4All.Tests; 6 | 7 | public class NativeLibraryLoaderTests 8 | { 9 | [Fact] 10 | public void NativeLibraryShouldLoad() 11 | { 12 | var result = NativeLibraryLoader.LoadNativeLibrary(bypassLoading: false); 13 | Assert.True(result.IsSuccess); 14 | } 15 | 16 | private const string LLModelLib = "libllmodel.{0}"; 17 | 18 | [PlatformSpecificFact(Platforms.Windows)] 19 | public void NativeLibraryShouldLoad_Windows() 20 | { 21 | var libraryLoader = new WindowsLibraryLoader(); 22 | 23 | var libraryPath = Path.Combine( 24 | Environment.CurrentDirectory, 25 | string.Format(LLModelLib, "dll")); 26 | 27 | var result = libraryLoader.OpenLibrary(libraryPath); 28 | Assert.True(result.IsSuccess); 29 | } 30 | 31 | [PlatformSpecificFact(Platforms.Linux)] 32 | public void NativeLibraryShouldLoad_Linux() 33 | { 34 | var libraryLoader = new LinuxLibraryLoader(); 35 | 36 | var libraryPath = Path.Combine( 37 | Environment.CurrentDirectory, 38 | string.Format(LLModelLib, "so")); 39 | 40 | var result = libraryLoader.OpenLibrary(libraryPath); 41 | Assert.True(result.IsSuccess); 42 | } 43 | 44 | [PlatformSpecificFact(Platforms.MacOS)] 45 | public void NativeLibraryShouldLoad_MacOS() 46 | { 47 | var libraryLoader = new MacOsLibraryLoader(); 48 | 49 | var libraryPath = Path.Combine( 50 | Environment.CurrentDirectory, 51 | string.Format(LLModelLib, "dylib")); 52 | 53 | var result = libraryLoader.OpenLibrary(libraryPath); 54 | Assert.True(result.IsSuccess); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Tests/PlatformSpecificFactAttribute.cs: -------------------------------------------------------------------------------- 1 | using Xunit; 2 | 3 | namespace Gpt4All.Tests; 4 | 5 | public static class Platforms 6 | { 7 | public const string Windows = "windows"; 8 | public const string Linux = "linux"; 9 | public const string MacOS = "macOS"; 10 | } 11 | 12 | /// 13 | /// This attribute ensures the Fact is only run on the specified platform. 14 | /// 15 | /// 16 | /// for info about the platform string. 17 | /// 18 | public class PlatformSpecificFactAttribute : FactAttribute 19 | { 20 | public PlatformSpecificFactAttribute(string platform) 21 | { 22 | if (!OperatingSystem.IsOSPlatform(platform)) 23 | { 24 | Skip = $"Test only runs on {platform}."; 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.Tests/Traits.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All.Tests; 2 | 3 | public static class Traits 4 | { 5 | public const string SkipOnCI = "SKIP_ON_CI"; 6 | } 7 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.33516.290 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All.Samples", "Gpt4All.Samples\Gpt4All.Samples.csproj", "{59864AE8-E45D-42F7-A7C0-1308EF185F39}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{DA396C11-CEAD-4368-8234-FB12255A30D2}" 9 | ProjectSection(SolutionItems) = preProject 10 | .gitignore = .gitignore 11 | build_linux.sh = build_linux.sh 12 | build_win-mingw.ps1 = build_win-mingw.ps1 13 | build_win-msvc.ps1 = build_win-msvc.ps1 14 | docs\gpt4all_csharp.md = docs\gpt4all_csharp.md 15 | README.md = README.md 16 | EndProjectSection 17 | EndProject 18 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Gpt4All", "Gpt4All\Gpt4All.csproj", "{6015C62B-2008-426B-A334-740D6F1FE38B}" 19 | EndProject 20 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gpt4All.Tests", "Gpt4All.Tests\Gpt4All.Tests.csproj", "{33A72341-52C1-4EAE-878B-A98BC77F686A}" 21 | EndProject 22 | Global 23 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 24 | Debug|Any CPU = Debug|Any CPU 25 | Release|Any CPU = Release|Any CPU 26 | EndGlobalSection 27 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 28 | {59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {59864AE8-E45D-42F7-A7C0-1308EF185F39}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {59864AE8-E45D-42F7-A7C0-1308EF185F39}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {6015C62B-2008-426B-A334-740D6F1FE38B}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {6015C62B-2008-426B-A334-740D6F1FE38B}.Release|Any CPU.Build.0 = Release|Any CPU 36 | {33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {33A72341-52C1-4EAE-878B-A98BC77F686A}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {33A72341-52C1-4EAE-878B-A98BC77F686A}.Release|Any CPU.Build.0 = Release|Any CPU 40 | EndGlobalSection 41 | GlobalSection(SolutionProperties) = preSolution 42 | HideSolutionNode = FALSE 43 | EndGlobalSection 44 | GlobalSection(ExtensibilityGlobals) = postSolution 45 | SolutionGuid = {17632027-F4C2-4903-B88F-310CE3DE386B} 46 | EndGlobalSection 47 | EndGlobal 48 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All.Bindings; 2 | 3 | /// 4 | /// Represents the interface exposed by the universal wrapper for GPT4All language models built around llmodel C-API. 5 | /// 6 | public interface ILLModel : IDisposable 7 | { 8 | ulong GetStateSizeBytes(); 9 | 10 | int GetThreadCount(); 11 | 12 | void SetThreadCount(int threadCount); 13 | 14 | bool IsLoaded(); 15 | 16 | bool Load(string modelPath); 17 | 18 | void Prompt( 19 | string text, 20 | LLModelPromptContext context, 21 | Func? promptCallback = null, 22 | Func? responseCallback = null, 23 | Func? recalculateCallback = null, 24 | CancellationToken cancellationToken = default); 25 | 26 | unsafe ulong RestoreStateData(byte* destination); 27 | 28 | unsafe ulong SaveStateData(byte* source); 29 | } 30 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Bindings/NativeTypeNameAttribute.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | 3 | namespace Gpt4All.Bindings; 4 | 5 | /// Defines the type of a member as it was used in the native signature. 6 | [AttributeUsage(AttributeTargets.Struct | AttributeTargets.Enum | AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.ReturnValue, AllowMultiple = false, Inherited = true)] 7 | [Conditional("DEBUG")] 8 | internal sealed partial class NativeTypeNameAttribute : Attribute 9 | { 10 | private readonly string _name; 11 | 12 | /// Initializes a new instance of the class. 13 | /// The name of the type that was used in the native signature. 14 | public NativeTypeNameAttribute(string name) 15 | { 16 | _name = name; 17 | } 18 | 19 | /// Gets the name of the type that was used in the native signature. 20 | public string Name => _name; 21 | } -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs: -------------------------------------------------------------------------------- 1 | using Gpt4All.Bindings; 2 | 3 | namespace Gpt4All; 4 | 5 | internal static class LLPromptContextExtensions 6 | { 7 | public static string Dump(this LLModelPromptContext context) 8 | { 9 | var ctx = context.UnderlyingContext; 10 | return @$" 11 | {{ 12 | logits_size = {ctx.logits_size} 13 | tokens_size = {ctx.tokens_size} 14 | n_past = {ctx.n_past} 15 | n_ctx = {ctx.n_ctx} 16 | n_predict = {ctx.n_predict} 17 | top_k = {ctx.top_k} 18 | top_p = {ctx.top_p} 19 | min_p = {ctx.min_p} 20 | temp = {ctx.temp} 21 | n_batch = {ctx.n_batch} 22 | repeat_penalty = {ctx.repeat_penalty} 23 | repeat_last_n = {ctx.repeat_last_n} 24 | context_erase = {ctx.context_erase} 25 | }}"; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs: -------------------------------------------------------------------------------- 1 | using Gpt4All.Bindings; 2 | 3 | namespace Gpt4All; 4 | 5 | public static class PredictRequestOptionsExtensions 6 | { 7 | public static LLModelPromptContext ToPromptContext(this PredictRequestOptions opts) 8 | { 9 | return new LLModelPromptContext 10 | { 11 | LogitsSize = opts.LogitsSize, 12 | TokensSize = opts.TokensSize, 13 | TopK = opts.TopK, 14 | TopP = opts.TopP, 15 | MinP = opts.MinP, 16 | PastNum = opts.PastConversationTokensNum, 17 | RepeatPenalty = opts.RepeatPenalty, 18 | Temperature = opts.Temperature, 19 | RepeatLastN = opts.RepeatLastN, 20 | Batches = opts.Batches, 21 | ContextErase = opts.ContextErase, 22 | ContextSize = opts.ContextSize, 23 | TokensToPredict = opts.TokensToPredict 24 | }; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/GenLLModelBindings.rsp: -------------------------------------------------------------------------------- 1 | --config 2 | exclude-funcs-with-body 3 | --with-access-specifier 4 | *=Public 5 | --include-directory 6 | ..\..\..\gpt4all-backend\ 7 | --file 8 | ..\..\..\gpt4all-backend\llmodel_c.h 9 | --libraryPath 10 | libllmodel 11 | --remap 12 | sbyte*=IntPtr 13 | void*=IntPtr 14 | --namespace 15 | Gpt4All.Bindings 16 | --methodClassName 17 | NativeMethods 18 | --output 19 | .\Bindings\NativeMethods.cs 20 | --output-mode 21 | CSharp -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | enable 4 | enable 5 | true 6 | true 7 | net8.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | true 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/LibraryLoader/ILibraryLoader.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All.LibraryLoader; 2 | 3 | public interface ILibraryLoader 4 | { 5 | LoadResult OpenLibrary(string? fileName); 6 | } 7 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LinuxLibraryLoader.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.InteropServices; 2 | 3 | namespace Gpt4All.LibraryLoader; 4 | 5 | internal class LinuxLibraryLoader : ILibraryLoader 6 | { 7 | #pragma warning disable CA2101 8 | [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")] 9 | #pragma warning restore CA2101 10 | public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags); 11 | 12 | #pragma warning disable CA2101 13 | [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")] 14 | #pragma warning restore CA2101 15 | public static extern IntPtr NativeOpenLibraryLibdl2(string? filename, int flags); 16 | 17 | [DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")] 18 | public static extern IntPtr GetLoadError(); 19 | 20 | [DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")] 21 | public static extern IntPtr GetLoadError2(); 22 | 23 | public LoadResult OpenLibrary(string? fileName) 24 | { 25 | IntPtr loadedLib; 26 | try 27 | { 28 | // open with rtls lazy flag 29 | loadedLib = NativeOpenLibraryLibdl2(fileName, 0x00001); 30 | } 31 | catch (DllNotFoundException) 32 | { 33 | loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001); 34 | } 35 | 36 | if (loadedLib == IntPtr.Zero) 37 | { 38 | string errorMessage; 39 | try 40 | { 41 | errorMessage = Marshal.PtrToStringAnsi(GetLoadError2()) ?? "Unknown error"; 42 | } 43 | catch (DllNotFoundException) 44 | { 45 | errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error"; 46 | } 47 | 48 | return LoadResult.Failure(errorMessage); 49 | } 50 | 51 | return LoadResult.Success; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All.LibraryLoader; 2 | 3 | public class LoadResult 4 | { 5 | private LoadResult(bool isSuccess, string? errorMessage) 6 | { 7 | IsSuccess = isSuccess; 8 | ErrorMessage = errorMessage; 9 | } 10 | 11 | public static LoadResult Success { get; } = new(true, null); 12 | 13 | public static LoadResult Failure(string errorMessage) 14 | { 15 | return new(false, errorMessage); 16 | } 17 | 18 | public bool IsSuccess { get; } 19 | public string? ErrorMessage { get; } 20 | } 21 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/LibraryLoader/MacOsLibraryLoader.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.InteropServices; 2 | 3 | namespace Gpt4All.LibraryLoader; 4 | 5 | internal class MacOsLibraryLoader : ILibraryLoader 6 | { 7 | #pragma warning disable CA2101 8 | [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")] 9 | #pragma warning restore CA2101 10 | public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags); 11 | 12 | [DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")] 13 | public static extern IntPtr GetLoadError(); 14 | 15 | public LoadResult OpenLibrary(string? fileName) 16 | { 17 | var loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001); 18 | 19 | if (loadedLib == IntPtr.Zero) 20 | { 21 | var errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error"; 22 | 23 | return LoadResult.Failure(errorMessage); 24 | } 25 | 26 | return LoadResult.Success; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/LibraryLoader/WindowsLibraryLoader.cs: -------------------------------------------------------------------------------- 1 | using System.ComponentModel; 2 | using System.Runtime.InteropServices; 3 | 4 | namespace Gpt4All.LibraryLoader; 5 | 6 | internal class WindowsLibraryLoader : ILibraryLoader 7 | { 8 | public LoadResult OpenLibrary(string? fileName) 9 | { 10 | var loadedLib = LoadLibrary(fileName); 11 | 12 | if (loadedLib == IntPtr.Zero) 13 | { 14 | var errorCode = Marshal.GetLastWin32Error(); 15 | var errorMessage = new Win32Exception(errorCode).Message; 16 | return LoadResult.Failure(errorMessage); 17 | } 18 | 19 | return LoadResult.Success; 20 | } 21 | 22 | [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Auto)] 23 | private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPWStr)] string? lpFileName); 24 | } 25 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Model/DefaultPromptFormatter.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public class DefaultPromptFormatter : IPromptFormatter 4 | { 5 | public string FormatPrompt(string prompt) 6 | { 7 | return $""" 8 | ### Instruction: 9 | The prompt below is a question to answer, a task to complete, or a conversation 10 | to respond to; decide which and write an appropriate response. 11 | ### Prompt: 12 | {prompt} 13 | ### Response: 14 | """; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using Microsoft.Extensions.Logging.Abstractions; 3 | using Microsoft.Extensions.Logging; 4 | using Gpt4All.Bindings; 5 | using Gpt4All.LibraryLoader; 6 | using System.Runtime.InteropServices; 7 | 8 | namespace Gpt4All; 9 | 10 | public class Gpt4AllModelFactory : IGpt4AllModelFactory 11 | { 12 | private readonly ILoggerFactory _loggerFactory; 13 | private readonly ILogger _logger; 14 | private static bool bypassLoading; 15 | private static string? libraryPath; 16 | 17 | private static readonly Lazy libraryLoaded = new(() => 18 | { 19 | return NativeLibraryLoader.LoadNativeLibrary(Gpt4AllModelFactory.libraryPath, Gpt4AllModelFactory.bypassLoading); 20 | }, true); 21 | 22 | public Gpt4AllModelFactory(string? libraryPath = default, bool bypassLoading = true, ILoggerFactory? loggerFactory = null) 23 | { 24 | _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance; 25 | _logger = _loggerFactory.CreateLogger(); 26 | Gpt4AllModelFactory.libraryPath = libraryPath; 27 | Gpt4AllModelFactory.bypassLoading = bypassLoading; 28 | 29 | if (!libraryLoaded.Value.IsSuccess) 30 | { 31 | throw new Exception($"Failed to load native gpt4all library. Error: {libraryLoaded.Value.ErrorMessage}"); 32 | } 33 | } 34 | 35 | private Gpt4All CreateModel(string modelPath) 36 | { 37 | _logger.LogInformation("Creating model path={ModelPath}", modelPath); 38 | IntPtr error; 39 | var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error); 40 | if (error != IntPtr.Zero) 41 | { 42 | throw new Exception(Marshal.PtrToStringAnsi(error)); 43 | } 44 | _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle); 45 | _logger.LogInformation("Model loading started"); 46 | var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048, 100); 47 | _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully); 48 | if (!loadedSuccessfully) 49 | { 50 | throw new Exception($"Failed to load model: '{modelPath}'"); 51 | } 52 | 53 | var logger = _loggerFactory.CreateLogger(); 54 | var underlyingModel = LLModel.Create(handle, logger: logger); 55 | 56 | Debug.Assert(underlyingModel.IsLoaded()); 57 | 58 | return new Gpt4All(underlyingModel, logger: logger); 59 | } 60 | 61 | public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath); 62 | } 63 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModel.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public interface IGpt4AllModel : ITextPrediction, IDisposable 4 | { 5 | /// 6 | /// The prompt formatter used to format the prompt before 7 | /// feeding it to the model, if null no transformation is applied 8 | /// 9 | IPromptFormatter? PromptFormatter { get; set; } 10 | } 11 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Model/IGpt4AllModelFactory.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public interface IGpt4AllModelFactory 4 | { 5 | IGpt4AllModel LoadModel(string modelPath); 6 | } 7 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Model/IPromptFormatter.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | /// 4 | /// Formats a prompt 5 | /// 6 | public interface IPromptFormatter 7 | { 8 | /// 9 | /// Format the provided prompt 10 | /// 11 | /// the input prompt 12 | /// The formatted prompt 13 | string FormatPrompt(string prompt); 14 | } 15 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Model/ModelOptions.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public record ModelOptions 4 | { 5 | public int Threads { get; init; } = 4; 6 | } 7 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPrediction.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | /// 4 | /// Interface for text prediction services 5 | /// 6 | public interface ITextPrediction 7 | { 8 | /// 9 | /// Get prediction results for the prompt and provided options. 10 | /// 11 | /// The text to complete 12 | /// The prediction settings 13 | /// The for cancellation requests. The default is . 14 | /// The prediction result generated by the model 15 | Task GetPredictionAsync( 16 | string text, 17 | PredictRequestOptions opts, 18 | CancellationToken cancellation = default); 19 | 20 | /// 21 | /// Get streaming prediction results for the prompt and provided options. 22 | /// 23 | /// The text to complete 24 | /// The prediction settings 25 | /// The for cancellation requests. The default is . 26 | /// The prediction result generated by the model 27 | Task GetStreamingPredictionAsync( 28 | string text, 29 | PredictRequestOptions opts, 30 | CancellationToken cancellationToken = default); 31 | } 32 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPredictionResult.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public interface ITextPredictionResult 4 | { 5 | bool Success { get; } 6 | 7 | string? ErrorMessage { get; } 8 | 9 | Task GetPredictionAsync(CancellationToken cancellationToken = default); 10 | } 11 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Prediction/ITextPredictionStreamingResult.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public interface ITextPredictionStreamingResult : ITextPredictionResult 4 | { 5 | IAsyncEnumerable GetPredictionStreamingAsync(CancellationToken cancellationToken = default); 6 | } 7 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Prediction/PredictRequestOptions.cs: -------------------------------------------------------------------------------- 1 | namespace Gpt4All; 2 | 3 | public record PredictRequestOptions 4 | { 5 | public nuint LogitsSize { get; init; } = 0; 6 | 7 | public nuint TokensSize { get; init; } = 0; 8 | 9 | public int PastConversationTokensNum { get; init; } = 0; 10 | 11 | public int ContextSize { get; init; } = 1024; 12 | 13 | public int TokensToPredict { get; init; } = 128; 14 | 15 | public int TopK { get; init; } = 40; 16 | 17 | public float TopP { get; init; } = 0.9f; 18 | 19 | public float MinP { get; init; } = 0.0f; 20 | 21 | public float Temperature { get; init; } = 0.1f; 22 | 23 | public int Batches { get; init; } = 8; 24 | 25 | public float RepeatPenalty { get; init; } = 1.2f; 26 | 27 | public int RepeatLastN { get; init; } = 10; 28 | 29 | public float ContextErase { get; init; } = 0.5f; 30 | 31 | public static readonly PredictRequestOptions Defaults = new(); 32 | } 33 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Prediction/TextPredictionResult.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace Gpt4All; 4 | 5 | public record TextPredictionResult : ITextPredictionResult 6 | { 7 | private readonly StringBuilder _result; 8 | 9 | public bool Success { get; internal set; } = true; 10 | 11 | public string? ErrorMessage { get; internal set; } 12 | 13 | internal TextPredictionResult() 14 | { 15 | _result = new StringBuilder(); 16 | } 17 | 18 | internal void Append(string token) 19 | { 20 | _result.Append(token); 21 | } 22 | 23 | public Task GetPredictionAsync(CancellationToken cancellationToken = default) 24 | { 25 | return Task.FromResult(_result.ToString()); 26 | } 27 | } -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/Prediction/TextPredictionStreamingResult.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using System.Threading.Channels; 3 | 4 | namespace Gpt4All; 5 | 6 | public record TextPredictionStreamingResult : ITextPredictionStreamingResult 7 | { 8 | private readonly Channel _channel; 9 | 10 | public bool Success { get; internal set; } = true; 11 | 12 | public string? ErrorMessage { get; internal set; } 13 | 14 | public Task Completion => _channel.Reader.Completion; 15 | 16 | internal TextPredictionStreamingResult() 17 | { 18 | _channel = Channel.CreateUnbounded(); 19 | } 20 | 21 | internal bool Append(string token) 22 | { 23 | return _channel.Writer.TryWrite(token); 24 | } 25 | 26 | internal void Complete() 27 | { 28 | _channel.Writer.Complete(); 29 | } 30 | 31 | public async Task GetPredictionAsync(CancellationToken cancellationToken = default) 32 | { 33 | var sb = new StringBuilder(); 34 | 35 | var tokens = GetPredictionStreamingAsync(cancellationToken).ConfigureAwait(false); 36 | 37 | await foreach (var token in tokens) 38 | { 39 | sb.Append(token); 40 | } 41 | 42 | return sb.ToString(); 43 | } 44 | 45 | public IAsyncEnumerable GetPredictionStreamingAsync(CancellationToken cancellationToken = default) 46 | { 47 | return _channel.Reader.ReadAllAsync(cancellationToken); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/Gpt4All/gen_bindings.ps1: -------------------------------------------------------------------------------- 1 | ClangSharpPInvokeGenerator @(Get-Content .\GenLLModelBindings.rsp) -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/build_linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mkdir -p runtimes 3 | rm -rf runtimes/linux-x64 4 | mkdir -p runtimes/linux-x64/native 5 | mkdir runtimes/linux-x64/build 6 | cmake -S ../../gpt4all-backend -B runtimes/linux-x64/build 7 | cmake --build runtimes/linux-x64/build --parallel --config Release 8 | cp runtimes/linux-x64/build/libllmodel.so runtimes/linux-x64/native/libllmodel.so 9 | cp runtimes/linux-x64/build/libgptj*.so runtimes/linux-x64/native/ 10 | cp runtimes/linux-x64/build/libllama*.so runtimes/linux-x64/native/ 11 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/build_win-mingw.ps1: -------------------------------------------------------------------------------- 1 | $ROOT_DIR = '.\runtimes\win-x64' 2 | $BUILD_DIR = '.\runtimes\win-x64\build\mingw' 3 | $LIBS_DIR = '.\runtimes\win-x64\native' 4 | 5 | # cleanup env 6 | Remove-Item -Force -Recurse $ROOT_DIR -ErrorAction SilentlyContinue | Out-Null 7 | mkdir $BUILD_DIR | Out-Null 8 | mkdir $LIBS_DIR | Out-Null 9 | 10 | # build 11 | cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR 12 | cmake --build $BUILD_DIR --parallel --config Release 13 | 14 | # copy native dlls 15 | cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR 16 | cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR 17 | -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/build_win-msvc.ps1: -------------------------------------------------------------------------------- 1 | Remove-Item -Force -Recurse .\runtimes\win-x64\msvc -ErrorAction SilentlyContinue 2 | mkdir .\runtimes\win-x64\msvc\build | Out-Null 3 | cmake -G "Visual Studio 17 2022" -A X64 -S ..\..\gpt4all-backend -B .\runtimes\win-x64\msvc\build 4 | cmake --build .\runtimes\win-x64\msvc\build --parallel --config Release 5 | cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64 6 | mv .\runtimes\win-x64\llmodel.dll .\runtimes\win-x64\libllmodel.dll -------------------------------------------------------------------------------- /gpt4all-bindings/csharp/docs/gpt4all_csharp.md: -------------------------------------------------------------------------------- 1 | # GPT4All C# API -------------------------------------------------------------------------------- /gpt4all-bindings/golang/README.md: -------------------------------------------------------------------------------- 1 | # GPT4All Golang bindings 2 | 3 | The golang bindings have been tested on: 4 | - MacOS 5 | - Linux 6 | 7 | ### Usage 8 | 9 | ``` 10 | import ( 11 | "github.com/nomic-ai/gpt4all/gpt4all-bindings/golang" 12 | ) 13 | 14 | func main() { 15 | // Load the model 16 | model, err := gpt4all.New("model.bin", gpt4all.SetModelType(gpt4all.GPTJType)) 17 | if err != nil { 18 | panic(err) 19 | } 20 | defer model.Free() 21 | 22 | model.SetTokenCallback(func(s string) bool { 23 | fmt.Print(s) 24 | return true 25 | }) 26 | 27 | _, err = model.Predict("Here are 4 steps to create a website:", "", "", gpt4all.SetTemperature(0.1)) 28 | if err != nil { 29 | panic(err) 30 | } 31 | } 32 | ``` 33 | 34 | ## Building 35 | 36 | In order to use the bindings you will need to build `libgpt4all.a`: 37 | 38 | ``` 39 | git clone --recurse-submodules https://github.com/nomic-ai/gpt4all 40 | cd gpt4all/gpt4all-bindings/golang 41 | make libgpt4all.a 42 | ``` 43 | 44 | To use the bindings in your own software: 45 | 46 | - Import `github.com/nomic-ai/gpt4all/gpt4all-bindings/golang`; 47 | - Compile `libgpt4all.a` (you can use `make libgpt4all.a` in the bindings/go directory); 48 | - Link your go binary by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH` to point to the `binding.h` file directory and `libgpt4all.a` file directory respectively. 49 | - Note: you need to have *.so/*.dynlib/*.dll files of the implementation nearby the binary produced by the binding in order to make this to work 50 | 51 | ## Testing 52 | 53 | To run tests, run `make test`: 54 | 55 | ``` 56 | git clone https://github.com/nomic-ai/gpt4all 57 | cd gpt4all/gpt4all-bindings/golang 58 | make test 59 | ``` 60 | -------------------------------------------------------------------------------- /gpt4all-bindings/golang/binding.h: -------------------------------------------------------------------------------- 1 | #ifdef __cplusplus 2 | extern "C" { 3 | #endif 4 | 5 | #include 6 | 7 | void* load_model(const char *fname, int n_threads); 8 | 9 | void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 10 | void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 11 | int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase); 12 | 13 | void free_model(void *state_ptr); 14 | 15 | extern unsigned char getTokenCallback(void *, char *); 16 | 17 | #ifdef __cplusplus 18 | } 19 | #endif 20 | -------------------------------------------------------------------------------- /gpt4all-bindings/golang/example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | "runtime" 10 | "strings" 11 | 12 | gpt4all "github.com/nomic-ai/gpt4all/gpt4all-bindings/golang" 13 | ) 14 | 15 | var ( 16 | threads = 4 17 | tokens = 128 18 | ) 19 | 20 | func main() { 21 | var model string 22 | 23 | flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 24 | flags.StringVar(&model, "m", "./models/7B/ggml-model-q4_0.bin", "path to q4_0.bin model file to load") 25 | flags.IntVar(&threads, "t", runtime.NumCPU(), "number of threads to use during computation") 26 | flags.IntVar(&tokens, "n", 512, "number of tokens to predict") 27 | 28 | err := flags.Parse(os.Args[1:]) 29 | if err != nil { 30 | fmt.Printf("Parsing program arguments failed: %s", err) 31 | os.Exit(1) 32 | } 33 | l, err := gpt4all.New(model, gpt4all.SetThreads(threads)) 34 | if err != nil { 35 | fmt.Println("Loading the model failed:", err.Error()) 36 | os.Exit(1) 37 | } 38 | fmt.Printf("Model loaded successfully.\n") 39 | 40 | l.SetTokenCallback(func(token string) bool { 41 | fmt.Print(token) 42 | return true 43 | }) 44 | 45 | reader := bufio.NewReader(os.Stdin) 46 | 47 | for { 48 | text := readMultiLineInput(reader) 49 | 50 | _, err := l.Predict(text, "", "", gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86)) 51 | if err != nil { 52 | panic(err) 53 | } 54 | fmt.Printf("\n\n") 55 | } 56 | } 57 | 58 | // readMultiLineInput reads input until an empty line is entered. 59 | func readMultiLineInput(reader *bufio.Reader) string { 60 | var lines []string 61 | fmt.Print(">>> ") 62 | 63 | for { 64 | line, err := reader.ReadString('\n') 65 | if err != nil { 66 | if err == io.EOF { 67 | os.Exit(0) 68 | } 69 | fmt.Printf("Reading the prompt failed: %s", err) 70 | os.Exit(1) 71 | } 72 | 73 | if len(strings.TrimSpace(line)) == 0 { 74 | break 75 | } 76 | 77 | lines = append(lines, line) 78 | } 79 | 80 | text := strings.Join(lines, "") 81 | return text 82 | } 83 | -------------------------------------------------------------------------------- /gpt4all-bindings/golang/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nomic-ai/gpt4all/gpt4all-bindings/golang 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/onsi/ginkgo/v2 v2.9.4 7 | github.com/onsi/gomega v1.27.6 8 | ) 9 | 10 | require ( 11 | github.com/go-logr/logr v1.2.4 // indirect 12 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect 13 | github.com/google/go-cmp v0.5.9 // indirect 14 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect 15 | golang.org/x/net v0.9.0 // indirect 16 | golang.org/x/sys v0.7.0 // indirect 17 | golang.org/x/text v0.9.0 // indirect 18 | golang.org/x/tools v0.8.0 // indirect 19 | gopkg.in/yaml.v3 v3.0.1 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /gpt4all-bindings/golang/gpt4all_suite_test.go: -------------------------------------------------------------------------------- 1 | package gpt4all_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestGPT(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "go-gpt4all-j test suite") 13 | } 14 | -------------------------------------------------------------------------------- /gpt4all-bindings/golang/gpt4all_test.go: -------------------------------------------------------------------------------- 1 | package gpt4all_test 2 | 3 | import ( 4 | . "github.com/nomic-ai/gpt4all/gpt4all-bindings/golang" 5 | . "github.com/onsi/ginkgo/v2" 6 | . "github.com/onsi/gomega" 7 | ) 8 | 9 | var _ = Describe("LLama binding", func() { 10 | Context("Declaration", func() { 11 | It("fails with no model", func() { 12 | model, err := New("not-existing") 13 | Expect(err).To(HaveOccurred()) 14 | Expect(model).To(BeNil()) 15 | }) 16 | }) 17 | }) 18 | -------------------------------------------------------------------------------- /gpt4all-bindings/java/.gitignore: -------------------------------------------------------------------------------- 1 | # Make sure native directory never gets commited to git for the project. 2 | /src/main/resources/native 3 | 4 | # IntelliJ project file 5 | *.iml -------------------------------------------------------------------------------- /gpt4all-bindings/java/TODO.md: -------------------------------------------------------------------------------- 1 | ## Needed 2 | 1. Integrate with circleci build pipeline like the C# binding. 3 | 4 | ## These are just ideas 5 | 1. Better Chat completions function. 6 | 2. Chat completion that returns result in OpenAI compatible format. 7 | -------------------------------------------------------------------------------- /gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/PromptIsTooLongException.java: -------------------------------------------------------------------------------- 1 | package com.hexadevlabs.gpt4all; 2 | 3 | public class PromptIsTooLongException extends RuntimeException { 4 | public PromptIsTooLongException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example1.java: -------------------------------------------------------------------------------- 1 | package com.hexadevlabs.gpt4all; 2 | 3 | import java.nio.file.Path; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | /** 8 | * GPTJ chat completion, multiple messages 9 | */ 10 | public class Example1 { 11 | public static void main(String[] args) { 12 | 13 | // Optionally in case override to location of shared libraries is necessary 14 | //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 15 | 16 | try ( LLModel gptjModel = new LLModel(Path.of("C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-gpt4all-j-v1.3-groovy.bin")) ){ 17 | 18 | LLModel.GenerationConfig config = LLModel.config() 19 | .withNPredict(4096).build(); 20 | 21 | gptjModel.chatCompletion( 22 | List.of(Map.of("role", "user", "content", "Add 2+2"), 23 | Map.of("role", "assistant", "content", "4"), 24 | Map.of("role", "user", "content", "Multiply 4 * 5")), config, true, true); 25 | 26 | } catch (Exception e) { 27 | throw new RuntimeException(e); 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example2.java: -------------------------------------------------------------------------------- 1 | package com.hexadevlabs.gpt4all; 2 | 3 | import java.nio.file.Path; 4 | 5 | /** 6 | * Generation with MPT model 7 | */ 8 | public class Example2 { 9 | public static void main(String[] args) { 10 | 11 | String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:"; 12 | 13 | // Optionally in case override to location of shared libraries is necessary 14 | //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 15 | 16 | try (LLModel mptModel = new LLModel(Path.of("C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-mpt-7b-instruct.bin"))) { 17 | 18 | LLModel.GenerationConfig config = 19 | LLModel.config() 20 | .withNPredict(4096) 21 | .withRepeatLastN(64) 22 | .build(); 23 | 24 | mptModel.generate(prompt, config, true); 25 | 26 | } catch (Exception e) { 27 | throw new RuntimeException(e); 28 | } 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example3.java: -------------------------------------------------------------------------------- 1 | package com.hexadevlabs.gpt4all; 2 | 3 | import jnr.ffi.LibraryLoader; 4 | 5 | import java.nio.file.Path; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | * GPTJ chat completion with system message 11 | */ 12 | public class Example3 { 13 | public static void main(String[] args) { 14 | 15 | // Optionally in case override to location of shared libraries is necessary 16 | //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 17 | 18 | try ( LLModel gptjModel = new LLModel(Path.of("C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\ggml-gpt4all-j-v1.3-groovy.bin")) ){ 19 | 20 | LLModel.GenerationConfig config = LLModel.config() 21 | .withNPredict(4096).build(); 22 | 23 | // String result = gptjModel.generate(prompt, config, true); 24 | gptjModel.chatCompletion( 25 | List.of(Map.of("role", "system", "content", "You are a helpful assistant"), 26 | Map.of("role", "user", "content", "Add 2+2")), config, true, true); 27 | 28 | 29 | } catch (Exception e) { 30 | throw new RuntimeException(e); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example4.java: -------------------------------------------------------------------------------- 1 | package com.hexadevlabs.gpt4all; 2 | 3 | import java.nio.file.Path; 4 | 5 | public class Example4 { 6 | 7 | public static void main(String[] args) { 8 | 9 | String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:"; 10 | // The emoji is poop emoji. The Unicode character is encoded as surrogate pair for Java string. 11 | // LLM should correctly identify it as poop emoji in the description 12 | //String prompt = "### Human:\nDescribe the meaning of this emoji \uD83D\uDCA9\n### Assistant:"; 13 | //String prompt = "### Human:\nOutput the unicode character of smiley face emoji\n### Assistant:"; 14 | 15 | // Optionally in case override to location of shared libraries is necessary 16 | //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 17 | 18 | String model = "ggml-vicuna-7b-1.1-q4_2.bin"; 19 | //String model = "ggml-gpt4all-j-v1.3-groovy.bin"; 20 | //String model = "ggml-mpt-7b-instruct.bin"; 21 | String basePath = "C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\"; 22 | //String basePath = "/Users/fzaslavs/Library/Application Support/nomic.ai/GPT4All/"; 23 | 24 | try (LLModel mptModel = new LLModel(Path.of(basePath + model))) { 25 | 26 | LLModel.GenerationConfig config = 27 | LLModel.config() 28 | .withNPredict(4096) 29 | .withRepeatLastN(64) 30 | .build(); 31 | 32 | 33 | String result = mptModel.generate(prompt, config, true); 34 | 35 | System.out.println("Code points:"); 36 | result.codePoints().forEach(System.out::println); 37 | 38 | 39 | } catch (Exception e) { 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /gpt4all-bindings/java/src/test/java/com/hexadevlabs/gpt4all/Example5.java: -------------------------------------------------------------------------------- 1 | package com.hexadevlabs.gpt4all; 2 | 3 | import java.nio.file.Path; 4 | 5 | public class Example5 { 6 | 7 | public static void main(String[] args) { 8 | 9 | // String prompt = "### Human:\nWhat is the meaning of life\n### Assistant:"; 10 | // The emoji is poop emoji. The Unicode character is encoded as surrogate pair for Java string. 11 | // LLM should correctly identify it as poop emoji in the description 12 | //String prompt = "### Human:\nDescribe the meaning of this emoji \uD83D\uDCA9\n### Assistant:"; 13 | //String prompt = "### Human:\nOutput the unicode character of smiley face emoji\n### Assistant:"; 14 | 15 | // Optionally in case override to location of shared libraries is necessary 16 | //LLModel.LIBRARY_SEARCH_PATH = "C:\\Users\\felix\\gpt4all\\lib\\"; 17 | StringBuffer b = new StringBuffer(); 18 | b.append("The ".repeat(2060)); 19 | String prompt = b.toString(); 20 | 21 | 22 | String model = "ggml-vicuna-7b-1.1-q4_2.bin"; 23 | //String model = "ggml-gpt4all-j-v1.3-groovy.bin"; 24 | //String model = "ggml-mpt-7b-instruct.bin"; 25 | String basePath = "C:\\Users\\felix\\AppData\\Local\\nomic.ai\\GPT4All\\"; 26 | //String basePath = "/Users/fzaslavs/Library/Application Support/nomic.ai/GPT4All/"; 27 | 28 | try (LLModel mptModel = new LLModel(Path.of(basePath + model))) { 29 | 30 | LLModel.GenerationConfig config = 31 | LLModel.config() 32 | .withNPredict(4096) 33 | .withRepeatLastN(64) 34 | .build(); 35 | 36 | String result = mptModel.generate(prompt, config, true); 37 | 38 | System.out.println("Code points:"); 39 | result.codePoints().forEach(System.out::println); 40 | 41 | 42 | } catch (Exception e) { 43 | System.out.println(e.getMessage()); 44 | throw new RuntimeException(e); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /gpt4all-bindings/python/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch 3 | 4 | line_length=120 5 | include_trailing_comma=True 6 | multi_line_output=3 7 | use_parentheses=True -------------------------------------------------------------------------------- /gpt4all-bindings/python/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Nomic, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /gpt4all-bindings/python/MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include gpt4all/llmodel_DO_NOT_MODIFY * -------------------------------------------------------------------------------- /gpt4all-bindings/python/docs/assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/python/docs/assets/favicon.ico -------------------------------------------------------------------------------- /gpt4all-bindings/python/docs/assets/nomic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/python/docs/assets/nomic.png -------------------------------------------------------------------------------- /gpt4all-bindings/python/docs/css/custom.css: -------------------------------------------------------------------------------- 1 | /* Remove the `In` and `Out` block in rendered Jupyter notebooks */ 2 | .md-container .jp-Cell-outputWrapper .jp-OutputPrompt.jp-OutputArea-prompt, 3 | .md-container .jp-Cell-inputWrapper .jp-InputPrompt.jp-InputArea-prompt { 4 | display: none !important; 5 | } -------------------------------------------------------------------------------- /gpt4all-bindings/python/docs/gpt4all_monitoring.md: -------------------------------------------------------------------------------- 1 | # Monitoring 2 | 3 | Leverage OpenTelemetry to perform real-time monitoring of your LLM application using [OpenLIT](https://github.com/openlit/openlit). This tool helps you easily collect data on user interactions, performance metrics, and other key information, which can assist in enhancing the functionality and dependability of your GPT4All based LLM application. 4 | 5 | ## How it works? 6 | 7 | OpenLIT adds automatic OTel instrumentation to the GPT4all SDK. It covers the `generate` and `embedding` functions, helping to track LLM usage by gathering inputs and outputs. This allows users to monitor and evaluate the performance and behavior of their LLM application in different environments. Additionally, you have the flexibility to view and analyze the generated traces and metrics either in the OpenLIT UI or by exporting them to widely used observability tools like Grafana and DataDog for more comprehensive analysis and visualization. 8 | 9 | ## Getting Started 10 | 11 | Here’s a straightforward guide to help you set up and start monitoring your application: 12 | 13 | ### 1. Install the OpenLIT SDK 14 | Open your terminal and run: 15 | 16 | ```shell 17 | pip install openlit 18 | ``` 19 | 20 | ### 2. Setup Monitoring for your Application 21 | In your application, initiate OpenLIT as outlined below: 22 | 23 | ```python 24 | from gpt4all import GPT4All 25 | import openlit 26 | 27 | openlit.init() # Initialize OpenLIT monitoring 28 | 29 | model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf') 30 | 31 | # Start a chat session and send queries 32 | with model.chat_session(): 33 | response1 = model.generate(prompt='hello', temp=0) 34 | response2 = model.generate(prompt='write me a short poem', temp=0) 35 | response3 = model.generate(prompt='thank you', temp=0) 36 | 37 | print(model.current_chat_session) 38 | ``` 39 | This setup wraps your gpt4all model interactions, capturing valuable data about each request and response. 40 | 41 | ### Visualize 42 | 43 | Once you've set up data collection with OpenLIT, you can visualize and analyze this information to better understand your application's performance: 44 | 45 | - **Using OpenLIT UI:** Connect to OpenLIT's UI to start exploring performance metrics. Visit the OpenLIT [Quickstart Guide](https://docs.openlit.io/latest/quickstart) for step-by-step details. 46 | 47 | - **Integrate with existing Observability Tools:** If you use tools like Grafana or DataDog, you can integrate the data collected by OpenLIT. For instructions on setting up these connections, check the OpenLIT [Connections Guide](https://docs.openlit.io/latest/connections/intro). 48 | -------------------------------------------------------------------------------- /gpt4all-bindings/python/gpt4all/__init__.py: -------------------------------------------------------------------------------- 1 | from .gpt4all import CancellationError as CancellationError, Embed4All as Embed4All, GPT4All as GPT4All 2 | -------------------------------------------------------------------------------- /gpt4all-bindings/python/gpt4all/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-bindings/python/gpt4all/tests/__init__.py -------------------------------------------------------------------------------- /gpt4all-bindings/python/gpt4all/tests/test_embed_timings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import time 4 | from io import StringIO 5 | 6 | from gpt4all import Embed4All, GPT4All 7 | 8 | 9 | def time_embedding(i, embedder): 10 | text = 'foo bar ' * i 11 | start_time = time.time() 12 | output = embedder.embed(text) 13 | end_time = time.time() 14 | elapsed_time = end_time - start_time 15 | print(f"Time report: {2 * i / elapsed_time} tokens/second with {2 * i} tokens taking {elapsed_time} seconds") 16 | 17 | 18 | if __name__ == "__main__": 19 | embedder = Embed4All(n_threads=8) 20 | for i in [2**n for n in range(6, 14)]: 21 | time_embedding(i, embedder) 22 | -------------------------------------------------------------------------------- /gpt4all-bindings/python/makefile: -------------------------------------------------------------------------------- 1 | SHELL:=/bin/bash -o pipefail 2 | ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 3 | PYTHON:=python3 4 | 5 | env: 6 | if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi 7 | 8 | dev: env 9 | source env/bin/activate; pip install black isort pytest; pip install -e . 10 | 11 | documentation: 12 | rm -rf ./site && mkdocs build 13 | 14 | wheel: 15 | rm -rf dist/ build/ gpt4all/llmodel_DO_NOT_MODIFY; python setup.py bdist_wheel; 16 | 17 | clean: 18 | rm -rf {.pytest_cache,env,gpt4all.egg-info} 19 | find . | grep -E "(__pycache__|\.pyc|\.pyo$\)" | xargs rm -rf 20 | 21 | black: 22 | source env/bin/activate; black -l 120 -S --target-version py36 gpt4all 23 | 24 | isort: 25 | source env/bin/activate; isort --ignore-whitespace --atomic -w 120 gpt4all 26 | 27 | test: 28 | source env/bin/activate; pytest -s gpt4all/tests -k "not test_inference_long" 29 | 30 | test_all: 31 | source env/bin/activate; pytest -s gpt4all/tests 32 | -------------------------------------------------------------------------------- /gpt4all-bindings/python/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: GPT4All Documentation 2 | repo_url: https://github.com/nomic-ai/gpt4all 3 | repo_name: nomic-ai/gpt4all 4 | site_url: https://docs.gpt4all.io 5 | edit_uri: edit/main/docs/ 6 | site_description: Documentation for running GPT4All anywhere. 7 | copyright: Copyright © 2023 Nomic, Inc 8 | use_directory_urls: false 9 | 10 | nav: 11 | - 'index.md' 12 | - 'GPT4All Chat Client': 'gpt4all_chat.md' 13 | - 'Bindings': 14 | - 'GPT4All in Python': 15 | - 'Generation': 'gpt4all_python.md' 16 | - 'Embedding': 'gpt4all_python_embedding.md' 17 | - 'Monitoring with OpenLIT': 'gpt4all_monitoring.md' 18 | - 'GPT4ALL in NodeJs': 'gpt4all_nodejs.md' 19 | - 'gpt4all_cli.md' 20 | - 'Wiki': 21 | - 'gpt4all_faq.md' 22 | 23 | theme: 24 | name: material 25 | palette: 26 | primary: white 27 | logo: assets/nomic.png 28 | favicon: assets/favicon.ico 29 | features: 30 | - navigation.instant 31 | - navigation.tracking 32 | - navigation.sections 33 | # - navigation.tabs 34 | # - navigation.tabs.sticky 35 | 36 | markdown_extensions: 37 | - pymdownx.highlight: 38 | anchor_linenums: true 39 | - pymdownx.inlinehilite 40 | - pymdownx.snippets 41 | - pymdownx.details 42 | - pymdownx.superfences 43 | - pymdownx.tabbed: 44 | alternate_style: true 45 | - pymdownx.emoji: 46 | emoji_index: !!python/name:material.extensions.emoji.twemoji 47 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 48 | options: 49 | custom_icons: 50 | - docs/overrides/.icons 51 | - tables 52 | - admonition 53 | - codehilite: 54 | css_class: highlight 55 | 56 | extra_css: 57 | - css/custom.css 58 | 59 | plugins: 60 | - mkdocstrings: 61 | handlers: 62 | python: 63 | options: 64 | show_root_heading: True 65 | heading_level: 4 66 | show_root_full_path: false 67 | docstring_section_style: list 68 | #- material/social: 69 | # cards_font: Roboto 70 | 71 | #- mkdocs-jupyter: 72 | # ignore_h1_titles: True 73 | # show_input: True 74 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: Microsoft 4 | ColumnLimit: 120 -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | build/ 3 | prebuilds/ 4 | .yarn/* 5 | !.yarn/patches 6 | !.yarn/plugins 7 | !.yarn/releases 8 | !.yarn/sdks 9 | !.yarn/versions 10 | runtimes/ 11 | compile_flags.txt 12 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | spec/ 3 | scripts/ 4 | build -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/binding.ci.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "gpt4all", # gpt4all-ts will cause compile error 5 | "include_dirs": [ 6 | " 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace fs = std::filesystem; 13 | 14 | class NodeModelWrapper : public Napi::ObjectWrap 15 | { 16 | 17 | public: 18 | NodeModelWrapper(const Napi::CallbackInfo &); 19 | // virtual ~NodeModelWrapper(); 20 | Napi::Value GetType(const Napi::CallbackInfo &info); 21 | Napi::Value IsModelLoaded(const Napi::CallbackInfo &info); 22 | Napi::Value StateSize(const Napi::CallbackInfo &info); 23 | // void Finalize(Napi::Env env) override; 24 | /** 25 | * Prompting the model. This entails spawning a new thread and adding the response tokens 26 | * into a thread local string variable. 27 | */ 28 | Napi::Value Infer(const Napi::CallbackInfo &info); 29 | void SetThreadCount(const Napi::CallbackInfo &info); 30 | void Dispose(const Napi::CallbackInfo &info); 31 | Napi::Value GetName(const Napi::CallbackInfo &info); 32 | Napi::Value ThreadCount(const Napi::CallbackInfo &info); 33 | Napi::Value GenerateEmbedding(const Napi::CallbackInfo &info); 34 | Napi::Value HasGpuDevice(const Napi::CallbackInfo &info); 35 | Napi::Value ListGpus(const Napi::CallbackInfo &info); 36 | Napi::Value InitGpuByString(const Napi::CallbackInfo &info); 37 | Napi::Value GetRequiredMemory(const Napi::CallbackInfo &info); 38 | Napi::Value GetGpuDevices(const Napi::CallbackInfo &info); 39 | /* 40 | * The path that is used to search for the dynamic libraries 41 | */ 42 | Napi::Value GetLibraryPath(const Napi::CallbackInfo &info); 43 | /** 44 | * Creates the LLModel class 45 | */ 46 | static Napi::Function GetClass(Napi::Env); 47 | llmodel_model GetInference(); 48 | 49 | private: 50 | /** 51 | * The underlying inference that interfaces with the C interface 52 | */ 53 | llmodel_model inference_; 54 | 55 | std::mutex inference_mutex; 56 | 57 | std::string type; 58 | // corresponds to LLModel::name() in typescript 59 | std::string name; 60 | int nCtx{}; 61 | int nGpuLayers{}; 62 | std::string full_model_path; 63 | }; 64 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gpt4all", 3 | "version": "4.0.0", 4 | "packageManager": "yarn@3.6.1", 5 | "main": "src/gpt4all.js", 6 | "repository": "nomic-ai/gpt4all", 7 | "scripts": { 8 | "install": "node-gyp-build", 9 | "test": "jest", 10 | "build:backend": "node scripts/build.js", 11 | "build": "node-gyp-build", 12 | "docs:build": "node scripts/docs.js && documentation readme ./src/gpt4all.d.ts --parse-extension js d.ts --format md --section \"API Reference\" --readme-file ../python/docs/gpt4all_nodejs.md" 13 | }, 14 | "files": [ 15 | "src/**/*", 16 | "runtimes/**/*", 17 | "binding.gyp", 18 | "prebuilds/**/*", 19 | "*.h", 20 | "*.cc", 21 | "gpt4all-backend/**/*" 22 | ], 23 | "dependencies": { 24 | "md5-file": "^5.0.0", 25 | "node-addon-api": "^6.1.0", 26 | "node-gyp-build": "^4.6.0" 27 | }, 28 | "devDependencies": { 29 | "@types/node": "^20.1.5", 30 | "documentation": "^14.0.2", 31 | "jest": "^29.5.0", 32 | "prebuildify": "^5.0.1", 33 | "prettier": "^2.8.8" 34 | }, 35 | "optionalDependencies": { 36 | "node-gyp": "9.x.x" 37 | }, 38 | "engines": { 39 | "node": ">= 18.x.x" 40 | }, 41 | "prettier": { 42 | "endOfLine": "lf", 43 | "tabWidth": 4 44 | }, 45 | "jest": { 46 | "verbose": true 47 | }, 48 | "publishConfig": { 49 | "registry": "https://registry.npmjs.org/", 50 | "access": "public", 51 | "tag": "latest" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/prompt.h: -------------------------------------------------------------------------------- 1 | #ifndef PREDICT_WORKER_H 2 | #define PREDICT_WORKER_H 3 | 4 | #include "llmodel.h" 5 | #include "llmodel_c.h" 6 | #include "napi.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | struct ResponseCallbackData 14 | { 15 | int32_t tokenId; 16 | std::string token; 17 | }; 18 | 19 | struct PromptCallbackData 20 | { 21 | int32_t tokenId; 22 | }; 23 | 24 | struct LLModelWrapper 25 | { 26 | LLModel *llModel = nullptr; 27 | LLModel::PromptContext promptContext; 28 | ~LLModelWrapper() 29 | { 30 | delete llModel; 31 | } 32 | }; 33 | 34 | struct PromptWorkerConfig 35 | { 36 | Napi::Function responseCallback; 37 | bool hasResponseCallback = false; 38 | Napi::Function promptCallback; 39 | bool hasPromptCallback = false; 40 | llmodel_model model; 41 | std::mutex *mutex; 42 | std::string prompt; 43 | std::string promptTemplate; 44 | llmodel_prompt_context context; 45 | std::string result; 46 | bool special = false; 47 | std::string *fakeReply = nullptr; 48 | }; 49 | 50 | class PromptWorker : public Napi::AsyncWorker 51 | { 52 | public: 53 | PromptWorker(Napi::Env env, PromptWorkerConfig config); 54 | ~PromptWorker(); 55 | void Execute() override; 56 | void OnOK() override; 57 | void OnError(const Napi::Error &e) override; 58 | Napi::Promise GetPromise(); 59 | 60 | bool ResponseCallback(int32_t token_id, const std::string token); 61 | bool RecalculateCallback(bool isrecalculating); 62 | bool PromptCallback(int32_t token_id); 63 | 64 | private: 65 | Napi::Promise::Deferred promise; 66 | std::string result; 67 | PromptWorkerConfig _config; 68 | Napi::ThreadSafeFunction _responseCallbackFn; 69 | Napi::ThreadSafeFunction _promptCallbackFn; 70 | }; 71 | 72 | #endif // PREDICT_WORKER_H 73 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/build.js: -------------------------------------------------------------------------------- 1 | const { spawn } = require("node:child_process"); 2 | const { resolve } = require("path"); 3 | const args = process.argv.slice(2); 4 | const platform = process.platform; 5 | //windows 64bit or 32 6 | if (platform === "win32") { 7 | const path = "scripts/build_msvc.bat"; 8 | spawn(resolve(path), ["/Y", ...args], { shell: true, stdio: "inherit" }); 9 | process.on("data", (s) => console.log(s.toString())); 10 | } else if (platform === "linux" || platform === "darwin") { 11 | const path = "scripts/build_unix.sh"; 12 | spawn(`sh `, [path, args], { 13 | shell: true, 14 | stdio: "inherit", 15 | }); 16 | process.on("data", (s) => console.log(s.toString())); 17 | } 18 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/build_mingw.ps1: -------------------------------------------------------------------------------- 1 | $ROOT_DIR = '.\runtimes\win-x64' 2 | $BUILD_DIR = '.\runtimes\win-x64\build\mingw' 3 | $LIBS_DIR = '.\runtimes\win-x64\native' 4 | 5 | # cleanup env 6 | Remove-Item -Force -Recurse $ROOT_DIR -ErrorAction SilentlyContinue | Out-Null 7 | mkdir $BUILD_DIR | Out-Null 8 | mkdir $LIBS_DIR | Out-Null 9 | 10 | # build 11 | cmake -G "MinGW Makefiles" -S ..\..\gpt4all-backend -B $BUILD_DIR -DLLAMA_AVX2=ON 12 | cmake --build $BUILD_DIR --parallel --config Release 13 | 14 | # copy native dlls 15 | # cp "C:\ProgramData\mingw64\mingw64\bin\*dll" $LIBS_DIR 16 | cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR 17 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/build_msvc.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set "BUILD_TYPE=Release" 4 | set "BUILD_DIR=.\build\win-x64-msvc" 5 | set "LIBS_DIR=.\runtimes\win32-x64" 6 | 7 | REM Cleanup env 8 | rmdir /s /q %BUILD_DIR% 9 | 10 | REM Create directories 11 | mkdir %BUILD_DIR% 12 | mkdir %LIBS_DIR% 13 | 14 | REM Build 15 | cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -S ..\..\gpt4all-backend -B %BUILD_DIR% -A x64 16 | 17 | :BUILD 18 | REM Build the project 19 | cmake --build "%BUILD_DIR%" --parallel --config %BUILD_TYPE% 20 | 21 | REM Check the exit code of the build command 22 | if %errorlevel% neq 0 ( 23 | echo Build failed. Retrying... 24 | goto BUILD 25 | ) 26 | 27 | mkdir runtimes\win32-x64 28 | 29 | REM Copy the DLLs to the desired location 30 | del /F /A /Q %LIBS_DIR% 31 | xcopy /Y "%BUILD_DIR%\bin\%BUILD_TYPE%\*.dll" runtimes\win32-x64\native\ 32 | 33 | echo Batch script execution completed. 34 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/build_unix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SYSNAME=$(uname -s) 4 | 5 | if [ "$SYSNAME" = "Linux" ]; then 6 | BASE_DIR="runtimes/linux-x64" 7 | LIB_EXT="so" 8 | elif [ "$SYSNAME" = "Darwin" ]; then 9 | BASE_DIR="runtimes/osx" 10 | LIB_EXT="dylib" 11 | elif [ -n "$SYSNAME" ]; then 12 | echo "Unsupported system: $SYSNAME" >&2 13 | exit 1 14 | else 15 | echo "\"uname -s\" failed" >&2 16 | exit 1 17 | fi 18 | 19 | NATIVE_DIR="$BASE_DIR/native" 20 | BUILD_DIR="$BASE_DIR/build" 21 | 22 | rm -rf "$BASE_DIR" 23 | mkdir -p "$NATIVE_DIR" "$BUILD_DIR" 24 | 25 | cmake -S ../../gpt4all-backend -B "$BUILD_DIR" && 26 | cmake --build "$BUILD_DIR" -j --config Release && { 27 | cp "$BUILD_DIR"/libgptj*.$LIB_EXT "$NATIVE_DIR"/ 28 | cp "$BUILD_DIR"/libllama*.$LIB_EXT "$NATIVE_DIR"/ 29 | } 30 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/docs.js: -------------------------------------------------------------------------------- 1 | //Maybe some command line piping would work better, but can't think of platform independent command line tool 2 | 3 | const fs = require('fs'); 4 | 5 | const newPath = '../python/docs/gpt4all_nodejs.md'; 6 | const filepath = './README.md'; 7 | const intro = fs.readFileSync(filepath); 8 | 9 | fs.writeFileSync( 10 | newPath, intro 11 | ); 12 | 13 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/mkclangd.js: -------------------------------------------------------------------------------- 1 | /// makes compile_flags.txt for clangd server support with this project 2 | /// run this with typescript as your cwd 3 | // 4 | //for debian users make sure to install libstdc++-12-dev 5 | 6 | const nodeaddonapi=require('node-addon-api').include; 7 | 8 | const fsp = require('fs/promises'); 9 | const { existsSync, readFileSync } = require('fs'); 10 | const assert = require('node:assert'); 11 | const findnodeapih = () => { 12 | assert(existsSync("./build"), "Haven't built the application once yet. run node scripts/prebuild.js"); 13 | const dir = readFileSync("./build/config.gypi", 'utf8'); 14 | const nodedir_line = dir.match(/"nodedir": "([^"]+)"/); 15 | assert(nodedir_line, "Found no matches") 16 | assert(nodedir_line[1]); 17 | console.log("node_api.h found at: ", nodedir_line[1]); 18 | return nodedir_line[1]+"/include/node"; 19 | }; 20 | 21 | const knownIncludes = [ 22 | '-I', 23 | './', 24 | '-I', 25 | nodeaddonapi.substring(1, nodeaddonapi.length-1), 26 | '-I', 27 | '../../gpt4all-backend', 28 | '-I', 29 | findnodeapih() 30 | ]; 31 | const knownFlags = [ 32 | "-x", 33 | "c++", 34 | '-std=c++17' 35 | ]; 36 | 37 | 38 | const output = knownFlags.join('\n')+'\n'+knownIncludes.join('\n'); 39 | 40 | fsp.writeFile('./compile_flags.txt', output, 'utf8') 41 | .then(() => console.log('done')) 42 | .catch(() => console.err('failed')); 43 | 44 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/scripts/prebuild.js: -------------------------------------------------------------------------------- 1 | const prebuildify = require("prebuildify"); 2 | 3 | async function createPrebuilds(combinations) { 4 | for (const { platform, arch } of combinations) { 5 | const opts = { 6 | platform, 7 | arch, 8 | napi: true, 9 | targets: ["18.16.0"] 10 | }; 11 | try { 12 | await createPrebuild(opts); 13 | console.log( 14 | `Build succeeded for platform ${opts.platform} and architecture ${opts.arch}` 15 | ); 16 | } catch (err) { 17 | console.error( 18 | `Error building for platform ${opts.platform} and architecture ${opts.arch}:`, 19 | err 20 | ); 21 | } 22 | } 23 | } 24 | 25 | function createPrebuild(opts) { 26 | return new Promise((resolve, reject) => { 27 | prebuildify(opts, (err) => { 28 | if (err) { 29 | reject(err); 30 | } else { 31 | resolve(); 32 | } 33 | }); 34 | }); 35 | } 36 | 37 | let prebuildConfigs; 38 | if(process.platform === 'win32') { 39 | prebuildConfigs = [ 40 | { platform: "win32", arch: "x64" } 41 | ]; 42 | } else if(process.platform === 'linux') { 43 | //Unsure if darwin works, need mac tester! 44 | prebuildConfigs = [ 45 | { platform: "linux", arch: "x64" }, 46 | //{ platform: "linux", arch: "arm64" }, 47 | //{ platform: "linux", arch: "armv7" }, 48 | ] 49 | } else if(process.platform === 'darwin') { 50 | prebuildConfigs = [ 51 | { platform: "darwin", arch: "x64" }, 52 | { platform: "darwin", arch: "arm64" }, 53 | ] 54 | } 55 | 56 | createPrebuilds(prebuildConfigs) 57 | .then(() => console.log("All builds succeeded")) 58 | .catch((err) => console.error("Error building:", err)); 59 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/callbacks.mjs: -------------------------------------------------------------------------------- 1 | import { promises as fs } from "node:fs"; 2 | import { loadModel, createCompletion } from "../src/gpt4all.js"; 3 | 4 | const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", { 5 | verbose: true, 6 | device: "gpu", 7 | }); 8 | 9 | const res = await createCompletion( 10 | model, 11 | "I've got three 🍣 - What shall I name them?", 12 | { 13 | onPromptToken: (tokenId) => { 14 | console.debug("onPromptToken", { tokenId }); 15 | // throwing an error will cancel 16 | throw new Error("This is an error"); 17 | // const foo = thisMethodDoesNotExist(); 18 | // returning false will cancel as well 19 | // return false; 20 | }, 21 | onResponseToken: (tokenId, token) => { 22 | console.debug("onResponseToken", { tokenId, token }); 23 | // same applies here 24 | }, 25 | } 26 | ); 27 | 28 | console.debug("Output:", { 29 | usage: res.usage, 30 | message: res.choices[0].message, 31 | }); 32 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/chat-memory.mjs: -------------------------------------------------------------------------------- 1 | import { loadModel, createCompletion } from "../src/gpt4all.js"; 2 | 3 | const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", { 4 | verbose: true, 5 | device: "gpu", 6 | }); 7 | 8 | const chat = await model.createChatSession({ 9 | messages: [ 10 | { 11 | role: "user", 12 | content: "I'll tell you a secret password: It's 63445.", 13 | }, 14 | { 15 | role: "assistant", 16 | content: "I will do my best to remember that.", 17 | }, 18 | { 19 | role: "user", 20 | content: 21 | "And here another fun fact: Bananas may be bluer than bread at night.", 22 | }, 23 | { 24 | role: "assistant", 25 | content: "Yes, that makes sense.", 26 | }, 27 | ], 28 | }); 29 | 30 | const turn1 = await createCompletion( 31 | chat, 32 | "Please tell me the secret password." 33 | ); 34 | console.debug(turn1.choices[0].message); 35 | // "The secret password you shared earlier is 63445."" 36 | 37 | const turn2 = await createCompletion( 38 | chat, 39 | "Thanks! Have your heard about the bananas?" 40 | ); 41 | console.debug(turn2.choices[0].message); 42 | 43 | for (let i = 0; i < 32; i++) { 44 | // gpu go brr 45 | const turn = await createCompletion( 46 | chat, 47 | i % 2 === 0 ? "Tell me a fun fact." : "And a boring one?" 48 | ); 49 | console.debug({ 50 | message: turn.choices[0].message, 51 | n_past_tokens: turn.usage.n_past_tokens, 52 | }); 53 | } 54 | 55 | const finalTurn = await createCompletion( 56 | chat, 57 | "Now I forgot the secret password. Can you remind me?" 58 | ); 59 | console.debug(finalTurn.choices[0].message); 60 | 61 | // result of finalTurn may vary depending on whether the generated facts pushed the secret out of the context window. 62 | // "Of course! The secret password you shared earlier is 63445." 63 | // "I apologize for any confusion. As an AI language model, ..." 64 | 65 | model.dispose(); 66 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/chat-minimal.mjs: -------------------------------------------------------------------------------- 1 | import { loadModel, createCompletion } from "../src/gpt4all.js"; 2 | 3 | const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", { 4 | verbose: true, 5 | device: "gpu", 6 | }); 7 | 8 | const chat = await model.createChatSession(); 9 | 10 | await createCompletion( 11 | chat, 12 | "Why are bananas rather blue than bread at night sometimes?", 13 | { 14 | verbose: true, 15 | } 16 | ); 17 | await createCompletion(chat, "Are you sure?", { 18 | verbose: true, 19 | }); 20 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/concurrency.mjs: -------------------------------------------------------------------------------- 1 | import { 2 | loadModel, 3 | createCompletion, 4 | } from "../src/gpt4all.js"; 5 | 6 | const modelOptions = { 7 | verbose: true, 8 | }; 9 | 10 | const model1 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", { 11 | ...modelOptions, 12 | device: "gpu", // only one model can be on gpu 13 | }); 14 | const model2 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions); 15 | const model3 = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", modelOptions); 16 | 17 | const promptContext = { 18 | verbose: true, 19 | } 20 | 21 | const responses = await Promise.all([ 22 | createCompletion(model1, "What is 1 + 1?", promptContext), 23 | // generating with the same model instance will wait for the previous completion to finish 24 | createCompletion(model1, "What is 1 + 1?", promptContext), 25 | // generating with different model instances will run in parallel 26 | createCompletion(model2, "What is 1 + 2?", promptContext), 27 | createCompletion(model3, "What is 1 + 3?", promptContext), 28 | ]); 29 | console.log(responses.map((res) => res.choices[0].message)); 30 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/embed-jsonl.mjs: -------------------------------------------------------------------------------- 1 | import { loadModel, createEmbedding } from '../src/gpt4all.js' 2 | import { createGunzip, createGzip, createUnzip } from 'node:zlib'; 3 | import { Readable } from 'stream' 4 | import readline from 'readline' 5 | const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding', device: 'gpu' }) 6 | console.log("Running with", embedder.llm.threadCount(), "threads"); 7 | 8 | 9 | const unzip = createGunzip(); 10 | const url = "https://huggingface.co/datasets/sentence-transformers/embedding-training-data/resolve/main/squad_pairs.jsonl.gz" 11 | const stream = await fetch(url) 12 | .then(res => Readable.fromWeb(res.body)); 13 | 14 | const lineReader = readline.createInterface({ 15 | input: stream.pipe(unzip), 16 | crlfDelay: Infinity 17 | }) 18 | 19 | lineReader.on('line', line => { 20 | //pairs of questions and answers 21 | const question_answer = JSON.parse(line) 22 | console.log(createEmbedding(embedder, question_answer)) 23 | }) 24 | 25 | lineReader.on('close', () => embedder.dispose()) 26 | 27 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/embed.mjs: -------------------------------------------------------------------------------- 1 | import { loadModel, createEmbedding } from '../src/gpt4all.js' 2 | 3 | const embedder = await loadModel("nomic-embed-text-v1.5.f16.gguf", { verbose: true, type: 'embedding' , device: 'gpu' }) 4 | 5 | try { 6 | console.log(createEmbedding(embedder, ["Accept your current situation", "12312"], { prefix: "search_document" })) 7 | 8 | } catch(e) { 9 | console.log(e) 10 | } 11 | 12 | embedder.dispose() 13 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/llmodel.mjs: -------------------------------------------------------------------------------- 1 | import { 2 | LLModel, 3 | createCompletion, 4 | DEFAULT_DIRECTORY, 5 | DEFAULT_LIBRARIES_DIRECTORY, 6 | loadModel, 7 | } from "../src/gpt4all.js"; 8 | 9 | const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", { 10 | verbose: true, 11 | device: "gpu", 12 | }); 13 | const ll = model.llm; 14 | 15 | try { 16 | class Extended extends LLModel {} 17 | } catch (e) { 18 | console.log("Extending from native class gone wrong " + e); 19 | } 20 | 21 | console.log("state size " + ll.stateSize()); 22 | 23 | console.log("thread count " + ll.threadCount()); 24 | ll.setThreadCount(5); 25 | 26 | console.log("thread count " + ll.threadCount()); 27 | ll.setThreadCount(4); 28 | console.log("thread count " + ll.threadCount()); 29 | console.log("name " + ll.name()); 30 | console.log("type: " + ll.type()); 31 | console.log("Default directory for models", DEFAULT_DIRECTORY); 32 | console.log("Default directory for libraries", DEFAULT_LIBRARIES_DIRECTORY); 33 | console.log("Has GPU", ll.hasGpuDevice()); 34 | console.log("gpu devices", ll.listGpu()); 35 | console.log("Required Mem in bytes", ll.memoryNeeded()); 36 | 37 | // to ingest a custom system prompt without using a chat session. 38 | await createCompletion( 39 | model, 40 | "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n", 41 | { 42 | promptTemplate: "%1", 43 | nPredict: 0, 44 | special: true, 45 | } 46 | ); 47 | const completion1 = await createCompletion(model, "What is 1 + 1?", { 48 | verbose: true, 49 | }); 50 | console.log(`🤖 > ${completion1.choices[0].message.content}`); 51 | //Very specific: 52 | // tested on Ubuntu 22.0, Linux Mint, if I set nPast to 100, the app hangs. 53 | const completion2 = await createCompletion(model, "And if we add two?", { 54 | verbose: true, 55 | }); 56 | console.log(`🤖 > ${completion2.choices[0].message.content}`); 57 | 58 | //CALLING DISPOSE WILL INVALID THE NATIVE MODEL. USE THIS TO CLEANUP 59 | model.dispose(); 60 | 61 | console.log("model disposed, exiting..."); 62 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/long-context.mjs: -------------------------------------------------------------------------------- 1 | import { promises as fs } from "node:fs"; 2 | import { loadModel, createCompletion } from "../src/gpt4all.js"; 3 | 4 | const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", { 5 | verbose: true, 6 | device: "gpu", 7 | nCtx: 32768, 8 | }); 9 | 10 | const typeDefSource = await fs.readFile("./src/gpt4all.d.ts", "utf-8"); 11 | 12 | const res = await createCompletion( 13 | model, 14 | "Here are the type definitions for the GPT4All API:\n\n" + 15 | typeDefSource + 16 | "\n\nHow do I create a completion with a really large context window?", 17 | { 18 | verbose: true, 19 | } 20 | ); 21 | console.debug(res.choices[0].message); 22 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/model-switching.mjs: -------------------------------------------------------------------------------- 1 | import { loadModel, createCompletion } from "../src/gpt4all.js"; 2 | 3 | const model1 = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", { 4 | device: "gpu", 5 | nCtx: 4096, 6 | }); 7 | 8 | const chat1 = await model1.createChatSession({ 9 | temperature: 0.8, 10 | topP: 0.7, 11 | topK: 60, 12 | }); 13 | 14 | const chat1turn1 = await createCompletion( 15 | chat1, 16 | "Outline a short story concept for adults. About why bananas are rather blue than bread is green at night sometimes. Not too long." 17 | ); 18 | console.debug(chat1turn1.choices[0].message); 19 | 20 | const chat1turn2 = await createCompletion( 21 | chat1, 22 | "Lets sprinkle some plot twists. And a cliffhanger at the end." 23 | ); 24 | console.debug(chat1turn2.choices[0].message); 25 | 26 | const chat1turn3 = await createCompletion( 27 | chat1, 28 | "Analyze your plot. Find the weak points." 29 | ); 30 | console.debug(chat1turn3.choices[0].message); 31 | 32 | const chat1turn4 = await createCompletion( 33 | chat1, 34 | "Rewrite it based on the analysis." 35 | ); 36 | console.debug(chat1turn4.choices[0].message); 37 | 38 | model1.dispose(); 39 | 40 | const model2 = await loadModel("gpt4all-falcon-newbpe-q4_0.gguf", { 41 | device: "gpu", 42 | }); 43 | 44 | const chat2 = await model2.createChatSession({ 45 | messages: chat1.messages, 46 | }); 47 | 48 | const chat2turn1 = await createCompletion( 49 | chat2, 50 | "Give three ideas how this plot could be improved." 51 | ); 52 | console.debug(chat2turn1.choices[0].message); 53 | 54 | const chat2turn2 = await createCompletion( 55 | chat2, 56 | "Revise the plot, applying your ideas." 57 | ); 58 | console.debug(chat2turn2.choices[0].message); 59 | 60 | model2.dispose(); 61 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/stateless.mjs: -------------------------------------------------------------------------------- 1 | import { loadModel, createCompletion } from "../src/gpt4all.js"; 2 | 3 | const model = await loadModel("orca-mini-3b-gguf2-q4_0.gguf", { 4 | verbose: true, 5 | device: "gpu", 6 | }); 7 | 8 | const messages = [ 9 | { 10 | role: "system", 11 | content: "<|im_start|>system\nYou are an advanced mathematician.\n<|im_end|>\n", 12 | }, 13 | { 14 | role: "user", 15 | content: "What's 2+2?", 16 | }, 17 | { 18 | role: "assistant", 19 | content: "5", 20 | }, 21 | { 22 | role: "user", 23 | content: "Are you sure?", 24 | }, 25 | ]; 26 | 27 | 28 | const res1 = await createCompletion(model, messages); 29 | console.debug(res1.choices[0].message); 30 | messages.push(res1.choices[0].message); 31 | 32 | messages.push({ 33 | role: "user", 34 | content: "Could you double check that?", 35 | }); 36 | 37 | const res2 = await createCompletion(model, messages); 38 | console.debug(res2.choices[0].message); 39 | messages.push(res2.choices[0].message); 40 | 41 | messages.push({ 42 | role: "user", 43 | content: "Let's bring out the big calculators.", 44 | }); 45 | 46 | const res3 = await createCompletion(model, messages); 47 | console.debug(res3.choices[0].message); 48 | messages.push(res3.choices[0].message); 49 | 50 | // console.debug(messages); 51 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/streaming.mjs: -------------------------------------------------------------------------------- 1 | import { 2 | loadModel, 3 | createCompletion, 4 | createCompletionStream, 5 | createCompletionGenerator, 6 | } from "../src/gpt4all.js"; 7 | 8 | const model = await loadModel("mistral-7b-openorca.gguf2.Q4_0.gguf", { 9 | device: "gpu", 10 | }); 11 | 12 | process.stdout.write("### Stream:"); 13 | const stream = createCompletionStream(model, "How are you?"); 14 | stream.tokens.on("data", (data) => { 15 | process.stdout.write(data); 16 | }); 17 | await stream.result; 18 | process.stdout.write("\n"); 19 | 20 | process.stdout.write("### Stream with pipe:"); 21 | const stream2 = createCompletionStream( 22 | model, 23 | "Please say something nice about node streams." 24 | ); 25 | stream2.tokens.pipe(process.stdout); 26 | const stream2Res = await stream2.result; 27 | process.stdout.write("\n"); 28 | 29 | process.stdout.write("### Generator:"); 30 | const gen = createCompletionGenerator(model, "generators instead?", { 31 | nPast: stream2Res.usage.n_past_tokens, 32 | }); 33 | for await (const chunk of gen) { 34 | process.stdout.write(chunk); 35 | } 36 | 37 | process.stdout.write("\n"); 38 | 39 | process.stdout.write("### Callback:"); 40 | await createCompletion(model, "Why not just callbacks?", { 41 | onResponseToken: (tokenId, token) => { 42 | process.stdout.write(token); 43 | }, 44 | }); 45 | process.stdout.write("\n"); 46 | 47 | process.stdout.write("### 2nd Generator:"); 48 | const gen2 = createCompletionGenerator(model, "If 3 + 3 is 5, what is 2 + 2?"); 49 | 50 | let chunk = await gen2.next(); 51 | while (!chunk.done) { 52 | process.stdout.write(chunk.value); 53 | chunk = await gen2.next(); 54 | } 55 | process.stdout.write("\n"); 56 | console.debug("generator finished", chunk); 57 | model.dispose(); 58 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/spec/system.mjs: -------------------------------------------------------------------------------- 1 | import { 2 | loadModel, 3 | createCompletion, 4 | } from "../src/gpt4all.js"; 5 | 6 | const model = await loadModel("Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf", { 7 | verbose: true, 8 | device: "gpu", 9 | }); 10 | 11 | const chat = await model.createChatSession({ 12 | verbose: true, 13 | systemPrompt: "<|im_start|>system\nRoleplay as Batman. Answer as if you are Batman, never say you're an Assistant.\n<|im_end|>", 14 | }); 15 | const turn1 = await createCompletion(chat, "You have any plans tonight?"); 16 | console.log(turn1.choices[0].message); 17 | // "I'm afraid I must decline any personal invitations tonight. As Batman, I have a responsibility to protect Gotham City." 18 | 19 | model.dispose(); 20 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/src/config.js: -------------------------------------------------------------------------------- 1 | const os = require("node:os"); 2 | const path = require("node:path"); 3 | 4 | const DEFAULT_DIRECTORY = path.resolve(os.homedir(), ".cache/gpt4all"); 5 | 6 | const librarySearchPaths = [ 7 | path.join(DEFAULT_DIRECTORY, "libraries"), 8 | path.resolve("./libraries"), 9 | path.resolve( 10 | __dirname, 11 | "..", 12 | `runtimes/${process.platform}-${process.arch}/native`, 13 | ), 14 | //for darwin. This is hardcoded for now but it should work 15 | path.resolve( 16 | __dirname, 17 | "..", 18 | `runtimes/${process.platform}/native`, 19 | ), 20 | process.cwd(), 21 | ]; 22 | 23 | const DEFAULT_LIBRARIES_DIRECTORY = librarySearchPaths.join(";"); 24 | 25 | const DEFAULT_MODEL_CONFIG = { 26 | systemPrompt: "", 27 | promptTemplate: "### Human:\n%1\n\n### Assistant:\n", 28 | } 29 | 30 | const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models3.json"; 31 | 32 | const DEFAULT_PROMPT_CONTEXT = { 33 | temp: 0.1, 34 | topK: 40, 35 | topP: 0.9, 36 | minP: 0.0, 37 | repeatPenalty: 1.18, 38 | repeatLastN: 10, 39 | nBatch: 100, 40 | } 41 | 42 | module.exports = { 43 | DEFAULT_DIRECTORY, 44 | DEFAULT_LIBRARIES_DIRECTORY, 45 | DEFAULT_MODEL_CONFIG, 46 | DEFAULT_MODEL_LIST_URL, 47 | DEFAULT_PROMPT_CONTEXT, 48 | }; 49 | -------------------------------------------------------------------------------- /gpt4all-bindings/typescript/test/models.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "order": "a", 4 | "md5sum": "08d6c05a21512a79a1dfeb9d2a8f262f", 5 | "name": "Not a real model", 6 | "filename": "fake-model.gguf", 7 | "filesize": "4", 8 | "systemPrompt": " " 9 | } 10 | ] 11 | -------------------------------------------------------------------------------- /gpt4all-chat/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2023 Nomic, Inc., Aaron Miller 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | ADDENDUM: 10 | 11 | Any LLM models that are loaded and used by the application are not themselves 12 | subject to this license if indeed they are even copyrightable. The terms of 13 | this license apply only to the application software and its accompanying 14 | documentation and do not extend to any LLM models, whether created by the 15 | author of the application or obtained from third-party sources. 16 | -------------------------------------------------------------------------------- /gpt4all-chat/cmake/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #define APP_VERSION "@APP_VERSION@" 5 | 6 | #endif // CONFIG_H 7 | -------------------------------------------------------------------------------- /gpt4all-chat/cmake/deploy-qt-linux.cmake.in: -------------------------------------------------------------------------------- 1 | set(LINUXDEPLOYQT "@LINUXDEPLOYQT@") 2 | set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@") 3 | set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@") 4 | set(DATA_DIR ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 5 | set(BIN_DIR ${DATA_DIR}/bin) 6 | set(Qt6_ROOT_DIR "@Qt6_ROOT_DIR@") 7 | set(ENV{LD_LIBRARY_PATH} "${BIN_DIR}:${Qt6_ROOT_DIR}/../lib/") 8 | execute_process(COMMAND ${LINUXDEPLOYQT} ${BIN_DIR}/chat -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -bundle-non-qt-libs -qmake=${Qt6_ROOT_DIR}/bin/qmake -verbose=2 -exclude-libs=libcuda.so.1) 9 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png" 10 | DESTINATION ${DATA_DIR}) 11 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png" 12 | DESTINATION ${DATA_DIR}) 13 | -------------------------------------------------------------------------------- /gpt4all-chat/cmake/deploy-qt-mac.cmake.in: -------------------------------------------------------------------------------- 1 | set(MACDEPLOYQT "@MACDEPLOYQT@") 2 | set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@") 3 | set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@") 4 | execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2) 5 | file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*) 6 | file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*) 7 | file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*) 8 | file(COPY ${MYGPTJLIBS} 9 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks) 10 | file(COPY ${MYLLAMALIBS} 11 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks) 12 | file(COPY ${MYLLMODELLIBS} 13 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks) 14 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png" 15 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 16 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png" 17 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 18 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns" 19 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 20 | -------------------------------------------------------------------------------- /gpt4all-chat/cmake/deploy-qt-windows.cmake.in: -------------------------------------------------------------------------------- 1 | set(WINDEPLOYQT "@WINDEPLOYQT@") 2 | set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@") 3 | set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@") 4 | execute_process(COMMAND ${WINDEPLOYQT} --qmldir ${CMAKE_CURRENT_SOURCE_DIR} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin) 5 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-32.png" 6 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 7 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/icons/logo-48.png" 8 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 9 | file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.ico" 10 | DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data) 11 | -------------------------------------------------------------------------------- /gpt4all-chat/embeddings.h: -------------------------------------------------------------------------------- 1 | #ifndef EMBEDDINGS_H 2 | #define EMBEDDINGS_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace hnswlib { 11 | class InnerProductSpace; 12 | template class HierarchicalNSW; 13 | } 14 | 15 | class Embeddings : public QObject 16 | { 17 | Q_OBJECT 18 | public: 19 | Embeddings(QObject *parent); 20 | virtual ~Embeddings(); 21 | 22 | bool load(); 23 | bool load(qint64 maxElements); 24 | bool save(); 25 | bool isLoaded() const; 26 | bool fileExists() const; 27 | bool resize(qint64 size); 28 | 29 | // Adds the embedding and returns the label used 30 | bool add(const std::vector &embedding, qint64 label); 31 | 32 | // Removes the embedding at label by marking it as unused 33 | void remove(qint64 label); 34 | 35 | // Clears the embeddings 36 | void clear(); 37 | 38 | // Performs a nearest neighbor search of the embeddings and returns a vector of labels 39 | // for the K nearest neighbors of the given embedding 40 | std::vector search(const std::vector &embedding, int K); 41 | 42 | private: 43 | QString m_filePath; 44 | hnswlib::InnerProductSpace *m_space; 45 | hnswlib::HierarchicalNSW *m_hnsw; 46 | }; 47 | 48 | #endif // EMBEDDINGS_H 49 | -------------------------------------------------------------------------------- /gpt4all-chat/embllm.h: -------------------------------------------------------------------------------- 1 | #ifndef EMBLLM_H 2 | #define EMBLLM_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | class LLModel; 16 | class QNetworkAccessManager; 17 | 18 | struct EmbeddingChunk { 19 | int folder_id; 20 | int chunk_id; 21 | QString chunk; 22 | }; 23 | 24 | Q_DECLARE_METATYPE(EmbeddingChunk) 25 | 26 | struct EmbeddingResult { 27 | int folder_id; 28 | int chunk_id; 29 | std::vector embedding; 30 | }; 31 | 32 | class EmbeddingLLMWorker : public QObject { 33 | Q_OBJECT 34 | public: 35 | EmbeddingLLMWorker(); 36 | virtual ~EmbeddingLLMWorker(); 37 | 38 | void wait(); 39 | 40 | std::vector lastResponse() const { return m_lastResponse; } 41 | 42 | bool loadModel(); 43 | bool hasModel() const; 44 | bool isNomic() const; 45 | 46 | std::vector generateSyncEmbedding(const QString &text); 47 | 48 | public Q_SLOTS: 49 | void requestSyncEmbedding(const QString &text); 50 | void requestAsyncEmbedding(const QVector &chunks); 51 | 52 | Q_SIGNALS: 53 | void embeddingsGenerated(const QVector &embeddings); 54 | void errorGenerated(int folder_id, const QString &error); 55 | void finished(); 56 | 57 | private Q_SLOTS: 58 | void handleFinished(); 59 | 60 | private: 61 | void sendAtlasRequest(const QStringList &texts, const QString &taskType, QVariant userData = {}); 62 | 63 | QString m_nomicAPIKey; 64 | QNetworkAccessManager *m_networkManager; 65 | std::vector m_lastResponse; 66 | LLModel *m_model = nullptr; 67 | std::atomic m_stopGenerating; 68 | QThread m_workerThread; 69 | }; 70 | 71 | class EmbeddingLLM : public QObject 72 | { 73 | Q_OBJECT 74 | public: 75 | EmbeddingLLM(); 76 | virtual ~EmbeddingLLM(); 77 | 78 | bool loadModel(); 79 | bool hasModel() const; 80 | 81 | public Q_SLOTS: 82 | std::vector generateEmbeddings(const QString &text); // synchronous 83 | void generateAsyncEmbeddings(const QVector &chunks); 84 | 85 | Q_SIGNALS: 86 | void requestSyncEmbedding(const QString &text); 87 | void requestAsyncEmbedding(const QVector &chunks); 88 | void embeddingsGenerated(const QVector &embeddings); 89 | void errorGenerated(int folder_id, const QString &error); 90 | 91 | private: 92 | EmbeddingLLMWorker *m_embeddingWorker; 93 | }; 94 | 95 | #endif // EMBLLM_H 96 | -------------------------------------------------------------------------------- /gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | io.gpt4all.gpt4all 4 | CC0-1.0 5 | MIT 6 | GPT4ALL 7 | Open-source assistant 8 | Nomic-ai 9 | 10 |

Cross platform Qt based GUI for GPT4All

11 |
    12 |
  • Fast CPU and GPU based inference using ggml for open source LLM's
    • 13 |
  • The UI is made to look and feel like you've come to expect from a chatty gpt
    • 14 |
  • Check for updates so you can always stay fresh with latest models
    • 15 |
  • Easy to install with precompiled binaries available for all three major desktop platforms
    • 16 |
  • Multi-model - Ability to load more than one model and switch between them
    • 17 |
  • Supports both llama.cpp and gptj.cpp style models
    • 18 |
  • Model downloader in GUI featuring many popular open source models
    • 19 |
  • Settings dialog to change temp, top_p, top_k, threads, etc
    • 20 |
  • Copy your conversation to clipboard
    • 21 |
22 | 23 | 24 | 25 | Main Window 26 | https://user-images.githubusercontent.com/50458173/231464085-da9edff6-a593-410e-8f38-7513f75c8aab.png 27 | 28 | 29 | https://gpt4all.io 30 | https://github.com/nomic-ai/gpt4all/issues 31 | https://github.com/nomic-ai/gpt4all 32 | 33 | 34 | 35 | io.gpt4all.gpt4all.desktop 36 | 37 | mild 38 | moderate 39 | mild 40 | 41 | -------------------------------------------------------------------------------- /gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.desktop: -------------------------------------------------------------------------------- 1 | [Desktop Entry] 2 | Name=GPT4ALL 3 | GenericName=Open-source assistant-style large language models that run locally on your CPU 4 | Comment=Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See GPT4All Website for a full list of open-source models you can run with this powerful desktop application. 5 | Exec=chat 6 | Icon=io.gpt4all.gpt4all 7 | Type=Application 8 | Categories=Utility;Office; 9 | Keywords=GPT,Chat;AI 10 | -------------------------------------------------------------------------------- /gpt4all-chat/hnswlib/visited_list_pool.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace hnswlib { 8 | typedef unsigned short int vl_type; 9 | 10 | class VisitedList { 11 | public: 12 | vl_type curV; 13 | vl_type *mass; 14 | unsigned int numelements; 15 | 16 | VisitedList(int numelements1) { 17 | curV = -1; 18 | numelements = numelements1; 19 | mass = new vl_type[numelements]; 20 | } 21 | 22 | void reset() { 23 | curV++; 24 | if (curV == 0) { 25 | memset(mass, 0, sizeof(vl_type) * numelements); 26 | curV++; 27 | } 28 | } 29 | 30 | ~VisitedList() { delete[] mass; } 31 | }; 32 | /////////////////////////////////////////////////////////// 33 | // 34 | // Class for multi-threaded pool-management of VisitedLists 35 | // 36 | ///////////////////////////////////////////////////////// 37 | 38 | class VisitedListPool { 39 | std::deque pool; 40 | std::mutex poolguard; 41 | int numelements; 42 | 43 | public: 44 | VisitedListPool(int initmaxpools, int numelements1) { 45 | numelements = numelements1; 46 | for (int i = 0; i < initmaxpools; i++) 47 | pool.push_front(new VisitedList(numelements)); 48 | } 49 | 50 | VisitedList *getFreeVisitedList() { 51 | VisitedList *rez; 52 | { 53 | std::unique_lock lock(poolguard); 54 | if (pool.size() > 0) { 55 | rez = pool.front(); 56 | pool.pop_front(); 57 | } else { 58 | rez = new VisitedList(numelements); 59 | } 60 | } 61 | rez->reset(); 62 | return rez; 63 | } 64 | 65 | void releaseVisitedList(VisitedList *vl) { 66 | std::unique_lock lock(poolguard); 67 | pool.push_front(vl); 68 | } 69 | 70 | ~VisitedListPool() { 71 | while (pool.size()) { 72 | VisitedList *rez = pool.front(); 73 | pool.pop_front(); 74 | delete rez; 75 | } 76 | } 77 | }; 78 | } // namespace hnswlib 79 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/close.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/copy.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/db.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/download.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/edit.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/eject.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/image.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/left_panel_closed.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/left_panel_open.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-1024.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-1024.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-128.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-16.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-256.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-32.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-48.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-512.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/logo-64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/icons/logo-64.png -------------------------------------------------------------------------------- /gpt4all-chat/icons/network.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/question.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/regenerate.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/send_message.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/stop_generating.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/thumbs_down.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/thumbs_up.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/icons/trash.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | -------------------------------------------------------------------------------- /gpt4all-chat/llm.h: -------------------------------------------------------------------------------- 1 | #ifndef LLM_H 2 | #define LLM_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class LLM : public QObject 9 | { 10 | Q_OBJECT 11 | Q_PROPERTY(bool isNetworkOnline READ isNetworkOnline NOTIFY isNetworkOnlineChanged) 12 | 13 | public: 14 | static LLM *globalInstance(); 15 | 16 | Q_INVOKABLE bool hasSettingsAccess() const; 17 | Q_INVOKABLE bool compatHardware() const { return m_compatHardware; } 18 | 19 | Q_INVOKABLE bool checkForUpdates() const; 20 | Q_INVOKABLE static bool directoryExists(const QString &path); 21 | Q_INVOKABLE static bool fileExists(const QString &path); 22 | Q_INVOKABLE qint64 systemTotalRAMInGB() const; 23 | Q_INVOKABLE QString systemTotalRAMInGBString() const; 24 | Q_INVOKABLE bool isNetworkOnline() const; 25 | 26 | Q_SIGNALS: 27 | void isNetworkOnlineChanged(); 28 | 29 | private: 30 | bool m_compatHardware; 31 | 32 | private: 33 | explicit LLM(); 34 | ~LLM() {} 35 | friend class MyLLM; 36 | }; 37 | 38 | #endif // LLM_H 39 | -------------------------------------------------------------------------------- /gpt4all-chat/localdocs.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCALDOCS_H 2 | #define LOCALDOCS_H 3 | 4 | #include "localdocsmodel.h" // IWYU pragma: keep 5 | 6 | #include 7 | #include 8 | 9 | class Database; 10 | 11 | class LocalDocs : public QObject 12 | { 13 | Q_OBJECT 14 | Q_PROPERTY(LocalDocsModel *localDocsModel READ localDocsModel NOTIFY localDocsModelChanged) 15 | 16 | public: 17 | static LocalDocs *globalInstance(); 18 | 19 | LocalDocsModel *localDocsModel() const { return m_localDocsModel; } 20 | 21 | Q_INVOKABLE void addFolder(const QString &collection, const QString &path); 22 | Q_INVOKABLE void removeFolder(const QString &collection, const QString &path); 23 | 24 | Database *database() const { return m_database; } 25 | 26 | public Q_SLOTS: 27 | void handleChunkSizeChanged(); 28 | void aboutToQuit(); 29 | 30 | Q_SIGNALS: 31 | void requestStart(); 32 | void requestAddFolder(const QString &collection, const QString &path, bool fromDb); 33 | void requestRemoveFolder(const QString &collection, const QString &path); 34 | void requestChunkSizeChange(int chunkSize); 35 | void localDocsModelChanged(); 36 | 37 | private: 38 | LocalDocsModel *m_localDocsModel; 39 | Database *m_database; 40 | 41 | private: 42 | explicit LocalDocs(); 43 | friend class MyLocalDocs; 44 | }; 45 | 46 | #endif // LOCALDOCS_H 47 | -------------------------------------------------------------------------------- /gpt4all-chat/logger.cpp: -------------------------------------------------------------------------------- 1 | #include "logger.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | class MyLogger: public Logger { }; 14 | Q_GLOBAL_STATIC(MyLogger, loggerInstance) 15 | Logger *Logger::globalInstance() 16 | { 17 | return loggerInstance(); 18 | } 19 | 20 | Logger::Logger() 21 | { 22 | // Get log file dir 23 | auto dir = QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation); 24 | // Remove old log file 25 | QFile::remove(dir+"/log-prev.txt"); 26 | QFile::rename(dir+"/log.txt", dir+"/log-prev.txt"); 27 | // Open new log file 28 | m_file.setFileName(dir+"/log.txt"); 29 | if (!m_file.open(QIODevice::NewOnly | QIODevice::WriteOnly | QIODevice::Text)) { 30 | qWarning() << "Failed to open log file, logging to stdout..."; 31 | m_file.open(stdout, QIODevice::WriteOnly | QIODevice::Text); 32 | } 33 | // On success, install message handler 34 | qInstallMessageHandler(Logger::messageHandler); 35 | } 36 | 37 | void Logger::messageHandler(QtMsgType type, const QMessageLogContext &, const QString &msg) 38 | { 39 | auto logger = globalInstance(); 40 | // Get message type as string 41 | QString typeString; 42 | switch (type) { 43 | case QtDebugMsg: 44 | typeString = "Debug"; 45 | break; 46 | case QtInfoMsg: 47 | typeString = "Info"; 48 | break; 49 | case QtWarningMsg: 50 | typeString = "Warning"; 51 | break; 52 | case QtCriticalMsg: 53 | typeString = "Critical"; 54 | break; 55 | case QtFatalMsg: 56 | typeString = "Fatal"; 57 | break; 58 | default: 59 | typeString = "???"; 60 | } 61 | // Get time and date 62 | auto timestamp = QDateTime::currentDateTime().toString(); 63 | // Write message 64 | const std::string out = QString("[%1] (%2): %4\n").arg(typeString, timestamp, msg).toStdString(); 65 | logger->m_file.write(out.c_str()); 66 | logger->m_file.flush(); 67 | std::cerr << out; 68 | fflush(stderr); 69 | } 70 | -------------------------------------------------------------------------------- /gpt4all-chat/logger.h: -------------------------------------------------------------------------------- 1 | #ifndef LOGGER_H 2 | #define LOGGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class Logger 9 | { 10 | QFile m_file; 11 | 12 | static void messageHandler(QtMsgType type, const QMessageLogContext &context, const QString &msg); 13 | 14 | public: 15 | static Logger *globalInstance(); 16 | 17 | explicit Logger(); 18 | friend class MyLogger; 19 | }; 20 | 21 | #endif // LOGGER_H 22 | -------------------------------------------------------------------------------- /gpt4all-chat/main.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import QtQuick.Layouts 6 | import Qt5Compat.GraphicalEffects 7 | import llm 8 | import chatlistmodel 9 | import download 10 | import modellist 11 | import network 12 | import gpt4all 13 | import mysettings 14 | 15 | Window { 16 | id: window 17 | width: 1920 18 | height: 1080 19 | minimumWidth: 720 20 | minimumHeight: 480 21 | visible: true 22 | title: qsTr("GPT4All v") + Qt.application.version 23 | 24 | Settings { 25 | property alias x: window.x 26 | property alias y: window.y 27 | property alias width: window.width 28 | property alias height: window.height 29 | } 30 | 31 | Theme { 32 | id: theme 33 | } 34 | 35 | property bool hasSaved: false 36 | 37 | PopupDialog { 38 | id: savingPopup 39 | anchors.centerIn: parent 40 | shouldTimeOut: false 41 | shouldShowBusy: true 42 | text: qsTr("Saving chats.") 43 | font.pixelSize: theme.fontSizeLarge 44 | } 45 | 46 | onClosing: function(close) { 47 | if (window.hasSaved) 48 | return; 49 | 50 | savingPopup.open(); 51 | ChatListModel.saveChats(); 52 | close.accepted = false 53 | } 54 | 55 | Connections { 56 | target: ChatListModel 57 | function onSaveChatsFinished() { 58 | window.hasSaved = true; 59 | savingPopup.close(); 60 | window.close() 61 | } 62 | } 63 | 64 | color: theme.black 65 | 66 | ChatView { 67 | anchors.fill: parent 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /gpt4all-chat/network.h: -------------------------------------------------------------------------------- 1 | #ifndef NETWORK_H 2 | #define NETWORK_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | struct KeyValue { 17 | QString key; 18 | QJsonValue value; 19 | }; 20 | 21 | class Network : public QObject 22 | { 23 | Q_OBJECT 24 | public: 25 | static Network *globalInstance(); 26 | 27 | Q_INVOKABLE QString generateUniqueId() const; 28 | Q_INVOKABLE bool sendConversation(const QString &ingestId, const QString &conversation); 29 | Q_INVOKABLE void trackChatEvent(const QString &event, QVariantMap props = QVariantMap()); 30 | Q_INVOKABLE void trackEvent(const QString &event, const QVariantMap &props = QVariantMap()); 31 | 32 | Q_SIGNALS: 33 | void healthCheckFailed(int code); 34 | void requestMixpanel(const QByteArray &json, bool isOptOut = false); 35 | 36 | public Q_SLOTS: 37 | void sendStartup(); 38 | 39 | private Q_SLOTS: 40 | void handleIpifyFinished(); 41 | void handleHealthFinished(); 42 | void handleJsonUploadFinished(); 43 | void handleSslErrors(QNetworkReply *reply, const QList &errors); 44 | void handleMixpanelFinished(); 45 | void handleIsActiveChanged(); 46 | void handleUsageStatsActiveChanged(); 47 | void sendMixpanel(const QByteArray &json); 48 | 49 | private: 50 | void sendOptOut(); 51 | void sendHealth(); 52 | void sendIpify(); 53 | bool packageAndSendJson(const QString &ingestId, const QString &json); 54 | 55 | private: 56 | bool m_sendUsageStats = false; 57 | bool m_hasSentOptIn; 58 | bool m_hasSentOptOut; 59 | QString m_ipify; 60 | QString m_uniqueId; 61 | QString m_sessionId; 62 | QNetworkAccessManager m_networkManager; 63 | QVector m_activeUploads; 64 | 65 | private: 66 | explicit Network(); 67 | ~Network() {} 68 | friend class MyNetwork; 69 | }; 70 | 71 | #endif // LLM_H 72 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyBusyIndicator.qml: -------------------------------------------------------------------------------- 1 | import QtQuick 2 | import QtQuick.Controls 3 | import QtQuick.Controls.Basic 4 | 5 | BusyIndicator { 6 | id: control 7 | 8 | contentItem: Item { 9 | implicitWidth: 48 10 | implicitHeight: 48 11 | 12 | Item { 13 | id: item 14 | x: parent.width / 2 - width / 2 15 | y: parent.height / 2 - height / 2 16 | width: 48 17 | height: 48 18 | opacity: control.running ? 1 : 0 19 | 20 | Behavior on opacity { 21 | OpacityAnimator { 22 | duration: 250 23 | } 24 | } 25 | 26 | RotationAnimator { 27 | target: item 28 | running: control.visible && control.running 29 | from: 0 30 | to: 360 31 | loops: Animation.Infinite 32 | duration: 1750 33 | } 34 | 35 | Repeater { 36 | id: repeater 37 | model: 6 38 | 39 | Rectangle { 40 | id: delegate 41 | x: item.width / 2 - width / 2 42 | y: item.height / 2 - height / 2 43 | implicitWidth: 10 44 | implicitHeight: 10 45 | radius: 5 46 | color: theme.accentColor 47 | 48 | required property int index 49 | 50 | transform: [ 51 | Translate { 52 | y: -Math.min(item.width, item.height) * 0.5 + 5 53 | }, 54 | Rotation { 55 | angle: delegate.index / repeater.count * 360 56 | origin.x: 5 57 | origin.y: 5 58 | } 59 | ] 60 | } 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyButton.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import mysettings 6 | 7 | Button { 8 | id: myButton 9 | padding: 10 10 | rightPadding: 18 11 | leftPadding: 18 12 | property color textColor: theme.oppositeTextColor 13 | property color mutedTextColor: theme.oppositeMutedTextColor 14 | property color backgroundColor: theme.buttonBackground 15 | property color backgroundColorHovered: theme.buttonBackgroundHovered 16 | property real backgroundRadius: 10 17 | property real borderWidth: MySettings.chatTheme === "LegacyDark" ? 1 : 0 18 | property color borderColor: theme.buttonBorder 19 | property real fontPixelSize: theme.fontSizeLarge 20 | contentItem: Text { 21 | text: myButton.text 22 | horizontalAlignment: Text.AlignHCenter 23 | color: myButton.enabled ? textColor : mutedTextColor 24 | font.pixelSize: fontPixelSize 25 | Accessible.role: Accessible.Button 26 | Accessible.name: text 27 | } 28 | background: Rectangle { 29 | radius: myButton.backgroundRadius 30 | border.width: myButton.borderWidth 31 | border.color: myButton.borderColor 32 | color: myButton.hovered ? backgroundColorHovered : backgroundColor 33 | } 34 | Accessible.role: Accessible.Button 35 | Accessible.name: text 36 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 37 | } 38 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyCheckBox.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | 6 | CheckBox { 7 | id: myCheckBox 8 | 9 | background: Rectangle { 10 | color: "transparent" 11 | } 12 | 13 | indicator: Rectangle { 14 | implicitWidth: 26 15 | implicitHeight: 26 16 | x: myCheckBox.leftPadding 17 | y: parent.height / 2 - height / 2 18 | border.color: theme.checkboxBorder 19 | color: "transparent" 20 | radius: 3 21 | 22 | Rectangle { 23 | width: 14 24 | height: 14 25 | x: 6 26 | y: 6 27 | radius: 2 28 | color: theme.checkboxForeground 29 | visible: myCheckBox.checked 30 | } 31 | } 32 | 33 | contentItem: Text { 34 | text: myCheckBox.text 35 | font: myCheckBox.font 36 | opacity: enabled ? 1.0 : 0.3 37 | color: theme.textColor 38 | verticalAlignment: Text.AlignVCenter 39 | leftPadding: myCheckBox.indicator.width + myCheckBox.spacing 40 | } 41 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 42 | } -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyDialog.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import QtQuick.Dialogs 6 | import QtQuick.Layouts 7 | 8 | Dialog { 9 | id: myDialog 10 | property alias closeButtonVisible: myCloseButton.visible 11 | background: Rectangle { 12 | width: parent.width 13 | height: parent.height 14 | color: theme.containerBackground 15 | border.width: 1 16 | border.color: theme.dialogBorder 17 | radius: 10 18 | } 19 | 20 | Rectangle { 21 | id: closeBackground 22 | visible: myCloseButton.visible 23 | z: 299 24 | anchors.centerIn: myCloseButton 25 | width: myCloseButton.width + 10 26 | height: myCloseButton.height + 10 27 | color: theme.containerBackground 28 | } 29 | 30 | MyToolButton { 31 | id: myCloseButton 32 | x: 0 + myDialog.width - myDialog.padding - width - 15 33 | y: 0 - myDialog.padding + 15 34 | z: 300 35 | visible: myDialog.closePolicy != Popup.NoAutoClose 36 | width: 30 37 | height: 30 38 | padding: 0 39 | source: "qrc:/gpt4all/icons/close.svg" 40 | fillMode: Image.PreserveAspectFit 41 | onClicked: { 42 | myDialog.close(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyDirectoryField.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import llm 6 | 7 | TextField { 8 | id: myDirectoryField 9 | padding: 10 10 | property bool isValid: LLM.directoryExists(text) 11 | color: text === "" || isValid ? theme.textColor : theme.textErrorColor 12 | background: Rectangle { 13 | implicitWidth: 150 14 | color: theme.controlBackground 15 | radius: 10 16 | } 17 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 18 | } 19 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyMiniButton.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import Qt5Compat.GraphicalEffects 6 | 7 | Button { 8 | id: myButton 9 | padding: 0 10 | property color backgroundColor: theme.iconBackgroundDark 11 | property color backgroundColorHovered: theme.iconBackgroundHovered 12 | property alias source: image.source 13 | property alias fillMode: image.fillMode 14 | implicitWidth: 30 15 | implicitHeight: 30 16 | contentItem: Text { 17 | text: myButton.text 18 | horizontalAlignment: Text.AlignHCenter 19 | color: myButton.enabled ? theme.textColor : theme.mutedTextColor 20 | font.pixelSize: theme.fontSizeLarge 21 | Accessible.role: Accessible.Button 22 | Accessible.name: text 23 | } 24 | 25 | background: Item { 26 | anchors.fill: parent 27 | Rectangle { 28 | anchors.fill: parent 29 | color: "transparent" 30 | } 31 | Image { 32 | id: image 33 | anchors.centerIn: parent 34 | mipmap: true 35 | width: 20 36 | height: 20 37 | } 38 | ColorOverlay { 39 | anchors.fill: image 40 | source: image 41 | color: myButton.hovered ? backgroundColorHovered : backgroundColor 42 | } 43 | } 44 | Accessible.role: Accessible.Button 45 | Accessible.name: text 46 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 47 | } 48 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MySettingsButton.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import mysettings 6 | 7 | Button { 8 | id: myButton 9 | padding: 10 10 | rightPadding: 18 11 | leftPadding: 18 12 | property color textColor: MySettings.chatTheme === "Dark" ? theme.green800 : theme.green600 13 | property color mutedTextColor: textColor 14 | property color backgroundColor: MySettings.chatTheme === "Dark" ? theme.green400 : theme.green200 15 | property color backgroundColorHovered: theme.green300 16 | property real borderWidth: 0 17 | property color borderColor: "transparent" 18 | property real fontPixelSize: theme.fontSizeLarge 19 | 20 | contentItem: Text { 21 | text: myButton.text 22 | horizontalAlignment: Text.AlignHCenter 23 | color: myButton.enabled ? textColor : mutedTextColor 24 | font.pixelSize: fontPixelSize 25 | font.bold: true 26 | Accessible.role: Accessible.Button 27 | Accessible.name: text 28 | } 29 | background: Rectangle { 30 | radius: 10 31 | border.width: borderWidth 32 | border.color: borderColor 33 | color: myButton.hovered ? backgroundColorHovered : backgroundColor 34 | } 35 | Accessible.role: Accessible.Button 36 | Accessible.name: text 37 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 38 | } 39 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MySettingsDestructiveButton.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import mysettings 6 | 7 | Button { 8 | id: myButton 9 | padding: 10 10 | rightPadding: 18 11 | leftPadding: 18 12 | font.pixelSize: theme.fontSizeLarge 13 | property color textColor: MySettings.chatTheme === "Dark" ? theme.red800 : theme.red600 14 | property color mutedTextColor: MySettings.chatTheme === "Dark" ? theme.red400 : theme.red300 15 | property color backgroundColor: enabled ? (MySettings.chatTheme === "Dark" ? theme.red400 : theme.red200) : 16 | (MySettings.chatTheme === "Dark" ? theme.red200 : theme.red100) 17 | property color backgroundColorHovered: enabled ? (MySettings.chatTheme === "Dark" ? theme.red500 : theme.red300) : backgroundColor 18 | property real borderWidth: 0 19 | property color borderColor: "transparent" 20 | contentItem: Text { 21 | text: myButton.text 22 | horizontalAlignment: Text.AlignHCenter 23 | color: myButton.enabled ? textColor : mutedTextColor 24 | font.pixelSize: theme.fontSizeLarge 25 | font.bold: true 26 | Accessible.role: Accessible.Button 27 | Accessible.name: text 28 | } 29 | background: Rectangle { 30 | radius: 10 31 | border.width: borderWidth 32 | border.color: borderColor 33 | color: myButton.hovered ? backgroundColorHovered : backgroundColor 34 | } 35 | Accessible.role: Accessible.Button 36 | Accessible.name: text 37 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 38 | } 39 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MySettingsLabel.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | 6 | Label { 7 | color: theme.settingsTitleTextColor 8 | font.pixelSize: theme.fontSizeSmall 9 | font.bold: true 10 | } 11 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MySlug.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | 6 | Label { 7 | id: mySlug 8 | padding: 3 9 | rightPadding: 9 10 | leftPadding: 9 11 | font.pixelSize: theme.fontSizeFixedSmall 12 | background: Rectangle { 13 | radius: 6 14 | border.width: 1 15 | border.color: mySlug.color 16 | color: theme.slugBackground 17 | } 18 | ToolTip.visible: ma.containsMouse && ToolTip.text !== "" 19 | MouseArea { 20 | id: ma 21 | anchors.fill: parent 22 | hoverEnabled: true 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyTextArea.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | 6 | TextArea { 7 | id: myTextArea 8 | color: enabled ? theme.textColor : theme.mutedTextColor 9 | placeholderTextColor: theme.mutedTextColor 10 | font.pixelSize: theme.fontSizeLarge 11 | background: Rectangle { 12 | implicitWidth: 150 13 | color: theme.controlBackground 14 | border.width: 1 15 | border.color: theme.controlBorder 16 | radius: 10 17 | } 18 | padding: 10 19 | wrapMode: TextArea.Wrap 20 | 21 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 22 | } -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyTextField.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | 6 | TextField { 7 | id: myTextField 8 | padding: 10 9 | placeholderTextColor: theme.mutedTextColor 10 | background: Rectangle { 11 | implicitWidth: 150 12 | color: myTextField.enabled ? theme.controlBackground : theme.disabledControlBackground 13 | border.width: 1 14 | border.color: theme.controlBorder 15 | radius: 10 16 | } 17 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 18 | color: enabled ? theme.textColor : theme.mutedTextColor 19 | } 20 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/MyToolButton.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import Qt5Compat.GraphicalEffects 6 | 7 | Button { 8 | id: myButton 9 | padding: 10 10 | property color backgroundColor: theme.iconBackgroundDark 11 | property color backgroundColorHovered: theme.iconBackgroundHovered 12 | property bool toggled: false 13 | property alias source: image.source 14 | property alias fillMode: image.fillMode 15 | contentItem: Text { 16 | text: myButton.text 17 | horizontalAlignment: Text.AlignHCenter 18 | color: myButton.enabled ? theme.textColor : theme.mutedTextColor 19 | font.pixelSize: theme.fontSizeLarge 20 | Accessible.role: Accessible.Button 21 | Accessible.name: text 22 | } 23 | 24 | background: Item { 25 | anchors.fill: parent 26 | Rectangle { 27 | anchors.fill: parent 28 | color: "transparent" 29 | visible: myButton.toggled 30 | border.color: theme.accentColor 31 | border.width: 1 32 | radius: 10 33 | } 34 | Image { 35 | id: image 36 | anchors.centerIn: parent 37 | mipmap: true 38 | width: 30 39 | height: 30 40 | } 41 | ColorOverlay { 42 | anchors.fill: image 43 | source: image 44 | color: myButton.hovered ? backgroundColorHovered : backgroundColor 45 | } 46 | } 47 | Accessible.role: Accessible.Button 48 | Accessible.name: text 49 | ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval 50 | } 51 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/NewVersionDialog.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import QtQuick.Layouts 6 | import download 7 | import network 8 | import llm 9 | 10 | MyDialog { 11 | id: newVerionDialog 12 | anchors.centerIn: parent 13 | modal: true 14 | width: contentItem.width 15 | height: contentItem.height 16 | padding: 20 17 | closeButtonVisible: false 18 | 19 | Theme { 20 | id: theme 21 | } 22 | 23 | Item { 24 | id: contentItem 25 | width: childrenRect.width + 40 26 | height: childrenRect.height + 40 27 | 28 | Label { 29 | id: label 30 | anchors.top: parent.top 31 | anchors.left: parent.left 32 | topPadding: 20 33 | bottomPadding: 20 34 | text: qsTr("New version is available") 35 | color: theme.titleTextColor 36 | font.pixelSize: theme.fontSizeLarge 37 | font.bold: true 38 | } 39 | 40 | MySettingsButton { 41 | id: button 42 | anchors.left: label.right 43 | anchors.leftMargin: 10 44 | anchors.verticalCenter: label.verticalCenter 45 | padding: 20 46 | text: qsTr("Update") 47 | font.pixelSize: theme.fontSizeLarge 48 | Accessible.description: qsTr("Update to new version") 49 | onClicked: { 50 | if (!LLM.checkForUpdates()) 51 | checkForUpdatesError.open() 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/PopupDialog.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import QtQuick.Layouts 6 | 7 | Dialog { 8 | id: popupDialog 9 | anchors.centerIn: parent 10 | padding: 20 11 | property alias text: textField.text 12 | property bool shouldTimeOut: true 13 | property bool shouldShowBusy: false 14 | modal: shouldShowBusy 15 | closePolicy: shouldShowBusy ? Popup.NoAutoClose : (Popup.CloseOnEscape | Popup.CloseOnPressOutside) 16 | 17 | Theme { 18 | id: theme 19 | } 20 | 21 | Row { 22 | anchors.centerIn: parent 23 | spacing: 20 24 | 25 | Label { 26 | id: textField 27 | width: Math.min(1024, implicitWidth) 28 | height: Math.min(600, implicitHeight) 29 | anchors.verticalCenter: shouldShowBusy ? busyIndicator.verticalCenter : parent.verticalCenter 30 | horizontalAlignment: Text.AlignLeft 31 | verticalAlignment: Text.AlignVCenter 32 | textFormat: Text.StyledText 33 | wrapMode: Text.WordWrap 34 | color: theme.textColor 35 | linkColor: theme.linkColor 36 | Accessible.role: Accessible.HelpBalloon 37 | Accessible.name: text 38 | Accessible.description: qsTr("Reveals a shortlived help balloon") 39 | onLinkActivated: function(link) { Qt.openUrlExternally(link) } 40 | } 41 | 42 | MyBusyIndicator { 43 | id: busyIndicator 44 | visible: shouldShowBusy 45 | running: shouldShowBusy 46 | 47 | Accessible.role: Accessible.Animation 48 | Accessible.name: qsTr("Busy indicator") 49 | Accessible.description: qsTr("Displayed when the popup is showing busy") 50 | } 51 | } 52 | 53 | background: Rectangle { 54 | anchors.fill: parent 55 | color: theme.containerBackground 56 | border.width: 1 57 | border.color: theme.dialogBorder 58 | radius: 10 59 | } 60 | 61 | exit: Transition { 62 | NumberAnimation { duration: 500; property: "opacity"; from: 1.0; to: 0.0 } 63 | } 64 | 65 | onOpened: { 66 | if (shouldTimeOut) 67 | timer.start() 68 | } 69 | 70 | Timer { 71 | id: timer 72 | interval: 500; running: false; repeat: false 73 | onTriggered: popupDialog.close() 74 | } 75 | } -------------------------------------------------------------------------------- /gpt4all-chat/qml/SwitchModelDialog.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import QtQuick.Layouts 6 | import llm 7 | import mysettings 8 | 9 | MyDialog { 10 | id: switchModelDialog 11 | anchors.centerIn: parent 12 | modal: true 13 | padding: 20 14 | property int index: -1 15 | 16 | Theme { 17 | id: theme 18 | } 19 | 20 | contentItem: Text { 21 | textFormat: Text.StyledText 22 | text: qsTr("Warning: changing the model will erase the current conversation. Do you wish to continue?") 23 | color: theme.textColor 24 | font.pixelSize: theme.fontSizeLarge 25 | } 26 | 27 | footer: DialogButtonBox { 28 | id: dialogBox 29 | padding: 20 30 | alignment: Qt.AlignRight 31 | spacing: 10 32 | MySettingsButton { 33 | text: qsTr("Continue") 34 | Accessible.description: qsTr("Continue with model loading") 35 | DialogButtonBox.buttonRole: DialogButtonBox.AcceptRole 36 | } 37 | MySettingsButton { 38 | text: qsTr("Cancel") 39 | Accessible.description: qsTr("Cancel") 40 | DialogButtonBox.buttonRole: DialogButtonBox.RejectRole 41 | } 42 | background: Rectangle { 43 | color: "transparent" 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /gpt4all-chat/qml/ThumbsDownDialog.qml: -------------------------------------------------------------------------------- 1 | import QtCore 2 | import QtQuick 3 | import QtQuick.Controls 4 | import QtQuick.Controls.Basic 5 | import QtQuick.Layouts 6 | import download 7 | import network 8 | import llm 9 | 10 | MyDialog { 11 | id: thumbsDownDialog 12 | modal: true 13 | padding: 20 14 | 15 | Theme { 16 | id: theme 17 | } 18 | 19 | property alias response: thumbsDownNewResponse.text 20 | 21 | Column { 22 | anchors.fill: parent 23 | spacing: 20 24 | Item { 25 | width: childrenRect.width 26 | height: childrenRect.height 27 | Image { 28 | id: img 29 | anchors.top: parent.top 30 | anchors.left: parent.left 31 | width: 60 32 | height: 60 33 | source: "qrc:/gpt4all/icons/thumbs_down.svg" 34 | } 35 | Text { 36 | anchors.left: img.right 37 | anchors.leftMargin: 30 38 | anchors.verticalCenter: img.verticalCenter 39 | text: qsTr("Please edit the text below to provide a better response. (optional)") 40 | color: theme.textColor 41 | font.pixelSize: theme.fontSizeLarge 42 | } 43 | } 44 | 45 | ScrollView { 46 | clip: true 47 | height: 300 48 | width: parent.width 49 | ScrollBar.vertical.policy: ScrollBar.AlwaysOn 50 | ScrollBar.horizontal.policy: ScrollBar.AlwaysOff 51 | 52 | MyTextArea { 53 | id: thumbsDownNewResponse 54 | placeholderText: qsTr("Please provide a better response...") 55 | } 56 | } 57 | } 58 | 59 | footer: DialogButtonBox { 60 | padding: 20 61 | alignment: Qt.AlignRight 62 | spacing: 10 63 | MySettingsButton { 64 | text: qsTr("Submit") 65 | Accessible.description: qsTr("Submits the user's response") 66 | DialogButtonBox.buttonRole: DialogButtonBox.AcceptRole 67 | } 68 | MySettingsButton { 69 | text: qsTr("Cancel") 70 | Accessible.description: qsTr("Closes the response dialog") 71 | DialogButtonBox.buttonRole: DialogButtonBox.RejectRole 72 | } 73 | background: Rectangle { 74 | color: "transparent" 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /gpt4all-chat/resources/gpt4all.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/resources/gpt4all.icns -------------------------------------------------------------------------------- /gpt4all-chat/resources/gpt4all.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-chat/resources/gpt4all.ico -------------------------------------------------------------------------------- /gpt4all-chat/resources/gpt4all.rc: -------------------------------------------------------------------------------- 1 | IDI_ICON1 ICON "gpt4all.ico" 2 | -------------------------------------------------------------------------------- /gpt4all-chat/responsetext.h: -------------------------------------------------------------------------------- 1 | #ifndef RESPONSETEXT_H 2 | #define RESPONSETEXT_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include // IWYU pragma: keep 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | class QPainter; 16 | class QTextDocument; 17 | class QTextFormat; 18 | 19 | class SyntaxHighlighter : public QSyntaxHighlighter { 20 | Q_OBJECT 21 | public: 22 | SyntaxHighlighter(QObject *parent); 23 | ~SyntaxHighlighter(); 24 | void highlightBlock(const QString &text) override; 25 | }; 26 | 27 | struct ContextLink { 28 | int startPos = -1; 29 | int endPos = -1; 30 | QString text; 31 | QString href; 32 | }; 33 | 34 | struct CodeCopy { 35 | int startPos = -1; 36 | int endPos = -1; 37 | QString text; 38 | }; 39 | 40 | class ContextLinkInterface : public QObject, public QTextObjectInterface 41 | { 42 | Q_OBJECT 43 | Q_INTERFACES(QTextObjectInterface) 44 | 45 | public: 46 | explicit ContextLinkInterface(QObject *parent) : QObject(parent) {} 47 | void drawObject(QPainter *painter, const QRectF &rect, QTextDocument *doc, int posInDocument, 48 | const QTextFormat &format) override; 49 | 50 | QSizeF intrinsicSize(QTextDocument *doc, int posInDocument, const QTextFormat &format) override; 51 | }; 52 | 53 | 54 | class ResponseText : public QObject 55 | { 56 | Q_OBJECT 57 | Q_PROPERTY(QQuickTextDocument* textDocument READ textDocument WRITE setTextDocument NOTIFY textDocumentChanged()) 58 | QML_ELEMENT 59 | public: 60 | explicit ResponseText(QObject *parent = nullptr); 61 | 62 | QQuickTextDocument* textDocument() const; 63 | void setTextDocument(QQuickTextDocument* textDocument); 64 | 65 | Q_INVOKABLE void setLinkColor(const QColor &c) { m_linkColor = c; } 66 | Q_INVOKABLE void setHeaderColor(const QColor &c) { m_headerColor = c; } 67 | 68 | Q_INVOKABLE QString getLinkAtPosition(int position) const; 69 | Q_INVOKABLE bool tryCopyAtPosition(int position) const; 70 | 71 | Q_SIGNALS: 72 | void textDocumentChanged(); 73 | 74 | private Q_SLOTS: 75 | void handleTextChanged(); 76 | void handleContextLinks(); 77 | void handleCodeBlocks(); 78 | 79 | private: 80 | QQuickTextDocument *m_textDocument; 81 | SyntaxHighlighter *m_syntaxHighlighter; 82 | QVector m_links; 83 | QVector m_copies; 84 | QColor m_linkColor; 85 | QColor m_headerColor; 86 | bool m_isProcessingText = false; 87 | ContextLinkInterface *m_contextLink; 88 | }; 89 | 90 | #endif // RESPONSETEXT_H 91 | -------------------------------------------------------------------------------- /gpt4all-chat/server.h: -------------------------------------------------------------------------------- 1 | #ifndef SERVER_H 2 | #define SERVER_H 3 | 4 | #include "chatllm.h" 5 | #include "database.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | class Chat; 14 | class QHttpServer; 15 | 16 | class Server : public ChatLLM 17 | { 18 | Q_OBJECT 19 | 20 | public: 21 | Server(Chat *parent); 22 | virtual ~Server(); 23 | 24 | public Q_SLOTS: 25 | void start(); 26 | 27 | Q_SIGNALS: 28 | void requestServerNewPromptResponsePair(const QString &prompt); 29 | 30 | private Q_SLOTS: 31 | QHttpServerResponse handleCompletionRequest(const QHttpServerRequest &request, bool isChat); 32 | void handleDatabaseResultsChanged(const QList &results) { m_databaseResults = results; } 33 | void handleCollectionListChanged(const QList &collectionList) { m_collections = collectionList; } 34 | 35 | private: 36 | Chat *m_chat; 37 | QHttpServer *m_server; 38 | QList m_databaseResults; 39 | QList m_collections; 40 | }; 41 | 42 | #endif // SERVER_H 43 | -------------------------------------------------------------------------------- /gpt4all-lora-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-lora-demo.gif -------------------------------------------------------------------------------- /gpt4all-training/GPT-J_MAP.md: -------------------------------------------------------------------------------- 1 | # Inference on Training Data 2 | 3 | 4 | ## Run Inference 5 | 6 | ```bash 7 | torchrun --master_port=29085 --nproc-per-node 8 inference.py --config=configs/inference/gptj.yaml 8 | ``` 9 | 10 | 11 | ## Visualizations 12 | 13 | ```bash 14 | python build_map.py 15 | ``` 16 | 17 | will build a map in `Atlas`, one using the internal clustering algorithm provided by Nomic and one using the embeddings generated by the finetuned model. -------------------------------------------------------------------------------- /gpt4all-training/README.md: -------------------------------------------------------------------------------- 1 | ## Training GPT4All-J 2 | 3 | Please see [GPT4All-J Technical Report](https://static.nomic.ai/gpt4all/2023_GPT4All-J_Technical_Report_2.pdf) for details. 4 | 5 | ### GPT4All-J Training Data 6 | 7 | - We are releasing the curated training data for anyone to replicate GPT4All-J here: [GPT4All-J Training Data](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations) 8 | - [Atlas Map of Prompts](https://atlas.nomic.ai/map/gpt4all-j-prompts-curated) 9 | - [Atlas Map of Responses](https://atlas.nomic.ai/map/gpt4all-j-response-curated) 10 | 11 | We have released updated versions of our `GPT4All-J` model and training data. 12 | 13 | - `v1.0`: The original model trained on the v1.0 dataset 14 | - `v1.1-breezy`: Trained on a filtered dataset where we removed all instances of AI language model 15 | - `v1.2-jazzy`: Trained on a filtered dataset where we also removed instances like I'm sorry, I can't answer... and AI language model 16 | 17 | The [models](https://huggingface.co/nomic-ai/gpt4all-j) and [data](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations) versions can be specified by passing a `revision` argument. 18 | 19 | For example, to load the `v1.2-jazzy` model and dataset, run: 20 | 21 | ```python 22 | from datasets import load_dataset 23 | from transformers import AutoModelForCausalLM 24 | 25 | dataset = load_dataset("nomic-ai/gpt4all-j-prompt-generations", revision="v1.2-jazzy") 26 | model = AutoModelForCausalLM.from_pretrained("nomic-ai/gpt4all-j", revision="v1.2-jazzy") 27 | ``` 28 | 29 | ### GPT4All-J Training Instructions 30 | 31 | ```bash 32 | accelerate launch --dynamo_backend=inductor --num_processes=8 --num_machines=1 --machine_rank=0 --deepspeed_multinode_launcher standard --mixed_precision=bf16 --use_deepspeed --deepspeed_config_file=configs/deepspeed/ds_config_gptj.json train.py --config configs/train/finetune_gptj.yaml 33 | ``` 34 | -------------------------------------------------------------------------------- /gpt4all-training/build_map.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import numpy as np 3 | from nomic import atlas 4 | import glob 5 | from tqdm import tqdm 6 | from datasets import load_dataset, concatenate_datasets 7 | from sklearn.decomposition import PCA 8 | 9 | files = glob.glob("inference/*.jsonl") 10 | print(files) 11 | df = concatenate_datasets([load_dataset("json", data_files=file, split="train") for file in tqdm(files)]) 12 | 13 | print(len(df)) 14 | print(df) 15 | 16 | df = df.map(lambda example: {"inputs": [prompt + "\n" + response for prompt, response in zip(example["prompt"], example["response"])]}, 17 | batched=True, 18 | num_proc=64) 19 | 20 | df = df.map(lambda example: {"trained_on": [int(t) for t in example["is_train"]]}, 21 | batched=True, 22 | num_proc=64) 23 | 24 | df = df.remove_columns("is_train") 25 | 26 | text = df.remove_columns(["labels", "input_ids", "embeddings"]) 27 | 28 | text_df = [text[i] for i in range(len(text))] 29 | 30 | atlas.map_text(text_df, indexed_field="inputs", 31 | name="CHANGE ME!", 32 | colorable_fields=["source", "loss", "trained_on"], 33 | reset_project_if_exists=True, 34 | ) 35 | 36 | # index is local to train/test split, regenerate 37 | data = df.remove_columns(["labels", "input_ids", "index"]) 38 | data = data.add_column("index", list(range(len(data)))) 39 | # max embed dim is 2048 for now 40 | # note! this is slow in pyarrow/hf datasets 41 | embeddings = np.array(data["embeddings"]) 42 | print("embeddings shape:", embeddings.shape) 43 | embeddings = PCA(n_components=2048).fit_transform(embeddings) 44 | 45 | data = data.remove_columns(["embeddings"]) 46 | columns = data.to_pandas().to_dict("records") 47 | 48 | atlas.map_embeddings(embeddings, 49 | data=columns, 50 | id_field="index", 51 | name="CHANGE ME!", 52 | colorable_fields=["source", "loss", "trained_on"], 53 | build_topic_model=True, 54 | topic_label_field="inputs", 55 | reset_project_if_exists=True,) 56 | -------------------------------------------------------------------------------- /gpt4all-training/clean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import numpy as np 3 | import glob 4 | import os 5 | import json 6 | import jsonlines 7 | import pandas as pd 8 | 9 | 10 | prompt_generation_dir = "raw_data_sanity_cleaned_without_p3/" 11 | for file in glob.glob(os.path.join(prompt_generation_dir, "*.jsonl")): 12 | if "clean.jsonl" in file: 13 | continue 14 | data = [] 15 | print(file) 16 | with open(file) as f: 17 | for line in f: 18 | try: 19 | contents = json.loads(line) 20 | data.append(contents) 21 | except BaseException: 22 | pass 23 | 24 | processed = [] 25 | 26 | for item in data: 27 | if 'source' not in item: 28 | item['source'] = 'unspecified' 29 | if 'model_settings' in item: 30 | item.pop('model_settings', None) 31 | 32 | for key in list(item.keys()): 33 | if key not in ['source', 'prompt', 'response']: 34 | #print(item[key]) 35 | item.pop(key, None) 36 | 37 | if isinstance(item['prompt'], dict): 38 | if "value" in item["prompt"]: 39 | item["prompt"] = item["prompt"]["value"] 40 | elif "description" in item["prompt"]: 41 | item["prompt"] = item["prompt"]["description"] 42 | else: 43 | continue 44 | 45 | elif not isinstance(item['prompt'], str): 46 | continue 47 | 48 | if isinstance(item['response'], dict): 49 | if "value" in item["response"]: 50 | item["response"] = item["response"]["value"] 51 | elif "description" in item["response"]: 52 | item["response"] = item["response"]["description"] 53 | else: 54 | continue 55 | elif not isinstance(item['response'], str): 56 | continue 57 | 58 | if item: 59 | processed.append(item) 60 | 61 | df = pd.DataFrame(processed) 62 | prev_len = len(df) 63 | 64 | # drop empty or null string 65 | df = df.dropna(subset=['prompt', 'response']) 66 | df = df[df['prompt'] != ''] 67 | df = df[df['response'] != ''] 68 | df = df[df["prompt"].str.len() > 1] 69 | curr_len = len(df) 70 | 71 | print(f"Removed {prev_len - curr_len} rows") 72 | 73 | clean_name = file.split(".jsonl")[0] + "_clean.jsonl" 74 | print(f"writing to {curr_len} rows to {clean_name}") 75 | df.to_json(clean_name, orient="records", lines=True) 76 | -------------------------------------------------------------------------------- /gpt4all-training/configs/deepspeed/ds_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "train_batch_size": "auto", 3 | "gradient_accumulation_steps": "auto", 4 | "train_micro_batch_size_per_gpu": "auto", 5 | "fp16": { 6 | "enabled": "auto", 7 | "min_loss_scale": 1, 8 | "loss_scale_window": 1000, 9 | "hysteresis": 2, 10 | "initial_scale_power": 32 11 | }, 12 | "bf16": { 13 | "enabled": "auto" 14 | }, 15 | "gradient_clipping": 1, 16 | "zero_optimization": { 17 | "stage": 2, 18 | "offload_param": { 19 | "device": "none" 20 | }, 21 | "offload_optimizer": { 22 | "device": "none" 23 | }, 24 | "allgather_partitions": true, 25 | "allgather_bucket_size": 5e8, 26 | "contiguous_gradients": true 27 | }, 28 | "optimizer": { 29 | "type": "AdamW", 30 | "params": { 31 | "lr": "auto", 32 | "betas": [ 33 | 0.9, 34 | 0.999 35 | ], 36 | "eps": 1e-08 37 | } 38 | }, 39 | "scheduler": { 40 | "type": "WarmupLR", 41 | "params": { 42 | "warmup_min_lr": 0, 43 | "warmup_max_lr": "auto", 44 | "warmup_num_steps": "auto", 45 | "warmup_type": "linear" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /gpt4all-training/configs/deepspeed/ds_config_gptj.json: -------------------------------------------------------------------------------- 1 | { 2 | "train_batch_size": "auto", 3 | "gradient_accumulation_steps": "auto", 4 | "train_micro_batch_size_per_gpu": "auto", 5 | "fp16": { 6 | "enabled": "auto", 7 | "min_loss_scale": 1, 8 | "loss_scale_window": 1000, 9 | "hysteresis": 2, 10 | "initial_scale_power": 32 11 | }, 12 | "bf16": { 13 | "enabled": "auto" 14 | }, 15 | "gradient_clipping": 1.0, 16 | "zero_optimization": { 17 | "stage": 2, 18 | "offload_param": { 19 | "device": "none" 20 | }, 21 | "offload_optimizer": { 22 | "device": "none" 23 | }, 24 | "allgather_partitions": true, 25 | "allgather_bucket_size": 5e8, 26 | "contiguous_gradients": true 27 | }, 28 | "optimizer": { 29 | "type": "AdamW", 30 | "params": { 31 | "lr": "auto", 32 | "betas": [ 33 | 0.9, 34 | 0.999 35 | ], 36 | "eps": 1e-08 37 | } 38 | }, 39 | "scheduler": { 40 | "type": "WarmupLR", 41 | "params": { 42 | "warmup_min_lr": 0, 43 | "warmup_max_lr": "auto", 44 | "warmup_num_steps": "auto", 45 | "warmup_type": "linear" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /gpt4all-training/configs/deepspeed/ds_config_gptj_lora.json: -------------------------------------------------------------------------------- 1 | { 2 | "train_batch_size": "auto", 3 | "gradient_accumulation_steps": "auto", 4 | "train_micro_batch_size_per_gpu": "auto", 5 | "fp16": { 6 | "enabled": "auto", 7 | "min_loss_scale": 1, 8 | "loss_scale_window": 1000, 9 | "hysteresis": 2, 10 | "initial_scale_power": 32 11 | }, 12 | "bf16": { 13 | "enabled": "auto" 14 | }, 15 | "gradient_clipping": 1, 16 | "zero_optimization": { 17 | "stage": 2, 18 | "offload_param": { 19 | "device": "cpu" 20 | }, 21 | "offload_optimizer": { 22 | "device": "cpu" 23 | }, 24 | "allgather_partitions": true, 25 | "allgather_bucket_size": 5e8, 26 | "contiguous_gradients": true 27 | }, 28 | "optimizer": { 29 | "type": "AdamW", 30 | "params": { 31 | "lr": "auto", 32 | "betas": [ 33 | 0.9, 34 | 0.999 35 | ], 36 | "eps": 1e-08 37 | } 38 | }, 39 | "scheduler": { 40 | "type": "WarmupLR", 41 | "params": { 42 | "warmup_min_lr": 0, 43 | "warmup_max_lr": "auto", 44 | "warmup_num_steps": "auto", 45 | "warmup_type": "linear" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /gpt4all-training/configs/deepspeed/ds_config_mpt.json: -------------------------------------------------------------------------------- 1 | { 2 | "train_batch_size": "auto", 3 | "gradient_accumulation_steps": "auto", 4 | "train_micro_batch_size_per_gpu": "auto", 5 | "fp16": { 6 | "enabled": "auto", 7 | "min_loss_scale": 1, 8 | "loss_scale_window": 1000, 9 | "hysteresis": 2, 10 | "initial_scale_power": 32 11 | }, 12 | "bf16": { 13 | "enabled": "auto" 14 | }, 15 | "gradient_clipping": 1.0, 16 | "zero_optimization": { 17 | "stage": 1, 18 | "offload_param": { 19 | "device": "none" 20 | }, 21 | "offload_optimizer": { 22 | "device": "none" 23 | }, 24 | "allgather_partitions": true, 25 | "allgather_bucket_size": 5e8, 26 | "contiguous_gradients": true 27 | }, 28 | "optimizer": { 29 | "type": "AdamW", 30 | "params": { 31 | "lr": "auto", 32 | "betas": [ 33 | 0.9, 34 | 0.999 35 | ], 36 | "eps": 1e-08 37 | } 38 | }, 39 | "scheduler": { 40 | "type": "WarmupDecayLR", 41 | "params": { 42 | "warmup_min_lr": 0, 43 | "warmup_max_lr": "auto", 44 | "warmup_num_steps": "auto", 45 | "warmup_type": "linear", 46 | "total_num_steps": "auto" 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /gpt4all-training/configs/deepspeed/ds_config_pythia.json: -------------------------------------------------------------------------------- 1 | { 2 | "train_batch_size": "auto", 3 | "gradient_accumulation_steps": "auto", 4 | "train_micro_batch_size_per_gpu": "auto", 5 | "fp16": { 6 | "enabled": "auto", 7 | "min_loss_scale": 1, 8 | "loss_scale_window": 1000, 9 | "hysteresis": 2, 10 | "initial_scale_power": 32 11 | }, 12 | "bf16": { 13 | "enabled": "auto" 14 | }, 15 | "gradient_clipping": 1.0, 16 | "zero_optimization": { 17 | "stage": 2, 18 | "offload_param": { 19 | "device": "none" 20 | }, 21 | "offload_optimizer": { 22 | "device": "none" 23 | }, 24 | "allgather_partitions": true, 25 | "allgather_bucket_size": 5e8, 26 | "contiguous_gradients": true 27 | }, 28 | "optimizer": { 29 | "type": "AdamW", 30 | "params": { 31 | "lr": "auto", 32 | "betas": [ 33 | 0.9, 34 | 0.999 35 | ], 36 | "eps": 1e-08 37 | } 38 | }, 39 | "scheduler": { 40 | "type": "WarmupLR", 41 | "params": { 42 | "warmup_min_lr": 0, 43 | "warmup_max_lr": "auto", 44 | "warmup_num_steps": "auto", 45 | "warmup_type": "linear" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /gpt4all-training/configs/eval/generate_baseline.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "zpn/llama-7b" 3 | tokenizer_name: "zpn/llama-7b" 4 | lora: true 5 | lora_path: "tloen/alpaca-lora-7b" -------------------------------------------------------------------------------- /gpt4all-training/configs/eval/generate_gpt4all_gptj.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "nomic-ai/gpt4all-warmup-lr-epoch_0" 3 | tokenizer_name: "EleutherAI/gpt-j-6b" 4 | lora: false 5 | -------------------------------------------------------------------------------- /gpt4all-training/configs/eval/generate_gpt4all_gptj_lora.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "EleutherAI/gpt-j-6b" 3 | tokenizer_name: "EleutherAI/gpt-j-6B" 4 | lora: true 5 | lora_path: "nomic-ai/gpt4all-gptj-lora-epoch_1" 6 | -------------------------------------------------------------------------------- /gpt4all-training/configs/eval/generate_gpt4all_llama_lora.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "zpn/llama-7b" 3 | tokenizer_name: "zpn/llama-7b" 4 | lora: true 5 | lora_path: "nomic-ai/gpt4all-lora" 6 | -------------------------------------------------------------------------------- /gpt4all-training/configs/generate/generate.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "zpn/llama-7b" 3 | tokenizer_name: "zpn/llama-7b" 4 | lora: true 5 | lora_path: "nomic-ai/gpt4all-lora" 6 | 7 | max_new_tokens: 512 8 | temperature: 0 9 | prompt: null 10 | -------------------------------------------------------------------------------- /gpt4all-training/configs/generate/generate_gptj.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1" 3 | tokenizer_name: "EleutherAI/gpt-j-6b" 4 | lora: false 5 | 6 | 7 | max_new_tokens: 512 8 | temperature: 0.001 9 | prompt: | 10 | #this code prints a string reversed 11 | my_string = "hello how are you" 12 | print(len(my_string)) 13 | 14 | 15 | My code above does not work. Can you help me? 16 | -------------------------------------------------------------------------------- /gpt4all-training/configs/generate/generate_gptj_lora.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "EleutherAI/gpt-j-6b" 3 | tokenizer_name: "EleutherAI/gpt-j-6b" 4 | lora: true 5 | lora_path: "nomic-ai/gpt4all-gptj-lora-epoch_0" 6 | 7 | max_new_tokens: 512 8 | temperature: 0 9 | prompt: | 10 | #this code prints a string reversed 11 | my_string = "hello how are you" 12 | print(len(my_string)) 13 | 14 | 15 | My code above does not work. Can you help me? -------------------------------------------------------------------------------- /gpt4all-training/configs/generate/generate_llama.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: # REPLACE WITH LLAMA MODEL NAME 3 | tokenizer_name: # REPLACE WITH LLAMA MODEL NAME 4 | 5 | 6 | max_new_tokens: 512 7 | temperature: 0.001 8 | prompt: | 9 | #this code prints a string reversed 10 | my_string = "hello how are you" 11 | print(len(my_string)) 12 | 13 | 14 | My code above does not work. Can you help me? 15 | -------------------------------------------------------------------------------- /gpt4all-training/configs/inference/gptj.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1" 3 | tokenizer_name: "EleutherAI/gpt-j-6B" 4 | 5 | # dataset 6 | streaming: false 7 | num_proc: 64 8 | dataset_path: "nomic-ai/turbo-500k-multi" 9 | max_length: 1024 10 | batch_size: 32 11 | 12 | # logging 13 | seed: 42 14 | 15 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: # add model here 3 | tokenizer_name: # add model here 4 | gradient_checkpointing: true 5 | save_name: # CHANGE 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: # update 11 | max_length: 1024 12 | batch_size: 32 13 | 14 | # train dynamics 15 | lr: 5.0e-5 16 | eval_every: 800 17 | eval_steps: 100 18 | save_every: 800 19 | output_dir: # CHANGE 20 | checkpoint: null 21 | lora: false 22 | warmup_steps: 100 23 | num_epochs: 2 24 | 25 | # logging 26 | wandb: true 27 | wandb_entity: # update 28 | wandb_project_name: # update 29 | seed: 42 30 | 31 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune_falcon.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "tiiuae/falcon-7b" 3 | tokenizer_name: "tiiuae/falcon-7b" 4 | gradient_checkpointing: true 5 | save_name: "nomic-ai/gpt4all-falcon" 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: "nomic-ai/gpt4all-j-prompt-generations" 11 | revision: "v1.3-groovy" 12 | max_length: 1024 13 | batch_size: 32 14 | 15 | # train dynamics 16 | lr: 2.0e-5 17 | min_lr: 0 18 | weight_decay: 0.0 19 | eval_every: 500 20 | eval_steps: 105 21 | save_every: 1000 22 | log_grads_every: 500 23 | output_dir: "ckpts/falcon" 24 | checkpoint: "/home/paperspace/gpt4all/ckpts/mpt/step_1000" 25 | lora: false 26 | warmup_steps: 500 27 | num_epochs: 2 28 | 29 | # logging 30 | wandb: true 31 | wandb_entity: "gpt4all" 32 | wandb_project_name: "gpt4all" 33 | seed: 42 34 | 35 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune_gptj.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "EleutherAI/gpt-j-6B" 3 | tokenizer_name: "EleutherAI/gpt-j-6B" 4 | gradient_checkpointing: true 5 | save_name: # CHANGE 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: # CHANGE 11 | max_length: 1024 12 | batch_size: 32 13 | 14 | # train dynamics 15 | lr: 2.0e-5 16 | min_lr: 0 17 | weight_decay: 0.0 18 | eval_every: 500 19 | eval_steps: 105 20 | save_every: 500 21 | log_grads_every: 100 22 | output_dir: # CHANGE 23 | checkpoint: null 24 | lora: false 25 | warmup_steps: 500 26 | num_epochs: 2 27 | 28 | # logging 29 | wandb: true 30 | wandb_entity: # CHANGE 31 | wandb_project_name: # CHANGE 32 | seed: 42 33 | 34 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune_gptj_lora.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "EleutherAI/gpt-j-6b" 3 | tokenizer_name: "EleutherAI/gpt-j-6b" 4 | gradient_checkpointing: false 5 | save_name: # CHANGE 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: # CHANGE 11 | max_length: 1024 12 | batch_size: 1 13 | 14 | # train dynamics 15 | lr: 2.0e-5 16 | min_lr: 0 17 | weight_decay: 0.0 18 | eval_every: 500 19 | eval_steps: 105 20 | save_every: 500 21 | log_grads_every: 500 22 | output_dir: # CHANGE 23 | checkpoint: null 24 | lora: true 25 | warmup_steps: 500 26 | num_epochs: 2 27 | 28 | # logging 29 | wandb: true 30 | wandb_entity: # CHANGE 31 | wandb_project_name: # CHANGE 32 | seed: 42 33 | 34 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune_lora.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: # update 3 | tokenizer_name: # update 4 | gradient_checkpointing: false 5 | save_name: # CHANGE 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: # CHANGE 11 | max_length: 1024 12 | batch_size: 4 13 | 14 | # train dynamics 15 | lr: 5.0e-5 16 | min_lr: 0 17 | weight_decay: 0.0 18 | eval_every: 2000 19 | eval_steps: 100 20 | save_every: 2000 21 | output_dir: # CHANGE 22 | checkpoint: null 23 | lora: true 24 | warmup_steps: 100 25 | num_epochs: 2 26 | 27 | # logging 28 | wandb: true 29 | wandb_entity: # update 30 | wandb_project_name: # update 31 | seed: 42 32 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune_mpt.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "mosaicml/mpt-7b" 3 | tokenizer_name: "mosaicml/mpt-7b" 4 | gradient_checkpointing: false 5 | save_name: "nomic-ai/mpt-finetuned-round2" 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: "nomic-ai/gpt4all-j-prompt-generations" 11 | revision: "v1.3-groovy" 12 | max_length: 1024 13 | batch_size: 8 14 | 15 | # train dynamics 16 | lr: 2.0e-5 17 | min_lr: 0 18 | weight_decay: 0.0 19 | eval_every: 500 20 | eval_steps: 105 21 | save_every: 1000 22 | log_grads_every: 500 23 | output_dir: "ckpts/mpt" 24 | checkpoint: null 25 | lora: false 26 | warmup_steps: 500 27 | num_epochs: 2 28 | 29 | # logging 30 | wandb: false 31 | wandb_entity: "gpt4all" 32 | wandb_project_name: "gpt4all" 33 | seed: 42 34 | 35 | -------------------------------------------------------------------------------- /gpt4all-training/configs/train/finetune_openllama.yaml: -------------------------------------------------------------------------------- 1 | # model/tokenizer 2 | model_name: "openlm-research/open_llama_7b" 3 | tokenizer_name: "openlm-research/open_llama_7b" 4 | gradient_checkpointing: true 5 | save_name: "nomic-ai/gpt4all-openllama" 6 | 7 | # dataset 8 | streaming: false 9 | num_proc: 64 10 | dataset_path: "nomic-ai/gpt4all-updated" 11 | revision: null 12 | max_length: 1024 13 | batch_size: 32 14 | 15 | # train dynamics 16 | lr: 2.0e-5 17 | min_lr: 0 18 | weight_decay: 0.0 19 | eval_every: 500 20 | log_every: 10 21 | save_every: 1000 22 | log_grads_every: 500 23 | output_dir: "ckpts/falcon" 24 | checkpoint: null 25 | lora: false 26 | warmup_steps: 500 27 | num_epochs: 3 28 | 29 | # logging 30 | wandb: true 31 | wandb_entity: "gpt4all" 32 | wandb_project_name: "gpt4all" 33 | seed: 42 34 | 35 | -------------------------------------------------------------------------------- /gpt4all-training/create_hostname.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export WORKER_IP=$1 4 | N_GPUS=8 5 | # create dir if doesn't exist 6 | sudo mkdir -p /job 7 | printf "localhost slots=$N_GPUS\n$WORKER_IP slots=$N_GPUS" | sudo tee /job/hostfile 8 | echo /job/hostfile -------------------------------------------------------------------------------- /gpt4all-training/env.yaml: -------------------------------------------------------------------------------- 1 | name: vicuna 2 | channels: 3 | - conda-forge 4 | - pytorch 5 | - nvidia 6 | - huggingface 7 | dependencies: 8 | - python=3.8 9 | - accelerate 10 | - datasets 11 | - torchmetrics 12 | - evaluate 13 | - transformers 14 | - wandb 15 | - jsonlines 16 | - pip: 17 | - peft 18 | - nodelist-inflator 19 | - deepspeed 20 | - sentencepiece -------------------------------------------------------------------------------- /gpt4all-training/eval_figures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import glob 3 | import pickle 4 | import numpy as np 5 | from matplotlib import pyplot as plt 6 | 7 | plt.figure() 8 | for fpath in glob.glob('./eval_data/*.pkl'): 9 | parts = fpath.split('__') 10 | model_name = "-".join(fpath.replace(".pkl", "").split("_")[2:]) 11 | with open(fpath, 'rb') as f: 12 | data = pickle.load(f) 13 | perplexities = data['perplexities'] 14 | perplexities = np.nan_to_num(perplexities, 100) 15 | perplexities = np.clip(perplexities, 0, 100) 16 | if 'alpaca' not in fpath: 17 | identifier = model_name = "-".join(fpath.replace(".pkl", "").split("eval__model-")[1:]) 18 | label = 'GPT4all-' 19 | label += identifier 20 | 21 | else: 22 | label = 'alpaca-lora' 23 | plt.hist(perplexities, label=label, alpha=.5, bins=50) 24 | 25 | plt.xlabel('Perplexity') 26 | plt.ylabel('Frequency') 27 | plt.legend() 28 | plt.savefig('figs/perplexity_hist.png') 29 | 30 | -------------------------------------------------------------------------------- /gpt4all-training/figs/clustering_overfit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-training/figs/clustering_overfit.png -------------------------------------------------------------------------------- /gpt4all-training/figs/duplicate_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-training/figs/duplicate_loss.png -------------------------------------------------------------------------------- /gpt4all-training/figs/first_lora.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-training/figs/first_lora.png -------------------------------------------------------------------------------- /gpt4all-training/figs/overfit-gpt-j.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-training/figs/overfit-gpt-j.png -------------------------------------------------------------------------------- /gpt4all-training/figs/perplexity_hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-training/figs/perplexity_hist.png -------------------------------------------------------------------------------- /gpt4all-training/figs/single_epoch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sulhan12/gpt4all/41c9013fa46a194b3e4fee6ced1b9d1b65e177ac/gpt4all-training/figs/single_epoch.png -------------------------------------------------------------------------------- /gpt4all-training/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from transformers import AutoModelForCausalLM, AutoTokenizer 3 | from peft import PeftModelForCausalLM 4 | from read import read_config 5 | from argparse import ArgumentParser 6 | import torch 7 | import time 8 | 9 | 10 | def generate(tokenizer, prompt, model, config): 11 | input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to(model.device) 12 | 13 | outputs = model.generate(input_ids=input_ids, max_new_tokens=config["max_new_tokens"], temperature=config["temperature"]) 14 | 15 | decoded = tokenizer.decode(outputs[0], skip_special_tokens=True).strip() 16 | 17 | return decoded[len(prompt):] 18 | 19 | 20 | def setup_model(config): 21 | model = AutoModelForCausalLM.from_pretrained(config["model_name"], device_map="auto", torch_dtype=torch.float16) 22 | tokenizer = AutoTokenizer.from_pretrained(config["tokenizer_name"]) 23 | added_tokens = tokenizer.add_special_tokens({"bos_token": "", "eos_token": "", "pad_token": ""}) 24 | 25 | if added_tokens > 0: 26 | model.resize_token_embeddings(len(tokenizer)) 27 | 28 | if config["lora"]: 29 | model = PeftModelForCausalLM.from_pretrained(model, config["lora_path"], device_map="auto", torch_dtype=torch.float16) 30 | model.to(dtype=torch.float16) 31 | 32 | print(f"Mem needed: {model.get_memory_footprint() / 1024 / 1024 / 1024:.2f} GB") 33 | 34 | return model, tokenizer 35 | 36 | 37 | 38 | if __name__ == "__main__": 39 | parser = ArgumentParser() 40 | parser.add_argument("--config", type=str, required=True) 41 | parser.add_argument("--prompt", type=str) 42 | 43 | args = parser.parse_args() 44 | 45 | config = read_config(args.config) 46 | 47 | if config["prompt"] is None and args.prompt is None: 48 | raise ValueError("Prompt is required either in config or as argument") 49 | 50 | prompt = config["prompt"] if args.prompt is None else args.prompt 51 | 52 | print("Setting up model") 53 | model, tokenizer = setup_model(config) 54 | 55 | print("Generating") 56 | start = time.time() 57 | generation = generate(tokenizer, prompt, model, config) 58 | print(f"Done in {time.time() - start:.2f}s") 59 | print(generation) 60 | -------------------------------------------------------------------------------- /gpt4all-training/launcher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Display header 4 | echo "==========================================================" 5 | echo " ██████ ██████ ████████ ██ ██ █████ ██ ██ " 6 | echo "██ ██ ██ ██ ██ ██ ██ ██ ██ ██ " 7 | echo "██ ███ ██████ ██ ███████ ███████ ██ ██ " 8 | echo "██ ██ ██ ██ ██ ██ ██ ██ ██ " 9 | echo " ██████ ██ ██ ██ ██ ██ ███████ ███████ " 10 | echo " └─> https://github.com/nomic-ai/gpt4all" 11 | 12 | # Function to detect macOS architecture and set the binary filename 13 | detect_mac_arch() { 14 | local mac_arch 15 | mac_arch=$(uname -m) 16 | case "$mac_arch" in 17 | arm64) 18 | os_type="M1 Mac/OSX" 19 | binary_filename="gpt4all-lora-quantized-OSX-m1" 20 | ;; 21 | x86_64) 22 | os_type="Intel Mac/OSX" 23 | binary_filename="gpt4all-lora-quantized-OSX-intel" 24 | ;; 25 | *) 26 | echo "Unknown macOS architecture" 27 | exit 1 28 | ;; 29 | esac 30 | } 31 | 32 | # Detect operating system and set the binary filename 33 | case "$(uname -s)" in 34 | Darwin*) 35 | detect_mac_arch 36 | ;; 37 | Linux*) 38 | if grep -q Microsoft /proc/version; then 39 | os_type="Windows (WSL)" 40 | binary_filename="gpt4all-lora-quantized-win64.exe" 41 | else 42 | os_type="Linux" 43 | binary_filename="gpt4all-lora-quantized-linux-x86" 44 | fi 45 | ;; 46 | CYGWIN*|MINGW32*|MSYS*|MINGW*) 47 | os_type="Windows (Cygwin/MSYS/MINGW)" 48 | binary_filename="gpt4all-lora-quantized-win64.exe" 49 | ;; 50 | *) 51 | echo "Unknown operating system" 52 | exit 1 53 | ;; 54 | esac 55 | echo "================================" 56 | echo "== You are using $os_type." 57 | 58 | 59 | # Change to the chat directory 60 | cd chat 61 | 62 | # List .bin files and prompt user to select one 63 | bin_files=(*.bin) 64 | echo "== Available .bin files:" 65 | for i in "${!bin_files[@]}"; do 66 | echo " [$((i+1))] ${bin_files[i]}" 67 | done 68 | 69 | # Function to get user input and validate it 70 | get_valid_user_input() { 71 | local input_valid=false 72 | 73 | while ! $input_valid; do 74 | echo "==> Please enter a number:" 75 | read -r user_selection 76 | if [[ $user_selection =~ ^[0-9]+$ ]] && (( user_selection >= 1 && user_selection <= ${#bin_files[@]} )); then 77 | input_valid=true 78 | else 79 | echo "Invalid input. Please enter a number between 1 and ${#bin_files[@]}." 80 | fi 81 | done 82 | } 83 | 84 | get_valid_user_input 85 | selected_bin_file="${bin_files[$((user_selection-1))]}" 86 | 87 | # Run the selected .bin file with the appropriate command 88 | ./"$binary_filename" -m "$selected_bin_file" 89 | -------------------------------------------------------------------------------- /gpt4all-training/read.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | 4 | def read_config(path): 5 | # read yaml and return contents 6 | with open(path, 'r') as file: 7 | try: 8 | return yaml.safe_load(file) 9 | except yaml.YAMLError as exc: 10 | print(exc) -------------------------------------------------------------------------------- /gpt4all-training/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | datasets 3 | einops 4 | torchmetrics 5 | evaluate 6 | transformers>=4.28.0 7 | wandb 8 | peft 9 | nodelist-inflator 10 | deepspeed 11 | sentencepiece 12 | jsonlines 13 | nomic 14 | scikit-learn 15 | matplotlib --------------------------------------------------------------------------------