├── LICENSE ├── README.md ├── chatbot-ui-legacy ├── 0001-next.config-Add-output-standalone.patch ├── 0002-Use-local-font.patch ├── 0003-Update-npm-packages.patch └── default.nix ├── chatbot-ui ├── 0001-Set-nextjs-output-to-standalone.patch ├── 0002-Use-local-font.patch └── default.nix ├── flake.lock ├── flake.nix ├── local-ai ├── default.nix ├── lib.nix ├── module.nix └── tests.nix ├── open-webui └── default.nix └── overlay.nix /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Christian Kögler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nix-local-ai 2 | 3 | Nix Flake for [LocalAI](https://github.com/go-skynet/LocalAI) 4 | 5 | ## Running 6 | 7 | ```sh 8 | nix run github:ck3d/nix-local-ai 9 | ``` 10 | -------------------------------------------------------------------------------- /chatbot-ui-legacy/0001-next.config-Add-output-standalone.patch: -------------------------------------------------------------------------------- 1 | From 4d76a976530fa3ad54087cfaff1d14fd33020eea Mon Sep 17 00:00:00 2001 2 | From: =?UTF-8?q?Christian=20K=C3=B6gler?= 3 | Date: Sun, 14 Apr 2024 08:11:15 +0200 4 | Subject: [PATCH 1/3] next.config: Add output standalone 5 | 6 | --- 7 | next.config.js | 1 + 8 | 1 file changed, 1 insertion(+) 9 | 10 | diff --git a/next.config.js b/next.config.js 11 | index 715516f..546e007 100644 12 | --- a/next.config.js 13 | +++ b/next.config.js 14 | @@ -4,6 +4,7 @@ const { i18n } = require('./next-i18next.config'); 15 | const nextConfig = { 16 | i18n, 17 | reactStrictMode: true, 18 | + output: "standalone", 19 | 20 | webpack(config, { isServer, dev }) { 21 | config.experiments = { 22 | -- 23 | 2.42.0 24 | 25 | -------------------------------------------------------------------------------- /chatbot-ui-legacy/0002-Use-local-font.patch: -------------------------------------------------------------------------------- 1 | From b82d00f856e24d17a798b30550c89100ea795f61 Mon Sep 17 00:00:00 2001 2 | From: =?UTF-8?q?Christian=20K=C3=B6gler?= 3 | Date: Sun, 14 Apr 2024 08:12:00 +0200 4 | Subject: [PATCH 2/3] Use local font 5 | 6 | --- 7 | pages/_app.tsx | 4 ++-- 8 | 1 file changed, 2 insertions(+), 2 deletions(-) 9 | 10 | diff --git a/pages/_app.tsx b/pages/_app.tsx 11 | index 0ef2d18..e60765c 100644 12 | --- a/pages/_app.tsx 13 | +++ b/pages/_app.tsx 14 | @@ -1,10 +1,10 @@ 15 | import '@/styles/globals.css'; 16 | import { appWithTranslation } from 'next-i18next'; 17 | import type { AppProps } from 'next/app'; 18 | -import { Inter } from 'next/font/google'; 19 | +import localFont from 'next/font/local'; 20 | import { Toaster } from 'react-hot-toast'; 21 | 22 | -const inter = Inter({ subsets: ['latin'] }); 23 | +const inter = localFont({ src: '../InterVariable.ttf' }); 24 | 25 | function App({ Component, pageProps }: AppProps<{}>) { 26 | return ( 27 | -- 28 | 2.42.0 29 | 30 | -------------------------------------------------------------------------------- /chatbot-ui-legacy/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | buildNpmPackage, 3 | nodejs, 4 | fetchFromGitHub, 5 | makeWrapper, 6 | inter, 7 | }: 8 | buildNpmPackage rec { 9 | pname = "chatbot-ui-legacy"; 10 | version = "20230404"; 11 | 12 | src = fetchFromGitHub { 13 | owner = "mckaywrigley"; 14 | repo = "chatbot-ui"; 15 | rev = "6bdbf1cbe38ff3a40c6d14df4610656ea26b174e"; # branch legacy 16 | hash = "sha256-b9qrlKUs2PH0uhQesdbWMbpX38S+CFSovxtYcKII6Kk="; 17 | }; 18 | 19 | patches = [ 20 | ./0001-next.config-Add-output-standalone.patch 21 | ./0002-Use-local-font.patch 22 | ./0003-Update-npm-packages.patch 23 | ]; 24 | 25 | npmDepsHash = "sha256-YF2TFXjdPyCVNWCJreP/GCAuVmKZf25m4gbyyX5OI+A="; 26 | 27 | nativeBuildInputs = [ makeWrapper ]; 28 | 29 | # https://nextjs.org/telemetry 30 | env.NEXT_TELEMETRY_DISABLED = 1; 31 | 32 | postConfigure = '' 33 | cp ${inter.src}/*.ttf . 34 | ''; 35 | 36 | # inspired by 37 | # https://github.com/vercel/next.js/blob/canary/examples/with-docker/Dockerfile 38 | postInstall = '' 39 | rm -r $out/lib 40 | dest=$out/share/${pname} 41 | mkdir -p $dest/.next $out/bin 42 | cp -r -t $dest .next/standalone/. public 43 | cp -r -t $dest/.next .next/static 44 | makeWrapper ${nodejs}/bin/node $out/bin/${pname} \ 45 | --set NEXT_TELEMETRY_DISABLED 1 \ 46 | --add-flags "$dest/server.js" 47 | ''; 48 | } 49 | -------------------------------------------------------------------------------- /chatbot-ui/0001-Set-nextjs-output-to-standalone.patch: -------------------------------------------------------------------------------- 1 | From 0a6b51daf116fde7496c364885287d36a635640b Mon Sep 17 00:00:00 2001 2 | From: =?UTF-8?q?Christian=20K=C3=B6gler?= 3 | Date: Wed, 17 Apr 2024 21:34:47 +0200 4 | Subject: [PATCH 1/2] Set nextjs output to standalone 5 | 6 | --- 7 | next.config.js | 1 + 8 | 1 file changed, 1 insertion(+) 9 | 10 | diff --git a/next.config.js b/next.config.js 11 | index 201356a..87633c0 100644 12 | --- a/next.config.js 13 | +++ b/next.config.js 14 | @@ -9,6 +9,7 @@ const withPWA = require("next-pwa")({ 15 | module.exports = withBundleAnalyzer( 16 | withPWA({ 17 | reactStrictMode: true, 18 | + output: "standalone", 19 | images: { 20 | remotePatterns: [ 21 | { 22 | -- 23 | 2.42.0 24 | 25 | -------------------------------------------------------------------------------- /chatbot-ui/0002-Use-local-font.patch: -------------------------------------------------------------------------------- 1 | From a4eef73d6e6608c169e9d2feef2d6f58ee28b5af Mon Sep 17 00:00:00 2001 2 | From: =?UTF-8?q?Christian=20K=C3=B6gler?= 3 | Date: Wed, 17 Apr 2024 21:57:57 +0200 4 | Subject: [PATCH 2/2] Use local font 5 | 6 | --- 7 | app/[locale]/layout.tsx | 4 ++-- 8 | 1 file changed, 2 insertions(+), 2 deletions(-) 9 | 10 | diff --git a/app/[locale]/layout.tsx b/app/[locale]/layout.tsx 11 | index 474058a..b9af9eb 100644 12 | --- a/app/[locale]/layout.tsx 13 | +++ b/app/[locale]/layout.tsx 14 | @@ -6,12 +6,12 @@ import initTranslations from "@/lib/i18n" 15 | import { Database } from "@/supabase/types" 16 | import { createServerClient } from "@supabase/ssr" 17 | import { Metadata, Viewport } from "next" 18 | -import { Inter } from "next/font/google" 19 | +import localFont from "next/font/local" 20 | import { cookies } from "next/headers" 21 | import { ReactNode } from "react" 22 | import "./globals.css" 23 | 24 | -const inter = Inter({ subsets: ["latin"] }) 25 | +const inter = localFont({ src: "../../InterVariable.ttf" }) 26 | const APP_NAME = "Chatbot UI" 27 | const APP_DEFAULT_TITLE = "Chatbot UI" 28 | const APP_TITLE_TEMPLATE = "%s - Chatbot UI" 29 | -- 30 | 2.42.0 31 | 32 | -------------------------------------------------------------------------------- /chatbot-ui/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | buildNpmPackage, 3 | nodejs, 4 | fetchFromGitHub, 5 | makeWrapper, 6 | fetchzip, 7 | vips, 8 | pkg-config, 9 | inter, 10 | }: 11 | buildNpmPackage rec { 12 | pname = "chatbot-ui"; 13 | version = "20240402"; 14 | 15 | src = fetchFromGitHub { 16 | owner = "mckaywrigley"; 17 | repo = pname; 18 | rev = "b865b0555f53957e96727bc0bbb369c9eaecd83b"; 19 | hash = "sha256-9xg9Q+0rwxakaDqf5pIw9EDte4pdLUnaRaXtScpLWFI="; 20 | }; 21 | 22 | patches = [ 23 | ./0001-Set-nextjs-output-to-standalone.patch 24 | ./0002-Use-local-font.patch 25 | ]; 26 | 27 | npmDepsHash = "sha256-708oOP26UOZ0MfclDg5doke8O1OnwbP4m62WXpty8Mc="; 28 | 29 | nativeBuildInputs = [ 30 | makeWrapper 31 | pkg-config 32 | ]; 33 | 34 | buildInputs = [ vips ]; 35 | 36 | # https://nextjs.org/telemetry 37 | env.NEXT_TELEMETRY_DISABLED = 1; 38 | 39 | postConfigure = '' 40 | cp ${inter.src}/*.ttf . 41 | ''; 42 | 43 | # inspired by 44 | # https://github.com/vercel/next.js/blob/canary/examples/with-docker/Dockerfile 45 | postInstall = '' 46 | rm -r $out/lib 47 | dest=$out/share/${pname} 48 | mkdir -p $dest/.next $out/bin 49 | cp -r -t $dest .next/standalone/. public 50 | cp -r -t $dest/.next .next/static 51 | makeWrapper ${nodejs}/bin/node $out/bin/${pname} \ 52 | --set NEXT_TELEMETRY_DISABLED 1 \ 53 | --add-flags "$dest/server.js" 54 | ''; 55 | } 56 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs-2411": { 4 | "locked": { 5 | "lastModified": 1745487689, 6 | "narHash": "sha256-FQoi3R0NjQeBAsEOo49b5tbDPcJSMWc3QhhaIi9eddw=", 7 | "owner": "nixos", 8 | "repo": "nixpkgs", 9 | "rev": "5630cf13cceac06cefe9fc607e8dfa8fb342dde3", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "nixos", 14 | "ref": "nixos-24.11", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "nixpkgs-unstable": { 20 | "locked": { 21 | "lastModified": 1745526057, 22 | "narHash": "sha256-ITSpPDwvLBZBnPRS2bUcHY3gZSwis/uTe255QgMtTLA=", 23 | "owner": "nixos", 24 | "repo": "nixpkgs", 25 | "rev": "f771eb401a46846c1aebd20552521b233dd7e18b", 26 | "type": "github" 27 | }, 28 | "original": { 29 | "owner": "nixos", 30 | "ref": "nixos-unstable", 31 | "repo": "nixpkgs", 32 | "type": "github" 33 | } 34 | }, 35 | "root": { 36 | "inputs": { 37 | "nixpkgs-2411": "nixpkgs-2411", 38 | "nixpkgs-unstable": "nixpkgs-unstable" 39 | } 40 | } 41 | }, 42 | "root": "root", 43 | "version": 7 44 | } 45 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Nix package of LocalAI"; 3 | 4 | inputs = { 5 | nixpkgs-2411.url = "github:nixos/nixpkgs/nixos-24.11"; 6 | nixpkgs-unstable.url = "github:nixos/nixpkgs/nixos-unstable"; 7 | }; 8 | 9 | outputs = 10 | { 11 | self, 12 | nixpkgs-2411, 13 | nixpkgs-unstable, 14 | }: 15 | let 16 | inherit (nixpkgs-unstable) lib; 17 | forAllSystems = lib.genAttrs lib.systems.flakeExposed; 18 | in 19 | { 20 | overlays.default = import ./overlay.nix; 21 | 22 | formatter = forAllSystems (system: nixpkgs-unstable.legacyPackages.${system}.nixfmt-rfc-style); 23 | 24 | packages = forAllSystems ( 25 | system: 26 | let 27 | pkgs-2411 = import nixpkgs-2411 { 28 | inherit system; 29 | overlays = builtins.attrValues self.overlays; 30 | config.allowUnfree = true; 31 | }; 32 | 33 | pkgs-unstable = import nixpkgs-unstable { 34 | inherit system; 35 | overlays = builtins.attrValues self.overlays; 36 | config.allowUnfree = true; 37 | }; 38 | in 39 | { 40 | local-ai-nixos2411 = pkgs-2411.nix-local-ai.local-ai; 41 | local-ai-openblas-nixos2411 = pkgs-2411.nix-local-ai.local-ai.override { with_openblas = true; }; 42 | local-ai-cublas-nixos2411 = pkgs-2411.nix-local-ai.local-ai.override { with_cublas = true; }; 43 | default = pkgs-unstable.nix-local-ai.local-ai; 44 | } 45 | // pkgs-unstable.nix-local-ai 46 | // builtins.listToAttrs ( 47 | map 48 | (type: { 49 | name = "local-ai-" + type; 50 | value = pkgs-unstable.nix-local-ai.local-ai.override { 51 | "with_${type}" = true; 52 | }; 53 | }) 54 | [ 55 | "cublas" 56 | "clblas" 57 | "openblas" 58 | "vulkan" 59 | ] 60 | ) 61 | ); 62 | 63 | checks = forAllSystems ( 64 | system: 65 | let 66 | packages = self.packages.${system}; 67 | in 68 | packages 69 | // (builtins.foldl' ( 70 | acc: package: 71 | acc 72 | // (lib.mapAttrs' (test: value: { 73 | name = package + "-test-" + test; 74 | inherit value; 75 | }) (packages.${package}.tests or { })) 76 | ) { } (builtins.attrNames packages)) 77 | ); 78 | }; 79 | } 80 | -------------------------------------------------------------------------------- /local-ai/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | callPackages, 4 | stdenv, 5 | lib, 6 | addDriverRunpath, 7 | fetchFromGitHub, 8 | protobuf, 9 | protoc-gen-go, 10 | protoc-gen-go-grpc, 11 | grpc, 12 | openssl, 13 | llama-cpp, 14 | # needed for audio-to-text 15 | ffmpeg, 16 | cmake, 17 | pkg-config, 18 | buildGo123Module, 19 | makeWrapper, 20 | ncurses, 21 | which, 22 | opencv, 23 | curl, 24 | 25 | enable_upx ? true, 26 | upx, 27 | 28 | # apply feature parameter names according to 29 | # https://github.com/NixOS/rfcs/pull/169 30 | 31 | # CPU extensions 32 | enable_avx ? stdenv.hostPlatform.isx86_64, 33 | enable_avx2 ? stdenv.hostPlatform.isx86_64, 34 | enable_avx512 ? stdenv.hostPlatform.avx512Support, 35 | enable_f16c ? stdenv.hostPlatform.isx86_64, 36 | enable_fma ? stdenv.hostPlatform.isx86_64, 37 | 38 | with_openblas ? false, 39 | openblas, 40 | 41 | with_cublas ? config.cudaSupport, 42 | cudaPackages, 43 | 44 | with_clblas ? false, 45 | clblast, 46 | ocl-icd, 47 | opencl-headers, 48 | 49 | with_vulkan ? false, 50 | 51 | with_tts ? true, 52 | onnxruntime, 53 | sonic, 54 | spdlog, 55 | fmt, 56 | espeak-ng, 57 | piper-tts, 58 | }: 59 | let 60 | BUILD_TYPE = 61 | assert 62 | (lib.count lib.id [ 63 | with_openblas 64 | with_cublas 65 | with_clblas 66 | with_vulkan 67 | ]) <= 1; 68 | if with_openblas then 69 | "openblas" 70 | else if with_cublas then 71 | "cublas" 72 | else if with_clblas then 73 | "clblas" 74 | else 75 | ""; 76 | 77 | inherit (cudaPackages) 78 | libcublas 79 | cuda_nvcc 80 | cuda_cccl 81 | cuda_cudart 82 | libcufft 83 | ; 84 | 85 | llama-cpp-rpc = 86 | (llama-cpp-grpc.overrideAttrs (prev: { 87 | name = "llama-cpp-rpc"; 88 | cmakeFlags = prev.cmakeFlags ++ [ 89 | (lib.cmakeBool "GGML_AVX" false) 90 | (lib.cmakeBool "GGML_AVX2" false) 91 | (lib.cmakeBool "GGML_AVX512" false) 92 | (lib.cmakeBool "GGML_FMA" false) 93 | (lib.cmakeBool "GGML_F16C" false) 94 | ]; 95 | })).override 96 | { 97 | cudaSupport = false; 98 | openclSupport = false; 99 | blasSupport = false; 100 | rpcSupport = true; 101 | vulkanSupport = false; 102 | }; 103 | 104 | llama-cpp-grpc = 105 | (llama-cpp.overrideAttrs ( 106 | final: prev: { 107 | name = "llama-cpp-grpc"; 108 | src = fetchFromGitHub { 109 | owner = "ggerganov"; 110 | repo = "llama.cpp"; 111 | rev = "d6d2c2ab8c8865784ba9fef37f2b2de3f2134d33"; 112 | hash = "sha256-b9B5I3EbBFrkWc6RLXMWcCRKayyWjlGuQrogUcrISrc="; 113 | fetchSubmodules = true; 114 | }; 115 | postPatch = 116 | prev.postPatch 117 | + '' 118 | cd examples 119 | cp -r --no-preserve=mode ${src}/backend/cpp/llama grpc-server 120 | cp llava/clip* llava/llava.* grpc-server 121 | printf "\nadd_subdirectory(grpc-server)" >> CMakeLists.txt 122 | 123 | cp ${src}/backend/backend.proto grpc-server 124 | sed -i grpc-server/CMakeLists.txt \ 125 | -e '/get_filename_component/ s;[.\/]*backend/;;' \ 126 | -e '$a\install(TARGETS ''${TARGET} RUNTIME)' 127 | cd .. 128 | ''; 129 | cmakeFlags = prev.cmakeFlags ++ [ 130 | (lib.cmakeBool "BUILD_SHARED_LIBS" false) 131 | (lib.cmakeBool "GGML_AVX" enable_avx) 132 | (lib.cmakeBool "GGML_AVX2" enable_avx2) 133 | (lib.cmakeBool "GGML_AVX512" enable_avx512) 134 | (lib.cmakeBool "GGML_FMA" enable_fma) 135 | (lib.cmakeBool "GGML_F16C" enable_f16c) 136 | ]; 137 | buildInputs = prev.buildInputs ++ [ 138 | protobuf # provides also abseil_cpp as propagated build input 139 | grpc 140 | openssl 141 | curl 142 | ]; 143 | } 144 | )).override 145 | { 146 | cudaSupport = with_cublas; 147 | rocmSupport = false; 148 | openclSupport = with_clblas; 149 | blasSupport = with_openblas; 150 | vulkanSupport = with_vulkan; 151 | }; 152 | 153 | espeak-ng' = espeak-ng.overrideAttrs (self: { 154 | name = "espeak-ng'"; 155 | inherit (go-piper) src; 156 | sourceRoot = "${go-piper.src.name}/espeak"; 157 | patches = [ ]; 158 | nativeBuildInputs = [ cmake ]; 159 | cmakeFlags = (self.cmakeFlags or [ ]) ++ [ 160 | (lib.cmakeBool "BUILD_SHARED_LIBS" true) 161 | (lib.cmakeBool "USE_ASYNC" false) 162 | (lib.cmakeBool "USE_MBROLA" false) 163 | (lib.cmakeBool "USE_LIBPCAUDIO" false) 164 | (lib.cmakeBool "USE_KLATT" false) 165 | (lib.cmakeBool "USE_SPEECHPLAYER" false) 166 | (lib.cmakeBool "USE_LIBSONIC" false) 167 | (lib.cmakeBool "CMAKE_POSITION_INDEPENDENT_CODE" true) 168 | ]; 169 | preConfigure = null; 170 | postInstall = null; 171 | }); 172 | 173 | piper-phonemize = stdenv.mkDerivation { 174 | name = "piper-phonemize"; 175 | inherit (go-piper) src; 176 | sourceRoot = "${go-piper.src.name}/piper-phonemize"; 177 | buildInputs = [ 178 | espeak-ng' 179 | onnxruntime 180 | ]; 181 | nativeBuildInputs = [ 182 | cmake 183 | pkg-config 184 | ]; 185 | cmakeFlags = [ 186 | (lib.cmakeFeature "ONNXRUNTIME_DIR" "${onnxruntime.dev}") 187 | (lib.cmakeFeature "ESPEAK_NG_DIR" "${espeak-ng'}") 188 | ]; 189 | passthru.espeak-ng = espeak-ng'; 190 | }; 191 | 192 | piper-tts' = (piper-tts.override { inherit piper-phonemize; }).overrideAttrs (self: { 193 | name = "piper-tts'"; 194 | inherit (go-piper) src; 195 | sourceRoot = "${go-piper.src.name}/piper"; 196 | installPhase = null; 197 | postInstall = '' 198 | cp CMakeFiles/piper.dir/src/cpp/piper.cpp.o $out/piper.o 199 | cd $out 200 | mkdir bin lib 201 | mv lib*so* lib/ 202 | mv piper piper_phonemize bin/ 203 | rm -rf cmake pkgconfig espeak-ng-data *.ort 204 | ''; 205 | }); 206 | 207 | go-piper = stdenv.mkDerivation { 208 | name = "go-piper"; 209 | src = fetchFromGitHub { 210 | owner = "mudler"; 211 | repo = "go-piper"; 212 | rev = "e10ca041a885d4a8f3871d52924b47792d5e5aa0"; 213 | hash = "sha256-Yv9LQkWwGpYdOS0FvtP0vZ0tRyBAx27sdmziBR4U4n8="; 214 | fetchSubmodules = true; 215 | }; 216 | postUnpack = '' 217 | cp -r --no-preserve=mode ${piper-tts'}/* source 218 | ''; 219 | postPatch = '' 220 | sed -i Makefile \ 221 | -e '/CXXFLAGS *= / s;$; -DSPDLOG_FMT_EXTERNAL=1;' 222 | ''; 223 | buildFlags = [ "libpiper_binding.a" ]; 224 | buildInputs = [ 225 | piper-tts' 226 | espeak-ng' 227 | piper-phonemize 228 | sonic 229 | fmt 230 | spdlog 231 | onnxruntime 232 | ]; 233 | installPhase = '' 234 | cp -r --no-preserve=mode $src $out 235 | mkdir -p $out/piper-phonemize/pi 236 | cp -r --no-preserve=mode ${piper-phonemize}/share $out/piper-phonemize/pi 237 | cp *.a $out 238 | ''; 239 | }; 240 | 241 | # try to merge with openai-whisper-cpp in future 242 | whisper-cpp = effectiveStdenv.mkDerivation { 243 | name = "whisper-cpp"; 244 | src = fetchFromGitHub { 245 | owner = "ggerganov"; 246 | repo = "whisper.cpp"; 247 | rev = "6266a9f9e56a5b925e9892acf650f3eb1245814d"; 248 | hash = "sha256-y30ZccpF3SCdRGa+P3ddF1tT1KnvlI4Fexx81wZxfTk="; 249 | }; 250 | 251 | nativeBuildInputs = [ 252 | cmake 253 | pkg-config 254 | ] ++ lib.optionals with_cublas [ cuda_nvcc ]; 255 | 256 | buildInputs = 257 | [ ] 258 | ++ lib.optionals with_cublas [ 259 | cuda_cccl 260 | cuda_cudart 261 | libcublas 262 | libcufft 263 | ] 264 | ++ lib.optionals with_clblas [ 265 | clblast 266 | ocl-icd 267 | opencl-headers 268 | ] 269 | ++ lib.optionals with_openblas [ openblas.dev ]; 270 | 271 | cmakeFlags = [ 272 | (lib.cmakeBool "WHISPER_CUDA" with_cublas) 273 | (lib.cmakeBool "WHISPER_CLBLAST" with_clblas) 274 | (lib.cmakeBool "WHISPER_OPENBLAS" with_openblas) 275 | (lib.cmakeBool "WHISPER_NO_AVX" (!enable_avx)) 276 | (lib.cmakeBool "WHISPER_NO_AVX2" (!enable_avx2)) 277 | (lib.cmakeBool "WHISPER_NO_FMA" (!enable_fma)) 278 | (lib.cmakeBool "WHISPER_NO_F16C" (!enable_f16c)) 279 | (lib.cmakeBool "BUILD_SHARED_LIBS" false) 280 | ]; 281 | postInstall = '' 282 | install -Dt $out/bin bin/* 283 | ''; 284 | }; 285 | 286 | bark = stdenv.mkDerivation { 287 | name = "bark"; 288 | src = fetchFromGitHub { 289 | owner = "PABannier"; 290 | repo = "bark.cpp"; 291 | tag = "v1.0.0"; 292 | hash = "sha256-wOcggRWe8lsUzEj/wqOAUlJVypgNFmit5ISs9fbwoCE="; 293 | fetchSubmodules = true; 294 | }; 295 | installPhase = '' 296 | mkdir -p $out/build 297 | cp -ra $src/* $out 298 | find . \( -name '*.a' -or -name '*.c.o' \) -print0 \ 299 | | tar cf - --null --files-from - \ 300 | | tar xf - -C $out/build 301 | ''; 302 | nativeBuildInputs = [ cmake ]; 303 | }; 304 | 305 | stable-diffusion = stdenv.mkDerivation { 306 | name = "stable-diffusion"; 307 | src = fetchFromGitHub { 308 | owner = "richiejp"; 309 | repo = "stable-diffusion.cpp"; 310 | rev = "53e3b17eb3d0b5760ced06a1f98320b68b34aaae"; # branch cuda-fix 311 | hash = "sha256-z56jafOdibpX+XhRsrc7ieGbeug4bf737/UobqkpBV0="; 312 | fetchSubmodules = true; 313 | }; 314 | installPhase = '' 315 | mkdir -p $out/build 316 | cp -ra $src/* $out 317 | find . \( -name '*.a' -or -name '*.c.o' \) -print0 \ 318 | | tar cf - --null --files-from - \ 319 | | tar xf - -C $out/build 320 | ''; 321 | cmakeFlags = [ 322 | (lib.cmakeFeature "GGML_BUILD_NUMBER" "1") 323 | ]; 324 | nativeBuildInputs = [ cmake ]; 325 | buildInputs = [ opencv ]; 326 | }; 327 | 328 | GO_TAGS = lib.optional with_tts "tts"; 329 | 330 | effectiveStdenv = 331 | if with_cublas then 332 | # It's necessary to consistently use backendStdenv when building with CUDA support, 333 | # otherwise we get libstdc++ errors downstream. 334 | cudaPackages.backendStdenv 335 | else 336 | stdenv; 337 | 338 | pname = "local-ai"; 339 | version = "2.28.0"; 340 | src = fetchFromGitHub { 341 | owner = "go-skynet"; 342 | repo = "LocalAI"; 343 | tag = "v${version}"; 344 | hash = "sha256-Hpz0dGkgasSY/FGO7mDzqsLjXut0LdQ9PUXGaURUOlY="; 345 | }; 346 | 347 | prepare-sources = 348 | let 349 | cp = "cp -r --no-preserve=mode,ownership"; 350 | in 351 | '' 352 | mkdir sources 353 | ${cp} ${if with_tts then go-piper else go-piper.src} sources/go-piper 354 | ${cp} ${whisper-cpp.src} sources/whisper.cpp 355 | cp ${whisper-cpp}/lib/lib*.a sources/whisper.cpp 356 | ${cp} ${bark} sources/bark.cpp 357 | ${cp} ${stable-diffusion} sources/stablediffusion-ggml.cpp 358 | ''; 359 | 360 | self = buildGo123Module.override { stdenv = effectiveStdenv; } { 361 | inherit pname version src; 362 | 363 | vendorHash = "sha256-1OY/y1AeL0K+vOU4Jk/cj7rToVLC9EkkNhgifB+icDM="; 364 | 365 | env.NIX_CFLAGS_COMPILE = " -isystem ${opencv}/include/opencv4"; 366 | 367 | postPatch = 368 | '' 369 | # TODO: add silero-vad 370 | sed -i Makefile \ 371 | -e '/mod download/ d' \ 372 | -e '/^ALL_GRPC_BACKENDS+=backend-assets\/grpc\/llama-cpp-avx/ d' \ 373 | -e '/^ALL_GRPC_BACKENDS+=backend-assets\/grpc\/llama-cpp-cuda/ d' \ 374 | -e '/^ALL_GRPC_BACKENDS+=backend-assets\/grpc\/silero-vad/ d' \ 375 | 376 | sed -i backend/go/image/stablediffusion-ggml/Makefile \ 377 | -e '/^libsd/ s,$, $(COMBINED_LIB),' 378 | 379 | '' 380 | + lib.optionalString with_cublas '' 381 | sed -i Makefile \ 382 | -e '/^CGO_LDFLAGS_WHISPER?=/ s;$;-L${libcufft}/lib -L${cuda_cudart}/lib;' 383 | ''; 384 | 385 | postConfigure = 386 | prepare-sources 387 | + '' 388 | shopt -s extglob 389 | mkdir -p backend-assets/grpc 390 | cp ${llama-cpp-grpc}/bin/grpc-server backend-assets/grpc/llama-cpp-fallback 391 | cp ${llama-cpp-rpc}/bin/grpc-server backend-assets/grpc/llama-cpp-grpc 392 | 393 | mkdir -p backend/cpp/llama/llama.cpp 394 | 395 | mkdir -p backend-assets/util 396 | cp ${llama-cpp-rpc}/bin/llama-rpc-server backend-assets/util/llama-cpp-rpc-server 397 | 398 | cp -r --no-preserve=mode,ownership ${stable-diffusion}/build backend/go/image/stablediffusion-ggml/build 399 | 400 | # avoid rebuild of prebuilt make targets 401 | touch backend-assets/grpc/* backend-assets/util/* 402 | find sources -name "lib*.a" -exec touch {} + 403 | ''; 404 | 405 | buildInputs = 406 | [ ] 407 | ++ lib.optionals with_cublas [ 408 | cuda_cudart 409 | libcublas 410 | libcufft 411 | ] 412 | ++ lib.optionals with_clblas [ 413 | clblast 414 | ocl-icd 415 | opencl-headers 416 | ] 417 | ++ lib.optionals with_openblas [ openblas.dev ] 418 | ++ lib.optionals with_tts go-piper.buildInputs; 419 | 420 | nativeBuildInputs = 421 | [ 422 | protobuf 423 | protoc-gen-go 424 | protoc-gen-go-grpc 425 | makeWrapper 426 | ncurses # tput 427 | which 428 | ] 429 | ++ lib.optional enable_upx upx 430 | ++ lib.optionals with_cublas [ cuda_nvcc ]; 431 | 432 | enableParallelBuilding = false; 433 | 434 | modBuildPhase = 435 | prepare-sources 436 | + '' 437 | make protogen-go 438 | go mod tidy -v 439 | ''; 440 | 441 | proxyVendor = true; 442 | 443 | # should be passed as makeFlags, but build system failes with strings 444 | # containing spaces 445 | env.GO_TAGS = builtins.concatStringsSep " " GO_TAGS; 446 | 447 | makeFlags = 448 | [ 449 | "VERSION=v${version}" 450 | "BUILD_TYPE=${BUILD_TYPE}" 451 | ] 452 | ++ lib.optional with_cublas "CUDA_LIBPATH=${cuda_cudart}/lib" 453 | ++ lib.optional with_tts "PIPER_CGO_CXXFLAGS=-DSPDLOG_FMT_EXTERNAL=1"; 454 | 455 | buildPhase = '' 456 | runHook preBuild 457 | 458 | local flagsArray=( 459 | ''${enableParallelBuilding:+-j''${NIX_BUILD_CORES}} 460 | SHELL=$SHELL 461 | ) 462 | 463 | # copy from Makefile:258 464 | make -C backend/go/image/stablediffusion-ggml libsd.a 465 | 466 | concatTo flagsArray makeFlags makeFlagsArray buildFlags buildFlagsArray 467 | echoCmd 'build flags' "''${flagsArray[@]}" 468 | make build "''${flagsArray[@]}" 469 | unset flagsArray 470 | 471 | runHook postBuild 472 | ''; 473 | 474 | installPhase = '' 475 | runHook preInstall 476 | 477 | install -Dt $out/bin ${pname} 478 | 479 | runHook postInstall 480 | ''; 481 | 482 | # patching rpath with patchelf doens't work. The executable 483 | # raises an segmentation fault 484 | postFixup = 485 | let 486 | LD_LIBRARY_PATH = 487 | [ ] 488 | ++ lib.optionals with_cublas [ 489 | # driverLink has to be first to avoid loading the stub version of libcuda.so 490 | # https://github.com/NixOS/nixpkgs/issues/320145#issuecomment-2190319327 491 | addDriverRunpath.driverLink 492 | (lib.getLib libcublas) 493 | cuda_cudart 494 | ] 495 | ++ lib.optionals with_clblas [ 496 | clblast 497 | ocl-icd 498 | ] 499 | ++ lib.optionals with_openblas [ openblas ] 500 | ++ lib.optionals with_tts [ piper-phonemize ] 501 | ++ lib.optionals (with_tts && enable_upx) [ 502 | fmt 503 | spdlog 504 | ]; 505 | in 506 | '' 507 | wrapProgram $out/bin/${pname} \ 508 | --prefix LD_LIBRARY_PATH : "${lib.makeLibraryPath LD_LIBRARY_PATH}" \ 509 | --prefix PATH : "${ffmpeg}/bin" 510 | ''; 511 | 512 | passthru.local-packages = { 513 | inherit 514 | go-piper 515 | llama-cpp-grpc 516 | whisper-cpp 517 | espeak-ng' 518 | piper-phonemize 519 | piper-tts' 520 | llama-cpp-rpc 521 | bark 522 | stable-diffusion 523 | ; 524 | }; 525 | 526 | passthru.features = { 527 | inherit 528 | with_cublas 529 | with_openblas 530 | with_vulkan 531 | with_tts 532 | with_clblas 533 | ; 534 | }; 535 | 536 | passthru.tests = callPackages ./tests.nix { inherit self; }; 537 | passthru.lib = callPackages ./lib.nix { }; 538 | 539 | meta = with lib; { 540 | description = "OpenAI alternative to run local LLMs, image and audio generation"; 541 | mainProgram = "local-ai"; 542 | homepage = "https://localai.io"; 543 | license = licenses.mit; 544 | maintainers = with maintainers; [ 545 | onny 546 | ck3d 547 | ]; 548 | platforms = platforms.linux; 549 | }; 550 | }; 551 | in 552 | self 553 | -------------------------------------------------------------------------------- /local-ai/lib.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, 3 | writers, 4 | writeText, 5 | linkFarmFromDrvs, 6 | }: 7 | { 8 | genModels = 9 | configs: 10 | let 11 | name = lib.strings.sanitizeDerivationName ( 12 | builtins.concatStringsSep "_" ([ "local-ai-models" ] ++ (builtins.attrNames configs)) 13 | ); 14 | 15 | genModelFiles = 16 | name: config: 17 | let 18 | templateName = type: name + "_" + type; 19 | 20 | config' = lib.recursiveUpdate config ( 21 | { 22 | inherit name; 23 | } 24 | // lib.optionalAttrs (lib.isDerivation config.parameters.model) { 25 | parameters.model = config.parameters.model.name; 26 | } 27 | // lib.optionalAttrs (config ? template) { 28 | template = builtins.mapAttrs (n: _: templateName n) config.template; 29 | } 30 | ); 31 | in 32 | [ (writers.writeYAML "${name}.yaml" config') ] 33 | ++ lib.optional (lib.isDerivation config.parameters.model) config.parameters.model 34 | ++ lib.optionals (config ? template) ( 35 | lib.mapAttrsToList (n: writeText "${templateName n}.tmpl") config.template 36 | ); 37 | in 38 | linkFarmFromDrvs name (lib.flatten (lib.mapAttrsToList genModelFiles configs)); 39 | } 40 | -------------------------------------------------------------------------------- /local-ai/module.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs, 3 | config, 4 | lib, 5 | ... 6 | }: 7 | let 8 | cfg = config.services.local-ai; 9 | inherit (lib) mkOption types; 10 | in 11 | { 12 | options.services.local-ai = { 13 | enable = lib.mkEnableOption "Enable service"; 14 | 15 | package = lib.mkPackageOption pkgs "local-ai" { }; 16 | 17 | extraArgs = mkOption { 18 | type = types.listOf types.str; 19 | default = [ ]; 20 | }; 21 | 22 | port = mkOption { 23 | type = types.port; 24 | default = 8080; 25 | }; 26 | 27 | threads = mkOption { 28 | type = types.int; 29 | default = 1; 30 | }; 31 | 32 | models = mkOption { 33 | type = types.either types.package types.str; 34 | default = "models"; 35 | }; 36 | 37 | parallelRequests = mkOption { 38 | type = types.int; 39 | default = 1; 40 | }; 41 | 42 | logLevel = mkOption { 43 | type = types.enum [ 44 | "error" 45 | "warn" 46 | "info" 47 | "debug" 48 | "trace" 49 | ]; 50 | default = "warn"; 51 | }; 52 | }; 53 | 54 | config = lib.mkIf cfg.enable { 55 | systemd.services.local-ai = { 56 | wantedBy = [ "multi-user.target" ]; 57 | environment.LLAMACPP_PARALLEL = toString cfg.parallelRequests; 58 | serviceConfig = { 59 | DynamicUser = true; 60 | ExecStart = lib.escapeShellArgs ( 61 | [ 62 | "${cfg.package}/bin/local-ai" 63 | "--address=:${toString cfg.port}" 64 | "--threads=${toString cfg.threads}" 65 | "--localai-config-dir=." 66 | "--models-path=${cfg.models}" 67 | "--log-level=${cfg.logLevel}" 68 | ] 69 | ++ lib.optional (cfg.parallelRequests > 1) "--parallel-requests" 70 | ++ cfg.extraArgs 71 | ); 72 | RuntimeDirectory = "local-ai"; 73 | WorkingDirectory = "%t/local-ai"; 74 | }; 75 | }; 76 | }; 77 | } 78 | -------------------------------------------------------------------------------- /local-ai/tests.nix: -------------------------------------------------------------------------------- 1 | { 2 | self, 3 | lib, 4 | testers, 5 | fetchzip, 6 | fetchurl, 7 | writers, 8 | symlinkJoin, 9 | jq, 10 | prom2json, 11 | }: 12 | let 13 | common-config = 14 | { config, ... }: 15 | { 16 | imports = [ ./module.nix ]; 17 | services.local-ai = { 18 | enable = true; 19 | package = self; 20 | threads = config.virtualisation.cores; 21 | logLevel = "debug"; 22 | }; 23 | }; 24 | 25 | inherit (self.lib) genModels; 26 | in 27 | { 28 | version = testers.testVersion { 29 | package = self; 30 | version = "v" + self.version; 31 | command = "local-ai --help"; 32 | }; 33 | 34 | health = testers.runNixOSTest { 35 | name = self.name + "-health"; 36 | nodes.machine = { 37 | imports = [ common-config ]; 38 | virtualisation.memorySize = 2048; 39 | }; 40 | testScript = 41 | let 42 | port = "8080"; 43 | in 44 | '' 45 | machine.wait_for_open_port(${port}) 46 | machine.succeed("curl -f http://localhost:${port}/readyz") 47 | machine.succeed("curl -f http://localhost:${port}/v1/models --output models.json") 48 | 49 | machine.succeed("${prom2json}/bin/prom2json http://localhost:${port}/metrics > metrics.json") 50 | # check if following issue is still valid 51 | # https://github.com/mudler/LocalAI/issues/2207 52 | machine.succeed("${jq}/bin/jq --exit-status '.[] | select(.name == \"api_call\").metrics | debug | any(.labels.path == \"/metricsls\" and .count == \"1\")' metrics.json") 53 | machine.copy_from_vm("metrics.json") 54 | ''; 55 | }; 56 | 57 | } 58 | // 59 | lib.optionalAttrs 60 | (!self.features.with_cublas && !self.features.with_clblas && !self.features.with_vulkan) 61 | { 62 | # https://localai.io/features/embeddings/#llamacpp-embeddings 63 | llamacpp-embeddings = 64 | let 65 | model = "embedding"; 66 | model-configs.${model} = { 67 | parameters.model = fetchurl { 68 | url = "https://huggingface.co/hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF/resolve/main/llama-3.2-1b-instruct-q4_k_m.gguf"; 69 | sha256 = "1d0e9419ec4e12aef73ccf4ffd122703e94c48344a96bc7c5f0f2772c2152ce3"; 70 | }; 71 | backend = "llama-cpp"; 72 | embeddings = true; 73 | }; 74 | 75 | models = genModels model-configs; 76 | 77 | requests.request = { 78 | inherit model; 79 | input = "Your text string goes here"; 80 | }; 81 | in 82 | testers.runNixOSTest { 83 | name = self.name + "-llamacpp-embeddings"; 84 | nodes.machine = { 85 | imports = [ common-config ]; 86 | virtualisation.cores = 2; 87 | virtualisation.memorySize = 3 * 4096; 88 | services.local-ai.models = models; 89 | }; 90 | passthru = { 91 | inherit models requests; 92 | }; 93 | testScript = 94 | let 95 | port = "8080"; 96 | in 97 | '' 98 | machine.wait_for_open_port(${port}) 99 | machine.succeed("curl -f http://localhost:${port}/readyz") 100 | machine.succeed("curl -f http://localhost:${port}/v1/models --output models.json") 101 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .data[].id == \"${model}\"' models.json") 102 | 103 | machine.succeed("curl -f http://localhost:${port}/embeddings --json @${writers.writeJSON "request.json" requests.request} --output embeddings.json") 104 | machine.copy_from_vm("embeddings.json") 105 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .model == \"${model}\"' embeddings.json") 106 | 107 | machine.succeed("${prom2json}/bin/prom2json http://localhost:${port}/metrics > metrics.json") 108 | machine.copy_from_vm("metrics.json") 109 | ''; 110 | }; 111 | 112 | # https://localai.io/docs/getting-started/manual/ 113 | llama = 114 | let 115 | model = "gpt-3.5-turbo"; 116 | 117 | # https://localai.io/advanced/#full-config-model-file-reference 118 | model-configs.${model} = rec { 119 | context_size = 16 * 1024; # 128kb is possible, but needs 16GB RAM 120 | backend = "llama-cpp"; 121 | parameters = { 122 | # https://ai.meta.com/blog/meta-llama-3-1/ 123 | model = fetchurl { 124 | url = "https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"; 125 | sha256 = "f2be3e1a239c12c9f3f01a962b11fb2807f8032fdb63b0a5502ea42ddef55e44"; 126 | }; 127 | # defaults from: 128 | # https://deepinfra.com/meta-llama/Meta-Llama-3.1-8B-Instruct 129 | temperature = 0.7; 130 | top_p = 0.9; 131 | top_k = 0; 132 | # following parameter leads to outputs like: !!!!!!!!!!!!!!!!!!! 133 | #repeat_penalty = 1; 134 | presence_penalty = 0; 135 | frequency_penalty = 0; 136 | max_tokens = 100; 137 | }; 138 | stopwords = [ "<|eot_id|>" ]; 139 | template = { 140 | # Templates implement following specifications 141 | # https://github.com/meta-llama/llama3/tree/main?tab=readme-ov-file#instruction-tuned-models 142 | # ... and are insprired by: 143 | # https://github.com/mudler/LocalAI/blob/master/embedded/models/llama3-instruct.yaml 144 | # 145 | # The rules for template evaluateion are defined here: 146 | # https://pkg.go.dev/text/template 147 | chat_message = '' 148 | <|start_header_id|>{{.RoleName}}<|end_header_id|> 149 | 150 | {{.Content}}${builtins.head stopwords}''; 151 | 152 | chat = "{{.Input}}<|start_header_id|>assistant<|end_header_id|>"; 153 | 154 | completion = "{{.Input}}"; 155 | }; 156 | }; 157 | 158 | models = genModels model-configs; 159 | 160 | requests = { 161 | # https://localai.io/features/text-generation/#chat-completions 162 | chat-completions = { 163 | inherit model; 164 | messages = [ 165 | { 166 | role = "user"; 167 | content = "1 + 2 = ?"; 168 | } 169 | ]; 170 | }; 171 | # https://localai.io/features/text-generation/#edit-completions 172 | edit-completions = { 173 | inherit model; 174 | instruction = "rephrase"; 175 | input = "Black cat jumped out of the window"; 176 | max_tokens = 50; 177 | }; 178 | # https://localai.io/features/text-generation/#completions 179 | completions = { 180 | inherit model; 181 | prompt = "A long time ago in a galaxy far, far away"; 182 | }; 183 | }; 184 | in 185 | testers.runNixOSTest { 186 | name = self.name + "-llama"; 187 | nodes.machine = { 188 | imports = [ common-config ]; 189 | virtualisation.cores = 4; 190 | virtualisation.memorySize = 8192; 191 | services.local-ai.models = models; 192 | # TODO: Add test case parallel requests 193 | services.local-ai.parallelRequests = 2; 194 | }; 195 | passthru = { 196 | inherit models requests; 197 | }; 198 | testScript = 199 | let 200 | port = "8080"; 201 | in 202 | '' 203 | machine.wait_for_open_port(${port}) 204 | machine.succeed("curl -f http://localhost:${port}/readyz") 205 | machine.succeed("curl -f http://localhost:${port}/v1/models --output models.json") 206 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .data[].id == \"${model}\"' models.json") 207 | 208 | machine.succeed("curl -f http://localhost:${port}/v1/chat/completions --json @${writers.writeJSON "request-chat-completions.json" requests.chat-completions} --output chat-completions.json") 209 | machine.copy_from_vm("chat-completions.json") 210 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .object == \"chat.completion\"' chat-completions.json") 211 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .choices | first.message.content | split(\" \") | last | tonumber == 3' chat-completions.json") 212 | 213 | machine.succeed("curl -f http://localhost:${port}/v1/edits --json @${writers.writeJSON "request-edit-completions.json" requests.edit-completions} --output edit-completions.json") 214 | machine.copy_from_vm("edit-completions.json") 215 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .object == \"edit\"' edit-completions.json") 216 | machine.succeed("${jq}/bin/jq --exit-status '.usage.completion_tokens | debug == ${toString requests.edit-completions.max_tokens}' edit-completions.json") 217 | 218 | machine.succeed("curl -f http://localhost:${port}/v1/completions --json @${writers.writeJSON "request-completions.json" requests.completions} --output completions.json") 219 | machine.copy_from_vm("completions.json") 220 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .object ==\"text_completion\"' completions.json") 221 | 222 | machine.succeed("${prom2json}/bin/prom2json http://localhost:${port}/metrics > metrics.json") 223 | machine.copy_from_vm("metrics.json") 224 | ''; 225 | }; 226 | 227 | } 228 | // 229 | lib.optionalAttrs 230 | (self.features.with_tts && !self.features.with_cublas && !self.features.with_clblas) 231 | { 232 | # https://localai.io/features/text-to-audio/#piper 233 | tts = 234 | let 235 | model-stt = "whisper-en"; 236 | model-configs.${model-stt} = { 237 | backend = "whisper"; 238 | parameters.model = fetchurl { 239 | url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin"; 240 | hash = "sha256-x3xXZvHO8JtrfUfyG1Rsvd1BV4hrO11tT3CekeZsfCs="; 241 | }; 242 | }; 243 | 244 | model-tts = "piper-en"; 245 | model-configs.${model-tts} = { 246 | backend = "piper"; 247 | parameters.model = "en-us-danny-low.onnx"; 248 | }; 249 | 250 | models = 251 | let 252 | models = genModels model-configs; 253 | in 254 | symlinkJoin { 255 | inherit (models) name; 256 | paths = [ 257 | models 258 | (fetchzip { 259 | url = "https://github.com/rhasspy/piper/releases/download/v0.0.2/voice-en-us-danny-low.tar.gz"; 260 | hash = "sha256-5wf+6H5HeQY0qgdqnAG1vSqtjIFM9lXH53OgouuPm0M="; 261 | stripRoot = false; 262 | }) 263 | ]; 264 | }; 265 | 266 | requests.request = { 267 | model = model-tts; 268 | input = "Hello, how are you?"; 269 | }; 270 | in 271 | testers.runNixOSTest { 272 | name = self.name + "-tts"; 273 | nodes.machine = { 274 | imports = [ common-config ]; 275 | virtualisation.cores = 2; 276 | services.local-ai.models = models; 277 | }; 278 | passthru = { 279 | inherit models requests; 280 | }; 281 | testScript = 282 | let 283 | port = "8080"; 284 | in 285 | '' 286 | machine.wait_for_open_port(${port}) 287 | machine.succeed("curl -f http://localhost:${port}/readyz") 288 | machine.succeed("curl -f http://localhost:${port}/v1/models --output models.json") 289 | machine.succeed("${jq}/bin/jq --exit-status 'debug' models.json") 290 | 291 | machine.succeed("curl -f http://localhost:${port}/tts --json @${writers.writeJSON "request.json" requests.request} --output out.wav") 292 | machine.copy_from_vm("out.wav") 293 | 294 | machine.succeed("curl -f http://localhost:${port}/v1/audio/transcriptions --header 'Content-Type: multipart/form-data' --form file=@out.wav --form model=${model-stt} --output transcription.json") 295 | machine.copy_from_vm("transcription.json") 296 | machine.succeed("${jq}/bin/jq --exit-status 'debug | .segments | first.text == \"${requests.request.input}\"' transcription.json") 297 | 298 | machine.succeed("${prom2json}/bin/prom2json http://localhost:${port}/metrics > metrics.json") 299 | machine.copy_from_vm("metrics.json") 300 | ''; 301 | }; 302 | } 303 | -------------------------------------------------------------------------------- /open-webui/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | buildNpmPackage, 3 | nodePackages, 4 | fetchFromGitHub, 5 | runtimeShell, 6 | }: 7 | buildNpmPackage rec { 8 | pname = "open-webui"; 9 | version = "0.1.117"; 10 | 11 | src = fetchFromGitHub { 12 | owner = pname; 13 | repo = pname; 14 | rev = "v${version}"; 15 | hash = "sha256-AmWLWzxt8+Nz7ahYTtMwtjxVzqTYZeMfKTF1IOi0lsA="; 16 | }; 17 | 18 | npmDepsHash = "sha256-VW89XnzputCWw5dOAKg09kve7IMNlxGS6ShYEo1ZC7s="; 19 | 20 | postInstall = '' 21 | mkdir -p $out/lib 22 | cp -R ./build/. $out/lib 23 | 24 | mkdir -p $out/bin 25 | cat <>$out/bin/${pname} 26 | #!${runtimeShell} 27 | ${nodePackages.http-server}/bin/http-server $out/lib 28 | EOF 29 | chmod +x $out/bin/${pname} 30 | ''; 31 | } 32 | -------------------------------------------------------------------------------- /overlay.nix: -------------------------------------------------------------------------------- 1 | final: prev: 2 | let 3 | inherit (final) callPackage; 4 | in 5 | { 6 | nix-local-ai = { 7 | local-ai = callPackage ./local-ai { }; 8 | chatbot-ui = callPackage ./chatbot-ui { }; 9 | chatbot-ui-legacy = callPackage ./chatbot-ui-legacy { }; 10 | open-webui = callPackage ./open-webui { }; 11 | }; 12 | } 13 | --------------------------------------------------------------------------------