├── .envrc ├── .gitignore ├── README.md ├── cache.sh ├── flake.lock ├── flake.nix ├── nvidia.nix └── pkgs ├── autogptq.nix ├── ava ├── default.nix ├── deps.nix └── prebuilt.nix ├── dadaptation.nix ├── exllamav2.nix ├── exllamav3.nix ├── flash-attention.nix ├── formatron.nix ├── general-sam.nix ├── kbnf.nix ├── kohya_ss ├── default.nix └── xformers-cuda.nix ├── lycoris-lora.nix ├── prodigyopt.nix ├── rouge.nix ├── tabbyapi.nix └── tensor_parallel.nix /.envrc: -------------------------------------------------------------------------------- 1 | use flake -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | result 2 | .direnv -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!NOTE] 2 | > Please note that most of the packages may not have binary cache, and building anything CUDA is very painful and slow. 3 | 4 | # nix-ai-stuff 5 | Nix flake for several AI projects focusing on nvidia/CUDA. 6 | 7 | ## Packages 8 | - [kbnf](https://github.com/Dan-Wanna-M/kbnf) 0.4.1 9 | - [general-sam](https://github.com/ModelTC/general-sam-py) 1.0.1 10 | - [formatron](https://github.com/Dan-wanna-M/formatron) 0.4.11 11 | - [tabbyapi](https://github.com/theroyallab/tabbyAPI) unstable-2025-04-16 12 | - [exllamav2](https://github.com/turboderp/exllamav2) 0.2.9 13 | - [autogptq](https://github.com/PanQiWei/AutoGPTQ) 0.7.1 14 | - [ava & ava-headless](https://www.avapls.com/) 2024-04-24 15 | - [ava-prebuilt](https://www.avapls.com/) 2024-04-21 16 | - [tensor_parallel](https://github.com/BlackSamorez/tensor_parallel) 2.0.0 17 | - [dadaptation](https://github.com/facebookresearch/dadaptation) 3.1 18 | - [prodigyopt](https://github.com/konstmish/prodigy) 1.0 19 | - [lycoris-lora](https://github.com/KohakuBlueleaf/LyCORIS) 2.0.2 20 | - [kohya_ss](https://github.com/bmaltais/kohya_ss) 22.6.1 (patched to run without having to include web files) 21 | - [flash-attn](https://github.com/Dao-AILab/flash-attention) 2.7.4 22 | - [rouge](https://github.com/pltrdy/rouge) 1.0.1 23 | 24 | ``` 25 | nix run github:BatteredBunny/nix-ai-stuff#tabbyapi 26 | ``` 27 | 28 | # Similar projects 29 | - [nixifed-ai](https://github.com/nixified-ai/flake) 30 | -------------------------------------------------------------------------------- /cache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | packages=( 3 | "exllamav2" 4 | "autogptq" 5 | "ava" 6 | "ava-headless" 7 | "ava-prebuilt" 8 | "tensor_parallel" 9 | "dadaptation" 10 | "prodigyopt" 11 | "lycoris-lora" 12 | "kohya_ss" 13 | "flash-attention" 14 | "rouge" 15 | ) 16 | 17 | for package in ${packages[*]}; do 18 | echo "nix build .#$package" 19 | nix build .#$package --json | jq -r '.[].outputs | to_entries[].value' | cachix push nix-ai-stuff 20 | done 21 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1744463964, 6 | "narHash": "sha256-LWqduOgLHCFxiTNYi3Uj5Lgz0SR+Xhw3kr/3Xd0GPTM=", 7 | "owner": "NixOS", 8 | "repo": "nixpkgs", 9 | "rev": "2631b0b7abcea6e640ce31cd78ea58910d31e650", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "NixOS", 14 | "ref": "nixos-unstable", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "root": { 20 | "inputs": { 21 | "nixpkgs": "nixpkgs" 22 | } 23 | } 24 | }, 25 | "root": "root", 26 | "version": 7 27 | } 28 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; 4 | }; 5 | 6 | nixConfig = { 7 | extra-substituters = [ 8 | "https://nix-ai-stuff.cachix.org" 9 | "https://nix-community.cachix.org" 10 | "https://ai.cachix.org" 11 | ]; 12 | 13 | extra-trusted-public-keys = [ 14 | "nix-ai-stuff.cachix.org-1:WlUGeVCs26w9xF0/rjyg32PujDqbVMlSHufpj1fqix8=" 15 | "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" 16 | "ai.cachix.org-1:N9dzRK+alWwoKXQlnn0H6aUx0lU/mspIoz8hMvGvbbc=" 17 | ]; 18 | }; 19 | 20 | outputs = 21 | { self 22 | , nixpkgs 23 | , ... 24 | }: 25 | let 26 | inherit (nixpkgs) lib; 27 | 28 | systems = lib.systems.flakeExposed; 29 | 30 | forAllSystems = lib.genAttrs systems; 31 | 32 | nixpkgsFor = forAllSystems (system: import nixpkgs { 33 | inherit system; 34 | config = { 35 | allowUnfree = true; 36 | cudaSupport = true; 37 | }; 38 | }); 39 | in 40 | { 41 | devShells = forAllSystems (system: 42 | let 43 | pkgs = nixpkgsFor.${system}; 44 | in 45 | { 46 | default = pkgs.mkShell { 47 | buildInputs = with pkgs; 48 | [ 49 | python3 50 | cudatoolkit 51 | ]; 52 | 53 | shellHook = '' 54 | export CUDA_PATH=${pkgs.cudatoolkit} 55 | ''; 56 | }; 57 | }); 58 | 59 | overlays.default = final: prev: 60 | rec { 61 | exllamav2 = final.callPackage ./pkgs/exllamav2.nix { inherit flash-attn; }; 62 | autogptq = final.callPackage ./pkgs/autogptq.nix { inherit rouge; }; 63 | ava-prebuilt = final.callPackage ./pkgs/ava/prebuilt.nix { }; 64 | ava = final.callPackage ./pkgs/ava { }; 65 | ava-headless = final.callPackage ./pkgs/ava { headless = true; }; 66 | tensor_parallel = final.callPackage ./pkgs/tensor_parallel.nix { }; 67 | lycoris-lora = final.callPackage ./pkgs/lycoris-lora.nix { }; 68 | open-clip-torch = final.callPackage ./pkgs/open-clip-torch.nix { }; 69 | dadaptation = final.callPackage ./pkgs/dadaptation.nix { }; 70 | prodigyopt = final.callPackage ./pkgs/prodigyopt.nix { }; 71 | kohya_ss = final.callPackage ./pkgs/kohya_ss { 72 | inherit dadaptation prodigyopt; 73 | }; 74 | rouge = final.callPackage ./pkgs/rouge.nix { }; 75 | flash-attn = final.callPackage ./pkgs/flash-attention.nix { }; 76 | kbnf = final.callPackage ./pkgs/kbnf.nix { }; 77 | general-sam = final.callPackage ./pkgs/general-sam.nix { }; 78 | formatron = final.callPackage ./pkgs/formatron.nix { kbnf = kbnf; general-sam = general-sam; exllamav2 = exllamav2; }; 79 | tabbyapi = final.callPackage ./pkgs/tabbyapi.nix { kbnf = kbnf; formatron = formatron; exllamav2 = exllamav2; }; 80 | }; 81 | 82 | packages = forAllSystems (system: 83 | let 84 | pkgs = nixpkgsFor.${system}; 85 | in 86 | lib.makeScope pkgs.newScope (final: self.overlays.default final pkgs) 87 | ); 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /nvidia.nix: -------------------------------------------------------------------------------- 1 | { symlinkJoin 2 | , cudaPackages 3 | , pkgs 4 | , cudaCapabilities ? pkgs.cudaPackages.cudaFlags.cudaCapabilities 5 | , lib 6 | , 7 | }: { 8 | BUILD_CUDA_EXT = "1"; 9 | 10 | CUDA_HOME = symlinkJoin { 11 | name = "cuda-redist"; 12 | paths = with cudaPackages; [ 13 | cuda_cudart # cuda_runtime.h 14 | cuda_nvcc 15 | ]; 16 | }; 17 | 18 | CUDA_VERSION = cudaPackages.cudaVersion; 19 | 20 | preBuild = '' 21 | export PATH=${pkgs.gcc11Stdenv.cc}/bin:$PATH 22 | export TORCH_CUDA_ARCH_LIST="${lib.concatStringsSep ";" cudaCapabilities}" 23 | ''; 24 | } 25 | -------------------------------------------------------------------------------- /pkgs/autogptq.nix: -------------------------------------------------------------------------------- 1 | { python3Packages 2 | , fetchFromGitHub 3 | , pkgs 4 | , lib 5 | , rouge 6 | , 7 | }: 8 | let 9 | nvidia = pkgs.callPackage ../nvidia.nix { }; 10 | in 11 | python3Packages.buildPythonPackage rec { 12 | inherit (nvidia) BUILD_CUDA_EXT CUDA_HOME CUDA_VERSION preBuild; 13 | 14 | pname = "auto_gptq"; 15 | version = "0.7.1"; 16 | pyproject = true; 17 | 18 | src = fetchFromGitHub { 19 | owner = "PanQiWei"; 20 | repo = "AutoGPTQ"; 21 | rev = "v${version}"; 22 | hash = "sha256-CkTWoLY1PLPHKTquwdpxN6L5jGTokK5BFV2owwoTlQ0="; 23 | }; 24 | 25 | buildInputs = with pkgs; [ 26 | python3Packages.pybind11 27 | cudatoolkit 28 | ]; 29 | 30 | nativeBuildInputs = with pkgs; [ 31 | which 32 | ninja 33 | ]; 34 | 35 | dependencies = with python3Packages; [ 36 | accelerate 37 | datasets 38 | sentencepiece 39 | numpy 40 | rouge 41 | gekko 42 | torch 43 | safetensors 44 | transformers 45 | peft 46 | tqdm 47 | ]; 48 | 49 | pythonImportsCheck = [ "auto_gptq" ]; 50 | 51 | meta = with lib; { 52 | homepage = "https://github.com/PanQiWei/AutoGPTQ"; 53 | description = "An easy-to-use LLMs quantization package"; 54 | changelog = "https://github.com/AutoGPTQ/AutoGPTQ/releases/tag/${src.rev}"; 55 | license = licenses.mit; 56 | }; 57 | } 58 | -------------------------------------------------------------------------------- /pkgs/ava/default.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , stdenv 3 | , zig_0_12 4 | , fetchFromGitHub 5 | , buildNpmPackage 6 | , callPackage 7 | , pkgs 8 | , headless ? stdenv.isLinux 9 | , 10 | }: 11 | let 12 | src = fetchFromGitHub { 13 | owner = "cztomsik"; 14 | repo = "ava"; 15 | rev = "06699ff862c58cbc15c965b58f605b17351ed01c"; 16 | fetchSubmodules = true; 17 | hash = "sha256-Actn/uJLFYOrA4klQAp0qArdmVuLJfCHOBJRnLVcH8o="; 18 | }; 19 | 20 | version = "2024-04-24"; 21 | 22 | web = buildNpmPackage { 23 | pname = "ava-web"; 24 | inherit src version; 25 | 26 | npmDepsHash = "sha256-B86igaQWHitdBRyA7lGbvrYruhrWsVZGKvWRsJVamdI="; 27 | 28 | nativeBuildInputs = with pkgs; [ 29 | git 30 | ]; 31 | 32 | postPatch = '' 33 | substituteInPlace package.json \ 34 | --replace "git submodule update --init --recursive" "" 35 | ''; 36 | 37 | buildPhase = '' 38 | npm run build 39 | ''; 40 | 41 | installPhase = '' 42 | cp zig-out/app/main.js $out 43 | ''; 44 | }; 45 | in 46 | stdenv.mkDerivation rec { 47 | pname = "ava"; 48 | inherit src version; 49 | 50 | nativeBuildInputs = [ 51 | zig_0_12.hook 52 | ]; 53 | 54 | postPatch = 55 | '' 56 | ln -s ${callPackage ./deps.nix {}} $ZIG_GLOBAL_CACHE_DIR/p 57 | '' 58 | + lib.optionals headless '' 59 | mkdir -p zig-out/app 60 | ln -s ${web} zig-out/app/main.js 61 | ''; 62 | 63 | zigBuildFlags = 64 | [ 65 | "-Dtarget=${stdenv.buildPlatform.qemuArch}-${ 66 | if stdenv.isLinux 67 | then "linux" 68 | else "macos" 69 | }" 70 | ] 71 | ++ lib.optionals headless [ 72 | "-Dheadless=true" 73 | ]; 74 | 75 | installPhase = '' 76 | mkdir -p $out/bin 77 | cp zig-out/bin/ava_${stdenv.buildPlatform.qemuArch} $out/bin/ava 78 | ''; 79 | 80 | meta = with lib; { 81 | homepage = "https://www.avapls.com/"; 82 | description = "A GUI macos LLM program"; 83 | license = licenses.mit; 84 | platforms = platforms.linux ++ platforms.darwin; 85 | mainProgram = pname; 86 | }; 87 | } 88 | -------------------------------------------------------------------------------- /pkgs/ava/deps.nix: -------------------------------------------------------------------------------- 1 | { linkFarm 2 | , fetchzip 3 | , 4 | }: 5 | linkFarm "zig-packages" [ 6 | { 7 | name = "1220491e02c1f645f41d46803d6fc039cbeef125a1d106f90a9cc718f6782df28a08"; 8 | path = fetchzip { 9 | url = "https://www.sqlite.org/2024/sqlite-autoconf-3450000.tar.gz"; 10 | hash = "sha256-0YUoGd7BAyQng1ev7MXJaefqVnWHkBXVc/D2bT21hWs="; 11 | }; 12 | } 13 | { 14 | name = "1220ed7d1e37d1b9e7cbf63e09fe17dc4cc9c6a07a867130c3cae33978cbd8d8da68"; 15 | path = fetchzip { 16 | url = "https://github.com/cztomsik/fridge/archive/29edd70.tar.gz"; 17 | hash = "sha256-7H6VhVu5kbeW6Iniyx+WIb8cl/PkZvau76ePSsj628c="; 18 | }; 19 | } 20 | { 21 | name = "122032d6c95983590d41ecf5c4c302ca6a986ac44468319d4cde675dee2f612f7444"; 22 | path = fetchzip { 23 | url = "https://github.com/cztomsik/tokamak/archive/68c80d6.tar.gz"; 24 | hash = "sha256-6RN+jXtMTrXpyRw29uhL1dJKo8L42ROqFP19cnIrm2M="; 25 | }; 26 | } 27 | ] 28 | -------------------------------------------------------------------------------- /pkgs/ava/prebuilt.nix: -------------------------------------------------------------------------------- 1 | { stdenv 2 | , fetchurl 3 | , undmg 4 | , lib 5 | , 6 | }: 7 | stdenv.mkDerivation rec { 8 | pname = "ava"; 9 | version = "2024-04-21"; 10 | 11 | src = fetchurl { 12 | url = "https://s3.amazonaws.com/www.avapls.com/Ava_${version}.dmg"; 13 | hash = "sha256-f9/fQJ1bmamPsWwR0Kz99Z4CgVzOC5X0gNUnSw4VMuQ="; 14 | }; 15 | 16 | nativeBuildInputs = [ undmg ]; 17 | 18 | sourceRoot = "Ava.app"; 19 | 20 | installPhase = '' 21 | mkdir -p $out/Applications/Ava.app $out/bin 22 | cp -R . $out/Applications/Ava.app 23 | ln -s ../Applications/Ava.app/Contents/MacOS/ava $out/bin 24 | ''; 25 | 26 | meta = with lib; { 27 | homepage = "https://www.avapls.com/"; 28 | description = "A GUI macos LLM program"; 29 | license = licenses.mit; 30 | platforms = platforms.darwin; 31 | }; 32 | } 33 | -------------------------------------------------------------------------------- /pkgs/dadaptation.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchPypi 4 | , 5 | }: 6 | python3Packages.buildPythonPackage rec { 7 | pname = "dadaptation"; 8 | version = "3.1"; 9 | pyproject = true; 10 | 11 | src = fetchPypi { 12 | inherit pname version; 13 | hash = "sha256-qkFZyZ8xtBA6Uwz556G3dpEuYcMX/2hLJxdR1NGDLfc="; 14 | }; 15 | 16 | build-system = with python3Packages; [ 17 | setuptools 18 | ]; 19 | 20 | dependencies = with python3Packages; [ 21 | torch 22 | ]; 23 | 24 | pythonImportsCheck = [ "dadaptation" ]; 25 | 26 | meta = with lib; { 27 | description = "Learning Rate Free Learning for Adam, SGD and AdaGrad"; 28 | homepage = "https://github.com/facebookresearch/dadaptation"; 29 | license = licenses.mit; 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /pkgs/exllamav2.nix: -------------------------------------------------------------------------------- 1 | { python3Packages 2 | , fetchFromGitHub 3 | , flash-attn 4 | , pkgs 5 | , lib 6 | , 7 | }: 8 | let 9 | inherit (python3Packages.torch) cudaCapabilities cudaPackages cudaSupport; 10 | inherit (cudaPackages) backendStdenv; 11 | in 12 | python3Packages.buildPythonPackage rec { 13 | pname = "exllamav2"; 14 | version = "0.2.9"; 15 | pyproject = true; 16 | 17 | src = fetchFromGitHub { 18 | owner = "turboderp"; 19 | repo = "exllamav2"; 20 | rev = "v${version}"; 21 | hash = "sha256-iaeo4D2I6J0/tDz1Q9kLLU6vHkdVayPhcQQAkYs/fDg="; 22 | }; 23 | 24 | preConfigure = '' 25 | export CC=${lib.getExe' backendStdenv.cc "cc"} 26 | export CXX=${lib.getExe' backendStdenv.cc "c++"} 27 | export TORCH_CUDA_ARCH_LIST="${lib.concatStringsSep ";" cudaCapabilities}" 28 | export FORCE_CUDA=1 29 | ''; 30 | 31 | buildInputs = with pkgs; [ 32 | python3Packages.pybind11 33 | cudatoolkit 34 | ]; 35 | 36 | env.CUDA_HOME = lib.optionalString cudaSupport (lib.getDev cudaPackages.cuda_nvcc); 37 | 38 | nativeBuildInputs = with pkgs; [ 39 | which 40 | ninja 41 | ]; 42 | 43 | postPatch = '' 44 | substituteInPlace requirements.txt --replace-fail "numpy~=1.26.4" "numpy" 45 | ''; 46 | 47 | dependencies = with python3Packages; [ 48 | flash-attn 49 | pandas 50 | fastparquet 51 | torch 52 | safetensors 53 | sentencepiece 54 | pygments 55 | websockets 56 | regex 57 | numpy 58 | tokenizers 59 | rich 60 | ninja 61 | pillow 62 | ]; 63 | 64 | pythonImportsCheck = [ "exllamav2" ]; 65 | 66 | meta = with lib; { 67 | homepage = "https://github.com/turboderp/exllamav2"; 68 | description = "A fast inference library for running LLMs locally on modern consumer-class GPUs"; 69 | changelog = "https://github.com/turboderp/exllamav2/releases/tag/${src.rev}"; 70 | license = licenses.mit; 71 | }; 72 | } 73 | -------------------------------------------------------------------------------- /pkgs/exllamav3.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , buildPythonPackage 3 | , fetchFromGitHub 4 | , flash-attn 5 | , python3Packages 6 | , 7 | }: 8 | 9 | buildPythonPackage { 10 | pname = "exllamav3"; 11 | version = "unstable-2025-04-18"; 12 | pyproject = true; 13 | 14 | src = fetchFromGitHub { 15 | owner = "turboderp-org"; 16 | repo = "exllamav3"; 17 | rev = "c44e56c73b2c67eee087c7195c9093520494d3bf"; 18 | hash = "sha256-NEIgBWJTCiwaKoq7R+6mMR7LcQ5enmzLGx64ortcjOo="; 19 | }; 20 | 21 | build-system = with python3Packages; [ 22 | setuptools 23 | wheel 24 | ]; 25 | 26 | dependencies = with python3Packages; [ 27 | flash-attn 28 | ninja 29 | numpy 30 | rich 31 | safetensors 32 | tokenizers 33 | torch 34 | typing-extensions 35 | ]; 36 | 37 | pythonImportsCheck = [ 38 | "exllamav3" 39 | ]; 40 | 41 | meta = { 42 | description = "An optimized quantization and inference library for running LLMs locally on modern consumer-class GPUs"; 43 | homepage = "https://github.com/turboderp-org/exllamav3"; 44 | license = lib.licenses.mit; 45 | }; 46 | } 47 | -------------------------------------------------------------------------------- /pkgs/flash-attention.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchFromGitHub 4 | , symlinkJoin 5 | , pkgs 6 | }: 7 | let 8 | inherit (python3Packages.torch) cudaCapabilities cudaPackages; 9 | inherit (cudaPackages) backendStdenv; 10 | 11 | nvidia = pkgs.callPackage ../nvidia.nix { }; 12 | in 13 | python3Packages.buildPythonPackage rec { 14 | inherit (nvidia) BUILD_CUDA_EXT CUDA_VERSION preBuild; 15 | pname = "flash-attn"; 16 | version = "2.7.4"; 17 | pyproject = true; 18 | 19 | src = fetchFromGitHub { 20 | owner = "Dao-AILab"; 21 | repo = "flash-attention"; 22 | rev = "v${version}"; 23 | hash = "sha256-dFJplpZojvtV1wbqNY0SGPw2c8kxMOF+qFo8I9asDJs="; 24 | fetchSubmodules = true; 25 | }; 26 | 27 | preConfigure = '' 28 | export CC=${lib.getExe' backendStdenv.cc "cc"} 29 | export CXX=${lib.getExe' backendStdenv.cc "c++"} 30 | export TORCH_CUDA_ARCH_LIST="${lib.concatStringsSep ";" cudaCapabilities}" 31 | export FORCE_CUDA=1 32 | ''; 33 | 34 | build-tools = with python3Packages; [ 35 | setuptools 36 | wheel 37 | ]; 38 | 39 | nativeBuildInputs = with pkgs; [ 40 | git 41 | which 42 | ninja 43 | ]; 44 | 45 | env.CUDA_HOME = symlinkJoin { 46 | name = "cuda-redist"; 47 | paths = buildInputs; 48 | }; 49 | 50 | buildInputs = with cudaPackages; [ 51 | cuda_cudart # cuda_runtime_api.h 52 | libcusparse # cusparse.h 53 | cuda_cccl # nv/target 54 | libcublas # cublas_v2.h 55 | libcusolver # cusolverDn.h 56 | libcurand # curand_kernel.h 57 | cuda_nvcc 58 | ]; 59 | 60 | dependencies = with python3Packages; [ 61 | torch 62 | psutil 63 | ninja 64 | einops 65 | ]; 66 | 67 | pythonImportsCheck = [ "flash_attn" ]; 68 | 69 | meta = with lib; { 70 | description = "Fast and memory-efficient exact attention"; 71 | homepage = "https://github.com/Dao-AILab/flash-attention"; 72 | license = licenses.bsd3; 73 | }; 74 | } 75 | -------------------------------------------------------------------------------- /pkgs/formatron.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchFromGitHub 4 | , exllamav2 5 | , general-sam 6 | , kbnf 7 | , 8 | }: 9 | python3Packages.buildPythonPackage rec { 10 | pname = "formatron"; 11 | version = "0.4.11"; 12 | pyproject = true; 13 | 14 | src = fetchFromGitHub { 15 | owner = "Dan-wanna-M"; 16 | repo = "formatron"; 17 | rev = "v${version}"; 18 | hash = "sha256-VXOZTSD9xNB9UaA84T1FrEN3WIYbaieSfwjf7J6P+KA="; 19 | fetchSubmodules = true; 20 | }; 21 | 22 | build-system = with python3Packages; [ 23 | setuptools 24 | ]; 25 | 26 | dependencies = with python3Packages; [ 27 | frozendict 28 | general-sam 29 | jsonschema 30 | kbnf 31 | pydantic 32 | ]; 33 | 34 | optional-dependencies = with python3Packages; { 35 | exllamav2 = [ 36 | exllamav2 37 | ]; 38 | rwkv = [ 39 | rwkv 40 | ]; 41 | transformers = [ 42 | transformers 43 | ]; 44 | vllm = [ 45 | vllm 46 | ]; 47 | }; 48 | 49 | pythonImportsCheck = [ 50 | "formatron" 51 | ]; 52 | 53 | meta = { 54 | description = "Formatron empowers everyone to control the format of language models' output with minimal overhead"; 55 | homepage = "https://github.com/Dan-wanna-M/formatron"; 56 | license = lib.licenses.mit; 57 | }; 58 | } 59 | -------------------------------------------------------------------------------- /pkgs/general-sam.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchFromGitHub 4 | , rustPlatform 5 | , pkgs 6 | , 7 | }: 8 | 9 | python3Packages.buildPythonPackage rec { 10 | pname = "general-sam"; 11 | version = "1.0.1"; 12 | pyproject = true; 13 | 14 | src = fetchFromGitHub { 15 | owner = "ModelTC"; 16 | repo = "general-sam-py"; 17 | rev = "v${version}"; 18 | hash = "sha256-II+pWrV2N3wo7z56VJH7D21QZLBXrx1VbIAcl3AnwwM="; 19 | }; 20 | 21 | cargoDeps = rustPlatform.fetchCargoVendor { 22 | inherit pname version src; 23 | hash = "sha256-XMaD6m1Y+CkMSx+Vn7Vc74DJCJ5qU12wr32+PLCiB0U="; 24 | }; 25 | 26 | build-system = with pkgs; [ 27 | cargo 28 | rustPlatform.cargoSetupHook 29 | rustPlatform.maturinBuildHook 30 | rustc 31 | ]; 32 | 33 | optional-dependencies = with python3Packages; { 34 | test = [ 35 | pytest 36 | ]; 37 | }; 38 | 39 | pythonImportsCheck = [ 40 | "general_sam" 41 | ]; 42 | 43 | meta = { 44 | description = "Python bindings for general-sam and some utilities"; 45 | homepage = "https://github.com/ModelTC/general-sam-py"; 46 | license = with lib.licenses; [ asl20 mit ]; 47 | }; 48 | } 49 | -------------------------------------------------------------------------------- /pkgs/kbnf.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchPypi 4 | , rustPlatform 5 | , pkgs 6 | , 7 | }: 8 | 9 | python3Packages.buildPythonPackage rec { 10 | pname = "kbnf"; 11 | version = "0.4.1"; 12 | pyproject = true; 13 | 14 | src = fetchPypi { 15 | inherit pname version; 16 | hash = "sha256-+CPHljZOLd0zPH+/N8KNhnpCRKRZuk6r6qx1X4sa2rk="; 17 | }; 18 | 19 | cargoDeps = rustPlatform.fetchCargoVendor { 20 | inherit pname version src; 21 | hash = "sha256-YZexO7/J9CBNVO1oVwUq/5Q7k5LyG6oDfidjUKiFGMg="; 22 | }; 23 | 24 | build-system = with pkgs; [ 25 | cargo 26 | rustPlatform.cargoSetupHook 27 | rustPlatform.maturinBuildHook 28 | rustc 29 | ]; 30 | 31 | dependencies = with python3Packages; [ 32 | numpy 33 | ]; 34 | 35 | optional-dependencies = with python3Packages; { 36 | torch = [ 37 | torch 38 | ]; 39 | }; 40 | 41 | pythonImportsCheck = [ 42 | "kbnf" 43 | ]; 44 | 45 | meta = { 46 | description = "A fast constrained decoding engine based on context free grammar"; 47 | homepage = "https://pypi.org/project/kbnf/"; 48 | license = with lib.licenses; [ asl20 mit ]; 49 | }; 50 | } 51 | -------------------------------------------------------------------------------- /pkgs/kohya_ss/default.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchFromGitHub 4 | , dadaptation 5 | , prodigyopt 6 | , pkgs 7 | , 8 | }: 9 | let 10 | xformers-cuda = pkgs.callPackage ./xformers-cuda.nix { }; 11 | diffusers = python3Packages.diffusers.overrideAttrs (f: p: { 12 | propagatedBuildInputs = 13 | p.propagatedBuildInputs 14 | ++ [ 15 | xformers-cuda 16 | ]; 17 | }); 18 | lycoris-lora = pkgs.callPackage ../lycoris-lora.nix { 19 | inherit diffusers; 20 | }; 21 | in 22 | python3Packages.buildPythonApplication rec { 23 | pname = "kohya_ss"; 24 | version = "22.6.1"; 25 | pyproject = true; 26 | 27 | src = fetchFromGitHub { 28 | owner = "bmaltais"; 29 | repo = pname; 30 | rev = "v${version}"; 31 | hash = "sha256-KS2tGXUk350lyflNd1Nr1euX91loMR4i8w9ZLbNBqyM="; 32 | }; 33 | 34 | nativeBuildInputs = with python3Packages; [ 35 | poetry-core 36 | ]; 37 | 38 | propagatedBuildInputs = with python3Packages; [ 39 | accelerate 40 | aiofiles 41 | altair 42 | dadaptation 43 | diffusers 44 | easygui 45 | einops 46 | fairscale 47 | ftfy 48 | gradio 49 | huggingface-hub 50 | invisible-watermark 51 | lion-pytorch 52 | lycoris-lora 53 | onnx 54 | onnxruntime # onnxruntime-gpu 55 | open-clip-torch 56 | opencv4 # opencv-python 57 | prodigyopt 58 | protobuf 59 | pytorch-lightning 60 | rich 61 | safetensors 62 | timm 63 | tkinter # tk 64 | toml 65 | transformers 66 | voluptuous 67 | wandb 68 | bitsandbytes 69 | ]; 70 | 71 | buildInputs = with pkgs; [ 72 | onnxruntime 73 | ]; 74 | 75 | # basically just replaces all the relative paths 76 | postPatch = '' 77 | substituteInPlace kohya_gui.py \ 78 | --replace "if os.path.exists(\"./.release\"):" "release = \"v${version}\" 79 | if False:" \ 80 | --replace "./style.css" "$out/lib/style.css" \ 81 | --replace "\"./README.md\"" "\"$out/lib/README.md\"" 82 | 83 | substituteInPlace lora_gui.py \ 84 | --replace "./presets/lora" "$out/lib/presets/lora" \ 85 | --replace "./style.css" "$out/lib/style.css" \ 86 | --replace "\"./docs/LoRA/top_level.md\"" "\"$out/lib/docs/LoRA/top_level.md\"" \ 87 | --replace "./sdxl_train_network.py" "$out/lib/sdxl_train_network.py" \ 88 | --replace "./train_network.py" "$out/lib/train_network.py" 89 | 90 | substituteInPlace finetune_gui.py \ 91 | --replace "./docs/Finetuning/top_level.md" "$out/lib/docs/Finetuning/top_level.md" \ 92 | --replace "./style.css" "$out/lib/style.css" \ 93 | --replace "./presets/finetune" "$out/lib/presets/finetune" \ 94 | --replace "./sdxl_train.py" "$out/lib/sdxl_train.py" \ 95 | --replace "./fine_tune.py" "$out/lib/fine_tune.py" 96 | 97 | substituteInPlace textual_inversion_gui.py \ 98 | --replace "./style.css" "$out/lib/style.css" \ 99 | --replace "./sdxl_train_textual_inversion.py" "$out/lib/sdxl_train_textual_inversion.py" \ 100 | --replace "./train_textual_inversion.py" "$out/lib/train_textual_inversion.py" 101 | 102 | substituteInPlace library/utilities.py \ 103 | --replace "'./style.css'" "'$out/lib/style.css'" 104 | 105 | substituteInPlace library/common_gui.py \ 106 | --replace "./v2_inference/" "$out/lib/v2_inference/" 107 | 108 | substituteInPlace library/convert_model_gui.py \ 109 | --replace "./v2_inference/" "$out/lib/v2_inference/" 110 | 111 | substituteInPlace library/localization.py \ 112 | --replace "'./localizations'" "'$out/lib/localizations'" 113 | 114 | substituteInPlace library/wd14_caption_gui.py \ 115 | --replace "./finetune/tag_images_by_wd14_tagger.py" "$out/lib/finetune/tag_images_by_wd14_tagger.py" 116 | 117 | substituteInPlace dreambooth_gui.py \ 118 | --replace "./style.css" "$out/lib/style.css" \ 119 | --replace "./train_db.py" "$out/lib/train_db.py" \ 120 | --replace "./sdxl_train.py" "$out/lib/sdxl_train.py" 121 | ''; 122 | 123 | postInstall = '' 124 | cp *.py $out/lib 125 | cp -r networks $out/lib/python3*/site-packages/ 126 | cp README.md $out/lib 127 | cp -r presets $out/lib 128 | cp -r docs $out/lib 129 | cp style.css $out/lib 130 | cp -r v2_inference $out/lib 131 | cp -r localizations $out/lib 132 | cp -r finetune $out/lib 133 | ''; 134 | 135 | postFixup = '' 136 | chmod +x $out/lib/kohya_gui.py 137 | sed -i -e '1i#!/usr/bin/python' $out/lib/kohya_gui.py 138 | patchShebangs $out/lib/kohya_gui.py 139 | 140 | mkdir -p $out/bin 141 | ln -s $out/lib/kohya_gui.py $out/bin/kohya_ss 142 | 143 | wrapProgram "$out/bin/kohya_ss" \ 144 | --prefix PYTHONPATH : "$PYTHONPATH" \ 145 | --prefix PATH : ${lib.makeBinPath [python3Packages.accelerate]} \ 146 | ''; 147 | 148 | pythonImportsCheck = [ "library" ]; 149 | 150 | meta = with lib; { 151 | homepage = "https://github.com/bmaltais/kohya_ss"; 152 | license = licenses.asl20; 153 | mainProgram = pname; 154 | }; 155 | } 156 | -------------------------------------------------------------------------------- /pkgs/kohya_ss/xformers-cuda.nix: -------------------------------------------------------------------------------- 1 | { config 2 | , cudaPackages 3 | , fetchFromGitHub 4 | , lib 5 | , symlinkJoin 6 | , pkgs 7 | , python3Packages 8 | , 9 | }: 10 | let 11 | cudaSupport = config.cudaSupport or false; 12 | 13 | # Build-time dependencies 14 | cuda-native-redist = symlinkJoin { 15 | name = "cuda-native-redist-${cudaPackages.cudaVersion}"; 16 | paths = with cudaPackages; [ 17 | cuda_cccl #include 18 | cuda_cudart #include 19 | cuda_nvcc 20 | libcublas #include 21 | libcurand #include 22 | libcusolver #include 23 | libcusparse #include 24 | ]; 25 | }; 26 | in 27 | # slightly modified version of https://github.com/NixOS/nixpkgs/pull/234557 28 | python3Packages.buildPythonPackage rec { 29 | pname = "xformers"; 30 | version = "0.0.22.post7"; 31 | pyproject = true; 32 | 33 | src = fetchFromGitHub { 34 | owner = "facebookresearch"; 35 | repo = pname; 36 | rev = "v${version}"; 37 | fetchSubmodules = true; 38 | leaveDotGit = true; 39 | hash = "sha256-PjauTf098pEj6d/WD4XgQnLI2xzr988XiHM0pfgP2j4="; 40 | }; 41 | 42 | nativeBuildInputs = with pkgs; [ 43 | cuda-native-redist 44 | git 45 | ninja 46 | which 47 | ]; 48 | 49 | dependencies = with python3Packages; [ 50 | numpy 51 | torch 52 | ]; 53 | 54 | postPatch = '' 55 | substituteInPlace xformers/__init__.py \ 56 | --replace "_is_functorch_available: bool = False" "_is_functorch_available: bool = True" 57 | ''; 58 | 59 | preBuild = '' 60 | export XFORMERS_BUILD_TYPE=Release 61 | export TORCH_CUDA_ARCH_LIST="${lib.strings.concatStringsSep ";" python3Packages.torch.cudaCapabilities}" 62 | export CC="${cudaPackages.backendStdenv.cc}/bin/cc" 63 | export CXX="${cudaPackages.backendStdenv.cc}/bin/c++" 64 | ''; 65 | 66 | # Note: Tests require ragged_inference, which is in the experimental module and is not built 67 | # by default. 68 | doCheck = false; 69 | pythonImportsCheck = [ pname ]; 70 | 71 | passthru = { 72 | inherit cudaSupport cudaPackages; 73 | cudaCapabilities = lib.lists.optionals cudaSupport python3Packages.torch.cudaCapabilities; 74 | }; 75 | 76 | meta = with lib; { 77 | description = "Hackable and optimized Transformers building blocks, supporting a composable construction"; 78 | homepage = "https://github.com/facebookresearch/xformers"; 79 | license = licenses.bsd3; 80 | platforms = platforms.linux; 81 | broken = cudaSupport != python3Packages.torch.cudaSupport; 82 | }; 83 | } 84 | -------------------------------------------------------------------------------- /pkgs/lycoris-lora.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , fetchPypi 3 | , python3Packages 4 | , diffusers ? python3Packages.diffusers 5 | , 6 | }: 7 | python3Packages.buildPythonPackage rec { 8 | pname = "lycoris-lora"; 9 | version = "2.0.2"; 10 | pyproject = true; 11 | 12 | src = fetchPypi { 13 | pname = "lycoris_lora"; 14 | inherit version; 15 | hash = "sha256-fkR0JPDJJK2sxn9QiYUjqzymQvi/kiqpcG+7rez8+4M="; 16 | }; 17 | 18 | build-system = with python3Packages; [ 19 | setuptools 20 | ]; 21 | 22 | dependencies = with python3Packages; [ 23 | toml 24 | einops 25 | diffusers 26 | safetensors 27 | torch 28 | transformers 29 | ]; 30 | 31 | pythonImportsCheck = [ "lycoris" ]; 32 | 33 | meta = with lib; { 34 | description = "Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion"; 35 | homepage = "https://github.com/KohakuBlueleaf/LyCORIS"; 36 | license = licenses.asl20; 37 | }; 38 | } 39 | -------------------------------------------------------------------------------- /pkgs/prodigyopt.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchPypi 4 | , 5 | }: 6 | python3Packages.buildPythonPackage rec { 7 | pname = "prodigyopt"; 8 | version = "1.0"; 9 | pyproject = true; 10 | 11 | src = fetchPypi { 12 | inherit pname version; 13 | hash = "sha256-zbvZnoNvpuuQr6SfXrGndg1jShWXbnfj6BFDSavpEKw="; 14 | }; 15 | 16 | build-system = with python3Packages; [ 17 | setuptools 18 | ]; 19 | 20 | dependencies = with python3Packages; [ 21 | torch 22 | ]; 23 | 24 | pythonImportsCheck = [ "prodigyopt" ]; 25 | 26 | meta = with lib; { 27 | description = "An Adam-like optimizer for neural networks with adaptive estimation of learning rate"; 28 | homepage = "https://github.com/konstmish/prodigy"; 29 | license = licenses.mit; 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /pkgs/rouge.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , fetchPypi 3 | , python3Packages 4 | }: 5 | 6 | python3Packages.buildPythonPackage rec { 7 | pname = "rouge"; 8 | version = "1.0.1"; 9 | pyproject = true; 10 | 11 | src = fetchPypi { 12 | inherit pname version; 13 | hash = "sha256-ErSDRspH1rzzxFBh8xVFK5zOwGIO6JXshbfvw9VKrjQ="; 14 | }; 15 | 16 | nativeBuildInputs = with python3Packages; [ 17 | setuptools 18 | wheel 19 | ]; 20 | 21 | propagatedBuildInputs = with python3Packages; [ 22 | six 23 | ]; 24 | 25 | pythonImportsCheck = [ "rouge" ]; 26 | 27 | meta = with lib; { 28 | description = "Full Python ROUGE Score Implementation"; 29 | homepage = "https://pypi.org/project/rouge/"; 30 | license = licenses.asl20; 31 | }; 32 | } 33 | -------------------------------------------------------------------------------- /pkgs/tabbyapi.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , fetchFromGitHub 3 | , exllamav2 4 | , python3Packages 5 | , formatron 6 | , kbnf 7 | , 8 | }: 9 | python3Packages.buildPythonApplication { 10 | pname = "tabbyapi"; 11 | version = "unstable-2025-04-16"; 12 | pyproject = true; 13 | 14 | src = fetchFromGitHub { 15 | owner = "theroyallab"; 16 | repo = "tabbyAPI"; 17 | rev = "6bb5f8f599d617f94af85e0818c8a841fc0ed806"; 18 | hash = "sha256-Ovj/8T98pSWuLtSbfGVWf++ywSsa+PgXHiHxg6W7/m4="; 19 | }; 20 | 21 | build-system = with python3Packages; [ 22 | packaging 23 | setuptools 24 | wheel 25 | ]; 26 | 27 | dependencies = with python3Packages; [ 28 | aiofiles 29 | aiohttp 30 | async-lru 31 | fastapi # fastapi-slim 32 | httptools 33 | huggingface-hub 34 | jinja2 35 | loguru 36 | numpy 37 | packaging 38 | pillow 39 | psutil 40 | pydantic 41 | rich 42 | ruamel-yaml 43 | setuptools 44 | sse-starlette 45 | tokenizers 46 | formatron 47 | kbnf 48 | uvicorn 49 | uvloop # linux and x86_64 50 | exllamav2 51 | ]; 52 | 53 | postPatch = '' 54 | substituteInPlace pyproject.toml --replace-fail 'fastapi-slim' 'fastapi' 55 | substituteInPlace pyproject.toml --replace-fail "numpy < 2.0.0" "numpy" 56 | ''; 57 | 58 | optional-dependencies = with python3Packages; { 59 | amd = [ 60 | pytorch-triton-rocm 61 | torch 62 | ]; 63 | cu118 = [ 64 | torch 65 | ]; 66 | cu121 = [ 67 | flash-attn 68 | torch 69 | ]; 70 | dev = [ 71 | ruff 72 | ]; 73 | extras = [ 74 | infinity-emb 75 | sentence-transformers 76 | ]; 77 | }; 78 | 79 | postInstall = '' 80 | cp *.py $out/lib/python3*/site-packages/ 81 | cp -r {common,endpoints,backends,templates} $out/lib/python3*/site-packages/ 82 | ''; 83 | 84 | postFixup = '' 85 | chmod +x $out/lib/python3*/site-packages/main.py 86 | sed -i -e '1i#!/usr/bin/python' $out/lib/python3*/site-packages/main.py 87 | patchShebangs $out/lib/python3*/site-packages/main.py 88 | 89 | mkdir $out/bin 90 | ln -s $out/lib/python3*/site-packages/main.py $out/bin/tabbyapi 91 | 92 | wrapProgram "$out/bin/tabbyapi" \ 93 | --prefix PYTHONPATH : "$PYTHONPATH" \ 94 | ''; 95 | 96 | meta = { 97 | description = "An OAI compatible exllamav2 API that's both lightweight and fast"; 98 | homepage = "https://github.com/theroyallab/tabbyAPI"; 99 | license = lib.licenses.agpl3Only; 100 | mainProgram = "tabbyapi"; 101 | }; 102 | } 103 | -------------------------------------------------------------------------------- /pkgs/tensor_parallel.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , python3Packages 3 | , fetchPypi 4 | , 5 | }: 6 | python3Packages.buildPythonPackage rec { 7 | pname = "tensor_parallel"; 8 | version = "2.0.0"; 9 | pyproject = true; 10 | 11 | src = fetchPypi { 12 | inherit pname version; 13 | hash = "sha256-Wcb9iUvtxYZSyHPeSvJdAkvyhAGv8yTVAVHorIVZnTc="; 14 | }; 15 | 16 | build-system = with python3Packages; [ 17 | setuptools 18 | ]; 19 | 20 | dependencies = with python3Packages; [ 21 | torch 22 | transformers 23 | ]; 24 | 25 | passthru.optional-dependencies = { 26 | dev = with python3Packages; [ 27 | accelerate 28 | black 29 | einops 30 | isort 31 | peft 32 | psutil 33 | pytest 34 | pytest-asyncio 35 | pytest-forked 36 | ]; 37 | }; 38 | 39 | pythonImportsCheck = [ "tensor_parallel" ]; 40 | 41 | meta = with lib; { 42 | description = "Automatically shard your large model between multiple GPUs, works without torch.distributed"; 43 | homepage = "https://github.com/BlackSamorez/tensor_parallel"; 44 | changelog = "https://github.com/BlackSamorez/tensor_parallel/releases/tag/${src.rev}"; 45 | license = licenses.mit; 46 | }; 47 | } 48 | --------------------------------------------------------------------------------