├── .envrc ├── .gitignore ├── .gitmodules ├── scripts └── start-kokoro.sh ├── shell.nix ├── README.md └── justfile /.envrc: -------------------------------------------------------------------------------- 1 | use nix 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .direnv/ 2 | .DS_Store 3 | .idea/ 4 | .vscode/ 5 | *.swp 6 | result 7 | bin/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "llama.cpp"] 2 | path = external/llama.cpp 3 | url = git@github.com:ggml-org/llama.cpp.git 4 | [submodule "ollama"] 5 | path = external/ollama 6 | url = git@github.com:ollama/ollama.git 7 | [submodule "ik_llama.cpp"] 8 | path = external/ik_llama.cpp 9 | url = https://github.com/ikawrakow/ik_llama.cpp 10 | [submodule "Kokoro-FastAPI"] 11 | path = external/Kokoro-FastAPI 12 | url = https://github.com/remsky/Kokoro-FastAPI.git 13 | [submodule "external/agent-cli"] 14 | path = external/agent-cli 15 | url = git@github.com:basnijholt/agent-cli.git 16 | [submodule "external/ComfyUI"] 17 | path = external/ComfyUI 18 | url = https://github.com/comfyanonymous/ComfyUI.git 19 | -------------------------------------------------------------------------------- /scripts/start-kokoro.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Ensure we are in the project root (one level up from scripts/) 5 | cd "$(dirname "$0")/.." 6 | PROJECT_ROOT=$(pwd) 7 | 8 | # Define the submodule path 9 | KOKORO_DIR="external/Kokoro-FastAPI" 10 | 11 | # Create virtual environment if it doesn't exist (in the root) 12 | if [ ! -d ".venv-kokoro" ]; then 13 | echo "Creating virtual environment in .venv-kokoro..." 14 | uv venv .venv-kokoro 15 | fi 16 | 17 | # Activate virtual environment 18 | source .venv-kokoro/bin/activate 19 | 20 | # Navigate to the submodule 21 | if [ ! -d "$KOKORO_DIR" ]; then 22 | echo "Error: Directory $KOKORO_DIR not found!" 23 | exit 1 24 | fi 25 | cd "$KOKORO_DIR" 26 | 27 | # Set environment variables for Kokoro 28 | # The original script used PWD, which is now the submodule dir 29 | export USE_GPU=true 30 | export USE_ONNX=false 31 | export PYTHONPATH=$(pwd):$(pwd)/api 32 | export MODEL_DIR=src/models 33 | export VOICES_DIR=src/voices/v1_0 34 | export WEB_PLAYER_PATH=$(pwd)/web 35 | 36 | echo "Installing dependencies..." 37 | uv pip install -e ".[gpu]" 38 | 39 | echo "Downloading models..." 40 | python docker/scripts/download_model.py --output api/src/models/v1_0 41 | 42 | echo "Starting Kokoro FastAPI..." 43 | uvicorn api.src.main:app --host 0.0.0.0 --port 8880 44 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { config.allowUnfree = true; } }: 2 | 3 | pkgs.mkShell { 4 | name = "ai-lab-env"; 5 | 6 | # Allow building with -march=native 7 | NIX_ENFORCE_NO_NATIVE = "0"; 8 | 9 | buildInputs = with pkgs; [ 10 | # Build tools 11 | cmake 12 | ninja 13 | pkg-config 14 | just 15 | 16 | # Compilers 17 | gcc 18 | go 19 | 20 | # Python & Tools 21 | python312 22 | uv 23 | cacert 24 | espeak-ng # Required for Kokoro TTS 25 | 26 | # CUDA packages 27 | cudaPackages.cudatoolkit 28 | cudaPackages.cuda_nvcc 29 | cudaPackages.cuda_cudart 30 | cudaPackages.cuda_cccl 31 | cudaPackages.libcublas 32 | cudaPackages.cudnn 33 | 34 | # System libraries (ComfyUI & others) 35 | stdenv.cc.cc.lib 36 | zlib 37 | libGL 38 | glib 39 | openssl 40 | glibc.bin # Provides ldconfig for Triton 41 | 42 | # Optional dependencies 43 | curl 44 | openblas 45 | ccache 46 | fd 47 | ripgrep 48 | ]; 49 | 50 | shellHook = '' 51 | # CUDA Environment 52 | export CUDA_PATH="${pkgs.cudaPackages.cudatoolkit}" 53 | export CUDA_HOME="${pkgs.cudaPackages.cudatoolkit}" 54 | export CUDA_TOOLKIT_ROOT_DIR="${pkgs.cudaPackages.cudatoolkit}" 55 | export CMAKE_CUDA_COMPILER="${pkgs.cudaPackages.cuda_nvcc}/bin/nvcc" 56 | 57 | # Setup library paths 58 | export LD_LIBRARY_PATH=${ 59 | pkgs.lib.makeLibraryPath [ 60 | "/run/opengl-driver" 61 | pkgs.stdenv.cc.cc.lib 62 | pkgs.zlib 63 | pkgs.cudaPackages.cudatoolkit 64 | pkgs.cudaPackages.cuda_cudart 65 | pkgs.cudaPackages.cudnn 66 | pkgs.libGL 67 | pkgs.glib.out 68 | pkgs.xorg.libX11 69 | pkgs.xorg.libXext 70 | pkgs.xorg.libXrender 71 | pkgs.xorg.libICE 72 | pkgs.xorg.libSM 73 | pkgs.openssl 74 | ] 75 | }:$LD_LIBRARY_PATH 76 | 77 | export LIBRARY_PATH=${ 78 | pkgs.lib.makeLibraryPath [ 79 | pkgs.cudaPackages.cudatoolkit 80 | pkgs.cudaPackages.cuda_cudart 81 | pkgs.zlib 82 | pkgs.openssl 83 | ] 84 | }:/run/opengl-driver/lib:$LIBRARY_PATH 85 | 86 | export PATH="${pkgs.cudaPackages.cuda_nvcc}/bin:${pkgs.glibc.bin}/bin:$PATH" 87 | 88 | # SSL Certificate for Python/uv 89 | export SSL_CERT_FILE="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" 90 | 91 | # Tell Triton where to find libcuda.so (avoids calling /sbin/ldconfig on NixOS) 92 | export TRITON_LIBCUDA_PATH=/run/opengl-driver/lib 93 | 94 | echo "Environment loaded. Use 'just' to run commands." 95 | echo "TRITON_LIBCUDA_PATH: $TRITON_LIBCUDA_PATH" 96 | ''; 97 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AI Lab Workspace 2 | 3 | This repository serves as a **meta-workspace** for managing, building, and running various local AI tools and inference engines. It centralizes dependencies using [Nix](https://nixos.org/) and orchestrates tasks using [Just](https://github.com/casey/just). 4 | 5 | The goal is to provide a reproducible, one-click setup for compiling high-performance inference backends (like `llama.cpp`) and running services (like TTS and ASR) without managing individual environments for each submodule. 6 | 7 | ## 📂 Included Projects 8 | 9 | All external projects are managed as git submodules in the `external/` directory: 10 | 11 | * **[llama.cpp](https://github.com/ggerganov/llama.cpp):** Inference of LLaMA model in pure C/C++. 12 | * **[ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp):** A fork of llama.cpp with optimizations. 13 | * **[Ollama](https://github.com/ollama/ollama):** Get up and running with large language models. 14 | * **[Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI):** A Dockerized/FastAPI wrapper for the Kokoro TTS model. 15 | * **[agent-cli](https://github.com/basnijholt/agent-cli):** CLI agent tool (used here for its `faster-whisper` server script). 16 | 17 | ## 🛠️ Prerequisites 18 | 19 | * **[Nix](https://nixos.org/download.html):** Required for the environment. 20 | * **[Direnv](https://direnv.net/)** (Recommended): Automatically loads the Nix environment when you enter the directory. 21 | * **Git:** To manage the repository and submodules. 22 | 23 | ## 🚀 Getting Started 24 | 25 | 1. **Clone the repository:** 26 | ```bash 27 | git clone --recursive git@github.com:basnijholt/ai.git 28 | cd ai 29 | ``` 30 | 31 | 2. **Enter the environment:** 32 | If you have `direnv` installed: 33 | ```bash 34 | direnv allow 35 | ``` 36 | Otherwise, drop into the Nix shell manually: 37 | ```bash 38 | nix-shell 39 | ``` 40 | *This provides `cmake`, `gcc`, `go`, `cuda`, `python`, `uv`, and `just` configured specifically for these projects.* 41 | 42 | 3. **Build everything:** 43 | ```bash 44 | just build 45 | ``` 46 | 47 | ## 🤖 Commands 48 | 49 | The `justfile` defines all available commands. 50 | 51 | ### Global Operations 52 | | Command | Alias | Description | 53 | | :--- | :--- | :--- | 54 | | `just build` | `just b` | Compiles `llama.cpp`, `ik_llama.cpp`, and `ollama` from scratch. | 55 | | `just rebuild` | `just r` | Incrementally recompiles all projects. | 56 | | `just sync` | `just s` | Pulls the latest changes for **all** submodules from their upstream remotes. | 57 | | `just commit-submodules` | `just cs` | Commits submodule updates with an auto-generated message (only updated modules). | 58 | | `just clean` | `just c` | Removes build artifacts for all projects. | 59 | 60 | ### Running Services 61 | | Command | Description | 62 | | :--- | :--- | 63 | | `just start-kokoro` | Starts the **Kokoro TTS** server (GPU accelerated).
*Automatically handles python venv and model downloads.* | 64 | | `just start-faster-whisper` | Starts the **Faster Whisper** ASR server on port 8811 (CUDA, float16). | 65 | 66 | ### Individual Project Commands 67 | You can also target specific projects: 68 | 69 | * **llama.cpp:** `build-llama`, `rebuild-llama`, `clean-llama`, `sync-llama` 70 | * **ik_llama.cpp:** `build-ik`, `rebuild-ik`, `clean-ik`, `sync-ik` 71 | * **Ollama:** `build-ollama`, `rebuild-ollama`, `clean-ollama`, `sync-ollama` 72 | * **Kokoro:** `sync-kokoro` 73 | * **Agent CLI:** `sync-agent-cli` 74 | 75 | ## ⚙️ Configuration 76 | 77 | > [!NOTE] 78 | > This setup is specifically tailored for a machine with **NVIDIA CUDA-compatible hardware**. 79 | 80 | * **Build Flags:** Configured in `justfile`. These include flags for CUDA support and hardware-specific architectures (e.g., targeting NVIDIA GPUs). 81 | * **Environment:** Defined in `shell.nix`. It ensures `LD_LIBRARY_PATH` includes necessary CUDA and C++ libraries for Python extensions. 82 | 83 | ## 🖥️ System Configuration 84 | 85 | My complete NixOS configuration, which powers this setup, can be found in my [dotfiles](https://github.com/basnijholt/dotfiles). 86 | 87 | ## 📝 License 88 | 89 | This meta-repository is for personal organization. Each submodule retains its own license. 90 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | # Configuration 2 | cmake_flags := '-DGGML_CUDA=ON -DGGML_BLAS=ON -DGGML_NATIVE=ON -DCMAKE_CUDA_ARCHITECTURES="86"' 3 | build_release := "cmake --build build --config Release -j 24" 4 | 5 | # Aliases 6 | alias b := build 7 | alias r := rebuild 8 | alias s := sync 9 | alias c := clean 10 | alias cs := commit-submodules 11 | 12 | default: 13 | @just --list 14 | 15 | # ========================================== 16 | # Aggregate Commands 17 | # ========================================== 18 | 19 | # Build all projects from scratch 20 | build: build-llama build-ik build-ollama install-bin 21 | 22 | # Rebuild all projects incrementally 23 | rebuild: rebuild-llama rebuild-ik rebuild-ollama install-bin 24 | 25 | # Update all repositories (git pull) 26 | sync: sync-llama sync-ik sync-ollama sync-kokoro sync-agent-cli sync-comfyui 27 | 28 | # Clean all build artifacts 29 | clean: clean-llama clean-ik clean-ollama 30 | rm -rf bin/ 31 | 32 | # Commit submodule updates after sync 33 | commit-submodules: 34 | #!/usr/bin/env bash 35 | set -euo pipefail 36 | git add external/ 37 | if git diff --cached --quiet -- external/; then 38 | echo "No submodule changes to commit" 39 | exit 0 40 | fi 41 | modules=$(git diff --cached --name-only -- external/ | xargs -n1 basename | paste -sd, -) 42 | git commit -m "chore: update submodules - $modules" 43 | 44 | # ========================================== 45 | # Helpers 46 | # ========================================== 47 | 48 | install-bin: 49 | #!/usr/bin/env bash 50 | set -euo pipefail 51 | mkdir -p bin 52 | echo "Linking binaries to bin/..." 53 | 54 | # Clean up old binaries/libraries 55 | rm -rf bin/* 56 | 57 | # Common exclusions for libraries 58 | EXCLUDE="-E *.so -E *.so.* -E *.dylib -E *.dll -E *.a" 59 | 60 | # Link llama.cpp binaries (standard names) 61 | fd --type x --absolute-path $EXCLUDE . external/llama.cpp/build/bin --exec ln -sf {} bin/ 2>/dev/null || true 62 | 63 | # Link ik_llama.cpp binaries (prefixed with ik-) 64 | fd --type x --absolute-path $EXCLUDE . external/ik_llama.cpp/build/bin 2>/dev/null | while read -r file; do 65 | ln -sf "$file" "bin/ik-$(basename "$file")" 66 | done || true 67 | 68 | # Link Ollama binary 69 | fd --type x --absolute-path --max-depth 1 ollama external/ollama --exec ln -sf {} bin/ 2>/dev/null || true 70 | 71 | 72 | # ========================================== 73 | # Agent CLI 74 | # ========================================== 75 | 76 | sync-agent-cli: 77 | cd external/agent-cli && git checkout main && git pull origin main 78 | 79 | # ========================================== 80 | # Kokoro TTS 81 | # ========================================== 82 | 83 | # Start the Kokoro FastAPI server (GPU) 84 | start-kokoro: 85 | nix-shell --run ./scripts/start-kokoro.sh 86 | 87 | sync-kokoro: 88 | cd external/Kokoro-FastAPI && git checkout master && git pull origin master 89 | 90 | # ========================================== 91 | # Faster Whisper 92 | # ========================================== 93 | 94 | # Start the faster-whisper server (GPU) 95 | start-faster-whisper: 96 | nix-shell --run "uv run --script external/agent-cli/scripts/run_faster_whisper_server.py --device cuda --compute-type float16" 97 | 98 | # ========================================== 99 | # llama.cpp 100 | # ========================================== 101 | 102 | build-llama: 103 | cd external/llama.cpp && cmake -B build {{cmake_flags}} && {{build_release}} 104 | 105 | rebuild-llama: 106 | cd external/llama.cpp && {{build_release}} 107 | 108 | clean-llama: 109 | rm -rf external/llama.cpp/build 110 | 111 | sync-llama: 112 | cd external/llama.cpp && git checkout master && git pull origin master 113 | 114 | # ========================================== 115 | # ik_llama.cpp 116 | # ========================================== 117 | 118 | build-ik: 119 | cd external/ik_llama.cpp && cmake -B build {{cmake_flags}} && {{build_release}} 120 | 121 | rebuild-ik: 122 | cd external/ik_llama.cpp && {{build_release}} 123 | 124 | clean-ik: 125 | rm -rf external/ik_llama.cpp/build 126 | 127 | sync-ik: 128 | cd external/ik_llama.cpp && git checkout main && git pull origin main 129 | 130 | # ========================================== 131 | # Ollama 132 | # ========================================== 133 | 134 | build-ollama: 135 | cd external/ollama && cmake -B build -DGGML_BLAS=ON -DGGML_NATIVE=ON -DCMAKE_CUDA_ARCHITECTURES="86" -DGGML_BLAS_VENDOR=OpenBLAS && {{build_release}} && go build . 136 | 137 | rebuild-ollama: 138 | cd external/ollama && {{build_release}} && go build . 139 | 140 | clean-ollama: 141 | rm -rf external/ollama/build external/ollama/ollama 142 | 143 | sync-ollama: 144 | cd external/ollama && git checkout main && git pull origin main 145 | 146 | # ========================================== 147 | # ComfyUI 148 | # ========================================== 149 | 150 | # Install ComfyUI environment and dependencies 151 | install-comfyui: 152 | #!/usr/bin/env bash 153 | set -e 154 | echo "Installing ComfyUI dependencies..." 155 | cd external/ComfyUI 156 | 157 | # Ensure Manager is present before installing requirements 158 | if [ ! -d custom_nodes/comfyui-manager ]; then 159 | echo "Cloning ComfyUI Manager..." 160 | git clone https://github.com/ltdrdata/ComfyUI-Manager.git custom_nodes/comfyui-manager 161 | fi 162 | 163 | if [ ! -d .venv-comfyui ]; then uv venv .venv-comfyui -p 3.12; fi 164 | source .venv-comfyui/bin/activate 165 | uv pip install pip # Ensure pip is installed for nodes that call it via subprocess 166 | uv pip install huggingface_hub # Required for authenticated downloads 167 | uv pip install opencv-python-headless # Avoid X11 dependencies and runtime reinstalls 168 | uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu128 169 | uv pip install --no-build-isolation sageattention # Used by ComfyUI-WanVideoWrapper 170 | uv pip install -r requirements.txt 171 | uv pip install -r custom_nodes/comfyui-manager/requirements.txt 172 | echo "ComfyUI installation complete in .venv-comfyui." 173 | 174 | # Login to Hugging Face (for downloading restricted models) 175 | login-hf: 176 | @echo "Please paste your Hugging Face token when prompted." 177 | @source external/ComfyUI/.venv-comfyui/bin/activate && huggingface-cli login 178 | 179 | # Start ComfyUI server 180 | start-comfyui: 181 | @echo "Starting ComfyUI..." 182 | cd external/ComfyUI && source .venv-comfyui/bin/activate && python main.py --listen 183 | 184 | # Update ComfyUI and Manager 185 | sync-comfyui: 186 | #!/usr/bin/env bash 187 | set -e 188 | cd external/ComfyUI 189 | git checkout master && git pull origin master 190 | if [ -d custom_nodes/comfyui-manager ]; then 191 | echo "Updating ComfyUI Manager..." 192 | cd custom_nodes/comfyui-manager && git checkout main && git pull origin main 193 | else 194 | echo "Cloning ComfyUI Manager..." 195 | git clone https://github.com/ltdrdata/ComfyUI-Manager.git custom_nodes/comfyui-manager 196 | fi 197 | --------------------------------------------------------------------------------