├── .env.cloudflared.example ├── .env.open-webui.example ├── .gitignore ├── Makefile ├── README.md ├── data └── openedai-speech │ └── config │ ├── pre_process_map.default.yaml │ └── voice_to_speaker.default.yaml ├── docker-compose.a1111-test.yaml ├── docker-compose.amdgpu.yaml ├── docker-compose.api.yaml ├── docker-compose.gpu.yaml ├── docker-compose.ollamadata.yaml ├── docker-compose.webuidata.yaml ├── docker-compose.yaml ├── initial-setup-configs ├── .env.open-webui └── openedai-speech │ └── config │ ├── pre_process_map.default.yaml │ └── voice_to_speaker.default.yaml └── run-compose.sh /.env.cloudflared.example: -------------------------------------------------------------------------------- 1 | TUNNEL_TOKEN=YourTunnelToken -------------------------------------------------------------------------------- /.env.open-webui.example: -------------------------------------------------------------------------------- 1 | OLLAMA_BASE_URL=http://ollama:11435 2 | WEBUI_SECRET_KEY= 3 | 4 | 5 | # Optional: Configure OpenAI API for models or to offload TTS 6 | # How to get an api key: https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key 7 | OPENAI_API_KEY=sk-proj-... 8 | 9 | ########################################### 10 | ### OpenAI Text-To-Speech Configuration ### 11 | ########################################### 12 | AUDIO_TTS_ENGINE=openai 13 | AUDIO_TTS_VOICE=fable 14 | AUDIO_TTS_MODEL=tts-1-hd # Much nicer audio quality 15 | 16 | ################################################ 17 | ### Local Model Text-To-Speech Configuration ### 18 | ################################################ 19 | # Audio Text-To-Speech Configuration 20 | # ToDo: Make some notes on how you could reconfigure this to point to OpenAI TTS 21 | # AUDIO_TTS_ENGINE=openai 22 | # AUDIO_TTS_OPENAI_API_BASE_URL=http://openedai-speech:8000/v1 23 | # AUDIO_TTS_OPENAI_API_KEY=sk-111111111 # Since we are using local TTS, set key to a fake value 24 | # AUDIO_TTS_MODEL=tts-1-hd # Much nicer audio quality 25 | # # AUDIO_TTS_MODEL=tts-1 # Faster, lower vRAM usage, much lower quality 26 | # AUDIO_TTS_VOICE=alloy -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | !/data/openedai-speech/config/ 3 | !/data/openedai-speech/config/* 4 | data/open-webui/ 5 | data/openedai-speech/voices/ 6 | data/ollama/ 7 | data/openedai-speech/config/* 8 | data/Extra\ Space/ 9 | data/ 10 | 11 | # Need this default config file for openedai-speech to work, it automatically gets copied to the right place on first container spin up 12 | !data/openedai-speech/config/*.default.yaml 13 | 14 | # Gotta make sure we don't commit any .env files with secrets 15 | /.env 16 | /.env.* 17 | 18 | # Inclide /.env.*.example files 19 | !/.env.*.example 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | ifneq ($(shell which docker-compose 2>/dev/null),) 3 | DOCKER_COMPOSE := docker-compose 4 | else 5 | DOCKER_COMPOSE := docker compose 6 | endif 7 | 8 | setup: 9 | @if [ ! -f .env.open-webui ]; then \ 10 | echo "no .env.open-webui file found, copying default .env.open-webui file"; \ 11 | cp initial-setup-configs/.env.open-webui .; \ 12 | fi 13 | @if [ ! -d data/initial-setup-configs ]; then \ 14 | echo "no data folder found, copying default data folder..."; \ 15 | cp -r initial-setup-configs/openedai-speech data/; \ 16 | fi 17 | 18 | install: 19 | $(DOCKER_COMPOSE) up -d 20 | 21 | remove: 22 | @chmod +x confirm_remove.sh 23 | @./confirm_remove.sh 24 | 25 | start run: setup 26 | ./run-compose.sh --webui[port=3030] --enable-gpu --enable-api --ollama-data[folder=./data/ollama] --open-webui-data[folder=./data/open-webui] 27 | 28 | down: 29 | @echo "Shutting down..." 30 | docker compose down --remove-orphans --volumes 31 | 32 | logs: 33 | $(DOCKER_COMPOSE) logs -f 34 | 35 | startAndBuild: 36 | $(DOCKER_COMPOSE) up -d --build 37 | 38 | stop: 39 | $(DOCKER_COMPOSE) stop 40 | 41 | clean: down 42 | $(DOCKER_COMPOSE) down --remove-orphans --volumes 43 | sudo rm -rf data/* 44 | 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Open WebUI Nvidia CUDA Setup 2 | 3 | This repo is a ready to set of infra for locally running Open WebUI and Ollama in docker along with other supporting software such as image generation and Text-To-Speech integrated out of the box and ready to go. 4 | 5 | The official Open WebUI repo has some good scripts and IaC but their need to support a broader audience means less of the bells and whistles are running out of the box. 6 | 7 | ## Configured out of the Box 8 | 9 | Services configured to run out of the box 10 | 11 | ### Implemented 12 | 13 | - [x] Open WebUI with local data persistence in `./data/open-webui` 14 | - [X] Ollama Container utilizing GPU with local data persistence in `./data/ollama` 15 | - [X] Local TTS with [Openedai-speech](https://github.com/matatonic/openedai-speech) configure out of the box 16 | - [X] Support CloudFlare Tunnel into OpenWeb UI 17 | 18 | ### ToDo 19 | 20 | - [ ] Basic GitHub actions workflow to test initial setup 21 | - [ ] Make CloudFlare tunnel optional 22 | - [ ] Local network HTTPS provided by a reverse proxy container + Certbot 23 | - [ ] Data backup functionality 24 | - [ ] Image generation with Stable Diffusion running locally 25 | - [ ] ... What else? 26 | 27 | ## Who is this for? 28 | 29 | - OS: Linux 30 | - GPU: Nvidia with CUDA enabled 31 | 32 | ## Spin it Up 33 | 34 | Setup initial configs 35 | ```bash 36 | make setup 37 | ``` 38 | 39 | The defaults are enough, but open up [.env.open-webui](./.env.open-webui) and see if you want to reconfigure anything 40 | 41 | Start up the docker compose stack 42 | 43 | ```bash 44 | make run 45 | ``` 46 | 47 | *Want to change how the stack spins up? Edit the Makefile or run ./run-compose.sh with your own flags.* 48 | 49 | 50 | View the logs 51 | 52 | ```bash 53 | make logs 54 | ``` 55 | 56 | Delete all the data for OpenWeb UI and Ollama 57 | ```bash 58 | make clean 59 | ``` 60 | 61 | ## Customizing Text-To-Speech Voice 62 | 63 | To select other text-to-speech voice or train your own read more at: 64 | 65 | ## Setup CloudFlare tunnel 66 | 67 | 1. Go setup your free tier tunnel 68 | 2. Get a CloudFlare tunnel token: 69 | 3. Copy the `.env.cloudflared.example` to create a `.env.cloudflared` and put your new token into it 70 | ``` 71 | cp .env.cloudflared.example .env.cloudflared 72 | ``` 73 | 74 | ## Contribute 75 | Contributions are welcome! If you have any ideas, suggestions, or improvements, feel free to open an issue or create a fork and submit a pull request against the [ GitHub repo](https://github.com/chrishart0/open-webui-nvidia-cuda-setup). We appreciate your contributions to make this project even better! 76 | 77 | ## Credit 78 | 79 | Most credit for all the local infra must go to the Open WebUI team, this repo is just a fork of their local infra with the configuration done for my specific needs (Ubuntu / 4090). 80 | -------------------------------------------------------------------------------- /data/openedai-speech/config/pre_process_map.default.yaml: -------------------------------------------------------------------------------- 1 | # regex pairs to clean the text before speaking 2 | - - ([^.])\.$ 3 | - \1 4 | - - '&' 5 | - '&' 6 | - - '<' 7 | - < 8 | - - '>' 9 | - '>' 10 | - - '"' 11 | - '"' 12 | - - ''' 13 | - '''' 14 | - - '©' 15 | - '©' 16 | - - '®' 17 | - '®' 18 | - - ' ' 19 | - ' ' 20 | - - '"' 21 | - '' 22 | - - ' biases ' 23 | - ' bias''s ' 24 | - - ex\. 25 | - for example 26 | - - e\.g\. 27 | - for example 28 | - - ' ESG ' 29 | - ' E.S.G. ' 30 | - - ' FY ' 31 | - ' F.Y. ' 32 | - - ([0-9]+)-([0-9]+) 33 | - \1 to \2 34 | - - '\*\*\*' 35 | - '*' 36 | - - '\*\*' 37 | - '*' 38 | -------------------------------------------------------------------------------- /data/openedai-speech/config/voice_to_speaker.default.yaml: -------------------------------------------------------------------------------- 1 | tts-1: 2 | some_other_voice_name_you_want: 3 | model: voices/choose your own model.onnx 4 | speaker: set your own speaker 5 | alloy: 6 | model: voices/en_US-libritts_r-medium.onnx 7 | speaker: 79 # 64, 79, 80, 101, 130 8 | echo: 9 | model: voices/en_US-libritts_r-medium.onnx 10 | speaker: 134 # 52, 102, 134 11 | echo-alt: 12 | model: voices/en_US-ryan-high.onnx 13 | speaker: # default speaker 14 | fable: 15 | model: voices/en_GB-northern_english_male-medium.onnx 16 | speaker: # default speaker 17 | onyx: 18 | model: voices/en_US-libritts_r-medium.onnx 19 | speaker: 159 # 55, 90, 132, 136, 137, 159 20 | nova: 21 | model: voices/en_US-libritts_r-medium.onnx 22 | speaker: 107 # 57, 61, 107, 150, 162 23 | shimmer: 24 | model: voices/en_US-libritts_r-medium.onnx 25 | speaker: 163 26 | tts-1-hd: 27 | alloy-alt: 28 | model: xtts 29 | speaker: voices/alloy-alt.wav 30 | alloy: 31 | model: xtts 32 | speaker: voices/alloy.wav 33 | echo: 34 | model: xtts 35 | speaker: voices/echo.wav 36 | fable: 37 | model: xtts 38 | speaker: voices/fable.wav 39 | onyx: 40 | model: xtts 41 | speaker: voices/onyx.wav 42 | nova: 43 | model: xtts 44 | speaker: voices/nova.wav 45 | shimmer: 46 | model: xtts 47 | speaker: voices/shimmer.wav 48 | me: 49 | model: xtts_v2.0.2 # you can specify different xtts version 50 | speaker: voices/me.wav # this could be you 51 | parler: 52 | model: parler-tts/parler_tts_mini_v0.1 53 | speaker: A female speaker with a slightly low-pitched voice delivers her words quite expressively, in a very confined sounding environment with clear audio quality. She speaks very fast. 54 | parler2: 55 | model: parler-tts/parler_tts_mini_v0.1 56 | speaker: A female voice with an Indian accent enunciates every word with precision. The speaker's voice is very close-sounding, and the recording is excellent, capturing her voice with crisp clarity. 57 | -------------------------------------------------------------------------------- /docker-compose.a1111-test.yaml: -------------------------------------------------------------------------------- 1 | # This is an overlay that spins up stable-diffusion-webui for integration testing 2 | # This is not designed to be used in production 3 | services: 4 | stable-diffusion-webui: 5 | # Not built for ARM64 6 | platform: linux/amd64 7 | image: ghcr.io/neggles/sd-webui-docker:latest 8 | restart: unless-stopped 9 | environment: 10 | CLI_ARGS: "--api --use-cpu all --precision full --no-half --skip-torch-cuda-test --ckpt /empty.pt --do-not-download-clip --disable-nan-check --disable-opt-split-attention" 11 | PYTHONUNBUFFERED: "1" 12 | TERM: "vt100" 13 | SD_WEBUI_VARIANT: "default" 14 | # Hack to get container working on Apple Silicon 15 | # Rosetta creates a conflict ${HOME}/.cache folder 16 | entrypoint: /bin/bash 17 | command: 18 | - -c 19 | - | 20 | export HOME=/root-home 21 | rm -rf $${HOME}/.cache 22 | /docker/entrypoint.sh python -u webui.py --listen --port $${WEBUI_PORT} --skip-version-check $${CLI_ARGS} 23 | volumes: 24 | - ./test/test_files/image_gen/sd-empty.pt:/empty.pt 25 | 26 | open-webui: 27 | environment: 28 | ENABLE_IMAGE_GENERATION: "true" 29 | AUTOMATIC1111_BASE_URL: http://stable-diffusion-webui:7860 30 | IMAGE_SIZE: "64x64" 31 | IMAGE_STEPS: "3" 32 | -------------------------------------------------------------------------------- /docker-compose.amdgpu.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | ollama: 3 | devices: 4 | - /dev/kfd:/dev/kfd 5 | - /dev/dri:/dev/dri 6 | image: ollama/ollama:${OLLAMA_DOCKER_TAG-rocm} 7 | environment: 8 | - 'HSA_OVERRIDE_GFX_VERSION=${HSA_OVERRIDE_GFX_VERSION-11.0.0}' -------------------------------------------------------------------------------- /docker-compose.api.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | ollama: 3 | # Expose Ollama API outside the container stack 4 | ports: 5 | - ${OLLAMA_WEBAPI_PORT-11434}:11434 6 | -------------------------------------------------------------------------------- /docker-compose.gpu.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | ollama: 3 | # GPU support 4 | deploy: 5 | resources: 6 | reservations: 7 | devices: 8 | - driver: ${OLLAMA_GPU_DRIVER-nvidia} 9 | count: ${OLLAMA_GPU_COUNT-1} 10 | capabilities: 11 | - gpu 12 | openedai-speech: 13 | # GPU support 14 | deploy: 15 | resources: 16 | reservations: 17 | devices: 18 | - driver: ${OLLAMA_GPU_DRIVER-nvidia} 19 | count: ${OLLAMA_GPU_COUNT-1} 20 | capabilities: 21 | - gpu 22 | 23 | -------------------------------------------------------------------------------- /docker-compose.ollamadata.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | ollama: 3 | volumes: 4 | - ${OLLAMA_DATA_DIR-./ollama-data}:/root/.ollama 5 | -------------------------------------------------------------------------------- /docker-compose.webuidata.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | open-webui: 3 | volumes: 4 | - ${OPEN_WEBUI_DATA_DIR-./webui-data}:/app/backend/data 5 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | # https://hub.docker.com/r/ollama/ollama 3 | ollama: 4 | volumes: 5 | - ollama:/root/.ollama 6 | container_name: ollama 7 | pull_policy: always 8 | tty: true 9 | restart: unless-stopped 10 | image: ollama/ollama:0.5.7 #${OLLAMA_DOCKER_TAG-latest} 11 | 12 | # Latest releases found here: https://github.com/open-webui/open-webui/releases 13 | open-webui: 14 | image: ghcr.io/open-webui/open-webui:v0.5.4 #${WEBUI_DOCKER_TAG-main} 15 | container_name: open-webui 16 | volumes: 17 | - open-webui:/app/backend/data 18 | depends_on: 19 | - ollama 20 | - openedai-speech 21 | ports: 22 | - ${OPEN_WEBUI_PORT-3000}:8080 23 | env_file: .env.open-webui 24 | healthcheck: 25 | test: ["CMD", "curl", "-f", "http://localhost:8080/health"] 26 | interval: 30s 27 | timeout: 10s 28 | retries: 3 29 | extra_hosts: 30 | - host.docker.internal:host-gateway 31 | restart: unless-stopped 32 | 33 | # Latest version found here: https://github.com/matatonic/openedai-speech/releases/tag/0.18.2 34 | openedai-speech: 35 | image: ghcr.io/matatonic/openedai-speech:0.18.2 36 | container_name: openedai-speech 37 | # env_file: .env 38 | environment: 39 | - 'TTS_HOME=voices' 40 | - 'HF_HOME=voices' 41 | - 'PRELOAD_MODEL=xtts' 42 | ports: # No need to expose ports, this is a backend service 43 | - "8087:8000" 44 | volumes: 45 | - ./data/openedai-speech/voices:/app/voices 46 | - ./data/openedai-speech/config:/app/config 47 | # labels: 48 | # - "com.centurylinklabs.watchtower.enable=true" 49 | restart: unless-stopped 50 | 51 | # https://hub.docker.com/r/cloudflare/cloudflared 52 | cloudflared-tunnel: 53 | container_name: cloudflared-tunnel 54 | image: cloudflare/cloudflared:2025.1.0 55 | restart: unless-stopped 56 | command: tunnel run 57 | env_file: .env.cloudflared 58 | 59 | volumes: 60 | ollama: {} 61 | open-webui: {} 62 | # tts-voices: {} 63 | # tts-config: {} 64 | -------------------------------------------------------------------------------- /initial-setup-configs/.env.open-webui: -------------------------------------------------------------------------------- 1 | OLLAMA_BASE_URL=http://ollama:11435 2 | WEBUI_SECRET_KEY= 3 | 4 | # Optional: Configure OpenAI API for models or to offload TTS 5 | # How to get an api key: https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key 6 | #OPENAI_API_KEY=sk-... 7 | 8 | 9 | # Audio Text-To-Speech Configuration 10 | # ToDo: Make some notes on how you could reconfigure this to point to OpenAI TTS 11 | AUDIO_TTS_ENGINE=openai 12 | AUDIO_TTS_OPENAI_API_BASE_URL=http://openedai-speech:8000/v1 13 | AUDIO_TTS_OPENAI_API_KEY=sk-111111111 # Since we are using local TTS, set key to a fake value 14 | AUDIO_TTS_MODEL=tts-1-hd # Much nicer audio quality 15 | # AUDIO_TTS_MODEL=tts-1 # Faster, lower vRAM usage, much lower quality 16 | AUDIO_TTS_VOICE=alloy -------------------------------------------------------------------------------- /initial-setup-configs/openedai-speech/config/pre_process_map.default.yaml: -------------------------------------------------------------------------------- 1 | # regex pairs to clean the text before speaking 2 | - - ([^.])\.$ 3 | - \1 4 | - - '&' 5 | - '&' 6 | - - '<' 7 | - < 8 | - - '>' 9 | - '>' 10 | - - '"' 11 | - '"' 12 | - - ''' 13 | - '''' 14 | - - '©' 15 | - '©' 16 | - - '®' 17 | - '®' 18 | - - ' ' 19 | - ' ' 20 | - - '"' 21 | - '' 22 | - - ' biases ' 23 | - ' bias''s ' 24 | - - ex\. 25 | - for example 26 | - - e\.g\. 27 | - for example 28 | - - ' ESG ' 29 | - ' E.S.G. ' 30 | - - ' FY ' 31 | - ' F.Y. ' 32 | - - ([0-9]+)-([0-9]+) 33 | - \1 to \2 34 | - - '\*\*\*' 35 | - '*' 36 | - - '\*\*' 37 | - '*' 38 | -------------------------------------------------------------------------------- /initial-setup-configs/openedai-speech/config/voice_to_speaker.default.yaml: -------------------------------------------------------------------------------- 1 | tts-1: 2 | some_other_voice_name_you_want: 3 | model: voices/choose your own model.onnx 4 | speaker: set your own speaker 5 | alloy: 6 | model: voices/en_US-libritts_r-medium.onnx 7 | speaker: 79 # 64, 79, 80, 101, 130 8 | echo: 9 | model: voices/en_US-libritts_r-medium.onnx 10 | speaker: 134 # 52, 102, 134 11 | echo-alt: 12 | model: voices/en_US-ryan-high.onnx 13 | speaker: # default speaker 14 | fable: 15 | model: voices/en_GB-northern_english_male-medium.onnx 16 | speaker: # default speaker 17 | onyx: 18 | model: voices/en_US-libritts_r-medium.onnx 19 | speaker: 159 # 55, 90, 132, 136, 137, 159 20 | nova: 21 | model: voices/en_US-libritts_r-medium.onnx 22 | speaker: 107 # 57, 61, 107, 150, 162 23 | shimmer: 24 | model: voices/en_US-libritts_r-medium.onnx 25 | speaker: 163 26 | tts-1-hd: 27 | alloy-alt: 28 | model: xtts 29 | speaker: voices/alloy-alt.wav 30 | alloy: 31 | model: xtts 32 | speaker: voices/alloy.wav 33 | echo: 34 | model: xtts 35 | speaker: voices/echo.wav 36 | fable: 37 | model: xtts 38 | speaker: voices/fable.wav 39 | onyx: 40 | model: xtts 41 | speaker: voices/onyx.wav 42 | nova: 43 | model: xtts 44 | speaker: voices/nova.wav 45 | shimmer: 46 | model: xtts 47 | speaker: voices/shimmer.wav 48 | me: 49 | model: xtts_v2.0.2 # you can specify different xtts version 50 | speaker: voices/me.wav # this could be you 51 | parler: 52 | model: parler-tts/parler_tts_mini_v0.1 53 | speaker: A female speaker with a slightly low-pitched voice delivers her words quite expressively, in a very confined sounding environment with clear audio quality. She speaks very fast. 54 | parler2: 55 | model: parler-tts/parler_tts_mini_v0.1 56 | speaker: A female voice with an Indian accent enunciates every word with precision. The speaker's voice is very close-sounding, and the recording is excellent, capturing her voice with crisp clarity. 57 | -------------------------------------------------------------------------------- /run-compose.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Define color and formatting codes 4 | BOLD='\033[1m' 5 | GREEN='\033[1;32m' 6 | WHITE='\033[1;37m' 7 | RED='\033[0;31m' 8 | NC='\033[0m' # No Color 9 | # Unicode character for tick mark 10 | TICK='\u2713' 11 | 12 | # Detect GPU driver 13 | get_gpu_driver() { 14 | # Detect NVIDIA GPUs using lspci or nvidia-smi 15 | if lspci | grep -i nvidia >/dev/null || nvidia-smi >/dev/null 2>&1; then 16 | echo "nvidia" 17 | return 18 | fi 19 | 20 | # Detect AMD GPUs (including GCN architecture check for amdgpu vs radeon) 21 | if lspci | grep -i amd >/dev/null; then 22 | # List of known GCN and later architecture cards 23 | # This is a simplified list, and in a real-world scenario, you'd want a more comprehensive one 24 | local gcn_and_later=("Radeon HD 7000" "Radeon HD 8000" "Radeon R5" "Radeon R7" "Radeon R9" "Radeon RX") 25 | 26 | # Get GPU information 27 | local gpu_info=$(lspci | grep -i 'vga.*amd') 28 | 29 | for model in "${gcn_and_later[@]}"; do 30 | if echo "$gpu_info" | grep -iq "$model"; then 31 | echo "amdgpu" 32 | return 33 | fi 34 | done 35 | 36 | # Default to radeon if no GCN or later architecture is detected 37 | echo "radeon" 38 | return 39 | fi 40 | 41 | # Detect Intel GPUs 42 | if lspci | grep -i intel >/dev/null; then 43 | echo "i915" 44 | return 45 | fi 46 | 47 | # If no known GPU is detected 48 | echo "Unknown or unsupported GPU driver" 49 | exit 1 50 | } 51 | 52 | # Function for rolling animation 53 | show_loading() { 54 | local spin='-\|/' 55 | local i=0 56 | 57 | printf " " 58 | 59 | while kill -0 $1 2>/dev/null; do 60 | i=$(( (i+1) %4 )) 61 | printf "\b${spin:$i:1}" 62 | sleep .1 63 | done 64 | 65 | # Replace the spinner with a tick 66 | printf "\b${GREEN}${TICK}${NC}" 67 | } 68 | 69 | # Usage information 70 | usage() { 71 | echo "Usage: $0 [OPTIONS]" 72 | echo "Options:" 73 | echo " --enable-gpu[count=COUNT] Enable GPU support with the specified count." 74 | echo " --enable-api[port=PORT] Enable API and expose it on the specified port." 75 | echo " --webui[port=PORT] Set the port for the web user interface." 76 | echo " --ollama-data[folder=PATH] Bind mount for ollama data folder (by default will create the 'ollama' volume)." 77 | echo " --webui-data[folder=PATH] Bind mount for open-webui data folder (by default will create the 'open-webui' volume)." 78 | echo " --build Build the docker image before running the compose project." 79 | echo " --drop Drop the compose project." 80 | echo " -q, --quiet Run script in headless mode." 81 | echo " -h, --help Show this help message." 82 | echo "" 83 | echo "Examples:" 84 | echo " $0 --drop" 85 | echo " $0 --enable-gpu[count=1]" 86 | echo " $0 --enable-gpu[count=all]" 87 | echo " $0 --enable-api[port=11435]" 88 | echo " $0 --enable-gpu[count=1] --enable-api[port=12345] --webui[port=3000]" 89 | echo " $0 --enable-gpu[count=1] --enable-api[port=12345] --webui[port=3000] --ollama-data[folder=./ollama-data]" 90 | echo " $0 --enable-gpu[count=1] --enable-api[port=12345] --webui[port=3000] --ollama-data[folder=./ollama-data] --webui-data[folder=./webui-data] --build" 91 | echo "" 92 | echo "This script configures and runs a docker-compose setup with optional GPU support, API exposure, and web UI configuration." 93 | echo "About the gpu to use, the script automatically detects it using the "lspci" command." 94 | echo "In this case the gpu detected is: $(get_gpu_driver)" 95 | } 96 | 97 | # Default values 98 | gpu_count=1 99 | api_port=11435 100 | webui_port=3000 101 | headless=false 102 | build_image=false 103 | kill_compose=false 104 | 105 | # Function to extract value from the parameter 106 | extract_value() { 107 | echo "$1" | sed -E 's/.*\[.*=(.*)\].*/\1/; t; s/.*//' 108 | } 109 | 110 | # Parse arguments 111 | while [[ $# -gt 0 ]]; do 112 | key="$1" 113 | 114 | case $key in 115 | --enable-gpu*) 116 | enable_gpu=true 117 | value=$(extract_value "$key") 118 | gpu_count=${value:-1} 119 | ;; 120 | --enable-api*) 121 | enable_api=true 122 | value=$(extract_value "$key") 123 | api_port=${value:-11435} 124 | ;; 125 | --webui*) 126 | value=$(extract_value "$key") 127 | webui_port=${value:-3000} 128 | ;; 129 | --data*) # Maintain old --data flag for backward compatibility but this should be removed in the future 130 | value=$(extract_value "$key") 131 | ollama_data_dir=${value:-"./ollama-data"} 132 | echo "WARNING: --data flag is deprecated. Please use --ollama-data instead." 133 | ;; 134 | --ollama-data*) 135 | value=$(extract_value "$key") 136 | ollama_data_dir=${value:-"./ollama-data"} 137 | ;; 138 | --open-webui-data*) # Used open-webui-data instead of webui-data because of the conflict with the webui flag catching webui-data 139 | value=$(extract_value "$key") 140 | webui_data_dir=${value:-"./webui-data"} 141 | ;; 142 | --drop) 143 | kill_compose=true 144 | ;; 145 | --build) 146 | build_image=true 147 | ;; 148 | -q|--quiet) 149 | headless=true 150 | ;; 151 | -h|--help) 152 | usage 153 | exit 154 | ;; 155 | *) 156 | # Unknown option 157 | echo "Unknown option: $key" 158 | usage 159 | exit 1 160 | ;; 161 | esac 162 | shift # past argument or value 163 | done 164 | 165 | if [[ $kill_compose == true ]]; then 166 | docker compose down --remove-orphans 167 | echo -e "${GREEN}${BOLD}Compose project dropped successfully.${NC}" 168 | exit 169 | else 170 | DEFAULT_COMPOSE_COMMAND="docker compose -f docker-compose.yaml" 171 | if [[ $enable_gpu == true ]]; then 172 | # Validate and process command-line arguments 173 | if [[ -n $gpu_count ]]; then 174 | if ! [[ $gpu_count =~ ^([0-9]+|all)$ ]]; then 175 | echo "Invalid GPU count: $gpu_count" 176 | exit 1 177 | fi 178 | echo "Enabling GPU with $gpu_count GPUs" 179 | # Add your GPU allocation logic here 180 | export OLLAMA_GPU_DRIVER=$(get_gpu_driver) 181 | export OLLAMA_GPU_COUNT=$gpu_count # Set OLLAMA_GPU_COUNT environment variable 182 | fi 183 | DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.gpu.yaml" 184 | fi 185 | if [[ $enable_api == true ]]; then 186 | DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.api.yaml" 187 | if [[ -n $api_port ]]; then 188 | export OLLAMA_WEBAPI_PORT=$api_port # Set OLLAMA_WEBAPI_PORT environment variable 189 | fi 190 | fi 191 | if [[ -n $ollama_data_dir ]]; then 192 | mkdir -p $ollama_data_dir 193 | DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.ollamadata.yaml" 194 | export OLLAMA_DATA_DIR=$ollama_data_dir # Set OLLAMA_DATA_DIR environment variable 195 | fi 196 | if [[ -n $webui_data_dir ]]; then 197 | echo "LOGGING: $webui_data_dir" 198 | DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.webuidata.yaml" 199 | export OPEN_WEBUI_DATA_DIR=$webui_data_dir # Set OPEN_WEBUI_DATA_DIR environment variable 200 | fi 201 | if [[ -n $webui_port ]]; then 202 | export OPEN_WEBUI_PORT=$webui_port # Set OPEN_WEBUI_PORT environment variable 203 | fi 204 | DEFAULT_COMPOSE_COMMAND+=" up -d" 205 | DEFAULT_COMPOSE_COMMAND+=" --remove-orphans" 206 | DEFAULT_COMPOSE_COMMAND+=" --force-recreate" 207 | if [[ $build_image == true ]]; then 208 | DEFAULT_COMPOSE_COMMAND+=" --build" 209 | fi 210 | fi 211 | 212 | # Recap of environment variables 213 | echo 214 | echo -e "${WHITE}${BOLD}Current Setup:${NC}" 215 | echo -e " ${GREEN}${BOLD}GPU Driver:${NC} ${OLLAMA_GPU_DRIVER:-Not Enabled}" 216 | echo -e " ${GREEN}${BOLD}GPU Count:${NC} ${OLLAMA_GPU_COUNT:-Not Enabled}" 217 | echo -e " ${GREEN}${BOLD}WebAPI Port:${NC} ${OLLAMA_WEBAPI_PORT:-Not Enabled}" 218 | echo -e " ${GREEN}${BOLD}Ollama Data Folder:${NC} ${ollama_data_dir:-Using ollama volume}" 219 | echo -e " ${GREEN}${BOLD}WebUI Data Folder:${NC} ${webui_data_dir:-Using open-webui volume}" 220 | echo -e " ${GREEN}${BOLD}WebUI Port:${NC} $webui_port" 221 | echo 222 | 223 | if [[ $headless == true ]]; then 224 | echo -ne "${WHITE}${BOLD}Running in headless mode... ${NC}" 225 | choice="y" 226 | else 227 | # Ask for user acceptance 228 | echo -ne "${WHITE}${BOLD}Do you want to proceed with current setup? (Y/n): ${NC}" 229 | read -n1 -s choice 230 | fi 231 | 232 | echo 233 | 234 | if [[ $choice == "" || $choice == "y" ]]; then 235 | # Execute the command with the current user 236 | eval "$DEFAULT_COMPOSE_COMMAND" & 237 | 238 | # Capture the background process PID 239 | PID=$! 240 | 241 | # Display the loading animation 242 | #show_loading $PID 243 | 244 | # Wait for the command to finish 245 | wait $PID 246 | 247 | echo 248 | # Check exit status 249 | if [ $? -eq 0 ]; then 250 | echo -e "${GREEN}${BOLD}Compose project started successfully.${NC}" 251 | else 252 | echo -e "${RED}${BOLD}There was an error starting the compose project.${NC}" 253 | fi 254 | else 255 | echo "Aborted." 256 | fi 257 | 258 | echo 259 | --------------------------------------------------------------------------------