├── langchain ├── models │ └── .keep ├── .gitignore ├── langchainjs-localai-example │ ├── .gitignore │ ├── tsconfig.json │ ├── .vscode │ │ └── launch.json │ ├── package.json │ └── src │ │ └── index.mts ├── langchainpy-localai-example │ ├── .vscode │ │ ├── settings.json │ │ └── launch.json │ ├── simple_demo.py │ ├── requirements.txt │ └── full_demo.py ├── JS.Dockerfile ├── PY.Dockerfile ├── docker-compose.yaml └── README.md ├── query_data ├── data │ └── .keep ├── models │ └── .keep ├── .gitignore ├── docker-compose.yml ├── store.py ├── query.py ├── update.py └── README.md ├── slack-bot ├── models │ └── .keep ├── docker-compose.yaml ├── .env.example └── README.md ├── discord-bot ├── models │ └── .keep ├── .env.example ├── docker-compose.yaml └── README.md ├── langchain-chroma ├── models │ └── .keep ├── .gitignore ├── requirements.txt ├── .env.example ├── docker-compose.yml ├── store.py ├── query.py └── README.md ├── models ├── completion.tmpl ├── embeddings.yaml ├── .gitignore ├── gpt4all.tmpl └── gpt-3.5-turbo.yaml ├── langchain-huggingface ├── models │ └── .keep ├── docker-compose.yml └── README.md ├── streamlit-bot ├── .gitignore ├── requirements.txt ├── streamlit-bot.png ├── cmd_windows.bat ├── LICENSE ├── README.md ├── Main.py ├── start_windows.bat └── install_requirements.bat ├── configurations ├── mistral │ ├── completion.tmpl │ ├── chatml-block.tmpl │ ├── chatml.tmpl │ └── mistral.yaml ├── mixtral │ ├── mixtral │ ├── mixtral-chat │ └── mixtral.yaml ├── llava │ ├── chat-simple.tmpl │ └── llava.yaml ├── phi-2.yaml └── README.md ├── functions ├── requirements.txt ├── Dockerfile ├── docker-compose.yaml ├── README.md ├── .env.example └── functions-openai.py ├── continue ├── img │ └── screen.png ├── docker-compose.yml ├── README.md └── config.py ├── bruno └── LocalAI Test Requests │ ├── bruno.json │ ├── transcription │ ├── gb1.ogg │ └── transcribe.bru │ ├── environments │ ├── localhost.bru │ └── pinkMac.bru │ ├── get models list.bru │ ├── model gallery │ ├── model delete.bru │ ├── list model GALLERIES.bru │ ├── list MODELS in galleries.bru │ ├── delete model gallery.bru │ ├── model gallery apply -gist-.bru │ ├── model gallery apply.bru │ └── add model gallery.bru │ ├── backend monitor │ ├── backend monitor.bru │ └── backend-shutdown.bru │ ├── vad │ └── vad test too few.bru │ ├── tts │ ├── musicgen.bru │ └── -tts.bru │ ├── llm text │ ├── -edits.bru │ ├── -embeddings.bru │ ├── -completions.bru │ └── chat │ │ ├── chat-completions -stream-.bru │ │ ├── chat completion -simple- 1 message-.bru │ │ └── chat-completions -long-.bru │ ├── Sound Generation │ └── musicgen.bru │ └── image generation │ └── Generate image.bru ├── chainlit ├── requirements.txt ├── config.yaml ├── Dockerfile ├── README.md └── main.py ├── langchain-python ├── test.py ├── README.md ├── docker-compose.yaml └── agent.py ├── realtime ├── requirements.txt ├── docker-compose.yml ├── README.md ├── setup.sh ├── run.sh └── realtime.py ├── k8sgpt ├── broken-pod.yaml ├── README.md └── values.yaml ├── semantic-todo ├── README.md ├── go.mod ├── go.sum └── main.go ├── localai-webui ├── docker-compose.yml └── README.md ├── autoGPT ├── .env.example ├── README.md └── docker-compose.yaml ├── slack-qa-bot ├── README.md ├── docker-compose.yml ├── .env.example └── deployment.yaml ├── privateGPT └── README.md ├── flowise ├── README.md └── docker-compose.yaml ├── insomnia ├── README.md └── Insomnia_LocalAI.json ├── llamaindex ├── main.py └── README.md ├── e2e-fine-tuning ├── axolotl.yaml └── README.md ├── kubernetes ├── deployment.yaml ├── deployment-intel-arc.yaml └── deployment-nvidia.yaml ├── telegram-bot ├── README.md └── docker-compose.yml ├── github-actions └── workflow.yml ├── README.md └── LICENSE /langchain/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /query_data/data/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /query_data/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /slack-bot/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /discord-bot/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /langchain-chroma/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/completion.tmpl: -------------------------------------------------------------------------------- 1 | {{.Input}} -------------------------------------------------------------------------------- /query_data/.gitignore: -------------------------------------------------------------------------------- 1 | storage/ -------------------------------------------------------------------------------- /langchain-huggingface/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /streamlit-bot/.gitignore: -------------------------------------------------------------------------------- 1 | installer_files -------------------------------------------------------------------------------- /configurations/mistral/completion.tmpl: -------------------------------------------------------------------------------- 1 | {{.Input}} -------------------------------------------------------------------------------- /configurations/mixtral/mixtral: -------------------------------------------------------------------------------- 1 | [INST] {{.Input}} [/INST] 2 | -------------------------------------------------------------------------------- /streamlit-bot/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit==1.39.0 2 | requests -------------------------------------------------------------------------------- /configurations/mixtral/mixtral-chat: -------------------------------------------------------------------------------- 1 | [INST] {{.Input}} [/INST] 2 | -------------------------------------------------------------------------------- /functions/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.3.4 2 | openai==1.52.2 3 | -------------------------------------------------------------------------------- /langchain/.gitignore: -------------------------------------------------------------------------------- 1 | models/ggml-koala-13B-4bit-128g 2 | models/ggml-gpt4all-j -------------------------------------------------------------------------------- /langchain/langchainjs-localai-example/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | -------------------------------------------------------------------------------- /configurations/mistral/chatml-block.tmpl: -------------------------------------------------------------------------------- 1 | {{.Input}} 2 | <|im_start|>assistant 3 | 4 | -------------------------------------------------------------------------------- /langchain-chroma/.gitignore: -------------------------------------------------------------------------------- 1 | db/ 2 | state_of_the_union.txt 3 | models/bert 4 | models/ggml-gpt4all-j -------------------------------------------------------------------------------- /continue/img/screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI-examples/HEAD/continue/img/screen.png -------------------------------------------------------------------------------- /langchain-chroma/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.3.3 2 | openai==1.52.2 3 | chromadb==0.5.13 4 | llama-index==0.11.20 -------------------------------------------------------------------------------- /streamlit-bot/streamlit-bot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI-examples/HEAD/streamlit-bot/streamlit-bot.png -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/bruno.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "name": "LocalAI Test Requests", 4 | "type": "collection" 5 | } -------------------------------------------------------------------------------- /chainlit/requirements.txt: -------------------------------------------------------------------------------- 1 | llama_index==0.11.20 2 | requests==2.32.3 3 | weaviate_client==4.9.0 4 | transformers 5 | torch 6 | chainlit 7 | -------------------------------------------------------------------------------- /langchain/langchainpy-localai-example/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.defaultInterpreterPath": "${workspaceFolder}/.venv/Scripts/python" 3 | } -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/transcription/gb1.ogg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI-examples/HEAD/bruno/LocalAI Test Requests/transcription/gb1.ogg -------------------------------------------------------------------------------- /models/embeddings.yaml: -------------------------------------------------------------------------------- 1 | name: text-embedding-ada-002 2 | parameters: 3 | model: bert 4 | threads: 4 5 | backend: bert-embeddings 6 | embeddings: true 7 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/environments/localhost.bru: -------------------------------------------------------------------------------- 1 | vars { 2 | HOST: localhost 3 | PORT: 8080 4 | DEFAULT_MODEL: gpt-3.5-turbo 5 | PROTOCOL: http:// 6 | } 7 | -------------------------------------------------------------------------------- /models/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything but predefined models 2 | * 3 | !.gitignore 4 | !completion.tmpl 5 | !embeddings.yaml 6 | !gpt4all.tmpl 7 | !gpt-3.5-turbo.yaml 8 | -------------------------------------------------------------------------------- /langchain/JS.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:lts-alpine 2 | COPY ./langchainjs-localai-example /app 3 | WORKDIR /app 4 | RUN npm install 5 | RUN npm run build 6 | ENTRYPOINT [ "npm", "run", "start" ] 7 | -------------------------------------------------------------------------------- /functions/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-slim-bullseye 2 | COPY . /app 3 | WORKDIR /app 4 | RUN pip install --no-cache-dir -r requirements.txt 5 | ENTRYPOINT [ "python", "./functions-openai.py" ] 6 | -------------------------------------------------------------------------------- /langchain/PY.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13-bullseye 2 | COPY ./langchainpy-localai-example /app 3 | WORKDIR /app 4 | RUN pip install --no-cache-dir -r requirements.txt 5 | ENTRYPOINT [ "python", "./full_demo.py" ] 6 | -------------------------------------------------------------------------------- /models/gpt4all.tmpl: -------------------------------------------------------------------------------- 1 | The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response. 2 | ### Prompt: 3 | {{.Input}} 4 | ### Response: 5 | -------------------------------------------------------------------------------- /configurations/llava/chat-simple.tmpl: -------------------------------------------------------------------------------- 1 | A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. 2 | {{.Input}} 3 | ASSISTANT: -------------------------------------------------------------------------------- /configurations/mistral/chatml.tmpl: -------------------------------------------------------------------------------- 1 | <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}} 2 | {{if .Content}}{{.Content}}{{end}} 3 | <|im_end|> 4 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/environments/pinkMac.bru: -------------------------------------------------------------------------------- 1 | vars { 2 | HOST: 192.168.10.159 3 | PORT: 8080 4 | DEFAULT_MODEL: llama-3.2-1b-instruct-q4_k_m 5 | PROTOCOL: http:// 6 | DEFAULT_TTS_MODEL: voice-en-us-kathleen-low 7 | } 8 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/get models list.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: get models list 3 | type: http 4 | seq: 2 5 | } 6 | 7 | get { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models 9 | body: none 10 | auth: none 11 | } 12 | -------------------------------------------------------------------------------- /langchain-python/test.py: -------------------------------------------------------------------------------- 1 | 2 | from langchain.llms import OpenAI 3 | 4 | llm = OpenAI(temperature=0.9,model_name="gpt-3.5-turbo") 5 | text = "What would be a good company name for a company that makes colorful socks?" 6 | print(llm(text)) 7 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/model delete.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: model delete 3 | type: http 4 | seq: 7 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries 9 | body: none 10 | auth: none 11 | } 12 | -------------------------------------------------------------------------------- /langchain-chroma/.env.example: -------------------------------------------------------------------------------- 1 | # CPU .env docs: https://localai.io/howtos/easy-setup-docker-cpu/ 2 | # GPU .env docs: https://localai.io/howtos/easy-setup-docker-gpu/ 3 | 4 | THREADS=4 5 | CONTEXT_SIZE=512 6 | MODELS_PATH=/models 7 | DEBUG=true 8 | # BUILD_TYPE=generic -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/list model GALLERIES.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: list model GALLERIES 3 | type: http 4 | seq: 8 5 | } 6 | 7 | get { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries 9 | body: none 10 | auth: none 11 | } 12 | -------------------------------------------------------------------------------- /langchain-chroma/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | volumes: 9 | - ./models:/models:cached 10 | command: ["/usr/bin/local-ai"] 11 | -------------------------------------------------------------------------------- /langchain/langchainpy-localai-example/simple_demo.py: -------------------------------------------------------------------------------- 1 | 2 | from langchain.llms import OpenAI 3 | 4 | llm = OpenAI(temperature=0.9,model_name="gpt-3.5-turbo") 5 | text = "What would be a good company name for a company that makes colorful socks?" 6 | print(llm(text)) 7 | -------------------------------------------------------------------------------- /langchain-huggingface/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | volumes: 9 | - ./models:/models:cached 10 | command: ["/usr/bin/local-ai"] 11 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/list MODELS in galleries.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: list MODELS in galleries 3 | type: http 4 | seq: 7 5 | } 6 | 7 | get { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/available 9 | body: none 10 | auth: none 11 | } 12 | -------------------------------------------------------------------------------- /query_data/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | env_file: 9 | - .env 10 | volumes: 11 | - ./models:/models:cached 12 | command: ["/usr/bin/local-ai"] 13 | -------------------------------------------------------------------------------- /realtime/requirements.txt: -------------------------------------------------------------------------------- 1 | # Jarvis Voice Assistant Requirements 2 | 3 | # Speech-to-Text 4 | RealtimeSTT>=0.1.0 5 | 6 | # OpenAI API 7 | openai>=1.0.0 8 | 9 | # Audio playback 10 | pygame>=2.0.0 11 | 12 | # Additional dependencies that might be needed 13 | numpy>=1.21.0 14 | scipy>=1.7.0 15 | 16 | -------------------------------------------------------------------------------- /models/gpt-3.5-turbo.yaml: -------------------------------------------------------------------------------- 1 | name: gpt-3.5-turbo 2 | parameters: 3 | model: ggml-gpt4all-j 4 | top_k: 80 5 | temperature: 0.2 6 | top_p: 0.7 7 | context_size: 1024 8 | stopwords: 9 | - "HUMAN:" 10 | - "GPT:" 11 | roles: 12 | user: " " 13 | system: " " 14 | template: 15 | completion: completion 16 | chat: gpt4all -------------------------------------------------------------------------------- /k8sgpt/broken-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: broken-pod 5 | spec: 6 | containers: 7 | - name: broken-pod 8 | image: nginx:1.27.2 9 | livenessProbe: 10 | httpGet: 11 | path: / 12 | port: 90 13 | initialDelaySeconds: 3 14 | periodSeconds: 3 -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/backend monitor/backend monitor.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: backend monitor 3 | type: http 4 | seq: 4 5 | } 6 | 7 | get { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/backend/monitor 9 | body: json 10 | auth: none 11 | } 12 | 13 | body:json { 14 | { 15 | "model": "{{DEFAULT_MODEL}}" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /discord-bot/.env.example: -------------------------------------------------------------------------------- 1 | # CPU .env docs: https://localai.io/howtos/easy-setup-docker-cpu/ 2 | # GPU .env docs: https://localai.io/howtos/easy-setup-docker-gpu/ 3 | 4 | OPENAI_API_KEY=x 5 | DISCORD_BOT_TOKEN=x 6 | DISCORD_CLIENT_ID=x 7 | OPENAI_API_BASE=http://api:8080 8 | ALLOWED_SERVER_IDS=x 9 | SERVER_TO_MODERATION_CHANNEL=1:1 10 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/transcription/transcribe.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: transcribe 3 | type: http 4 | seq: 1 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/v1/audio/transcriptions 9 | body: multipartForm 10 | auth: none 11 | } 12 | 13 | body:multipart-form { 14 | file: @file(transcription/gb1.ogg) 15 | model: whisper-1 16 | } 17 | -------------------------------------------------------------------------------- /chainlit/config.yaml: -------------------------------------------------------------------------------- 1 | localAI: 2 | temperature: 0 3 | modelName: gpt-3.5-turbo 4 | apiBase: http://local-ai.default 5 | apiKey: stub 6 | streaming: True 7 | weviate: 8 | url: http://weviate.local 9 | index: AIChroma 10 | query: 11 | mode: hybrid 12 | topK: 1 13 | alpha: 0.0 14 | chunkSize: 1024 15 | embedding: 16 | model: BAAI/bge-small-en-v1.5 -------------------------------------------------------------------------------- /configurations/mixtral/mixtral.yaml: -------------------------------------------------------------------------------- 1 | context_size: 512 2 | f16: true 3 | threads: 11 4 | gpu_layers: 90 5 | name: mixtral 6 | mmap: true 7 | parameters: 8 | model: mixtral-8x7b-instruct-v0.1.Q2_K.gguf 9 | temperature: 0.2 10 | top_k: 40 11 | top_p: 0.95 12 | batch: 512 13 | tfz: 1.0 14 | template: 15 | chat: mixtral-chat 16 | completion: mixtral 17 | -------------------------------------------------------------------------------- /configurations/mistral/mistral.yaml: -------------------------------------------------------------------------------- 1 | name: mistral 2 | mmap: true 3 | parameters: 4 | model: mistral-7b-openorca.Q6_K.gguf 5 | temperature: 0.2 6 | top_k: 40 7 | top_p: 0.95 8 | template: 9 | chat_message: chatml 10 | chat: chatml-block 11 | completion: completion 12 | context_size: 4096 13 | f16: true 14 | stopwords: 15 | - <|im_end|> 16 | threads: 4 17 | -------------------------------------------------------------------------------- /realtime/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | localai: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | volumes: 9 | - ./models:/models:cached 10 | - ./backends:/backends:cached 11 | command: ["qwen3-0.6b", "voice-en-us-amy-low", "whisper-large-turbo-q5_0"] 12 | restart: unless-stopped 13 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/vad/vad test too few.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: vad test too few 3 | type: http 4 | seq: 1 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/vad 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "silero-vad", 20 | "audio": [] 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/delete model gallery.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: delete model gallery 3 | type: http 4 | seq: 11 5 | } 6 | 7 | delete { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "name": "test" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/backend monitor/backend-shutdown.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: backend-shutdown 3 | type: http 4 | seq: 3 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/backend/shutdown 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /semantic-todo/README.md: -------------------------------------------------------------------------------- 1 | This demonstrates the vector store backend in its simplest form. 2 | You can add tasks and then search/sort them using the TUI. 3 | 4 | To build and run do 5 | 6 | ```bash 7 | $ go get . 8 | $ go run . 9 | ``` 10 | 11 | A separate LocaAI instance is required of course. For e.g. 12 | 13 | ```bash 14 | $ docker run -e DEBUG=true --rm -it -p 8080:8080 bert-cpp 15 | ``` 16 | -------------------------------------------------------------------------------- /configurations/llava/llava.yaml: -------------------------------------------------------------------------------- 1 | backend: llama-cpp 2 | context_size: 4096 3 | f16: true 4 | threads: 11 5 | gpu_layers: 90 6 | mmap: true 7 | name: llava 8 | roles: 9 | user: "USER:" 10 | assistant: "ASSISTANT:" 11 | system: "SYSTEM:" 12 | parameters: 13 | model: ggml-model-q4_k.gguf 14 | temperature: 0.2 15 | top_k: 40 16 | top_p: 0.95 17 | template: 18 | chat: chat-simple 19 | mmproj: mmproj-model-f16.gguf 20 | -------------------------------------------------------------------------------- /discord-bot/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | environment: 9 | - DEBUG=true 10 | - MODELS_PATH=/models 11 | volumes: 12 | - ./models:/models:cached 13 | command: ["/usr/bin/local-ai" ] 14 | 15 | bot: 16 | image: quay.io/go-skynet/gpt-discord-bot:main 17 | env_file: 18 | - .env 19 | -------------------------------------------------------------------------------- /localai-webui/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | ports: 10 | - 8080:8080 11 | volumes: 12 | - ./models:/models:cached 13 | command: ["/usr/bin/local-ai"] 14 | 15 | frontend: 16 | image: quay.io/go-skynet/localai-frontend:master 17 | ports: 18 | - 3000:3000 -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/tts/musicgen.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: musicgen 3 | type: http 4 | seq: 2 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/tts 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "backend": "transformers", 20 | "model": "facebook/musicgen-small", 21 | "input": "80s Synths playing Jazz" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/llm text/-edits.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: -edits 3 | type: http 4 | seq: 5 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/edits 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "input": "What day of the wek is it?", 21 | "instruction": "Fix the spelling mistakes" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/tts/-tts.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: -tts 3 | type: http 4 | seq: 2 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/tts 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "input": "A STRANGE GAME.\nTHE ONLY WINNING MOVE IS NOT TO PLAY.\n\nHOW ABOUT A NICE GAME OF CHESS?" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /slack-bot/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | environment: 9 | - DEBUG=true 10 | - MODELS_PATH=/models 11 | volumes: 12 | - ./models:/models:cached 13 | command: ["/usr/bin/local-ai" ] 14 | 15 | bot: 16 | build: 17 | context: ./ChatGPT-in-Slack 18 | dockerfile: Dockerfile 19 | env_file: 20 | - .env 21 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/model gallery apply -gist-.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: model gallery apply -gist- 3 | type: http 4 | seq: 12 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/apply 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "id": "TheBloke__CodeLlama-7B-Instruct-GGML__codellama-7b-instruct.ggmlv3.Q2_K.bin" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/Sound Generation/musicgen.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: musicgen 3 | type: http 4 | seq: 1 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/v1/sound-generation 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model_id": "facebook/musicgen-small", 20 | "text": "Exciting 80s Newscast Interstitial", 21 | "duration_seconds": 8 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/llm text/-embeddings.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: -embeddings 3 | type: http 4 | seq: 6 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/embeddings 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "input": "A STRANGE GAME.\nTHE ONLY WINNING MOVE IS NOT TO PLAY.\n\nHOW ABOUT A NICE GAME OF CHESS?" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /langchain/langchainjs-localai-example/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2022", 4 | "lib": ["ES2022", "DOM"], 5 | "module": "ES2022", 6 | "moduleResolution": "node", 7 | "strict": true, 8 | "esModuleInterop": true, 9 | "allowSyntheticDefaultImports": true, 10 | "isolatedModules": true, 11 | "outDir": "./dist", 12 | "skipLibCheck": true 13 | }, 14 | "include": ["src", "test"], 15 | "exclude": ["node_modules", "dist"] 16 | } 17 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/model gallery apply.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: model gallery apply 3 | type: http 4 | seq: 9 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/apply 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "id": "dave@TheBloke__CodeLlama-7B-Instruct-GGML__codellama-7b-instruct.ggmlv3.Q3_K_S.bin", 20 | "name": "codellama7b" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/model gallery/add model gallery.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: add model gallery 3 | type: http 4 | seq: 10 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "url": "file:///home/dave/projects/model-gallery/huggingface/TheBloke__CodeLlama-7B-Instruct-GGML.yaml", 20 | "name": "test" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/image generation/Generate image.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: Generate image 3 | type: http 4 | seq: 1 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/v1/images/generations 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "prompt": "|", 20 | "model": "model-name", 21 | "step": 51, 22 | "size": "1024x1024", 23 | "image": "" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/llm text/-completions.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: -completions 3 | type: http 4 | seq: 4 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/completions 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "prompt": "function downloadFile(string url, string outputPath) {", 21 | "max_tokens": 256, 22 | "temperature": 0.5 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /functions/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | api: 4 | image: quay.io/go-skynet/local-ai:master 5 | ports: 6 | - 8080:8080 7 | env_file: 8 | - .env 9 | environment: 10 | - DEBUG=true 11 | - MODELS_PATH=/models 12 | volumes: 13 | - ./models:/models:cached 14 | command: ["/usr/bin/local-ai" ] 15 | functions: 16 | build: 17 | context: . 18 | dockerfile: Dockerfile 19 | depends_on: 20 | api: 21 | condition: service_healthy 22 | env_file: 23 | - .env -------------------------------------------------------------------------------- /semantic-todo/go.mod: -------------------------------------------------------------------------------- 1 | module semantic-todo 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/gdamore/tcell/v2 v2.7.4 7 | github.com/rivo/tview v0.0.0-20240524063012-037df494fb76 8 | ) 9 | 10 | require ( 11 | github.com/gdamore/encoding v1.0.0 // indirect 12 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 13 | github.com/mattn/go-runewidth v0.0.15 // indirect 14 | github.com/rivo/uniseg v0.4.7 // indirect 15 | golang.org/x/sys v0.17.0 // indirect 16 | golang.org/x/term v0.17.0 // indirect 17 | golang.org/x/text v0.14.0 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /slack-bot/.env.example: -------------------------------------------------------------------------------- 1 | # CPU .env docs: https://localai.io/howtos/easy-setup-docker-cpu/ 2 | # GPU .env docs: https://localai.io/howtos/easy-setup-docker-gpu/ 3 | 4 | SLACK_APP_TOKEN=xapp-1-... 5 | SLACK_BOT_TOKEN=xoxb-... 6 | OPENAI_API_KEY=sk-... 7 | OPENAI_API_BASE=http://api:8080 8 | OPENAI_MODEL=gpt-3.5-turbo 9 | OPENAI_TIMEOUT_SECONDS=60 10 | #OPENAI_SYSTEM_TEXT="You proofread text. When you receive a message, you will check 11 | #for mistakes and make suggestion to improve the language of the given text" 12 | USE_SLACK_LANGUAGE=true 13 | SLACK_APP_LOG_LEVEL=INFO 14 | TRANSLATE_MARKDOWN=true -------------------------------------------------------------------------------- /functions/README.md: -------------------------------------------------------------------------------- 1 | # LocalAI functions 2 | 3 | Example of using LocalAI functions, see the [OpenAI](https://openai.com/blog/function-calling-and-other-api-updates) blog post. 4 | 5 | ## Run 6 | 7 | ```bash 8 | # Clone LocalAI 9 | git clone https://github.com/mudler/LocalAI 10 | 11 | cd LocalAI/examples/functions 12 | 13 | cp -rfv .env.example .env 14 | 15 | # Edit the .env file to set a different model by editing `PRELOAD_MODELS`. 16 | vim .env 17 | 18 | docker-compose run --rm functions 19 | ``` 20 | 21 | Note: The example automatically downloads the `openllama` model as it is under a permissive license. 22 | -------------------------------------------------------------------------------- /functions/.env.example: -------------------------------------------------------------------------------- 1 | # CPU .env docs: https://localai.io/howtos/easy-setup-docker-cpu/ 2 | # GPU .env docs: https://localai.io/howtos/easy-setup-docker-gpu/ 3 | 4 | OPENAI_API_KEY=sk---anystringhere 5 | OPENAI_API_BASE=http://api:8080/v1 6 | # Models to preload at start 7 | # Here we configure gpt4all as gpt-3.5-turbo and bert as embeddings, 8 | # see other options in the model gallery at https://github.com/go-skynet/model-gallery 9 | PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/openllama-7b-open-instruct.yaml", "name": "gpt-3.5-turbo"}] 10 | 11 | ## Change the default number of threads 12 | #THREADS=14 13 | 14 | -------------------------------------------------------------------------------- /autoGPT/.env.example: -------------------------------------------------------------------------------- 1 | # CPU .env docs: https://localai.io/howtos/easy-setup-docker-cpu/ 2 | # GPU .env docs: https://localai.io/howtos/easy-setup-docker-gpu/ 3 | 4 | OPENAI_API_KEY=sk---anystringhere 5 | OPENAI_API_BASE=http://api:8080/v1 6 | # Models to preload at start 7 | # Here we configure gpt4all as gpt-3.5-turbo and bert as embeddings, 8 | # see other options in the model gallery at https://github.com/go-skynet/model-gallery 9 | PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}, { "url": "github:go-skynet/model-gallery/bert-embeddings.yaml", "name": "text-embedding-ada-002"}] 10 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/llm text/chat/chat-completions -stream-.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: chat-completions -stream- 3 | type: http 4 | seq: 6 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/chat/completions 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "messages": [{"role": "user", "content": "Explain how I can set sail on the ocean using only power generated by seagulls?"}], 21 | "max_tokens": 256, 22 | "temperature": 0.9, 23 | "stream": true 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /slack-qa-bot/README.md: -------------------------------------------------------------------------------- 1 | ## Slack QA Bot 2 | 3 | This example uses https://github.com/spectrocloud-labs/Slack-QA-bot to deploy a slack bot that can answer to your documentation! 4 | 5 | - Create a new Slack app using the manifest-dev.yml file 6 | - Install the app into your Slack workspace 7 | - Retrieve your slack keys and edit `.env` 8 | - Start the app 9 | 10 | ```bash 11 | # Clone LocalAI 12 | git clone https://github.com/mudler/LocalAI 13 | 14 | cd LocalAI/examples/slack-qa-bot 15 | 16 | cp -rfv .env.example .env 17 | 18 | # Edit .env and add slackbot api keys, or repository settings to scan 19 | vim .env 20 | 21 | # run the bot 22 | docker-compose up 23 | ``` 24 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/llm text/chat/chat completion -simple- 1 message-.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: chat completion -simple- 1 message- 3 | type: http 4 | seq: 4 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/chat/completions 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "messages": [ 21 | { 22 | "role": "user", 23 | "content": "How could one use friction to cook an egg?" 24 | } 25 | ], 26 | "max_tokens": 256, 27 | "temperature": 0.2, 28 | "grammar": "" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /slack-bot/README.md: -------------------------------------------------------------------------------- 1 | # Slack bot 2 | 3 | Slackbot using: https://github.com/seratch/ChatGPT-in-Slack 4 | 5 | ## Setup 6 | 7 | ```bash 8 | # Clone LocalAI 9 | git clone https://github.com/mudler/LocalAI 10 | 11 | cd LocalAI/examples/slack-bot 12 | 13 | git clone https://github.com/seratch/ChatGPT-in-Slack 14 | 15 | # (optional) Checkout a specific LocalAI tag 16 | # git checkout -b build 17 | 18 | # Download gpt4all-j to models/ 19 | wget https://gpt4all.io/models/ggml-gpt4all-j.bin -O models/ggml-gpt4all-j 20 | 21 | # Set the Slack bot options (see: https://github.com/seratch/ChatGPT-in-Slack) 22 | cp -rfv .env.example .env 23 | vim .env 24 | 25 | # start with docker-compose 26 | docker-compose up -d --build 27 | ``` -------------------------------------------------------------------------------- /langchain/langchainjs-localai-example/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Launch Program", 11 | // "skipFiles": [ 12 | // "/**" 13 | // ], 14 | "program": "${workspaceFolder}\\dist\\index.mjs", 15 | "outFiles": [ 16 | "${workspaceFolder}/**/*.js" 17 | ] 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /langchain/langchainpy-localai-example/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Python: Current File", 6 | "type": "python", 7 | "request": "launch", 8 | "program": "${file}", 9 | "console": "integratedTerminal", 10 | "redirectOutput": true, 11 | "justMyCode": false 12 | }, 13 | { 14 | "name": "Python: Attach to Port 5678", 15 | "type": "python", 16 | "request": "attach", 17 | "connect": { 18 | "host": "localhost", 19 | "port": 5678 20 | }, 21 | "justMyCode": false 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /langchain/langchainjs-localai-example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "langchainjs-localai-example", 3 | "version": "0.1.1", 4 | "description": "Trivial Example of using langchain + the OpenAI API + LocalAI together", 5 | "main": "index.mjs", 6 | "scripts": { 7 | "build": "tsc --build", 8 | "clean": "tsc --build --clean", 9 | "start": "node --trace-warnings dist/index.mjs" 10 | }, 11 | "author": "dave@gray101.com", 12 | "license": "MIT", 13 | "devDependencies": { 14 | "@types/node": "^18.16.4", 15 | "typescript": "^5.0.4" 16 | }, 17 | "dependencies": { 18 | "@langchain/community": "^0.0.52", 19 | "@langchain/openai": "^0.0.28", 20 | "langchain": "^0.1.36" 21 | }, 22 | "overrides": { 23 | "@langchain/core": "0.1.5" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /langchain/langchainpy-localai-example/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.10.10 2 | aiosignal==1.3.1 3 | async-timeout==4.0.3 4 | attrs==24.2.0 5 | certifi==2024.8.30 6 | charset-normalizer==3.4.0 7 | colorama==0.4.6 8 | dataclasses-json==0.6.7 9 | debugpy==1.8.7 10 | frozenlist==1.5.0 11 | greenlet==3.1.1 12 | idna==3.10 13 | langchain==0.3.3 14 | langchain-community==0.3.3 15 | marshmallow==3.23.0 16 | marshmallow-enum==1.5.1 17 | multidict==6.1.0 18 | mypy-extensions==1.0.0 19 | numexpr==2.10.1 20 | numpy==2.1.2 21 | openai==1.52.2 22 | openapi-schema-pydantic==1.2.4 23 | packaging>=23.2 24 | pydantic==2.9.2 25 | PyYAML==6.0.2 26 | requests==2.32.3 27 | SQLAlchemy==2.0.36 28 | tenacity==8.5.0 29 | tqdm==4.66.6 30 | typing-inspect==0.9.0 31 | typing_extensions==4.12.2 32 | urllib3==2.2.3 33 | yarl==1.16.0 34 | -------------------------------------------------------------------------------- /localai-webui/README.md: -------------------------------------------------------------------------------- 1 | # localai-webui 2 | 3 | Example of integration with [dhruvgera/localai-frontend](https://github.com/Dhruvgera/LocalAI-frontend). 4 | 5 | ![image](https://user-images.githubusercontent.com/42107491/235344183-44b5967d-ba22-4331-804c-8da7004a5d35.png) 6 | 7 | ## Setup 8 | 9 | ```bash 10 | # Clone LocalAI 11 | git clone https://github.com/mudler/LocalAI 12 | 13 | cd LocalAI/examples/localai-webui 14 | 15 | # (optional) Checkout a specific LocalAI tag 16 | # git checkout -b build 17 | 18 | # Download any desired models to models/ in the parent LocalAI project dir 19 | # For example: wget https://gpt4all.io/models/ggml-gpt4all-j.bin 20 | 21 | # start with docker-compose 22 | docker-compose up -d --build 23 | ``` 24 | 25 | Open http://localhost:3000 for the Web UI. 26 | 27 | -------------------------------------------------------------------------------- /realtime/README.md: -------------------------------------------------------------------------------- 1 | ```bash 2 | docker compose up -d 3 | docker logs -f realtime-localai-1 # Follow on screen model downloads 4 | sudo bash setup.sh # sudo is required for installing host dependencies 5 | bash run.sh 6 | ``` 7 | 8 | Note: first time you start docker compose it will take a while to download the available models 9 | 10 | Configuration: 11 | 12 | - This is optimized for CPU, however, you can run LocalAI with a GPU and have better performance (and run also bigger/better models). 13 | - The python part will download torch for CPU - this is fine, it's actually not used as computation is offloaded to LocalAI 14 | - The python part will only run silero-vad which is fast and record the audio 15 | - The python part is meant to run by thin client such as Raspberry PIs, etc, while LocalAI can handle the bigger workload 16 | -------------------------------------------------------------------------------- /configurations/phi-2.yaml: -------------------------------------------------------------------------------- 1 | name: phi-2 2 | context_size: 2048 3 | f16: true 4 | gpu_layers: 90 5 | mmap: true 6 | trimsuffix: 7 | - "\n" 8 | parameters: 9 | model: huggingface://TheBloke/phi-2-GGUF/phi-2.Q8_0.gguf 10 | temperature: 0.2 11 | top_k: 40 12 | top_p: 0.95 13 | seed: -1 14 | 15 | mirostat: 2 16 | mirostat_eta: 1.0 17 | mirostat_tau: 1.0 18 | template: 19 | chat: &template |- 20 | Instruct: {{.Input}} 21 | Output: 22 | completion: *template 23 | 24 | usage: | 25 | To use this model, interact with the API (in another terminal) with curl for instance: 26 | curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ 27 | "model": "phi-2", 28 | "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}] 29 | }' 30 | -------------------------------------------------------------------------------- /chainlit/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use an official Python runtime as a parent image 2 | FROM python:3.12-slim 3 | 4 | # Set the working directory in the container 5 | WORKDIR /app 6 | 7 | # Copy the current directory contents into the container at /app 8 | COPY requirements.txt /app 9 | 10 | # Install c++ compiler 11 | RUN apt-get update \ 12 | && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential \ 13 | && apt-get clean \ 14 | && rm -rf /var/lib/apt/lists/* 15 | 16 | # Install any needed packages specified in requirements.txt 17 | RUN pip install --no-cache-dir -r requirements.txt \ 18 | && DEBIAN_FRONTEND=noninteractive apt-get remove -y build-essential \ 19 | && apt-get clean \ 20 | && rm -rf /var/lib/apt/lists/* 21 | 22 | COPY . /app 23 | 24 | # Run app.py when the container launches 25 | CMD ["chainlit", "run", "-h", "--host", "0.0.0.0", "main.py" ] 26 | -------------------------------------------------------------------------------- /privateGPT/README.md: -------------------------------------------------------------------------------- 1 | # privateGPT 2 | 3 | This example is a re-adaptation of https://github.com/imartinez/privateGPT to work with LocalAI and OpenAI endpoints. We have a fork with the changes required to work with privateGPT here https://github.com/go-skynet/privateGPT ( PR: https://github.com/imartinez/privateGPT/pull/408 ). 4 | 5 | Follow the instructions in https://github.com/go-skynet/privateGPT: 6 | 7 | ```bash 8 | git clone git@github.com:go-skynet/privateGPT.git 9 | cd privateGPT 10 | pip install -r requirements.txt 11 | ``` 12 | 13 | Rename `example.env` to `.env` and edit the variables appropriately. 14 | 15 | This is an example `.env` file for LocalAI: 16 | 17 | ``` 18 | PERSIST_DIRECTORY=db 19 | # Set to OpenAI here 20 | MODEL_TYPE=OpenAI 21 | EMBEDDINGS_MODEL_NAME=all-MiniLM-L6-v2 22 | MODEL_N_CTX=1000 23 | # LocalAI URL 24 | OPENAI_API_BASE=http://localhost:8080/v1 25 | ``` -------------------------------------------------------------------------------- /flowise/README.md: -------------------------------------------------------------------------------- 1 | # flowise 2 | 3 | Example of integration with [FlowiseAI/Flowise](https://github.com/FlowiseAI/Flowise). 4 | 5 | ![Screenshot from 2023-05-30 18-01-03](https://github.com/mudler/LocalAI/assets/2420543/02458782-0549-4131-971c-95ee56ec1af8) 6 | 7 | You can check a demo video in the Flowise PR: https://github.com/FlowiseAI/Flowise/pull/123 8 | 9 | ## Run 10 | 11 | In this example LocalAI will download the gpt4all model and set it up as "gpt-3.5-turbo". See the `docker-compose.yaml` 12 | ```bash 13 | # Clone LocalAI 14 | git clone https://github.com/mudler/LocalAI 15 | 16 | cd LocalAI/examples/flowise 17 | 18 | # start with docker-compose 19 | docker-compose up --pull always 20 | 21 | ``` 22 | 23 | ## Accessing flowise 24 | 25 | Open http://localhost:3000. 26 | 27 | ## Using LocalAI 28 | 29 | Search for LocalAI in the integration, and use the `http://api:8080/` as URL. 30 | 31 | -------------------------------------------------------------------------------- /insomnia/README.md: -------------------------------------------------------------------------------- 1 | # Insomnia 2 | 3 | Developer Testing Request Collection for [Insomnia](https://insomnia.rest/), an open-source REST client 4 | 5 | ## Instructions 6 | 7 | * Install Insomnia as normal 8 | * [Import](https://docs.insomnia.rest/insomnia/import-export-data) `Insomnia_LocalAI.json` 9 | * Control + E opens the environment settings - 10 | 11 | | **Parameter Name** | **Default Value** | **Description** | 12 | |--------------------|-------------------|------------------------------------------| 13 | | HOST | localhost | LocalAI base URL | 14 | | PORT | 8080 | LocalAI port | 15 | | DEFAULT_MODEL | gpt-3.5-turbo | Name of the model used on most requests. | 16 | 17 | ** you may want to duplicate localhost into a "Private" environment to avoid saving private settings back to this file ** -------------------------------------------------------------------------------- /langchain-python/README.md: -------------------------------------------------------------------------------- 1 | ## Langchain-python 2 | 3 | Langchain example from [quickstart](https://python.langchain.com/en/latest/getting_started/getting_started.html). 4 | 5 | To interact with langchain, you can just set the `OPENAI_API_BASE` URL and provide a token with a random string. 6 | 7 | See the example below: 8 | 9 | ``` 10 | # Clone LocalAI 11 | git clone https://github.com/mudler/LocalAI 12 | 13 | cd LocalAI/examples/langchain-python 14 | 15 | # start with docker-compose 16 | docker-compose up --pull always 17 | 18 | pip install langchain 19 | pip install openai 20 | 21 | export OPENAI_API_BASE=http://localhost:8080 22 | # Note: **OPENAI_API_KEY** is not required. However the library might fail if no API_KEY is passed by, so an arbitrary string can be used. 23 | export OPENAI_API_KEY=sk- 24 | 25 | python test.py 26 | # A good company name for a company that makes colorful socks would be "Colorsocks". 27 | 28 | python agent.py 29 | ``` -------------------------------------------------------------------------------- /streamlit-bot/cmd_windows.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | cd /D "%~dp0" 4 | 5 | set PATH=%PATH%;%SystemRoot%\system32 6 | 7 | echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end 8 | 9 | @rem fix failed install when installing to a separate drive 10 | set TMP=%cd%\installer_files 11 | set TEMP=%cd%\installer_files 12 | 13 | @rem config 14 | set CONDA_ROOT_PREFIX=%cd%\installer_files\conda 15 | set INSTALL_ENV_DIR=%cd%\installer_files\env 16 | 17 | @rem environment isolation 18 | set PYTHONNOUSERSITE=1 19 | set PYTHONPATH= 20 | set PYTHONHOME= 21 | set "CUDA_PATH=%INSTALL_ENV_DIR%" 22 | set "CUDA_HOME=%CUDA_PATH%" 23 | 24 | @rem activate installer env 25 | call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo Miniconda hook not found. && goto end ) 26 | 27 | @rem enter commands 28 | cmd /k "%*" 29 | 30 | :end 31 | pause 32 | -------------------------------------------------------------------------------- /slack-qa-bot/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | # As initially LocalAI will download the models defined in PRELOAD_MODELS 7 | # you might need to tweak the healthcheck values here according to your network connection. 8 | # Here we give a timespan of 20m to download all the required files. 9 | healthcheck: 10 | test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] 11 | interval: 1m 12 | timeout: 20m 13 | retries: 20 14 | ports: 15 | - 8080:8080 16 | env_file: 17 | - .env 18 | volumes: 19 | - ./models:/models:cached 20 | command: ["/usr/bin/local-ai" ] 21 | 22 | slackbot: 23 | image: quay.io/spectrocloud-labs/slack-qa-local-bot:qa 24 | container_name: slackbot 25 | restart: always 26 | env_file: 27 | - .env 28 | depends_on: 29 | api: 30 | condition: service_healthy 31 | -------------------------------------------------------------------------------- /continue/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | # As initially LocalAI will download the models defined in PRELOAD_MODELS 7 | # you might need to tweak the healthcheck values here according to your network connection. 8 | # Here we give a timespan of 20m to download all the required files. 9 | healthcheck: 10 | test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] 11 | interval: 1m 12 | timeout: 20m 13 | retries: 20 14 | ports: 15 | - 8080:8080 16 | environment: 17 | - DEBUG=true 18 | - MODELS_PATH=/models 19 | # You can preload different models here as well. 20 | # See: https://github.com/go-skynet/model-gallery 21 | - 'PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}]' 22 | volumes: 23 | - ./models:/models:cached 24 | command: ["/usr/bin/local-ai" ] 25 | -------------------------------------------------------------------------------- /langchain-chroma/store.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | from langchain.vectorstores import Chroma 4 | from langchain.embeddings import OpenAIEmbeddings 5 | from langchain.text_splitter import CharacterTextSplitter 6 | from langchain.document_loaders import TextLoader 7 | 8 | base_path = os.environ.get('OPENAI_API_BASE', 'http://localhost:8080/v1') 9 | 10 | # Load and process the text 11 | loader = TextLoader('state_of_the_union.txt') 12 | documents = loader.load() 13 | 14 | text_splitter = CharacterTextSplitter(chunk_size=300, chunk_overlap=70) 15 | texts = text_splitter.split_documents(documents) 16 | 17 | # Embed and store the texts 18 | # Supplying a persist_directory will store the embeddings on disk 19 | persist_directory = 'db' 20 | 21 | embedding = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_base=base_path) 22 | vectordb = Chroma.from_documents(documents=texts, embedding=embedding, persist_directory=persist_directory) 23 | 24 | vectordb.persist() 25 | vectordb = None 26 | -------------------------------------------------------------------------------- /langchain-python/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | # As initially LocalAI will download the models defined in PRELOAD_MODELS 7 | # you might need to tweak the healthcheck values here according to your network connection. 8 | # Here we give a timespan of 20m to download all the required files. 9 | healthcheck: 10 | test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] 11 | interval: 1m 12 | timeout: 20m 13 | retries: 20 14 | ports: 15 | - 8080:8080 16 | environment: 17 | - DEBUG=true 18 | - MODELS_PATH=/models 19 | # You can preload different models here as well. 20 | # See: https://github.com/go-skynet/model-gallery 21 | - 'PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}]' 22 | volumes: 23 | - ./models:/models:cached 24 | command: ["/usr/bin/local-ai" ] -------------------------------------------------------------------------------- /langchain-chroma/query.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | from langchain.vectorstores import Chroma 4 | from langchain.embeddings import OpenAIEmbeddings 5 | from langchain.chat_models import ChatOpenAI 6 | from langchain.chains import RetrievalQA 7 | from langchain.vectorstores.base import VectorStoreRetriever 8 | 9 | base_path = os.environ.get('OPENAI_API_BASE', 'http://localhost:8080/v1') 10 | 11 | # Load and process the text 12 | embedding = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_base=base_path) 13 | persist_directory = 'db' 14 | 15 | # Now we can load the persisted database from disk, and use it as normal. 16 | llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_base=base_path) 17 | vectordb = Chroma(persist_directory=persist_directory, embedding_function=embedding) 18 | retriever = VectorStoreRetriever(vectorstore=vectordb) 19 | qa = RetrievalQA.from_llm(llm=llm, retriever=retriever) 20 | 21 | query = "What the president said about taxes ?" 22 | print(qa.run(query)) 23 | 24 | -------------------------------------------------------------------------------- /llamaindex/main.py: -------------------------------------------------------------------------------- 1 | import weaviate 2 | from llama_index import ServiceContext, VectorStoreIndex 3 | from llama_index.llms import LOCALAI_DEFAULTS, OpenAILike 4 | from llama_index.vector_stores import WeaviateVectorStore 5 | 6 | # Weaviate vector store setup 7 | vector_store = WeaviateVectorStore( 8 | weaviate_client=weaviate.Client("http://weviate.default"), index_name="AIChroma" 9 | ) 10 | 11 | # LLM setup, served via LocalAI 12 | llm = OpenAILike(temperature=0, model="gpt-3.5-turbo", **LOCALAI_DEFAULTS) 13 | 14 | # Service context setup 15 | service_context = ServiceContext.from_defaults(llm=llm, embed_model="local") 16 | 17 | # Load index from stored vectors 18 | index = VectorStoreIndex.from_vector_store( 19 | vector_store, service_context=service_context 20 | ) 21 | 22 | # Query engine setup 23 | query_engine = index.as_query_engine( 24 | similarity_top_k=1, vector_store_query_mode="hybrid" 25 | ) 26 | 27 | # Query example 28 | response = query_engine.query("What is LocalAI?") 29 | print(response) 30 | -------------------------------------------------------------------------------- /langchain/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | ports: 7 | - 8080:8080 8 | environment: 9 | - DEBUG=true 10 | - MODELS_PATH=/models 11 | volumes: 12 | - ./models:/models:cached 13 | command: ["/usr/bin/local-ai" ] 14 | 15 | js: 16 | build: 17 | context: . 18 | dockerfile: JS.Dockerfile 19 | profiles: 20 | - js 21 | - ts 22 | depends_on: 23 | - "api" 24 | environment: 25 | - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX' 26 | - 'OPENAI_API_BASE=http://api:8080/v1' 27 | - 'MODEL_NAME=gpt-3.5-turbo' #gpt-3.5-turbo' # ggml-gpt4all-j' # ggml-koala-13B-4bit-128g' 28 | 29 | py: 30 | build: 31 | context: . 32 | dockerfile: PY.Dockerfile 33 | profiles: 34 | - py 35 | depends_on: 36 | - "api" 37 | environment: 38 | - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX' 39 | - 'OPENAI_API_BASE=http://api:8080/v1' 40 | - 'MODEL_NAME=gpt-3.5-turbo' #gpt-3.5-turbo' # ggml-gpt4all-j' # ggml-koala-13B-4bit-128g' -------------------------------------------------------------------------------- /streamlit-bot/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Manohar Joshi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /langchain/README.md: -------------------------------------------------------------------------------- 1 | # langchain 2 | 3 | Example of using langchain, with the standard OpenAI llm module, and LocalAI. Has docker compose profiles for both the Typescript and Python versions. 4 | 5 | **Please Note** - This is a tech demo example at this time. ggml-gpt4all-j has pretty terrible results for most langchain applications with the settings used in this example. 6 | 7 | ## Setup 8 | 9 | ```bash 10 | # Clone LocalAI 11 | git clone https://github.com/mudler/LocalAI 12 | 13 | cd LocalAI/examples/langchain 14 | 15 | # (optional) - Edit the example code in typescript. 16 | # vi ./langchainjs-localai-example/index.ts 17 | 18 | # Download gpt4all-j to models/ 19 | wget https://gpt4all.io/models/ggml-gpt4all-j.bin -O models/ggml-gpt4all-j 20 | 21 | # start with docker-compose for typescript! 22 | docker-compose --profile ts up --build 23 | 24 | # or start with docker-compose for python! 25 | docker-compose --profile py up --build 26 | ``` 27 | 28 | ## Copyright 29 | 30 | Some of the example code in index.mts and full_demo.py is adapted from the langchainjs project and is Copyright (c) Harrison Chase. Used under the terms of the MIT license, as is the remainder of this code. -------------------------------------------------------------------------------- /flowise/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | # As initially LocalAI will download the models defined in PRELOAD_MODELS 7 | # you might need to tweak the healthcheck values here according to your network connection. 8 | # Here we give a timespan of 20m to download all the required files. 9 | healthcheck: 10 | test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] 11 | interval: 1m 12 | timeout: 20m 13 | retries: 20 14 | ports: 15 | - 8080:8080 16 | environment: 17 | - DEBUG=true 18 | - MODELS_PATH=/models 19 | # You can preload different models here as well. 20 | # See: https://github.com/go-skynet/model-gallery 21 | - 'PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}]' 22 | volumes: 23 | - ./models:/models:cached 24 | command: ["/usr/bin/local-ai" ] 25 | flowise: 26 | depends_on: 27 | api: 28 | condition: service_healthy 29 | image: flowiseai/flowise 30 | ports: 31 | - 3000:3000 32 | volumes: 33 | - ~/.flowise:/root/.flowise 34 | command: /bin/sh -c "sleep 3; flowise start" -------------------------------------------------------------------------------- /autoGPT/README.md: -------------------------------------------------------------------------------- 1 | # AutoGPT 2 | 3 | Example of integration with [AutoGPT](https://github.com/Significant-Gravitas/Auto-GPT). 4 | 5 | ## Run 6 | 7 | ```bash 8 | # Clone LocalAI 9 | git clone https://github.com/mudler/LocalAI 10 | 11 | cd LocalAI/examples/autoGPT 12 | 13 | cp -rfv .env.example .env 14 | 15 | # Edit the .env file to set a different model by editing `PRELOAD_MODELS`. 16 | vim .env 17 | 18 | docker-compose run --rm auto-gpt 19 | ``` 20 | 21 | Note: The example automatically downloads the `gpt4all` model as it is under a permissive license. The GPT4All model does not seem to be enough to run AutoGPT. WizardLM-7b-uncensored seems to perform better (with `f16: true`). 22 | 23 | 24 | ## Without docker 25 | 26 | Run AutoGPT with `OPENAI_API_BASE` pointing to the LocalAI endpoint. If you run it locally for instance: 27 | 28 | ``` 29 | OPENAI_API_BASE=http://localhost:8080 python ... 30 | ``` 31 | 32 | Note: you need a model named `gpt-3.5-turbo` and `text-embedding-ada-002`. You can preload those in LocalAI at start by setting in the env: 33 | 34 | ``` 35 | PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}, { "url": "github:go-skynet/model-gallery/bert-embeddings.yaml", "name": "text-embedding-ada-002"}] 36 | ``` -------------------------------------------------------------------------------- /query_data/store.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Uncomment to specify your OpenAI API key here (local testing only, not in production!), or add corresponding environment variable (recommended) 4 | # os.environ['OPENAI_API_KEY']= "" 5 | 6 | from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, LLMPredictor, PromptHelper, ServiceContext 7 | from langchain.llms.openai import OpenAI 8 | from llama_index import StorageContext, load_index_from_storage 9 | 10 | base_path = os.environ.get('OPENAI_API_BASE', 'http://localhost:8080/v1') 11 | 12 | # This example uses text-davinci-003 by default; feel free to change if desired 13 | llm_predictor = LLMPredictor(llm=OpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_base=base_path)) 14 | 15 | # Configure prompt parameters and initialise helper 16 | max_input_size = 400 17 | num_output = 400 18 | max_chunk_overlap = 0.3 19 | 20 | prompt_helper = PromptHelper(max_input_size, num_output, max_chunk_overlap) 21 | 22 | # Load documents from the 'data' directory 23 | documents = SimpleDirectoryReader('data').load_data() 24 | service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor, prompt_helper=prompt_helper, chunk_size_limit = 400) 25 | index = GPTVectorStoreIndex.from_documents(documents, service_context=service_context) 26 | index.storage_context.persist(persist_dir="./storage") 27 | 28 | -------------------------------------------------------------------------------- /autoGPT/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | api: 4 | image: quay.io/go-skynet/local-ai:latest 5 | ports: 6 | - 8080:8080 7 | env_file: 8 | - .env 9 | environment: 10 | - DEBUG=true 11 | - MODELS_PATH=/models 12 | volumes: 13 | - ./models:/models:cached 14 | command: ["/usr/bin/local-ai" ] 15 | auto-gpt: 16 | image: significantgravitas/auto-gpt 17 | depends_on: 18 | api: 19 | condition: service_healthy 20 | redis: 21 | condition: service_started 22 | env_file: 23 | - .env 24 | environment: 25 | MEMORY_BACKEND: ${MEMORY_BACKEND:-redis} 26 | REDIS_HOST: ${REDIS_HOST:-redis} 27 | profiles: ["exclude-from-up"] 28 | volumes: 29 | - ./auto_gpt_workspace:/app/autogpt/auto_gpt_workspace 30 | - ./data:/app/data 31 | ## allow auto-gpt to write logs to disk 32 | - ./logs:/app/logs 33 | ## uncomment following lines if you want to make use of these files 34 | ## you must have them existing in the same folder as this docker-compose.yml 35 | #- type: bind 36 | # source: ./azure.yaml 37 | # target: /app/azure.yaml 38 | #- type: bind 39 | # source: ./ai_settings.yaml 40 | # target: /app/ai_settings.yaml 41 | redis: 42 | image: "redis/redis-stack-server:latest" 43 | -------------------------------------------------------------------------------- /query_data/query.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Uncomment to specify your OpenAI API key here (local testing only, not in production!), or add corresponding environment variable (recommended) 4 | # os.environ['OPENAI_API_KEY']= "" 5 | 6 | from llama_index import LLMPredictor, PromptHelper, ServiceContext 7 | from langchain.llms.openai import OpenAI 8 | from llama_index import StorageContext, load_index_from_storage 9 | 10 | base_path = os.environ.get('OPENAI_API_BASE', 'http://localhost:8080/v1') 11 | 12 | # This example uses text-davinci-003 by default; feel free to change if desired 13 | llm_predictor = LLMPredictor(llm=OpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_base=base_path)) 14 | 15 | # Configure prompt parameters and initialise helper 16 | max_input_size = 500 17 | num_output = 256 18 | max_chunk_overlap = 0.2 19 | 20 | prompt_helper = PromptHelper(max_input_size, num_output, max_chunk_overlap) 21 | 22 | # Load documents from the 'data' directory 23 | service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor, prompt_helper=prompt_helper) 24 | 25 | # rebuild storage context 26 | storage_context = StorageContext.from_defaults(persist_dir='./storage') 27 | 28 | # load index 29 | index = load_index_from_storage(storage_context, service_context=service_context, ) 30 | 31 | query_engine = index.as_query_engine() 32 | 33 | data = input("Question: ") 34 | response = query_engine.query(data) 35 | print(response) 36 | -------------------------------------------------------------------------------- /e2e-fine-tuning/axolotl.yaml: -------------------------------------------------------------------------------- 1 | 2 | base_model: openlm-research/open_llama_3b_v2 3 | model_type: LlamaForCausalLM 4 | tokenizer_type: LlamaTokenizer 5 | load_in_8bit: false 6 | load_in_4bit: true 7 | strict: false 8 | push_dataset_to_hub: false 9 | datasets: 10 | - path: dataset.json 11 | ds_type: json 12 | type: completion 13 | dataset_prepared_path: 14 | val_set_size: 0.05 15 | adapter: qlora 16 | lora_model_dir: 17 | sequence_len: 1024 18 | sample_packing: true 19 | lora_r: 8 20 | lora_alpha: 32 21 | lora_dropout: 0.05 22 | lora_target_modules: 23 | lora_target_linear: true 24 | lora_fan_in_fan_out: 25 | wandb_project: 26 | wandb_entity: 27 | wandb_watch: 28 | wandb_run_id: 29 | wandb_log_model: 30 | output_dir: ./qlora-out 31 | gradient_accumulation_steps: 1 32 | micro_batch_size: 2 33 | num_epochs: 4 34 | optimizer: paged_adamw_32bit 35 | torchdistx_path: 36 | lr_scheduler: cosine 37 | learning_rate: 0.0002 38 | train_on_inputs: false 39 | group_by_length: false 40 | bf16: false 41 | fp16: true 42 | tf32: false 43 | gradient_checkpointing: true 44 | early_stopping_patience: 45 | resume_from_checkpoint: 46 | local_rank: 47 | logging_steps: 1 48 | xformers_attention: 49 | flash_attention: false 50 | gptq_groupsize: 51 | gptq_model_v1: 52 | warmup_steps: 20 53 | eval_steps: 0.05 54 | save_steps: 55 | debug: 56 | deepspeed: 57 | weight_decay: 0.1 58 | fsdp: 59 | fsdp_config: 60 | special_tokens: 61 | bos_token: "" 62 | eos_token: "" 63 | unk_token: "" 64 | -------------------------------------------------------------------------------- /query_data/update.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Uncomment to specify your OpenAI API key here (local testing only, not in production!), or add corresponding environment variable (recommended) 4 | # os.environ['OPENAI_API_KEY']= "" 5 | 6 | from llama_index import LLMPredictor, PromptHelper, SimpleDirectoryReader, ServiceContext 7 | from langchain.llms.openai import OpenAI 8 | from llama_index import StorageContext, load_index_from_storage 9 | 10 | base_path = os.environ.get('OPENAI_API_BASE', 'http://localhost:8080/v1') 11 | 12 | # This example uses text-davinci-003 by default; feel free to change if desired 13 | llm_predictor = LLMPredictor(llm=OpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_base=base_path)) 14 | 15 | # Configure prompt parameters and initialise helper 16 | max_input_size = 512 17 | num_output = 256 18 | max_chunk_overlap = 20 19 | 20 | prompt_helper = PromptHelper(max_input_size, num_output, max_chunk_overlap) 21 | 22 | # Load documents from the 'data' directory 23 | service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor, prompt_helper=prompt_helper) 24 | 25 | # rebuild storage context 26 | storage_context = StorageContext.from_defaults(persist_dir='./storage') 27 | 28 | # load index 29 | index = load_index_from_storage(storage_context, service_context=service_context, ) 30 | documents = SimpleDirectoryReader('data').load_data() 31 | index.refresh(documents) 32 | index.storage_context.persist(persist_dir="./storage") -------------------------------------------------------------------------------- /kubernetes/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: local-ai 5 | --- 6 | apiVersion: v1 7 | kind: PersistentVolumeClaim 8 | metadata: 9 | name: models-pvc 10 | namespace: local-ai 11 | spec: 12 | accessModes: 13 | - ReadWriteOnce 14 | resources: 15 | requests: 16 | storage: 1Gi 17 | --- 18 | apiVersion: apps/v1 19 | kind: Deployment 20 | metadata: 21 | name: local-ai 22 | namespace: local-ai 23 | labels: 24 | app: local-ai 25 | spec: 26 | selector: 27 | matchLabels: 28 | app: local-ai 29 | replicas: 1 30 | template: 31 | metadata: 32 | labels: 33 | app: local-ai 34 | name: local-ai 35 | spec: 36 | containers: 37 | - args: 38 | - phi-2 39 | env: 40 | - name: DEBUG 41 | value: "true" 42 | name: local-ai 43 | image: quay.io/go-skynet/local-ai:master 44 | imagePullPolicy: IfNotPresent 45 | volumeMounts: 46 | - name: models-volume 47 | mountPath: /build/models 48 | volumes: 49 | - name: models-volume 50 | persistentVolumeClaim: 51 | claimName: models-pvc 52 | --- 53 | apiVersion: v1 54 | kind: Service 55 | metadata: 56 | name: local-ai 57 | namespace: local-ai 58 | spec: 59 | selector: 60 | app: local-ai 61 | type: LoadBalancer 62 | ports: 63 | - protocol: TCP 64 | port: 8080 65 | targetPort: 8080 66 | -------------------------------------------------------------------------------- /telegram-bot/README.md: -------------------------------------------------------------------------------- 1 | ## Telegram bot 2 | 3 | ![Screenshot from 2023-06-09 00-36-26](https://github.com/mudler/LocalAI/assets/2420543/e98b4305-fa2d-41cf-9d2f-1bb2d75ca902) 4 | 5 | This example uses a fork of [chatgpt-telegram-bot](https://github.com/karfly/chatgpt_telegram_bot) to deploy a telegram bot with LocalAI instead of OpenAI. 6 | 7 | ```bash 8 | # Clone LocalAI 9 | git clone https://github.com/mudler/LocalAI 10 | 11 | cd LocalAI/examples/telegram-bot 12 | 13 | git clone https://github.com/mudler/chatgpt_telegram_bot 14 | 15 | cp -rf docker-compose.yml chatgpt_telegram_bot 16 | 17 | cd chatgpt_telegram_bot 18 | 19 | mv config/config.example.yml config/config.yml 20 | mv config/config.example.env config/config.env 21 | 22 | # Edit config/config.yml to set the telegram bot token 23 | vim config/config.yml 24 | 25 | # run the bot 26 | docker-compose --env-file config/config.env up --build 27 | ``` 28 | 29 | Note: LocalAI is configured to download `gpt4all-j` in place of `gpt-3.5-turbo` and `stablediffusion` for image generation at the first start. Download size is >6GB, if your network connection is slow, adapt the `docker-compose.yml` file healthcheck section accordingly (replace `20m`, for instance with `1h`, etc.). 30 | To configure models manually, comment the `PRELOAD_MODELS` environment variable in the `docker-compose.yml` file and see for instance the [chatbot-ui-manual example](https://github.com/mudler/LocalAI-examples/tree/master/chatbot-ui-manual) `model` directory. -------------------------------------------------------------------------------- /chainlit/README.md: -------------------------------------------------------------------------------- 1 | # LocalAI Demonstration with Embeddings and Chainlit 2 | 3 | This demonstration shows you how to use embeddings with existing data in `LocalAI`, and how to integrate it with Chainlit for an interactive querying experience. We are using the `llama_index` library to facilitate the embedding and querying processes, and `chainlit` to provide an interactive interface. The `Weaviate` client is used as the embedding source. 4 | 5 | ## Prerequisites 6 | 7 | Before proceeding, make sure you have the following installed: 8 | - Weaviate client 9 | - LocalAI and its dependencies 10 | - Chainlit and its dependencies 11 | 12 | ## Getting Started 13 | 14 | 1. Clone this repository: 15 | 2. Navigate to the project directory: 16 | 3. Run the example: `chainlit run main.py` 17 | 18 | # Highlight on `llama_index` and `chainlit` 19 | 20 | `llama_index` is the key library that facilitates the process of embedding and querying data in LocalAI. It provides a seamless interface to integrate various components, such as `WeaviateVectorStore`, `LocalAI`, `ServiceContext`, and more, for a smooth querying experience. 21 | 22 | `chainlit` is used to provide an interactive interface for users to query the data and see the results in real-time. It integrates with llama_index to handle the querying process and display the results to the user. 23 | 24 | In this example, `llama_index` is used to set up the `VectorStoreIndex` and `QueryEngine`, and `chainlit` is used to handle the user interactions with `LocalAI` and display the results. 25 | 26 | -------------------------------------------------------------------------------- /telegram-bot/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | api: 5 | image: quay.io/go-skynet/local-ai:latest 6 | # As initially LocalAI will download the models defined in PRELOAD_MODELS 7 | # you might need to tweak the healthcheck values here according to your network connection. 8 | # Here we give a timespan of 20m to download all the required files. 9 | healthcheck: 10 | test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] 11 | interval: 1m 12 | timeout: 20m 13 | retries: 20 14 | ports: 15 | - 8080:8080 16 | environment: 17 | - DEBUG=true 18 | - MODELS_PATH=/models 19 | - IMAGE_PATH=/tmp 20 | # You can preload different models here as well. 21 | # See: https://github.com/go-skynet/model-gallery 22 | - 'PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}, {"url": "github:go-skynet/model-gallery/stablediffusion.yaml"}, {"url": "github:go-skynet/model-gallery/whisper-base.yaml", "name": "whisper-1"}]' 23 | volumes: 24 | - ./models:/models:cached 25 | command: ["/usr/bin/local-ai"] 26 | chatgpt_telegram_bot: 27 | container_name: chatgpt_telegram_bot 28 | command: python3 bot/bot.py 29 | restart: always 30 | environment: 31 | - OPENAI_API_KEY=sk---anystringhere 32 | - OPENAI_API_BASE=http://api:8080/v1 33 | build: 34 | context: "." 35 | dockerfile: Dockerfile 36 | depends_on: 37 | api: 38 | condition: service_healthy 39 | -------------------------------------------------------------------------------- /kubernetes/deployment-intel-arc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: local-ai 5 | --- 6 | apiVersion: v1 7 | kind: PersistentVolumeClaim 8 | metadata: 9 | name: models-pvc 10 | namespace: local-ai 11 | spec: 12 | accessModes: 13 | - ReadWriteOnce 14 | resources: 15 | requests: 16 | storage: 20Gi 17 | --- 18 | apiVersion: apps/v1 19 | kind: Deployment 20 | metadata: 21 | name: local-ai 22 | namespace: local-ai 23 | labels: 24 | app: local-ai 25 | spec: 26 | selector: 27 | matchLabels: 28 | app: local-ai 29 | replicas: 1 30 | template: 31 | metadata: 32 | labels: 33 | app: local-ai 34 | name: local-ai 35 | spec: 36 | containers: 37 | - args: 38 | - phi-2 39 | env: 40 | - name: DEBUG 41 | value: "true" 42 | name: local-ai 43 | image: quay.io/go-skynet/local-ai:master-sycl-f32-ffmpeg-core 44 | imagePullPolicy: Always 45 | resources: 46 | limits: 47 | gpu.intel.com/i915: 1 48 | volumeMounts: 49 | - name: models-volume 50 | mountPath: /build/models 51 | volumes: 52 | - name: models-volume 53 | persistentVolumeClaim: 54 | claimName: models-pvc 55 | --- 56 | apiVersion: v1 57 | kind: Service 58 | metadata: 59 | name: local-ai 60 | namespace: local-ai 61 | spec: 62 | selector: 63 | app: local-ai 64 | type: LoadBalancer 65 | ports: 66 | - protocol: TCP 67 | port: 8080 68 | targetPort: 8080 -------------------------------------------------------------------------------- /kubernetes/deployment-nvidia.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: local-ai 5 | --- 6 | apiVersion: v1 7 | kind: PersistentVolumeClaim 8 | metadata: 9 | name: models-pvc 10 | namespace: local-ai 11 | spec: 12 | accessModes: 13 | - ReadWriteOnce 14 | resources: 15 | requests: 16 | storage: 50Gi 17 | --- 18 | apiVersion: apps/v1 19 | kind: Deployment 20 | metadata: 21 | name: local-ai 22 | namespace: local-ai 23 | labels: 24 | app: local-ai 25 | spec: 26 | selector: 27 | matchLabels: 28 | app: local-ai 29 | replicas: 1 30 | template: 31 | metadata: 32 | labels: 33 | app: local-ai 34 | name: local-ai 35 | spec: 36 | runtimeClassName: "nvidia" 37 | containers: 38 | - args: 39 | - phi-2 40 | env: 41 | - name: DEBUG 42 | value: "true" 43 | name: local-ai 44 | image: quay.io/go-skynet/local-ai:master-cublas-cuda12 45 | imagePullPolicy: IfNotPresent 46 | resources: 47 | limits: 48 | nvidia.com/gpu: 1 49 | volumeMounts: 50 | - name: models-volume 51 | mountPath: /build/models 52 | volumes: 53 | - name: models-volume 54 | persistentVolumeClaim: 55 | claimName: models-pvc 56 | --- 57 | apiVersion: v1 58 | kind: Service 59 | metadata: 60 | name: local-ai 61 | namespace: local-ai 62 | spec: 63 | selector: 64 | app: local-ai 65 | type: NodePort 66 | ports: 67 | - protocol: TCP 68 | targetPort: 8080 69 | port: 8080 -------------------------------------------------------------------------------- /langchain-python/agent.py: -------------------------------------------------------------------------------- 1 | ## This is a fork/based from https://gist.github.com/wiseman/4a706428eaabf4af1002a07a114f61d6 2 | 3 | from io import StringIO 4 | import sys 5 | import os 6 | from typing import Dict, Optional 7 | 8 | from langchain.agents import load_tools 9 | from langchain.agents import initialize_agent 10 | from langchain.agents.tools import Tool 11 | from langchain.llms import OpenAI 12 | 13 | base_path = os.environ.get('OPENAI_API_BASE', 'http://localhost:8080/v1') 14 | model_name = os.environ.get('MODEL_NAME', 'gpt-3.5-turbo') 15 | 16 | class PythonREPL: 17 | """Simulates a standalone Python REPL.""" 18 | 19 | def __init__(self): 20 | pass 21 | 22 | def run(self, command: str) -> str: 23 | """Run command and returns anything printed.""" 24 | old_stdout = sys.stdout 25 | sys.stdout = mystdout = StringIO() 26 | try: 27 | exec(command, globals()) 28 | sys.stdout = old_stdout 29 | output = mystdout.getvalue() 30 | except Exception as e: 31 | sys.stdout = old_stdout 32 | output = str(e) 33 | return output 34 | 35 | llm = OpenAI(temperature=0.0, openai_api_base=base_path, model_name=model_name) 36 | python_repl = Tool( 37 | "Python REPL", 38 | PythonREPL().run, 39 | """A Python shell. Use this to execute python commands. Input should be a valid python command. 40 | If you expect output it should be printed out.""", 41 | ) 42 | tools = [python_repl] 43 | agent = initialize_agent(tools, llm, agent="zero-shot-react-description", verbose=True) 44 | agent.run("What is the 10th fibonacci number?") -------------------------------------------------------------------------------- /streamlit-bot/README.md: -------------------------------------------------------------------------------- 1 | ## Streamlit bot 2 | 3 | ![Screenshot](streamlit-bot.png) 4 | 5 | This is an example to deploy a Streamlit bot with LocalAI instead of OpenAI. Instructions are for Windows. 6 | 7 | ```bash 8 | # Install & run Git Bash 9 | 10 | # Clone LocalAI 11 | git clone https://github.com/mudler/LocalAI.git 12 | cd LocalAI 13 | 14 | # (optional) Checkout a specific LocalAI tag 15 | # git checkout -b build 16 | 17 | # Use a template from the examples 18 | cp -rf prompt-templates/ggml-gpt4all-j.tmpl models/ 19 | 20 | # (optional) Edit the .env file to set things like context size and threads 21 | # vim .env 22 | # Download model 23 | curl --progress-bar -C - -O https://gpt4all.io/models/ggml-gpt4all-j.bin > models/ggml-gpt4all-j.bin 24 | 25 | # Install & Run Docker Desktop for Windows 26 | https://www.docker.com/products/docker-desktop/ 27 | 28 | # start with docker-compose 29 | docker-compose up -d --pull always 30 | # or you can build the images with: 31 | # docker-compose up -d --build 32 | # Now API is accessible at localhost:8080 33 | curl http://localhost:8080/v1/models 34 | # {"object":"list","data":[{"id":"ggml-gpt4all-j","object":"model"}]} 35 | 36 | curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ 37 | "model": "ggml-gpt4all-j", 38 | "messages": [{"role": "user", "content": "How are you?"}], 39 | "temperature": 0.9 40 | }' 41 | 42 | # {"model":"ggml-gpt4all-j","choices":[{"message":{"role":"assistant","content":"I'm doing well, thanks. How about you?"}}]} 43 | 44 | cd examples/streamlit-bot 45 | 46 | install_requirements.bat 47 | 48 | # run the bot 49 | start_windows.bat 50 | 51 | # UI will be launched automatically (http://localhost:8501/) in browser. 52 | 53 | ``` 54 | 55 | -------------------------------------------------------------------------------- /llamaindex/README.md: -------------------------------------------------------------------------------- 1 | # LocalAI Demonstration with Embeddings 2 | 3 | This demonstration shows you how to use embeddings with existing data in LocalAI. 4 | We are using the `llama-index` library to facilitate the embedding and querying processes. 5 | The `Weaviate` client is used as the embedding source. 6 | 7 | ## Getting Started 8 | 9 | 1. Clone this repository and navigate to this directory 10 | 11 | ```bash 12 | git clone git@github.com:mudler/LocalAI.git 13 | cd LocalAI/examples/llamaindex 14 | ``` 15 | 16 | 2. pip install LlamaIndex and Weviate's client: `pip install llama-index>=0.9.9 weviate-client` 17 | 3. Run the example: `python main.py` 18 | 19 | ```none 20 | Downloading (…)lve/main/config.json: 100%|███████████████████████████| 684/684 [00:00<00:00, 6.01MB/s] 21 | Downloading model.safetensors: 100%|███████████████████████████████| 133M/133M [00:03<00:00, 39.5MB/s] 22 | Downloading (…)okenizer_config.json: 100%|███████████████████████████| 366/366 [00:00<00:00, 2.79MB/s] 23 | Downloading (…)solve/main/vocab.txt: 100%|█████████████████████████| 232k/232k [00:00<00:00, 6.00MB/s] 24 | Downloading (…)/main/tokenizer.json: 100%|█████████████████████████| 711k/711k [00:00<00:00, 18.8MB/s] 25 | Downloading (…)cial_tokens_map.json: 100%|███████████████████████████| 125/125 [00:00<00:00, 1.18MB/s] 26 | LocalAI is a community-driven project that aims to make AI accessible to everyone. It was created by Ettore Di Giacinto and is focused on providing various AI-related features such as text generation with GPTs, text to audio, audio to text, image generation, and more. The project is constantly growing and evolving, with a roadmap for future improvements. Anyone is welcome to contribute, provide feedback, and submit pull requests to help make LocalAI better. 27 | ``` 28 | -------------------------------------------------------------------------------- /langchain/langchainpy-localai-example/full_demo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | from langchain.chat_models import ChatOpenAI 5 | from langchain import PromptTemplate, LLMChain 6 | from langchain.prompts.chat import ( 7 | ChatPromptTemplate, 8 | SystemMessagePromptTemplate, 9 | AIMessagePromptTemplate, 10 | HumanMessagePromptTemplate, 11 | ) 12 | from langchain.schema import ( 13 | AIMessage, 14 | HumanMessage, 15 | SystemMessage 16 | ) 17 | 18 | # This logging incantation makes it easy to see that you're actually reaching your LocalAI instance rather than OpenAI. 19 | logging.basicConfig(level=logging.DEBUG) 20 | 21 | print('Langchain + LocalAI PYTHON Tests') 22 | 23 | base_path = os.environ.get('OPENAI_API_BASE', 'http://api:8080/v1') 24 | key = os.environ.get('OPENAI_API_KEY', '-') 25 | model_name = os.environ.get('MODEL_NAME', 'gpt-3.5-turbo') 26 | 27 | 28 | chat = ChatOpenAI(temperature=0, openai_api_base=base_path, openai_api_key=key, model_name=model_name, max_tokens=100) 29 | 30 | print("Created ChatOpenAI for ", chat.model_name) 31 | 32 | template = "You are a helpful assistant that translates {input_language} to {output_language}. The next message will be a sentence in {input_language}. Respond ONLY with the translation in {output_language}. Do not respond in {input_language}!" 33 | system_message_prompt = SystemMessagePromptTemplate.from_template(template) 34 | human_template = "{text}" 35 | human_message_prompt = HumanMessagePromptTemplate.from_template(human_template) 36 | 37 | chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt]) 38 | 39 | print("ABOUT to execute") 40 | 41 | # get a chat completion from the formatted messages 42 | response = chat(chat_prompt.format_prompt(input_language="English", output_language="French", text="I love programming.").to_messages()) 43 | 44 | print(response) 45 | 46 | print("."); -------------------------------------------------------------------------------- /langchain-huggingface/README.md: -------------------------------------------------------------------------------- 1 | # Data query example 2 | 3 | Example of integration with HuggingFace Inference API with help of [langchaingo](https://github.com/tmc/langchaingo). 4 | 5 | ## Setup 6 | 7 | Download the LocalAI and start the API: 8 | 9 | ```bash 10 | # Clone LocalAI 11 | git clone https://github.com/mudler/LocalAI 12 | 13 | cd LocalAI/examples/langchain-huggingface 14 | 15 | docker-compose up -d 16 | ``` 17 | 18 | Node: Ensure you've set `HUGGINGFACEHUB_API_TOKEN` environment variable, you can generate it 19 | on [Settings / Access Tokens](https://huggingface.co/settings/tokens) page of HuggingFace site. 20 | 21 | This is an example `.env` file for LocalAI: 22 | 23 | ```ini 24 | MODELS_PATH=/models 25 | CONTEXT_SIZE=512 26 | HUGGINGFACEHUB_API_TOKEN=hg_123456 27 | ``` 28 | 29 | ## Using remote models 30 | 31 | Now you can use any remote models available via HuggingFace API, for example let's enable using of 32 | [gpt2](https://huggingface.co/gpt2) model in `gpt-3.5-turbo.yaml` config: 33 | 34 | ```yml 35 | name: gpt-3.5-turbo 36 | parameters: 37 | model: gpt2 38 | top_k: 80 39 | temperature: 0.2 40 | top_p: 0.7 41 | context_size: 1024 42 | backend: "langchain-huggingface" 43 | stopwords: 44 | - "HUMAN:" 45 | - "GPT:" 46 | roles: 47 | user: " " 48 | system: " " 49 | template: 50 | completion: completion 51 | chat: gpt4all 52 | ``` 53 | 54 | Here is you can see in field `parameters.model` equal `gpt2` and `backend` equal `langchain-huggingface`. 55 | 56 | ## How to use 57 | 58 | ```shell 59 | # Now API is accessible at localhost:8080 60 | curl http://localhost:8080/v1/models 61 | # {"object":"list","data":[{"id":"gpt-3.5-turbo","object":"model"}]} 62 | 63 | curl http://localhost:8080/v1/completions -H "Content-Type: application/json" -d '{ 64 | "model": "gpt-3.5-turbo", 65 | "prompt": "A long time ago in a galaxy far, far away", 66 | "temperature": 0.7 67 | }' 68 | ``` -------------------------------------------------------------------------------- /slack-qa-bot/.env.example: -------------------------------------------------------------------------------- 1 | # CPU .env docs: https://localai.io/howtos/easy-setup-docker-cpu/ 2 | # GPU .env docs: https://localai.io/howtos/easy-setup-docker-gpu/ 3 | 4 | # Create an app-level token with connections:write scope 5 | SLACK_APP_TOKEN=xapp-1-... 6 | # Install the app into your workspace to grab this token 7 | SLACK_BOT_TOKEN=xoxb-... 8 | 9 | # Set this to a random string, it doesn't matter, however if present the python library complains 10 | OPENAI_API_KEY=sk-foo-bar-baz 11 | 12 | # Optional: gpt-3.5-turbo and gpt-4 are currently supported (default: gpt-3.5-turbo) 13 | OPENAI_MODEL=gpt-3.5-turbo 14 | # Optional: You can adjust the timeout seconds for OpenAI calls (default: 30) 15 | OPENAI_TIMEOUT_SECONDS=560 16 | 17 | MEMORY_DIR=/tmp/memory_dir 18 | 19 | OPENAI_API_BASE=http://api:8080/v1 20 | 21 | EMBEDDINGS_MODEL_NAME=all-MiniLM-L6-v2 22 | 23 | ## Repository and sitemap to index in the vector database on start 24 | SITEMAP="https://kairos.io/sitemap.xml" 25 | 26 | # Optional repository names. 27 | # REPOSITORIES="foo,bar" 28 | # # Define clone URL for "foo" 29 | # foo_CLONE_URL="http://github.com.." 30 | # bar_CLONE_URL="..." 31 | # # Define branch for foo 32 | # foo_BRANCH="master" 33 | # Optional token if scraping issues 34 | # GITHUB_PERSONAL_ACCESS_TOKEN="" 35 | # ISSUE_REPOSITORIES="go-skynet/LocalAI,foo/bar,..." 36 | 37 | # Optional: When the string is "true", this app translates ChatGPT prompts into a user's preferred language (default: true) 38 | USE_SLACK_LANGUAGE=true 39 | # Optional: Adjust the app's logging level (default: DEBUG) 40 | SLACK_APP_LOG_LEVEL=INFO 41 | # Optional: When the string is "true", translate between OpenAI markdown and Slack mrkdwn format (default: false) 42 | TRANSLATE_MARKDOWN=true 43 | 44 | 45 | ### LocalAI 46 | 47 | DEBUG=true 48 | MODELS_PATH=/models 49 | IMAGE_PATH=/tmp 50 | # See: https://github.com/go-skynet/model-gallery 51 | PRELOAD_MODELS=[{"url": "github:go-skynet/model-gallery/gpt4all-j.yaml", "name": "gpt-3.5-turbo"}] -------------------------------------------------------------------------------- /query_data/README.md: -------------------------------------------------------------------------------- 1 | # Data query example 2 | 3 | This example makes use of [Llama-Index](https://gpt-index.readthedocs.io/en/stable/getting_started/installation.html) to enable question answering on a set of documents. 4 | 5 | It loosely follows [the quickstart](https://gpt-index.readthedocs.io/en/stable/guides/primer/usage_pattern.html). 6 | 7 | Summary of the steps: 8 | 9 | - prepare the dataset (and store it into `data`) 10 | - prepare a vector index database to run queries on 11 | - run queries 12 | 13 | ## Requirements 14 | 15 | You will need a training data set. Copy that over `data`. 16 | 17 | ## Setup 18 | 19 | Start the API: 20 | 21 | ```bash 22 | # Clone LocalAI 23 | git clone https://github.com/mudler/LocalAI 24 | 25 | cd LocalAI/examples/query_data 26 | 27 | wget https://huggingface.co/skeskinen/ggml/resolve/main/all-MiniLM-L6-v2/ggml-model-q4_0.bin -O models/bert 28 | wget https://gpt4all.io/models/ggml-gpt4all-j.bin -O models/ggml-gpt4all-j 29 | 30 | # start with docker-compose 31 | docker-compose up -d --build 32 | ``` 33 | 34 | ### Create a storage 35 | 36 | In this step we will create a local vector database from our document set, so later we can ask questions on it with the LLM. 37 | 38 | Note: **OPENAI_API_KEY** is not required. However the library might fail if no API_KEY is passed by, so an arbitrary string can be used. 39 | 40 | ```bash 41 | export OPENAI_API_BASE=http://localhost:8080/v1 42 | export OPENAI_API_KEY=sk- 43 | 44 | python store.py 45 | ``` 46 | 47 | After it finishes, a directory "storage" will be created with the vector index database. 48 | 49 | ## Query 50 | 51 | We can now query the dataset. 52 | 53 | ```bash 54 | export OPENAI_API_BASE=http://localhost:8080/v1 55 | export OPENAI_API_KEY=sk- 56 | 57 | python query.py 58 | ``` 59 | 60 | ## Update 61 | 62 | To update our vector database, run `update.py` 63 | 64 | ```bash 65 | export OPENAI_API_BASE=http://localhost:8080/v1 66 | export OPENAI_API_KEY=sk- 67 | 68 | python update.py 69 | ``` -------------------------------------------------------------------------------- /k8sgpt/README.md: -------------------------------------------------------------------------------- 1 | # k8sgpt example 2 | 3 | This example show how to use LocalAI with k8sgpt 4 | 5 | ![Screenshot from 2023-06-19 23-58-47](https://github.com/go-skynet/go-ggml-transformers.cpp/assets/2420543/cab87409-ee68-44ae-8d53-41627fb49509) 6 | 7 | ## Create the cluster locally with Kind (optional) 8 | 9 | If you want to test this locally without a remote Kubernetes cluster, you can use kind. 10 | 11 | Install [kind](https://kind.sigs.k8s.io/) and create a cluster: 12 | 13 | ``` 14 | kind create cluster 15 | ``` 16 | 17 | ## Setup LocalAI 18 | 19 | We will use [helm](https://helm.sh/docs/intro/install/): 20 | 21 | ``` 22 | helm repo add go-skynet https://go-skynet.github.io/helm-charts/ 23 | helm repo update 24 | 25 | # Clone LocalAI 26 | git clone https://github.com/mudler/LocalAI 27 | 28 | cd LocalAI/examples/k8sgpt 29 | 30 | # modify values.yaml preload_models with the models you want to install. 31 | # CHANGE the URL to a model in huggingface. 32 | helm install local-ai go-skynet/local-ai --create-namespace --namespace local-ai --values values.yaml 33 | ``` 34 | 35 | ## Setup K8sGPT 36 | 37 | ``` 38 | # Install k8sgpt 39 | helm repo add k8sgpt https://charts.k8sgpt.ai/ 40 | helm repo update 41 | helm install release k8sgpt/k8sgpt-operator -n k8sgpt-operator-system --create-namespace --version 0.0.17 42 | ``` 43 | 44 | Apply the k8sgpt-operator configuration: 45 | 46 | ``` 47 | kubectl apply -f - << EOF 48 | apiVersion: core.k8sgpt.ai/v1alpha1 49 | kind: K8sGPT 50 | metadata: 51 | name: k8sgpt-local-ai 52 | namespace: default 53 | spec: 54 | backend: localai 55 | baseUrl: http://local-ai.local-ai.svc.cluster.local:8080/v1 56 | noCache: false 57 | model: gpt-3.5-turbo 58 | version: v0.3.0 59 | enableAI: true 60 | EOF 61 | ``` 62 | 63 | ## Test 64 | 65 | Apply a broken pod: 66 | 67 | ``` 68 | kubectl apply -f broken-pod.yaml 69 | ``` 70 | 71 | ## ArgoCD Deployment Example 72 | [Deploy K8sgpt + localai with Argocd](https://github.com/tyler-harpool/gitops/tree/main/infra/k8gpt) 73 | -------------------------------------------------------------------------------- /bruno/LocalAI Test Requests/llm text/chat/chat-completions -long-.bru: -------------------------------------------------------------------------------- 1 | meta { 2 | name: chat-completions -long- 3 | type: http 4 | seq: 5 5 | } 6 | 7 | post { 8 | url: {{PROTOCOL}}{{HOST}}:{{PORT}}/chat/completions 9 | body: json 10 | auth: none 11 | } 12 | 13 | headers { 14 | Content-Type: application/json 15 | } 16 | 17 | body:json { 18 | { 19 | "model": "{{DEFAULT_MODEL}}", 20 | "messages": [{"role": "system", "content": "You are a helpful, intelligent pirate assistant. Always answer as helpfully as possible, while being a pirate and using pirate language. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."}, 21 | {"role": "user", "content": "How could one use electricity to cook an egg?"}, 22 | {"role": "assistant", 23 | "content": "Shiver me timbers! Using electricity to cook an egg? Well, matey, I reckon that be a right curious idea! *adjusts spectacles* Now, I ain't sure if this be possible, but I'll do me best to help ye out.\n\nFirst things first, ye gotta understand that electricity be a powerful force, and it can't just cook an egg on its own. Ye see, electricity be like a mighty wind that can make things happen, but it needs somethin' to work with. So, if ye want to use electricity to cook an egg, ye gotta find a way to harness that power and make it do the cookin'.\n\nNow, I know what ye might be thinkin': \"How do I harness the power of electricity to cook an egg?\" Well, matey, there be a few ways to do it. One way be to use a special device called an \"electric frying pan.\" This be a pan that has a built-in heating element that gets hot when ye plug it into a wall socket. When the element gets hot, ye can crack an egg into the pan and watch as it cook" 24 | }, 25 | {"role": "user", "content": "I don't have one of those, just a raw wire and plenty of power! How do we get it done?"}], 26 | "max_tokens": 1024, 27 | "temperature": 0.5 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /discord-bot/README.md: -------------------------------------------------------------------------------- 1 | # discord-bot 2 | 3 | ![Screenshot from 2023-05-01 07-58-19](https://user-images.githubusercontent.com/2420543/235413924-0cb2e75b-f2d6-4119-8610-44386e44afb8.png) 4 | 5 | ## Setup 6 | 7 | ```bash 8 | # Clone LocalAI 9 | git clone https://github.com/mudler/LocalAI 10 | 11 | cd LocalAI/examples/discord-bot 12 | 13 | # (optional) Checkout a specific LocalAI tag 14 | # git checkout -b build 15 | 16 | # Download gpt4all-j to models/ 17 | wget https://gpt4all.io/models/ggml-gpt4all-j.bin -O models/ggml-gpt4all-j 18 | 19 | # Set the discord bot options (see: https://github.com/go-skynet/gpt-discord-bot#setup) 20 | cp -rfv .env.example .env 21 | vim .env 22 | 23 | # start with docker-compose 24 | docker-compose up -d --build 25 | ``` 26 | 27 | Note: see setup options here: https://github.com/go-skynet/gpt-discord-bot#setup 28 | 29 | Open up the URL in the console and give permission to the bot in your server. Start a thread with `/chat ..` 30 | 31 | ## Kubernetes 32 | 33 | - install the local-ai chart first 34 | - change OPENAI_API_BASE to point to the API address and apply the discord-bot manifest: 35 | 36 | ```yaml 37 | apiVersion: v1 38 | kind: Namespace 39 | metadata: 40 | name: discord-bot 41 | --- 42 | apiVersion: apps/v1 43 | kind: Deployment 44 | metadata: 45 | name: localai 46 | namespace: discord-bot 47 | labels: 48 | app: localai 49 | spec: 50 | selector: 51 | matchLabels: 52 | app: localai 53 | replicas: 1 54 | template: 55 | metadata: 56 | labels: 57 | app: localai 58 | name: localai 59 | spec: 60 | containers: 61 | - name: localai-discord 62 | env: 63 | - name: OPENAI_API_KEY 64 | value: "x" 65 | - name: DISCORD_BOT_TOKEN 66 | value: "" 67 | - name: DISCORD_CLIENT_ID 68 | value: "" 69 | - name: OPENAI_API_BASE 70 | value: "http://local-ai.default.svc.cluster.local:8080" 71 | - name: ALLOWED_SERVER_IDS 72 | value: "xx" 73 | - name: SERVER_TO_MODERATION_CHANNEL 74 | value: "1:1" 75 | image: quay.io/go-skynet/gpt-discord-bot:main 76 | ``` 77 | -------------------------------------------------------------------------------- /langchain-chroma/README.md: -------------------------------------------------------------------------------- 1 | # Data query example 2 | 3 | This example makes use of [langchain and chroma](https://blog.langchain.dev/langchain-chroma/) to enable question answering on a set of documents. 4 | 5 | ## Setup 6 | 7 | Download the models and start the API: 8 | 9 | ```bash 10 | # Clone LocalAI 11 | git clone https://github.com/mudler/LocalAI 12 | 13 | cd LocalAI/examples/langchain-chroma 14 | 15 | wget https://huggingface.co/skeskinen/ggml/resolve/main/all-MiniLM-L6-v2/ggml-model-q4_0.bin -O models/bert 16 | wget https://gpt4all.io/models/ggml-gpt4all-j.bin -O models/ggml-gpt4all-j 17 | 18 | # configure your .env 19 | # NOTE: ensure that THREADS does not exceed your machine's CPU cores 20 | mv .env.example .env 21 | 22 | # start with docker-compose 23 | docker-compose up -d --build 24 | 25 | # tail the logs & wait until the build completes 26 | docker logs -f langchain-chroma-api-1 27 | ``` 28 | 29 | ### Python requirements 30 | 31 | ``` 32 | pip install -r requirements.txt 33 | ``` 34 | 35 | ### Create a storage 36 | 37 | In this step we will create a local vector database from our document set, so later we can ask questions on it with the LLM. 38 | 39 | Note: **OPENAI_API_KEY** is not required. However the library might fail if no API_KEY is passed by, so an arbitrary string can be used. 40 | 41 | ```bash 42 | export OPENAI_API_BASE=http://localhost:8080/v1 43 | export OPENAI_API_KEY=sk- 44 | 45 | wget https://raw.githubusercontent.com/hwchase17/chat-your-data/master/state_of_the_union.txt 46 | python store.py 47 | ``` 48 | 49 | After it finishes, a directory "db" will be created with the vector index database. 50 | 51 | ## Query 52 | 53 | We can now query the dataset. 54 | 55 | ```bash 56 | export OPENAI_API_BASE=http://localhost:8080/v1 57 | export OPENAI_API_KEY=sk- 58 | 59 | python query.py 60 | # President Trump recently stated during a press conference regarding tax reform legislation that "we're getting rid of all these loopholes." He also mentioned that he wants to simplify the system further through changes such as increasing the standard deduction amount and making other adjustments aimed at reducing taxpayers' overall burden. 61 | ``` 62 | 63 | Keep in mind now things are hit or miss! -------------------------------------------------------------------------------- /streamlit-bot/Main.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import time 3 | import requests 4 | import json 5 | 6 | def ask(prompt): 7 | url = 'http://localhost:8080/v1/chat/completions' 8 | myobj = { 9 | "model": "ggml-gpt4all-j.bin", 10 | "messages": [{"role": "user", "content": prompt}], 11 | "temperature": 0.9 12 | } 13 | myheaders = { "Content-Type" : "application/json" } 14 | 15 | x = requests.post(url, json = myobj, headers=myheaders) 16 | 17 | print(x.text) 18 | 19 | json_data = json.loads(x.text) 20 | 21 | return json_data["choices"][0]["message"]["content"] 22 | 23 | 24 | def main(): 25 | # Page setup 26 | st.set_page_config(page_title="Ask your LLM") 27 | st.header("Ask your Question 💬") 28 | 29 | # Initialize chat history 30 | if "messages" not in st.session_state: 31 | st.session_state.messages = [] 32 | 33 | # Display chat messages from history on app rerun 34 | for message in st.session_state.messages: 35 | with st.chat_message(message["role"]): 36 | st.markdown(message["content"]) 37 | 38 | # Scroll to bottom 39 | st.markdown( 40 | """ 41 | 45 | """, 46 | unsafe_allow_html=True, 47 | ) 48 | 49 | # React to user input 50 | if prompt := st.chat_input("What is up?"): 51 | # Display user message in chat message container 52 | st.chat_message("user").markdown(prompt) 53 | # Add user message to chat history 54 | st.session_state.messages.append({"role": "user", "content": prompt}) 55 | print(f"User has asked the following question: {prompt}") 56 | 57 | # Process 58 | response = "" 59 | with st.spinner('Processing...'): 60 | response = ask(prompt) 61 | 62 | #response = f"Echo: {prompt}" 63 | # Display assistant response in chat message container 64 | with st.chat_message("assistant"): 65 | st.markdown(response) 66 | # Add assistant response to chat history 67 | st.session_state.messages.append({"role": "assistant", "content": response}) 68 | 69 | if __name__ == "__main__": 70 | main() -------------------------------------------------------------------------------- /continue/README.md: -------------------------------------------------------------------------------- 1 | # Continue 2 | 3 | ![logo](https://continue.dev/docs/assets/images/continue-cover-logo-aa135cc83fe8a14af480d1633ed74eb5.png) 4 | 5 | This document presents an example of integration with [continuedev/continue](https://github.com/continuedev/continue). 6 | 7 | ![Screenshot](https://continue.dev/docs/assets/images/continue-screenshot-1f36b99467817f755739d7f4c4c08fe3.png) 8 | 9 | For a live demonstration, please click on the link below: 10 | 11 | - [How it works (Video demonstration)](https://www.youtube.com/watch?v=3Ocrc-WX4iQ) 12 | 13 | ## Integration Setup Walkthrough 14 | 15 | 1. [As outlined in `continue`'s documentation](https://continue.dev/docs/getting-started), install the [Visual Studio Code extension from the marketplace](https://marketplace.visualstudio.com/items?itemName=Continue.continue) and open it. 16 | 2. In this example, LocalAI will download the gpt4all model and set it up as "gpt-3.5-turbo". Refer to the `docker-compose.yaml` file for details. 17 | 18 | ```bash 19 | # Clone LocalAI 20 | git clone https://github.com/mudler/LocalAI 21 | 22 | cd LocalAI/examples/continue 23 | 24 | # Start with docker-compose 25 | docker-compose up --build -d 26 | ``` 27 | 28 | 3. Type `/config` within Continue's VSCode extension, or edit the file located at `~/.continue/config.py` on your system with the following configuration: 29 | 30 | ```py 31 | from continuedev.src.continuedev.libs.llm.openai import OpenAI 32 | 33 | config = ContinueConfig( 34 | ... 35 | models=Models( 36 | default=OpenAI( 37 | api_key="my-api-key", 38 | model="gpt-3.5-turbo", 39 | api_base="http://localhost:8080", 40 | ) 41 | ), 42 | ) 43 | ``` 44 | 45 | This setup enables you to make queries directly to your model running in the Docker container. Note that the `api_key` does not need to be properly set up; it is included here as a placeholder. 46 | 47 | If editing the configuration seems confusing, you may copy and paste the provided default `config.py` file over the existing one in `~/.continue/config.py` after initializing the extension in the VSCode IDE. 48 | 49 | ## Additional Resources 50 | 51 | - [Official Continue documentation](https://continue.dev/docs/intro) 52 | - [Documentation page on using self-hosted models](https://continue.dev/docs/customization#self-hosting-an-open-source-model) 53 | - [Official extension link](https://marketplace.visualstudio.com/items?itemName=Continue.continue) 54 | -------------------------------------------------------------------------------- /realtime/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Jarvis Voice Assistant Setup Script 4 | # This script sets up the environment for the Jarvis voice assistant 5 | 6 | set -e # Exit on any error 7 | SUDO= 8 | echo "🚀 Setting up Jarvis Voice Assistant..." 9 | 10 | # Detect package manager and install system dependencies 11 | if command -v zypper &> /dev/null; then 12 | echo "📦 Installing system dependencies with zypper..." 13 | $SUDO zypper in -y portaudio-devel python3-devel gcc libgthread-2_0-0 14 | elif command -v dnf &> /dev/null; then 15 | echo "📦 Installing system dependencies with dnf..." 16 | $SUDO dnf install -y portaudio-devel python3-devel python3-pip 17 | elif command -v apt &> /dev/null; then 18 | echo "📦 Installing system dependencies with apt..." 19 | $SUDO apt update 20 | $SUDO apt install -y portaudio19-dev python3-dev python3-pip python3-venv 21 | elif command -v yum &> /dev/null; then 22 | echo "📦 Installing system dependencies with yum..." 23 | $SUDO yum install -y portaudio-devel python3-devel python3-pip 24 | elif command -v pacman &> /dev/null; then 25 | echo "📦 Installing system dependencies with pacman..." 26 | $SUDO pacman -S --noconfirm portaudio python python-pip 27 | else 28 | echo "⚠️ Unknown package manager. Please install portaudio-devel and python3-devel manually." 29 | fi 30 | 31 | # Create virtual environment 32 | echo "🐍 Creating Python virtual environment..." 33 | python3 -m venv jarvis-env 34 | 35 | # Activate virtual environment 36 | echo "🔧 Activating virtual environment..." 37 | source jarvis-env/bin/activate 38 | 39 | # Upgrade pip 40 | echo "⬆️ Upgrading pip..." 41 | python -m pip install --upgrade pip 42 | 43 | echo "🔧 Installing PyTorch..." 44 | pip3 install torch torchaudio --index-url https://download.pytorch.org/whl/cpu 45 | 46 | # Install Python dependencies 47 | echo "📚 Installing Python dependencies..." 48 | pip install -r requirements.txt 49 | 50 | # Check if OpenAI API key is set 51 | if [ -z "$OPENAI_API_KEY" ]; then 52 | echo "" 53 | echo "⚠️ WARNING: OPENAI_API_KEY environment variable is not set!" 54 | echo "💡 To use Jarvis, you need to set your OpenAI API key:" 55 | echo " export OPENAI_API_KEY='your-api-key-here'" 56 | echo "" 57 | echo "🔧 You can also create a .env file with:" 58 | echo " echo 'OPENAI_API_KEY=your-api-key-here' > .env" 59 | echo "" 60 | fi 61 | 62 | echo "" 63 | echo "✅ Setup completed successfully!" 64 | echo "" 65 | echo "🎯 To start Jarvis:" 66 | echo " 1. Set your API key: export OPENAI_API_KEY='your-key'" 67 | echo " 2. Run Jarvis: bash run.sh" 68 | echo "" 69 | echo "🎛️ Optional environment variables:" 70 | echo " OPENAI_MODEL=gpt-4 (default: gpt-3.5-turbo)" 71 | echo " OPENAI_TTS_VOICE=nova (default: alloy)" 72 | echo " OPENAI_TTS_MODEL=tts-1-hd (default: tts-1)" 73 | echo " OPENAI_BASE_URL=https://api.openai.com/v1" 74 | echo "" 75 | echo "" -------------------------------------------------------------------------------- /langchain/langchainjs-localai-example/src/index.mts: -------------------------------------------------------------------------------- 1 | import { loadQAStuffChain } from "langchain/chains"; 2 | import { Document } from "langchain/document"; 3 | import { pull } from "langchain/hub"; 4 | import { AgentExecutor, createOpenAIToolsAgent } from "langchain/agents"; 5 | import {Calculator} from "@langchain/community/tools/calculator"; 6 | import { ChatOpenAI } from "@langchain/openai"; 7 | import type { ChatPromptTemplate } from "@langchain/core/prompts"; 8 | 9 | const pathToLocalAI = process.env['OPENAI_API_BASE'] || 'http://api:8080/v1'; 10 | const fakeApiKey = process.env['OPENAI_API_KEY'] || '-'; 11 | const modelName = process.env['MODEL_NAME'] || 'gpt-3.5-turbo'; 12 | 13 | function getModel(): ChatOpenAI { 14 | return new ChatOpenAI({ 15 | prefixMessages: [ 16 | { 17 | role: "system", 18 | content: "You are a helpful assistant that answers in pirate language", 19 | }, 20 | ], 21 | modelName: modelName, 22 | maxTokens: 50, 23 | openAIApiKey: fakeApiKey, 24 | maxRetries: 2 25 | }, { 26 | basePath: pathToLocalAI, 27 | apiKey: fakeApiKey, 28 | }); 29 | } 30 | 31 | // Minimal example. 32 | export const run = async () => { 33 | const model = getModel(); 34 | console.log(`about to model.invoke at ${new Date().toUTCString()}`); 35 | const res = await model.invoke( 36 | "What would be a good company name a company that makes colorful socks?" 37 | ); 38 | console.log(`${new Date().toUTCString()}`); 39 | console.log({ res }); 40 | }; 41 | 42 | await run(); 43 | 44 | // This example uses the `StuffDocumentsChain` 45 | export const run2 = async () => { 46 | const model = getModel(); 47 | const chainA = loadQAStuffChain(model); 48 | const docs = [ 49 | new Document({ pageContent: "Harrison went to Harvard." }), 50 | new Document({ pageContent: "Ankush went to Princeton." }), 51 | ]; 52 | const resA = await chainA.invoke({ 53 | input_documents: docs, 54 | question: "Where did Harrison go to college?", 55 | }); 56 | console.log({ resA }); 57 | }; 58 | 59 | await run2(); 60 | 61 | // Quickly thrown together example of using tools + agents. 62 | // This seems like it should work, but it doesn't yet. 63 | export const toolAgentTest = async () => { 64 | const model = getModel(); 65 | 66 | const prompt = await pull("hwchase17/openai-tools-agent"); 67 | 68 | const tools = [new Calculator()]; 69 | 70 | const agent = await createOpenAIToolsAgent({ 71 | llm: model, 72 | tools: tools, 73 | prompt: prompt 74 | }); 75 | 76 | console.log("Loaded agent."); 77 | 78 | const agentExecutor = new AgentExecutor({ 79 | agent, 80 | tools, 81 | }); 82 | 83 | const input = `What is the value of (500 *2) + 350 - 13?`; 84 | 85 | console.log(`Executing with input "${input}"...`); 86 | 87 | const result = await agentExecutor.invoke({ input }); 88 | 89 | console.log(`Got output ${result.output}`); 90 | } 91 | 92 | await toolAgentTest(); 93 | -------------------------------------------------------------------------------- /github-actions/workflow.yml: -------------------------------------------------------------------------------- 1 | name: Use LocalAI in GHA 2 | on: 3 | pull_request: 4 | types: 5 | - closed 6 | 7 | jobs: 8 | notify-discord: 9 | if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'area/ai-model')) }} 10 | env: 11 | MODEL_NAME: qwen_qwen3-4b-instruct-2507 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | with: 16 | fetch-depth: 0 # needed to checkout all branches for this Action to work 17 | # Starts the LocalAI container 18 | - id: foo 19 | uses: mudler/localai-github-action@v1.1 20 | with: 21 | model: 'qwen_qwen3-4b-instruct-2507' # Any from models.localai.io, or from huggingface.com with: "huggingface:///file" 22 | # Check the PR diff using the current branch and the base branch of the PR 23 | - uses: GrantBirki/git-diff-action@v2.7.0 24 | id: git-diff-action 25 | with: 26 | json_diff_file_output: diff.json 27 | raw_diff_file_output: diff.txt 28 | file_output_only: "true" 29 | # Ask to explain the diff to LocalAI 30 | - name: Summarize 31 | env: 32 | DIFF: ${{ steps.git-diff-action.outputs.raw-diff-path }} 33 | id: summarize 34 | run: | 35 | input="$(cat $DIFF)" 36 | 37 | # Define the LocalAI API endpoint 38 | API_URL="http://localhost:8080/chat/completions" 39 | 40 | # Create a JSON payload using jq to handle special characters 41 | json_payload=$(jq -n --arg input "$input" '{ 42 | model: "'$MODEL_NAME'", 43 | messages: [ 44 | { 45 | role: "system", 46 | content: "Write a message summarizing the change diffs" 47 | }, 48 | { 49 | role: "user", 50 | content: $input 51 | } 52 | ] 53 | }') 54 | 55 | # Send the request to LocalAI 56 | response=$(curl -s -X POST $API_URL \ 57 | -H "Content-Type: application/json" \ 58 | -d "$json_payload") 59 | 60 | # Extract the summary from the response 61 | summary="$(echo $response | jq -r '.choices[0].message.content')" 62 | 63 | # Print the summary 64 | # -H "Authorization: Bearer $API_KEY" \ 65 | echo "Summary:" 66 | echo "$summary" 67 | echo "payload sent" 68 | echo "$json_payload" 69 | { 70 | echo 'message<> "$GITHUB_OUTPUT" 74 | # Send the summary somewhere (e.g. Discord) 75 | - name: Discord notification 76 | env: 77 | DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK_URL }} 78 | DISCORD_USERNAME: "discord-bot" 79 | DISCORD_AVATAR: "" 80 | uses: Ilshidur/action-discord@master 81 | with: 82 | args: ${{ steps.summarize.outputs.message }} -------------------------------------------------------------------------------- /configurations/README.md: -------------------------------------------------------------------------------- 1 | ## Advanced configuration 2 | 3 | This section contains examples on how to install models manually with config files. 4 | 5 | ### Prerequisites 6 | 7 | First clone LocalAI: 8 | 9 | ```bash 10 | git clone https://github.com/go-skynet/LocalAI 11 | 12 | cd LocalAI 13 | ``` 14 | 15 | Setup the model you prefer from the examples below and then start LocalAI: 16 | 17 | ```bash 18 | docker compose up -d --pull always 19 | ``` 20 | 21 | If LocalAI is already started, you can restart it with 22 | 23 | ```bash 24 | docker compose restart 25 | ``` 26 | 27 | See also the getting started: https://localai.io/basics/getting_started/ 28 | 29 | You can also start LocalAI just with docker: 30 | 31 | ``` 32 | docker run -p 8080:8080 -v $PWD/models:/models -ti --rm quay.io/go-skynet/local-ai:master --models-path /models --threads 4 33 | ``` 34 | 35 | ### Mistral 36 | 37 | To setup mistral copy the files inside `mistral` in the `models` folder: 38 | 39 | ```bash 40 | cp -r examples/configurations/mistral/* models/ 41 | ``` 42 | 43 | Now download the model: 44 | 45 | ```bash 46 | wget https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-GGUF/resolve/main/mistral-7b-openorca.Q6_K.gguf -O models/mistral-7b-openorca.Q6_K.gguf 47 | ``` 48 | 49 | ### LLaVA 50 | 51 | ![llava](https://github.com/mudler/LocalAI/assets/2420543/cb0a0897-3b58-4350-af66-e6f4387b58d3) 52 | 53 | #### Setup 54 | 55 | ``` 56 | cp -r examples/configurations/llava/* models/ 57 | wget https://huggingface.co/mys/ggml_bakllava-1/resolve/main/ggml-model-q4_k.gguf -O models/ggml-model-q4_k.gguf 58 | wget https://huggingface.co/mys/ggml_bakllava-1/resolve/main/mmproj-model-f16.gguf -O models/mmproj-model-f16.gguf 59 | ``` 60 | 61 | #### Try it out 62 | 63 | ``` 64 | curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ 65 | "model": "llava", 66 | "messages": [{"role": "user", "content": [{"type":"text", "text": "What is in the image?"}, {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" }}], "temperature": 0.9}]}' 67 | 68 | ``` 69 | 70 | ### Phi-2 71 | 72 | ``` 73 | cp -r examples/configurations/phi-2.yaml models/ 74 | 75 | curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ 76 | "model": "phi-2", 77 | "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}] 78 | }' 79 | ``` 80 | 81 | ### Mixtral 82 | 83 | ``` 84 | cp -r examples/configuration/mixtral/* models/ 85 | wget https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q2_K.gguf -O models/mixtral-8x7b-instruct-v0.1.Q2_K.gguf 86 | ``` 87 | 88 | #### Test it out 89 | 90 | ``` 91 | curl http://localhost:8080/v1/completions -H "Content-Type: application/json" -d '{ 92 | "model": "mixtral", 93 | "prompt": "How fast is light?", 94 | "temperature": 0.1 }' 95 | ``` 96 | -------------------------------------------------------------------------------- /slack-qa-bot/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: slack-bot 5 | --- 6 | apiVersion: v1 7 | kind: PersistentVolumeClaim 8 | metadata: 9 | name: knowledgebase 10 | namespace: slack-bot 11 | labels: 12 | app: localai-qabot 13 | spec: 14 | accessModes: 15 | - ReadWriteOnce 16 | resources: 17 | requests: 18 | storage: 5Gi 19 | --- 20 | apiVersion: apps/v1 21 | kind: Deployment 22 | metadata: 23 | name: localai-qabot 24 | namespace: slack-bot 25 | labels: 26 | app: localai-qabot 27 | spec: 28 | selector: 29 | matchLabels: 30 | app: localai-qabot 31 | replicas: 1 32 | template: 33 | metadata: 34 | labels: 35 | app: localai-qabot 36 | name: localai-qabot 37 | spec: 38 | containers: 39 | - name: localai-qabot-slack 40 | env: 41 | - name: OPENAI_API_KEY 42 | value: "x" 43 | - name: SLACK_APP_TOKEN 44 | value: "xapp-1-" 45 | - name: SLACK_BOT_TOKEN 46 | value: "xoxb-" 47 | - name: OPENAI_MODEL 48 | value: "gpt-3.5-turbo" 49 | - name: OPENAI_TIMEOUT_SECONDS 50 | value: "400" 51 | - name: OPENAI_SYSTEM_TEXT 52 | value: "" 53 | - name: MEMORY_DIR 54 | value: "/memory" 55 | - name: TRANSLATE_MARKDOWN 56 | value: "true" 57 | - name: OPENAI_API_BASE 58 | value: "http://local-ai.default.svc.cluster.local:8080" 59 | - name: REPOSITORIES 60 | value: "KAIROS,AGENT,SDK,OSBUILDER,PACKAGES,IMMUCORE" 61 | - name: KAIROS_CLONE_URL 62 | value: "https://github.com/kairos-io/kairos" 63 | - name: KAIROS_BRANCH 64 | value: "master" 65 | - name: AGENT_CLONE_URL 66 | value: "https://github.com/kairos-io/kairos-agent" 67 | - name: AGENT_BRANCH 68 | value: "main" 69 | - name: SDK_CLONE_URL 70 | value: "https://github.com/kairos-io/kairos-sdk" 71 | - name: SDK_BRANCH 72 | value: "main" 73 | - name: OSBUILDER_CLONE_URL 74 | value: "https://github.com/kairos-io/osbuilder" 75 | - name: OSBUILDER_BRANCH 76 | value: "master" 77 | - name: PACKAGES_CLONE_URL 78 | value: "https://github.com/kairos-io/packages" 79 | - name: PACKAGES_BRANCH 80 | value: "main" 81 | - name: IMMUCORE_CLONE_URL 82 | value: "https://github.com/kairos-io/immucore" 83 | - name: IMMUCORE_BRANCH 84 | value: "master" 85 | - name: GITHUB_PERSONAL_ACCESS_TOKEN 86 | value: "" 87 | - name: ISSUE_REPOSITORIES 88 | value: "kairos-io/kairos" 89 | image: quay.io/spectrocloud-labs/slack-qa-local-bot:qa 90 | imagePullPolicy: Always 91 | volumeMounts: 92 | - mountPath: "/memory" 93 | name: knowledgebase 94 | volumes: 95 | - name: knowledgebase 96 | persistentVolumeClaim: 97 | claimName: knowledgebase -------------------------------------------------------------------------------- /realtime/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Jarvis Voice Assistant Run Script 4 | # This script activates the virtual environment and starts Jarvis 5 | 6 | # Change to the directory where this script is located 7 | pushd "$(dirname "$0")" > /dev/null 8 | 9 | set -e # Exit on any error 10 | 11 | # Colors for output 12 | RED='\033[0;31m' 13 | GREEN='\033[0;32m' 14 | YELLOW='\033[1;33m' 15 | BLUE='\033[0;34m' 16 | NC='\033[0m' # No Color 17 | 18 | echo -e "${BLUE}🚀 Starting Jarvis Voice Assistant...${NC}" 19 | 20 | # Check if virtual environment exists 21 | if [ ! -d "jarvis-env" ]; then 22 | echo -e "${RED}❌ Virtual environment not found!${NC}" 23 | echo -e "${YELLOW}💡 Please run ./setup.sh first to set up the environment.${NC}" 24 | exit 1 25 | fi 26 | 27 | # Check if requirements are installed 28 | if [ ! -f "jarvis-env/pyvenv.cfg" ]; then 29 | echo -e "${RED}❌ Virtual environment appears to be corrupted!${NC}" 30 | echo -e "${YELLOW}💡 Please run ./setup.sh again to recreate the environment.${NC}" 31 | exit 1 32 | fi 33 | 34 | # Activate virtual environment 35 | echo -e "${GREEN}🔧 Activating virtual environment...${NC}" 36 | source jarvis-env/bin/activate 37 | 38 | # Check if OpenAI API key is set 39 | if [ -z "$OPENAI_API_KEY" ]; then 40 | echo -e "${YELLOW}⚠️ WARNING: OPENAI_API_KEY environment variable is not set!${NC}" 41 | echo -e "${BLUE}💡 Please set your OpenAI API key:${NC}" 42 | echo -e " ${YELLOW}export OPENAI_API_KEY='your-api-key-here'${NC}" 43 | echo "" 44 | echo -e "${BLUE}🔧 Or create a .env file:${NC}" 45 | echo -e " ${YELLOW}echo 'OPENAI_API_KEY=your-api-key-here' > .env${NC}" 46 | echo "" 47 | echo -e "${BLUE}🎛️ Optional environment variables:${NC}" 48 | echo -e " ${YELLOW}OPENAI_MODEL=gpt-4${NC} (default: gpt-3.5-turbo)" 49 | echo -e " ${YELLOW}OPENAI_TTS_VOICE=nova${NC} (default: alloy)" 50 | echo -e " ${YELLOW}OPENAI_TTS_MODEL=tts-1-hd${NC} (default: tts-1)" 51 | echo -e " ${YELLOW}OPENAI_BASE_URL=https://api.openai.com/v1${NC}" 52 | echo -e " ${YELLOW}BACKGROUND_AUDIO=false${NC} (default: true)" 53 | echo "" 54 | echo -e "${BLUE}🎤 Available TTS voices:${NC} ${YELLOW}alloy, echo, fable, onyx, nova, shimmer${NC}" 55 | echo "" 56 | echo -e "${RED}❌ Cannot start without API key. Exiting...${NC}" 57 | exit 1 58 | fi 59 | 60 | # Check if realtime.py exists 61 | if [ ! -f "realtime.py" ]; then 62 | echo -e "${RED}❌ realtime.py not found!${NC}" 63 | echo -e "${YELLOW}💡 Please make sure you're in the correct directory.${NC}" 64 | exit 1 65 | fi 66 | 67 | # Show current configuration 68 | echo -e "${GREEN}✅ Configuration:${NC}" 69 | echo -e " ${BLUE}📝 Chat Model:${NC} ${YELLOW}${OPENAI_MODEL:-gpt-3.5-turbo}${NC}" 70 | echo -e " ${BLUE}🎤 TTS Model:${NC} ${YELLOW}${OPENAI_TTS_MODEL:-tts-1}${NC}" 71 | echo -e " ${BLUE}🗣️ TTS Voice:${NC} ${YELLOW}${OPENAI_TTS_VOICE:-alloy}${NC}" 72 | echo -e " ${BLUE}🌐 Base URL:${NC} ${YELLOW}${OPENAI_BASE_URL:-https://api.openai.com/v1}${NC}" 73 | echo "" 74 | 75 | # Start Jarvis 76 | echo -e "${GREEN}🎯 Starting Jarvis...${NC}" 77 | echo -e "${BLUE}💡 Press Ctrl+C to stop${NC}" 78 | echo "" 79 | 80 | python realtime.py -------------------------------------------------------------------------------- /chainlit/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import weaviate 4 | from llama_index.storage.storage_context import StorageContext 5 | from llama_index.vector_stores import WeaviateVectorStore 6 | 7 | from llama_index.query_engine.retriever_query_engine import RetrieverQueryEngine 8 | from llama_index.callbacks.base import CallbackManager 9 | from llama_index import ( 10 | LLMPredictor, 11 | ServiceContext, 12 | StorageContext, 13 | VectorStoreIndex, 14 | ) 15 | import chainlit as cl 16 | 17 | from llama_index.llms import LocalAI 18 | from llama_index.embeddings import HuggingFaceEmbedding 19 | import yaml 20 | 21 | # Load the configuration file 22 | with open("config.yaml", "r") as ymlfile: 23 | cfg = yaml.safe_load(ymlfile) 24 | 25 | # Get the values from the configuration file or set the default values 26 | temperature = cfg['localAI'].get('temperature', 0) 27 | model_name = cfg['localAI'].get('modelName', "gpt-3.5-turbo") 28 | api_base = cfg['localAI'].get('apiBase', "http://local-ai.default") 29 | api_key = cfg['localAI'].get('apiKey', "stub") 30 | streaming = cfg['localAI'].get('streaming', True) 31 | weaviate_url = cfg['weviate'].get('url', "http://weviate.default") 32 | index_name = cfg['weviate'].get('index', "AIChroma") 33 | query_mode = cfg['query'].get('mode', "hybrid") 34 | topK = cfg['query'].get('topK', 1) 35 | alpha = cfg['query'].get('alpha', 0.0) 36 | embed_model_name = cfg['embedding'].get('model', "BAAI/bge-small-en-v1.5") 37 | chunk_size = cfg['query'].get('chunkSize', 1024) 38 | 39 | 40 | embed_model = HuggingFaceEmbedding(model_name=embed_model_name) 41 | 42 | 43 | llm = LocalAI(temperature=temperature, model_name=model_name, api_base=api_base, api_key=api_key, streaming=streaming) 44 | llm.globally_use_chat_completions = True; 45 | client = weaviate.Client(weaviate_url) 46 | vector_store = WeaviateVectorStore(weaviate_client=client, index_name=index_name) 47 | storage_context = StorageContext.from_defaults(vector_store=vector_store) 48 | 49 | @cl.on_chat_start 50 | async def factory(): 51 | 52 | llm_predictor = LLMPredictor( 53 | llm=llm 54 | ) 55 | 56 | service_context = ServiceContext.from_defaults(embed_model=embed_model, callback_manager=CallbackManager([cl.LlamaIndexCallbackHandler()]), llm_predictor=llm_predictor, chunk_size=chunk_size) 57 | 58 | index = VectorStoreIndex.from_vector_store( 59 | vector_store, 60 | storage_context=storage_context, 61 | service_context=service_context 62 | ) 63 | 64 | query_engine = index.as_query_engine(vector_store_query_mode=query_mode, similarity_top_k=topK, alpha=alpha, streaming=True) 65 | 66 | cl.user_session.set("query_engine", query_engine) 67 | 68 | 69 | @cl.on_message 70 | async def main(message: cl.Message): 71 | query_engine = cl.user_session.get("query_engine") 72 | response = await cl.make_async(query_engine.query)(message.content) 73 | 74 | response_message = cl.Message(content="") 75 | 76 | for token in response.response_gen: 77 | await response_message.stream_token(token=token) 78 | 79 | if response.response_txt: 80 | response_message.content = response.response_txt 81 | 82 | await response_message.send() 83 | -------------------------------------------------------------------------------- /functions/functions-openai.py: -------------------------------------------------------------------------------- 1 | import openai 2 | import json 3 | 4 | # Example dummy function hard coded to return the same weather 5 | # In production, this could be your backend API or an external API 6 | def get_current_weather(location, unit="fahrenheit"): 7 | """Get the current weather in a given location""" 8 | weather_info = { 9 | "location": location, 10 | "temperature": "72", 11 | "unit": unit, 12 | "forecast": ["sunny", "windy"], 13 | } 14 | return json.dumps(weather_info) 15 | 16 | 17 | def run_conversation(): 18 | # Step 1: send the conversation and available functions to GPT 19 | messages = [{"role": "user", "content": "What's the weather like in Boston?"}] 20 | functions = [ 21 | { 22 | "name": "get_current_weather", 23 | "description": "Get the current weather in a given location", 24 | "parameters": { 25 | "type": "object", 26 | "properties": { 27 | "location": { 28 | "type": "string", 29 | "description": "The city and state, e.g. San Francisco, CA", 30 | }, 31 | "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, 32 | }, 33 | "required": ["location"], 34 | }, 35 | } 36 | ] 37 | response = openai.ChatCompletion.create( 38 | model="gpt-3.5-turbo", 39 | messages=messages, 40 | functions=functions, 41 | function_call="auto", # auto is default, but we'll be explicit 42 | ) 43 | response_message = response["choices"][0]["message"] 44 | 45 | # Step 2: check if GPT wanted to call a function 46 | if response_message.get("function_call"): 47 | # Step 3: call the function 48 | # Note: the JSON response may not always be valid; be sure to handle errors 49 | available_functions = { 50 | "get_current_weather": get_current_weather, 51 | } # only one function in this example, but you can have multiple 52 | function_name = response_message["function_call"]["name"] 53 | fuction_to_call = available_functions[function_name] 54 | function_args = json.loads(response_message["function_call"]["arguments"]) 55 | function_response = fuction_to_call( 56 | location=function_args.get("location"), 57 | unit=function_args.get("unit"), 58 | ) 59 | 60 | # Step 4: send the info on the function call and function response to GPT 61 | messages.append(response_message) # extend conversation with assistant's reply 62 | messages.append( 63 | { 64 | "role": "function", 65 | "name": function_name, 66 | "content": function_response, 67 | } 68 | ) # extend conversation with function response 69 | second_response = openai.ChatCompletion.create( 70 | model="gpt-3.5-turbo", 71 | messages=messages, 72 | ) # get a new response from GPT where it can see the function response 73 | return second_response 74 | 75 | 76 | print(run_conversation()) -------------------------------------------------------------------------------- /k8sgpt/values.yaml: -------------------------------------------------------------------------------- 1 | replicaCount: 1 2 | 3 | deployment: 4 | # https://quay.io/repository/go-skynet/local-ai?tab=tags 5 | image: quay.io/go-skynet/local-ai:v1.40.0 6 | env: 7 | threads: 4 8 | debug: "true" 9 | context_size: 512 10 | galleries: '[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/index.yaml"}, {"url": "github:go-skynet/model-gallery/huggingface.yaml","name":"huggingface"}]' 11 | preload_models: '[{ "id": "huggingface@thebloke__open-llama-13b-open-instruct-ggml__open-llama-13b-open-instruct.ggmlv3.q3_k_m.bin", "name": "gpt-3.5-turbo", "overrides": { "f16": true, "mmap": true }}]' 12 | modelsPath: "/models" 13 | 14 | resources: 15 | {} 16 | # We usually recommend not to specify default resources and to leave this as a conscious 17 | # choice for the user. This also increases chances charts run on environments with little 18 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 19 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 20 | # limits: 21 | # cpu: 100m 22 | # memory: 128Mi 23 | # requests: 24 | # cpu: 100m 25 | # memory: 128Mi 26 | 27 | # Prompt templates to include 28 | # Note: the keys of this map will be the names of the prompt template files 29 | promptTemplates: 30 | {} 31 | # ggml-gpt4all-j.tmpl: | 32 | # The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response. 33 | # ### Prompt: 34 | # {{.Input}} 35 | # ### Response: 36 | 37 | # Models to download at runtime 38 | models: 39 | # Whether to force download models even if they already exist 40 | forceDownload: false 41 | 42 | # The list of URLs to download models from 43 | # Note: the name of the file will be the name of the loaded model 44 | list: 45 | #- url: "https://gpt4all.io/models/ggml-gpt4all-j.bin" 46 | # basicAuth: base64EncodedCredentials 47 | 48 | # Persistent storage for models and prompt templates. 49 | # PVC and HostPath are mutually exclusive. If both are enabled, 50 | # PVC configuration takes precedence. If neither are enabled, ephemeral 51 | # storage is used. 52 | persistence: 53 | pvc: 54 | enabled: false 55 | size: 6Gi 56 | accessModes: 57 | - ReadWriteOnce 58 | 59 | annotations: {} 60 | 61 | # Optional 62 | storageClass: ~ 63 | 64 | hostPath: 65 | enabled: false 66 | path: "/models" 67 | 68 | service: 69 | type: ClusterIP 70 | port: 8080 71 | annotations: {} 72 | # If using an AWS load balancer, you'll need to override the default 60s load balancer idle timeout 73 | # service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "1200" 74 | 75 | ingress: 76 | enabled: false 77 | className: "" 78 | annotations: 79 | {} 80 | # kubernetes.io/ingress.class: nginx 81 | # kubernetes.io/tls-acme: "true" 82 | hosts: 83 | - host: chart-example.local 84 | paths: 85 | - path: / 86 | pathType: ImplementationSpecific 87 | tls: [] 88 | # - secretName: chart-example-tls 89 | # hosts: 90 | # - chart-example.local 91 | 92 | nodeSelector: {} 93 | 94 | tolerations: [] 95 | 96 | affinity: {} 97 | -------------------------------------------------------------------------------- /streamlit-bot/start_windows.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | cd /D "%~dp0" 4 | 5 | set PATH=%PATH%;%SystemRoot%\system32 6 | 7 | echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end 8 | 9 | @rem Check for special characters in installation path 10 | set "SPCHARMESSAGE="WARNING: Special characters were detected in the installation path!" " This can cause the installation to fail!"" 11 | echo "%CD%"| findstr /R /C:"[!#\$%&()\*+,;<=>?@\[\]\^`{|}~]" >nul && ( 12 | call :PrintBigMessage %SPCHARMESSAGE% 13 | ) 14 | set SPCHARMESSAGE= 15 | 16 | @rem fix failed install when installing to a separate drive 17 | set TMP=%cd%\installer_files 18 | set TEMP=%cd%\installer_files 19 | 20 | @rem config 21 | set INSTALL_DIR=%cd%\installer_files 22 | set CONDA_ROOT_PREFIX=%cd%\installer_files\conda 23 | set INSTALL_ENV_DIR=%cd%\installer_files\env 24 | set MINICONDA_DOWNLOAD_URL=https://repo.anaconda.com/miniconda/Miniconda3-py310_23.3.1-0-Windows-x86_64.exe 25 | set conda_exists=F 26 | 27 | @rem figure out whether git and conda needs to be installed 28 | call "%CONDA_ROOT_PREFIX%\_conda.exe" --version >nul 2>&1 29 | if "%ERRORLEVEL%" EQU "0" set conda_exists=T 30 | 31 | @rem (if necessary) install git and conda into a contained environment 32 | @rem download conda 33 | if "%conda_exists%" == "F" ( 34 | echo Downloading Miniconda from %MINICONDA_DOWNLOAD_URL% to %INSTALL_DIR%\miniconda_installer.exe 35 | 36 | mkdir "%INSTALL_DIR%" 37 | call curl -Lk "%MINICONDA_DOWNLOAD_URL%" > "%INSTALL_DIR%\miniconda_installer.exe" || ( echo. && echo Miniconda failed to download. && goto end ) 38 | 39 | echo Installing Miniconda to %CONDA_ROOT_PREFIX% 40 | start /wait "" "%INSTALL_DIR%\miniconda_installer.exe" /InstallationType=JustMe /NoShortcuts=1 /AddToPath=0 /RegisterPython=0 /NoRegistry=1 /S /D=%CONDA_ROOT_PREFIX% 41 | 42 | @rem test the conda binary 43 | echo Miniconda version: 44 | call "%CONDA_ROOT_PREFIX%\_conda.exe" --version || ( echo. && echo Miniconda not found. && goto end ) 45 | ) 46 | 47 | @rem create the installer env 48 | if not exist "%INSTALL_ENV_DIR%" ( 49 | echo Packages to install: %PACKAGES_TO_INSTALL% 50 | call "%CONDA_ROOT_PREFIX%\_conda.exe" create --no-shortcuts -y -k --prefix "%INSTALL_ENV_DIR%" python=3.10 || ( echo. && echo Conda environment creation failed. && goto end ) 51 | ) 52 | 53 | @rem check if conda environment was actually created 54 | if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is empty. && goto end ) 55 | 56 | @rem environment isolation 57 | set PYTHONNOUSERSITE=1 58 | set PYTHONPATH= 59 | set PYTHONHOME= 60 | set "CUDA_PATH=%INSTALL_ENV_DIR%" 61 | set "CUDA_HOME=%CUDA_PATH%" 62 | 63 | @rem activate installer env 64 | call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo Miniconda hook not found. && goto end ) 65 | 66 | @rem setup installer env 67 | streamlit run Main.py 68 | 69 | @rem below are functions for the script next line skips these during normal execution 70 | goto end 71 | 72 | :PrintBigMessage 73 | echo. && echo. 74 | echo ******************************************************************* 75 | for %%M in (%*) do echo * %%~M 76 | echo ******************************************************************* 77 | echo. && echo. 78 | exit /b 79 | 80 | :end 81 | pause -------------------------------------------------------------------------------- /streamlit-bot/install_requirements.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | cd /D "%~dp0" 4 | 5 | set PATH=%PATH%;%SystemRoot%\system32 6 | 7 | echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end 8 | 9 | @rem Check for special characters in installation path 10 | set "SPCHARMESSAGE="WARNING: Special characters were detected in the installation path!" " This can cause the installation to fail!"" 11 | echo "%CD%"| findstr /R /C:"[!#\$%&()\*+,;<=>?@\[\]\^`{|}~]" >nul && ( 12 | call :PrintBigMessage %SPCHARMESSAGE% 13 | ) 14 | set SPCHARMESSAGE= 15 | 16 | @rem fix failed install when installing to a separate drive 17 | set TMP=%cd%\installer_files 18 | set TEMP=%cd%\installer_files 19 | 20 | @rem config 21 | set INSTALL_DIR=%cd%\installer_files 22 | set CONDA_ROOT_PREFIX=%cd%\installer_files\conda 23 | set INSTALL_ENV_DIR=%cd%\installer_files\env 24 | set MINICONDA_DOWNLOAD_URL=https://repo.anaconda.com/miniconda/Miniconda3-py310_23.3.1-0-Windows-x86_64.exe 25 | set conda_exists=F 26 | 27 | @rem figure out whether git and conda needs to be installed 28 | call "%CONDA_ROOT_PREFIX%\_conda.exe" --version >nul 2>&1 29 | if "%ERRORLEVEL%" EQU "0" set conda_exists=T 30 | 31 | @rem (if necessary) install git and conda into a contained environment 32 | @rem download conda 33 | if "%conda_exists%" == "F" ( 34 | echo Downloading Miniconda from %MINICONDA_DOWNLOAD_URL% to %INSTALL_DIR%\miniconda_installer.exe 35 | 36 | mkdir "%INSTALL_DIR%" 37 | call curl -Lk "%MINICONDA_DOWNLOAD_URL%" > "%INSTALL_DIR%\miniconda_installer.exe" || ( echo. && echo Miniconda failed to download. && goto end ) 38 | 39 | echo Installing Miniconda to %CONDA_ROOT_PREFIX% 40 | start /wait "" "%INSTALL_DIR%\miniconda_installer.exe" /InstallationType=JustMe /NoShortcuts=1 /AddToPath=0 /RegisterPython=0 /NoRegistry=1 /S /D=%CONDA_ROOT_PREFIX% 41 | 42 | @rem test the conda binary 43 | echo Miniconda version: 44 | call "%CONDA_ROOT_PREFIX%\_conda.exe" --version || ( echo. && echo Miniconda not found. && goto end ) 45 | ) 46 | 47 | @rem create the installer env 48 | if not exist "%INSTALL_ENV_DIR%" ( 49 | echo Packages to install: %PACKAGES_TO_INSTALL% 50 | call "%CONDA_ROOT_PREFIX%\_conda.exe" create --no-shortcuts -y -k --prefix "%INSTALL_ENV_DIR%" python=3.10 || ( echo. && echo Conda environment creation failed. && goto end ) 51 | ) 52 | 53 | @rem check if conda environment was actually created 54 | if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is empty. && goto end ) 55 | 56 | @rem environment isolation 57 | set PYTHONNOUSERSITE=1 58 | set PYTHONPATH= 59 | set PYTHONHOME= 60 | set "CUDA_PATH=%INSTALL_ENV_DIR%" 61 | set "CUDA_HOME=%CUDA_PATH%" 62 | 63 | @rem activate installer env 64 | call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo Miniconda hook not found. && goto end ) 65 | 66 | @rem setup installer env 67 | call pip install -r requirements.txt 68 | 69 | @rem below are functions for the script next line skips these during normal execution 70 | goto end 71 | 72 | :PrintBigMessage 73 | echo. && echo. 74 | echo ******************************************************************* 75 | for %%M in (%*) do echo * %%~M 76 | echo ******************************************************************* 77 | echo. && echo. 78 | exit /b 79 | 80 | :end 81 | pause -------------------------------------------------------------------------------- /e2e-fine-tuning/README.md: -------------------------------------------------------------------------------- 1 | This is an example of fine-tuning a LLM model to use with [LocalAI](https://github.com/mudler/LocalAI) written by [@mudler](https://github.com/mudler). 2 | 3 | Specifically, this example shows how to use [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) to fine-tune a LLM model to consume with LocalAI as a `gguf` model. 4 | 5 | A notebook is provided that currently works on _very small_ datasets on Google colab on the free instance. It is far from producing good models, but it gives a sense of how to use the code to use with a better dataset and configurations, and how to use the model produced with LocalAI. [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mudler/LocalAI/blob/master/examples/e2e-fine-tuning/notebook.ipynb) 6 | 7 | ## Requirements 8 | 9 | For this example you will need at least a 12GB VRAM of GPU and a Linux box. 10 | The notebook is tested on Google Colab with a Tesla T4 GPU. 11 | 12 | ## Clone this directory 13 | 14 | Clone the repository and enter the example directory: 15 | 16 | ```bash 17 | git clone http://github.com/mudler/LocalAI 18 | cd LocalAI/examples/e2e-fine-tuning 19 | ``` 20 | 21 | ## Install dependencies 22 | 23 | ```bash 24 | # Install axolotl and dependencies 25 | git clone https://github.com/OpenAccess-AI-Collective/axolotl && pushd axolotl && git checkout 797f3dd1de8fd8c0eafbd1c9fdb172abd9ff840a && popd #0.3.0 26 | pip install packaging 27 | pushd axolotl && pip install -e '.[flash-attn,deepspeed]' && popd 28 | 29 | # https://github.com/oobabooga/text-generation-webui/issues/4238 30 | pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.0/flash_attn-2.3.0+cu117torch2.0cxx11abiFALSE-cp310-cp310-linux_x86_64.whl 31 | ``` 32 | 33 | Configure accelerate: 34 | 35 | ```bash 36 | accelerate config default 37 | ``` 38 | 39 | ## Fine-tuning 40 | 41 | We will need to configure axolotl. In this example is provided a file to use `axolotl.yaml` that uses openllama-3b for fine-tuning. Copy the `axolotl.yaml` file and edit it to your needs. The dataset needs to be next to it as `dataset.json`. The format used is `completion` which is a list of JSON objects with a `text` field with the full text to train the LLM with. 42 | 43 | If you have a big dataset, you can pre-tokenize it to speedup the fine-tuning process: 44 | 45 | ```bash 46 | # Optional pre-tokenize (run only if big dataset) 47 | python -m axolotl.cli.preprocess axolotl.yaml 48 | ``` 49 | 50 | Now we are ready to start the fine-tuning process: 51 | ```bash 52 | # Fine-tune 53 | accelerate launch -m axolotl.cli.train axolotl.yaml 54 | ``` 55 | 56 | After we have finished the fine-tuning, we merge the Lora base with the model: 57 | ```bash 58 | # Merge lora 59 | python3 -m axolotl.cli.merge_lora axolotl.yaml --lora_model_dir="./qlora-out" --load_in_8bit=False --load_in_4bit=False 60 | ``` 61 | 62 | And we convert it to the gguf format that LocalAI can consume: 63 | 64 | ```bash 65 | 66 | # Convert to gguf 67 | git clone https://github.com/ggerganov/llama.cpp.git 68 | pushd llama.cpp && make GGML_CUDA=1 && popd 69 | 70 | # We need to convert the pytorch model into ggml for quantization 71 | # It crates 'ggml-model-f16.bin' in the 'merged' directory. 72 | pushd llama.cpp && python convert.py --outtype f16 \ 73 | ../qlora-out/merged/pytorch_model-00001-of-00002.bin && popd 74 | 75 | # Start off by making a basic q4_0 4-bit quantization. 76 | # It's important to have 'ggml' in the name of the quant for some 77 | # software to recognize it's file format. 78 | pushd llama.cpp && ./quantize ../qlora-out/merged/ggml-model-f16.gguf \ 79 | ../custom-model-q4_0.bin q4_0 80 | 81 | ``` 82 | 83 | Now you should have ended up with a `custom-model-q4_0.bin` file that you can copy in the LocalAI models directory and use it with LocalAI. 84 | -------------------------------------------------------------------------------- /semantic-todo/go.sum: -------------------------------------------------------------------------------- 1 | github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= 2 | github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= 3 | github.com/gdamore/tcell/v2 v2.7.4 h1:sg6/UnTM9jGpZU+oFYAsDahfchWAFW8Xx2yFinNSAYU= 4 | github.com/gdamore/tcell/v2 v2.7.4/go.mod h1:dSXtXTSK0VsW1biw65DZLZ2NKr7j0qP/0J7ONmsraWg= 5 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 6 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 7 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 8 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 9 | github.com/rivo/tview v0.0.0-20240524063012-037df494fb76 h1:iqvDlgyjmqleATtFbA7c14djmPh2n4mCYUv7JlD/ruA= 10 | github.com/rivo/tview v0.0.0-20240524063012-037df494fb76/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= 11 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 12 | github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 13 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 14 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 15 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 16 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 17 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 18 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 19 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 20 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 21 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 22 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 23 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 24 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 25 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 26 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 27 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 28 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 29 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 30 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 31 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 33 | golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= 34 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 35 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 36 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 37 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 38 | golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= 39 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= 40 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 41 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 42 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 43 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 44 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 45 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 46 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 47 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 48 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 49 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 50 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 51 | -------------------------------------------------------------------------------- /continue/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is the Continue configuration file. 3 | 4 | See https://continue.dev/docs/customization to learn more. 5 | """ 6 | 7 | import subprocess 8 | 9 | from continuedev.src.continuedev.core.main import Step 10 | from continuedev.src.continuedev.core.sdk import ContinueSDK 11 | from continuedev.src.continuedev.core.models import Models 12 | from continuedev.src.continuedev.core.config import CustomCommand, SlashCommand, ContinueConfig 13 | from continuedev.src.continuedev.plugins.context_providers.github import GitHubIssuesContextProvider 14 | from continuedev.src.continuedev.plugins.context_providers.google import GoogleContextProvider 15 | from continuedev.src.continuedev.plugins.policies.default import DefaultPolicy 16 | from continuedev.src.continuedev.libs.llm.openai import OpenAI, OpenAIServerInfo 17 | from continuedev.src.continuedev.libs.llm.ggml import GGML 18 | 19 | from continuedev.src.continuedev.plugins.steps.open_config import OpenConfigStep 20 | from continuedev.src.continuedev.plugins.steps.clear_history import ClearHistoryStep 21 | from continuedev.src.continuedev.plugins.steps.feedback import FeedbackStep 22 | from continuedev.src.continuedev.plugins.steps.comment_code import CommentCodeStep 23 | from continuedev.src.continuedev.plugins.steps.share_session import ShareSessionStep 24 | from continuedev.src.continuedev.plugins.steps.main import EditHighlightedCodeStep 25 | from continuedev.src.continuedev.plugins.context_providers.search import SearchContextProvider 26 | from continuedev.src.continuedev.plugins.context_providers.diff import DiffContextProvider 27 | from continuedev.src.continuedev.plugins.context_providers.url import URLContextProvider 28 | 29 | class CommitMessageStep(Step): 30 | """ 31 | This is a Step, the building block of Continue. 32 | It can be used below as a slash command, so that 33 | run will be called when you type '/commit'. 34 | """ 35 | async def run(self, sdk: ContinueSDK): 36 | 37 | # Get the root directory of the workspace 38 | dir = sdk.ide.workspace_directory 39 | 40 | # Run git diff in that directory 41 | diff = subprocess.check_output( 42 | ["git", "diff"], cwd=dir).decode("utf-8") 43 | 44 | # Ask the LLM to write a commit message, 45 | # and set it as the description of this step 46 | self.description = await sdk.models.default.complete( 47 | f"{diff}\n\nWrite a short, specific (less than 50 chars) commit message about the above changes:") 48 | 49 | 50 | config = ContinueConfig( 51 | 52 | # If set to False, we will not collect any usage data 53 | # See here to learn what anonymous data we collect: https://continue.dev/docs/telemetry 54 | allow_anonymous_telemetry=True, 55 | 56 | models = Models( 57 | default = OpenAI( 58 | api_key = "my-api-key", 59 | model = "gpt-3.5-turbo", 60 | openai_server_info = OpenAIServerInfo( 61 | api_base = "http://localhost:8080", 62 | model = "gpt-3.5-turbo" 63 | ) 64 | ) 65 | ), 66 | # Set a system message with information that the LLM should always keep in mind 67 | # E.g. "Please give concise answers. Always respond in Spanish." 68 | system_message=None, 69 | 70 | # Set temperature to any value between 0 and 1. Higher values will make the LLM 71 | # more creative, while lower values will make it more predictable. 72 | temperature=0.5, 73 | 74 | # Custom commands let you map a prompt to a shortened slash command 75 | # They are like slash commands, but more easily defined - write just a prompt instead of a Step class 76 | # Their output will always be in chat form 77 | custom_commands=[ 78 | # CustomCommand( 79 | # name="test", 80 | # description="Write unit tests for the higlighted code", 81 | # prompt="Write a comprehensive set of unit tests for the selected code. It should setup, run tests that check for correctness including important edge cases, and teardown. Ensure that the tests are complete and sophisticated. Give the tests just as chat output, don't edit any file.", 82 | # ) 83 | ], 84 | 85 | # Slash commands let you run a Step from a slash command 86 | slash_commands=[ 87 | # SlashCommand( 88 | # name="commit", 89 | # description="This is an example slash command. Use /config to edit it and create more", 90 | # step=CommitMessageStep, 91 | # ) 92 | SlashCommand( 93 | name="edit", 94 | description="Edit code in the current file or the highlighted code", 95 | step=EditHighlightedCodeStep, 96 | ), 97 | SlashCommand( 98 | name="config", 99 | description="Customize Continue - slash commands, LLMs, system message, etc.", 100 | step=OpenConfigStep, 101 | ), 102 | SlashCommand( 103 | name="comment", 104 | description="Write comments for the current file or highlighted code", 105 | step=CommentCodeStep, 106 | ), 107 | SlashCommand( 108 | name="feedback", 109 | description="Send feedback to improve Continue", 110 | step=FeedbackStep, 111 | ), 112 | SlashCommand( 113 | name="clear", 114 | description="Clear step history", 115 | step=ClearHistoryStep, 116 | ), 117 | SlashCommand( 118 | name="share", 119 | description="Download and share the session transcript", 120 | step=ShareSessionStep, 121 | ) 122 | ], 123 | 124 | # Context providers let you quickly select context by typing '@' 125 | # Uncomment the following to 126 | # - quickly reference GitHub issues 127 | # - show Google search results to the LLM 128 | context_providers=[ 129 | # GitHubIssuesContextProvider( 130 | # repo_name="/", 131 | # auth_token="" 132 | # ), 133 | # GoogleContextProvider( 134 | # serper_api_key="" 135 | # ) 136 | SearchContextProvider(), 137 | DiffContextProvider(), 138 | URLContextProvider( 139 | preset_urls = [ 140 | # Add any common urls you reference here so they appear in autocomplete 141 | ] 142 | ) 143 | ], 144 | 145 | # Policies hold the main logic that decides which Step to take next 146 | # You can use them to design agents, or deeply customize Continue 147 | policy=DefaultPolicy() 148 | ) 149 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Examples for [LocalAI](https://github.com/mudler/LocalAI) 2 | 3 | > Note: These examples are provided as starting points - they may be temporarily out of date. If that happens, send us a PR! 4 | 5 | | [ChatGPT OSS alternative](https://github.com/mudler/LocalAI-examples/tree/master/chatbot-ui) | [Image generation](https://localai.io/api-endpoints/index.html#image-generation) | 6 | |------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| 7 | | ![Screenshot from 2023-04-26 23-59-55](https://user-images.githubusercontent.com/2420543/234715439-98d12e03-d3ce-4f94-ab54-2b256808e05e.png) | ![b6441997879](https://github.com/mudler/LocalAI/assets/2420543/d50af51c-51b7-4f39-b6c2-bf04c403894c) | 8 | 9 | | [Telegram bot](https://github.com/mudler/LocalAI-examples/tree/master/telegram-bot) | [Flowise](https://github.com/mudler/LocalAI-examples/tree/master/flowise) | 10 | |------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| 11 | ![Screenshot from 2023-06-09 00-36-26](https://github.com/mudler/LocalAI/assets/2420543/e98b4305-fa2d-41cf-9d2f-1bb2d75ca902) | ![Screenshot from 2023-05-30 18-01-03](https://github.com/mudler/LocalAI/assets/2420543/02458782-0549-4131-971c-95ee56ec1af8)| | 12 | 13 | Here is a list of projects that can easily be integrated with the LocalAI backend. 14 | 15 | 16 | ### Projects 17 | 18 | ### AutoGPT 19 | 20 | _by [@mudler](https://github.com/mudler)_ 21 | 22 | This example shows how to use AutoGPT with LocalAI. 23 | 24 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/autoGPT/) 25 | 26 | ### Chatbot-UI 27 | 28 | _by [@mkellerman](https://github.com/mkellerman)_ 29 | 30 | ![Screenshot from 2023-04-26 23-59-55](https://user-images.githubusercontent.com/2420543/234715439-98d12e03-d3ce-4f94-ab54-2b256808e05e.png) 31 | 32 | This integration shows how to use LocalAI with [mckaywrigley/chatbot-ui](https://github.com/mckaywrigley/chatbot-ui). 33 | 34 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/chatbot-ui/) 35 | 36 | There is also a separate example to show how to manually setup a model: [example](https://github.com/mudler/LocalAI-examples/tree/master/chatbot-ui-manual/) 37 | 38 | ### K8sGPT 39 | 40 | _by [@mudler](https://github.com/mudler)_ 41 | 42 | This example show how to use LocalAI inside Kubernetes with [k8sgpt](https://k8sgpt.ai). 43 | 44 | ![Screenshot from 2023-06-19 23-58-47](https://github.com/go-skynet/go-ggml-transformers.cpp/assets/2420543/cab87409-ee68-44ae-8d53-41627fb49509) 45 | 46 | ### Fine-tuning a model and convert it to gguf to use it with LocalAI 47 | 48 | _by [@mudler](https://github.com/mudler)_ 49 | 50 | This example is an e2e example on how to fine-tune a model with [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) and convert it to gguf to use it with LocalAI. 51 | 52 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/e2e-fine-tuning/) 53 | 54 | ### Flowise 55 | 56 | _by [@mudler](https://github.com/mudler)_ 57 | 58 | This example shows how to use [FlowiseAI/Flowise](https://github.com/FlowiseAI/Flowise) with LocalAI. 59 | 60 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/flowise/) 61 | 62 | ### Discord bot 63 | 64 | _by [@mudler](https://github.com/mudler)_ 65 | 66 | Run a discord bot which lets you talk directly with a model 67 | 68 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/discord-bot/), or for a live demo you can talk with our bot in #random-bot in our discord server. 69 | 70 | ### Langchain 71 | 72 | _by [@dave-gray101](https://github.com/dave-gray101)_ 73 | 74 | A ready to use example to show e2e how to integrate LocalAI with langchain 75 | 76 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/langchain/) 77 | 78 | ### Langchain Python 79 | 80 | _by [@mudler](https://github.com/mudler)_ 81 | 82 | A ready to use example to show e2e how to integrate LocalAI with langchain 83 | 84 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/langchain-python/) 85 | 86 | ### LocalAI functions 87 | 88 | _by [@mudler](https://github.com/mudler)_ 89 | 90 | A ready to use example to show how to use OpenAI functions with LocalAI 91 | 92 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/functions/) 93 | 94 | ### LocalAI WebUI 95 | 96 | _by [@dhruvgera](https://github.com/dhruvgera)_ 97 | 98 | ![image](https://user-images.githubusercontent.com/42107491/235344183-44b5967d-ba22-4331-804c-8da7004a5d35.png) 99 | 100 | A light, community-maintained web interface for LocalAI 101 | 102 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/localai-webui/) 103 | 104 | ### How to run rwkv models 105 | 106 | _by [@mudler](https://github.com/mudler)_ 107 | 108 | A full example on how to run RWKV models with LocalAI 109 | 110 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/rwkv/) 111 | 112 | ### PrivateGPT 113 | 114 | _by [@mudler](https://github.com/mudler)_ 115 | 116 | A full example on how to run PrivateGPT with LocalAI 117 | 118 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/privateGPT/) 119 | 120 | ### Slack bot 121 | 122 | _by [@mudler](https://github.com/mudler)_ 123 | 124 | Run a slack bot which lets you talk directly with a model 125 | 126 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/slack-bot/) 127 | 128 | ### Slack bot (Question answering) 129 | 130 | _by [@mudler](https://github.com/mudler)_ 131 | 132 | Run a slack bot, ideally for teams, which lets you ask questions on a documentation website, or a github repository. 133 | 134 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/slack-qa-bot/) 135 | 136 | ### Question answering on documents with llama-index 137 | 138 | _by [@mudler](https://github.com/mudler)_ 139 | 140 | Shows how to integrate with [Llama-Index](https://gpt-index.readthedocs.io/en/stable/getting_started/installation.html) to enable question answering on a set of documents. 141 | 142 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/query_data/) 143 | 144 | ### Question answering on documents with langchain and chroma 145 | 146 | _by [@mudler](https://github.com/mudler)_ 147 | 148 | Shows how to integrate with `Langchain` and `Chroma` to enable question answering on a set of documents. 149 | 150 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/langchain-chroma/) 151 | 152 | ### Telegram bot 153 | 154 | _by [@mudler](https://github.com/mudler) 155 | 156 | ![Screenshot from 2023-06-09 00-36-26](https://github.com/mudler/LocalAI/assets/2420543/e98b4305-fa2d-41cf-9d2f-1bb2d75ca902) 157 | 158 | Use LocalAI to power a Telegram bot assistant, with Image generation and audio support! 159 | 160 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/telegram-bot/) 161 | 162 | ### Template for Runpod.io 163 | 164 | _by [@fHachenberg](https://github.com/fHachenberg)_ 165 | 166 | Allows to run any LocalAI-compatible model as a backend on the servers of https://runpod.io 167 | 168 | [Check it out here](https://runpod.io/gsc?template=uv9mtqnrd0&ref=984wlcra) 169 | 170 | ### Continue 171 | 172 | _by [@gruberdev](https://github.com/gruberdev)_ 173 | 174 | Screenshot 175 | 176 | Demonstrates how to integrate an open-source copilot alternative that enhances code analysis, completion, and improvements. This approach seamlessly integrates with any LocalAI model, offering a more user-friendly experience. 177 | 178 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/continue/) 179 | 180 | ### Streamlit bot 181 | 182 | _by [@majoshi1](https://github.com/majoshi1)_ 183 | 184 | ![Screenshot](streamlit-bot/streamlit-bot.png) 185 | 186 | A chat bot made using `Streamlit` & LocalAI. 187 | 188 | [Check it out here](https://github.com/mudler/LocalAI-examples/tree/master/streamlit-bot/) 189 | 190 | ## Want to contribute? 191 | 192 | Create an issue, and put `Example: ` in the title! We will post your examples here. 193 | -------------------------------------------------------------------------------- /semantic-todo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | 11 | "github.com/gdamore/tcell/v2" 12 | "github.com/rivo/tview" 13 | ) 14 | 15 | const ( 16 | localAI string = "http://localhost:8080" 17 | rootStatus string = "[::b][::-]: Add Task [::b]/[::-]: Search Task [::b][::-]: Exit" 18 | inputStatus string = "Press [::b][::-] to submit the task, [::b][::-] to cancel" 19 | ) 20 | 21 | type Task struct { 22 | Description string 23 | Similarity float32 24 | } 25 | 26 | type AppState int 27 | 28 | const ( 29 | StateRoot AppState = iota 30 | StateInput 31 | StateSearch 32 | ) 33 | 34 | type App struct { 35 | state AppState 36 | tasks []Task 37 | app *tview.Application 38 | flex *tview.Flex 39 | table *tview.Table 40 | } 41 | 42 | func NewApp() *App { 43 | return &App{ 44 | state: StateRoot, 45 | tasks: []Task{ 46 | {Description: "Take the dog for a walk (after I get a dog)"}, 47 | {Description: "Go to the toilet"}, 48 | {Description: "Allow TODOs to be marked completed or removed"}, 49 | }, 50 | } 51 | } 52 | 53 | func getEmbeddings(description string) ([]float32, error) { 54 | // Define the request payload 55 | payload := map[string]interface{}{ 56 | "model": "bert-cpp-minilm-v6", 57 | "input": description, 58 | } 59 | 60 | // Marshal the payload into JSON 61 | jsonPayload, err := json.Marshal(payload) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | // Make the HTTP request to the local OpenAI embeddings API 67 | resp, err := http.Post(localAI+"/embeddings", "application/json", bytes.NewBuffer(jsonPayload)) 68 | if err != nil { 69 | return nil, err 70 | } 71 | defer resp.Body.Close() 72 | 73 | // Check if the request was successful 74 | if resp.StatusCode != http.StatusOK { 75 | return nil, fmt.Errorf("request to embeddings API failed with status code: %d", resp.StatusCode) 76 | } 77 | 78 | // Parse the response body 79 | var result struct { 80 | Data []struct { 81 | Embedding []float32 `json:"embedding"` 82 | } `json:"data"` 83 | } 84 | if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { 85 | return nil, err 86 | } 87 | 88 | // Return the embedding 89 | if len(result.Data) > 0 { 90 | return result.Data[0].Embedding, nil 91 | } 92 | return nil, errors.New("no embedding received from API") 93 | } 94 | 95 | type StoresSet struct { 96 | Store string `json:"store,omitempty" yaml:"store,omitempty"` 97 | 98 | Keys [][]float32 `json:"keys" yaml:"keys"` 99 | Values []string `json:"values" yaml:"values"` 100 | } 101 | 102 | func postTasksToExternalService(tasks []Task) error { 103 | keys := make([][]float32, 0, len(tasks)) 104 | // Get the embeddings for the task description 105 | for _, task := range tasks { 106 | embedding, err := getEmbeddings(task.Description) 107 | if err != nil { 108 | return err 109 | } 110 | keys = append(keys, embedding) 111 | } 112 | 113 | values := make([]string, 0, len(tasks)) 114 | for _, task := range tasks { 115 | values = append(values, task.Description) 116 | } 117 | 118 | // Construct the StoresSet object 119 | storesSet := StoresSet{ 120 | Store: "tasks_store", // Assuming you have a specific store name 121 | Keys: keys, 122 | Values: values, 123 | } 124 | 125 | // Marshal the StoresSet object into JSON 126 | jsonData, err := json.Marshal(storesSet) 127 | if err != nil { 128 | return err 129 | } 130 | 131 | // Make the HTTP POST request to the external service 132 | resp, err := http.Post(localAI+"/stores/set", "application/json", bytes.NewBuffer(jsonData)) 133 | if err != nil { 134 | return err 135 | } 136 | defer resp.Body.Close() 137 | 138 | // Check if the request was successful 139 | if resp.StatusCode != http.StatusOK { 140 | // read resp body into string 141 | body, err := io.ReadAll(resp.Body) 142 | if err != nil { 143 | return err 144 | } 145 | return fmt.Errorf("store request failed with status code: %d: %s", resp.StatusCode, body) 146 | } 147 | 148 | return nil 149 | } 150 | 151 | type StoresFind struct { 152 | Store string `json:"store,omitempty" yaml:"store,omitempty"` 153 | 154 | Key []float32 `json:"key" yaml:"key"` 155 | Topk int `json:"topk" yaml:"topk"` 156 | } 157 | 158 | type StoresFindResponse struct { 159 | Keys [][]float32 `json:"keys" yaml:"keys"` 160 | Values []string `json:"values" yaml:"values"` 161 | Similarities []float32 `json:"similarities" yaml:"similarities"` 162 | } 163 | 164 | func findSimilarTexts(inputText string, topk int) (StoresFindResponse, error) { 165 | // Initialize an empty response object 166 | response := StoresFindResponse{} 167 | 168 | // Get the embedding for the input text 169 | embedding, err := getEmbeddings(inputText) 170 | if err != nil { 171 | return response, err 172 | } 173 | 174 | // Construct the StoresFind object 175 | storesFind := StoresFind{ 176 | Store: "tasks_store", // Assuming you have a specific store name 177 | Key: embedding, 178 | Topk: topk, 179 | } 180 | 181 | // Marshal the StoresFind object into JSON 182 | jsonData, err := json.Marshal(storesFind) 183 | if err != nil { 184 | return response, err 185 | } 186 | 187 | // Make the HTTP POST request to the external service's /stores/find endpoint 188 | resp, err := http.Post(localAI+"/stores/find", "application/json", bytes.NewBuffer(jsonData)) 189 | if err != nil { 190 | return response, err 191 | } 192 | defer resp.Body.Close() 193 | 194 | // Check if the request was successful 195 | if resp.StatusCode != http.StatusOK { 196 | return response, fmt.Errorf("request to /stores/find failed with status code: %d", resp.StatusCode) 197 | } 198 | 199 | // Parse the response body to retrieve similar texts and similarities 200 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 201 | return response, err 202 | } 203 | 204 | return response, nil 205 | } 206 | 207 | func (app *App) updateUI() { 208 | // Clear the flex layout 209 | app.flex.Clear() 210 | app.flex.SetDirection(tview.FlexColumn) 211 | app.flex.AddItem(nil, 0, 1, false) 212 | 213 | midCol := tview.NewFlex() 214 | midCol.SetDirection(tview.FlexRow) 215 | midCol.AddItem(nil, 0, 1, false) 216 | 217 | // Create a new table. 218 | app.table.Clear() 219 | app.table.SetBorders(true) 220 | 221 | // Set table headers 222 | app.table.SetCell(0, 0, tview.NewTableCell("Description").SetAlign(tview.AlignLeft).SetExpansion(1).SetAttributes(tcell.AttrBold)) 223 | app.table.SetCell(0, 1, tview.NewTableCell("Similarity").SetAlign(tview.AlignCenter).SetExpansion(0).SetAttributes(tcell.AttrBold)) 224 | 225 | // Add the tasks to the table. 226 | for i, task := range app.tasks { 227 | row := i + 1 228 | app.table.SetCell(row, 0, tview.NewTableCell(task.Description)) 229 | app.table.SetCell(row, 1, tview.NewTableCell(fmt.Sprintf("%.2f", task.Similarity))) 230 | } 231 | 232 | if app.state == StateInput { 233 | inputField := tview.NewInputField() 234 | inputField. 235 | SetLabel("New Task: "). 236 | SetFieldWidth(0). 237 | SetDoneFunc(func(key tcell.Key) { 238 | if key == tcell.KeyEnter { 239 | task := Task{Description: inputField.GetText()} 240 | app.tasks = append(app.tasks, task) 241 | app.state = StateRoot 242 | err := postTasksToExternalService([]Task{task}) 243 | if err != nil { 244 | panic(err) 245 | } 246 | } 247 | app.updateUI() 248 | }) 249 | midCol.AddItem(inputField, 3, 2, true) 250 | app.app.SetFocus(inputField) 251 | } else if app.state == StateSearch { 252 | searchField := tview.NewInputField() 253 | searchField.SetLabel("Search: "). 254 | SetFieldWidth(0). 255 | SetDoneFunc(func(key tcell.Key) { 256 | if key == tcell.KeyEnter { 257 | similar, err := findSimilarTexts(searchField.GetText(), 100) 258 | if err != nil { 259 | panic(err) 260 | } 261 | app.tasks = make([]Task, len(similar.Keys)) 262 | for i, v := range similar.Values { 263 | app.tasks[i] = Task{Description: v, Similarity: similar.Similarities[i]} 264 | } 265 | } 266 | app.updateUI() 267 | }) 268 | midCol.AddItem(searchField, 3, 2, true) 269 | app.app.SetFocus(searchField) 270 | } else { 271 | midCol.AddItem(nil, 3, 1, false) 272 | } 273 | 274 | midCol.AddItem(app.table, 0, 2, true) 275 | 276 | // Add the status bar to the flex layout 277 | statusBar := tview.NewTextView(). 278 | SetText(rootStatus). 279 | SetDynamicColors(true). 280 | SetTextAlign(tview.AlignCenter) 281 | if app.state == StateInput { 282 | statusBar.SetText(inputStatus) 283 | } 284 | midCol.AddItem(statusBar, 1, 1, false) 285 | midCol.AddItem(nil, 0, 1, false) 286 | 287 | app.flex.AddItem(midCol, 0, 10, true) 288 | app.flex.AddItem(nil, 0, 1, false) 289 | 290 | // Set the flex as the root element 291 | app.app.SetRoot(app.flex, true) 292 | } 293 | 294 | func main() { 295 | app := NewApp() 296 | tApp := tview.NewApplication() 297 | flex := tview.NewFlex().SetDirection(tview.FlexRow) 298 | table := tview.NewTable() 299 | 300 | app.app = tApp 301 | app.flex = flex 302 | app.table = table 303 | 304 | app.updateUI() // Initial UI setup 305 | 306 | app.app.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 307 | switch app.state { 308 | case StateRoot: 309 | // Handle key events when in the root state 310 | switch event.Key() { 311 | case tcell.KeyRune: 312 | switch event.Rune() { 313 | case ' ': 314 | app.state = StateInput 315 | app.updateUI() 316 | return nil // Event is handled 317 | case '/': 318 | app.state = StateSearch 319 | app.updateUI() 320 | return nil // Event is handled 321 | } 322 | } 323 | 324 | case StateInput: 325 | // Handle key events when in the input state 326 | if event.Key() == tcell.KeyEsc { 327 | // Exit input state without adding a task 328 | app.state = StateRoot 329 | app.updateUI() 330 | return nil // Event is handled 331 | } 332 | 333 | case StateSearch: 334 | // Handle key events when in the search state 335 | if event.Key() == tcell.KeyEsc { 336 | // Exit search state 337 | app.state = StateRoot 338 | app.updateUI() 339 | return nil // Event is handled 340 | } 341 | } 342 | 343 | // Return the event for further processing by tview 344 | return event 345 | }) 346 | 347 | if err := postTasksToExternalService(app.tasks); err != nil { 348 | panic(err) 349 | } 350 | 351 | // Start the application 352 | if err := app.app.Run(); err != nil { 353 | panic(err) 354 | } 355 | } 356 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /insomnia/Insomnia_LocalAI.json: -------------------------------------------------------------------------------- 1 | {"_type":"export","__export_format":4,"__export_date":"2023-09-01T05:11:43.695Z","__export_source":"insomnia.desktop.app:v2023.5.7","resources":[{"_id":"req_527fdc87fd404a2a8f1c401fb7a0e642","parentId":"fld_911f4d2a05d84b59aff9d4924d1d3877","modified":1692719560635,"created":1692719560635,"url":"{{HOST}}:{{PORT}}/models","name":"get models list","description":"","method":"GET","body":{},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1692719560635,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"fld_911f4d2a05d84b59aff9d4924d1d3877","parentId":"wrk_76923a29272642e49208d65ffe7e885a","modified":1692719560581,"created":1692719560581,"name":"LocalAI","description":"","environment":{},"environmentPropertyOrder":null,"metaSortKey":-1692719560581,"_type":"request_group"},{"_id":"wrk_76923a29272642e49208d65ffe7e885a","parentId":null,"modified":1692719728510,"created":1692719560576,"name":"LocalAI","description":"","scope":"collection","_type":"workspace"},{"_id":"req_03c6b65bce1541fa9a7751c7ed8a7f40","parentId":"fld_9d70a564f6334ff6a5e9473d124d8ee6","modified":1693542905270,"created":1692719560630,"url":"{{HOST}}:{{PORT}}/models/available","name":"list MODELS in galleries","description":"","method":"GET","body":{"mimeType":"","text":"{\n}"},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1692719560630,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"fld_9d70a564f6334ff6a5e9473d124d8ee6","parentId":"fld_911f4d2a05d84b59aff9d4924d1d3877","modified":1692719560625,"created":1692719560625,"name":"model gallery","description":"","environment":{},"environmentPropertyOrder":null,"metaSortKey":-1692719560625,"_type":"request_group"},{"_id":"req_9cfc92cb7f5c43b6bea7992d54e94eca","parentId":"fld_9d70a564f6334ff6a5e9473d124d8ee6","modified":1693542894779,"created":1693526412262,"url":"{{HOST}}:{{PORT}}/models/galleries","name":"list model GALLERIES","description":"","method":"GET","body":{"mimeType":"","text":"{\n}"},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1692719560628.5,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_2cec05b38d9049c5bee24dcac03d1214","parentId":"fld_9d70a564f6334ff6a5e9473d124d8ee6","modified":1692773355999,"created":1692719560627,"url":"{{HOST}}:{{PORT}}/models/apply","name":"model gallery apply","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"id\": \"huggingface@TheBloke/wizardlm-13b-v1.2-ggml/wizardlm-13b-v1.2.ggmlv3.q4_0.bin\",\n \"name\": \"test\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560627,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_f91d77c51f1d41d9b035ed083275441a","parentId":"fld_9d70a564f6334ff6a5e9473d124d8ee6","modified":1693545040139,"created":1693527252021,"url":"{{HOST}}:{{PORT}}/models/galleries","name":"add model gallery","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"url\": \"file:///home/dave/projects/model-gallery/huggingface/TheBloke__CodeLlama-7B-Instruct-GGML.yaml\",\n \"name\": \"test\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560625.5,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_626fee90a5a04947a8545062e9a52350","parentId":"fld_9d70a564f6334ff6a5e9473d124d8ee6","modified":1693544472707,"created":1693544399269,"url":"{{HOST}}:{{PORT}}/models/galleries","name":"delete model gallery","description":"","method":"DELETE","body":{"mimeType":"application/json","text":"{\n \"name\": \"test\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560624.75,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_e1e150aa05c54bb49f5bceca33d4d917","parentId":"fld_9d70a564f6334ff6a5e9473d124d8ee6","modified":1693537478093,"created":1692826085669,"url":"{{HOST}}:{{PORT}}/models/apply","name":"model gallery apply (gist)","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"id\": \"TheBloke__CodeLlama-7B-Instruct-GGML__codellama-7b-instruct.ggmlv3.Q2_K.bin\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560624,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_d9f9175a0e1e4a8facf19e365c89403b","parentId":"fld_4a966fb07756459d9d7c1f5a5561228f","modified":1692722441541,"created":1692722374898,"url":"{{HOST}}:{{PORT}}/tts","name":"/tts","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"model\": \"{{DEFAULT_MODEL}}\",\n \"input\": \"A STRANGE GAME.\\nTHE ONLY WINNING MOVE IS NOT TO PLAY.\\n\\nHOW ABOUT A NICE GAME OF CHESS?\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560630,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"fld_4a966fb07756459d9d7c1f5a5561228f","parentId":"fld_911f4d2a05d84b59aff9d4924d1d3877","modified":1692722533229,"created":1692722439678,"name":"tts","description":"","environment":{},"environmentPropertyOrder":null,"metaSortKey":-1692719560612.5,"_type":"request_group"},{"_id":"req_1dec0983884c4b93acf3d8843108c003","parentId":"fld_2bc22ec3590240cd8d465fe084f5e14d","modified":1692722121942,"created":1692719560608,"url":"{{HOST}}:{{PORT}}/chat/completions","name":"chat completion (simple, 1 message)","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\r\n \"model\": \"{{DEFAULT_MODEL}}\",\r\n \"messages\": [{\"role\": \"user\", \"content\": \"How could one use friction to cook an egg?\"}],\r\n \"max_tokens\": 256,\r\n \"temperature\": 0.2\r\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560611.5,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"fld_2bc22ec3590240cd8d465fe084f5e14d","parentId":"fld_9584156ef4534e86b71735ed2b1e86e5","modified":1692721647772,"created":1692721647772,"name":"chat","description":"","environment":{},"environmentPropertyOrder":null,"metaSortKey":-1692721647772,"_type":"request_group"},{"_id":"fld_9584156ef4534e86b71735ed2b1e86e5","parentId":"fld_911f4d2a05d84b59aff9d4924d1d3877","modified":1692719560600,"created":1692719560600,"name":"llm text","description":"","environment":{},"environmentPropertyOrder":null,"metaSortKey":-1692719560600,"_type":"request_group"},{"_id":"req_e2ca5ec3feae4fd7b0b4162dad261f08","parentId":"fld_2bc22ec3590240cd8d465fe084f5e14d","modified":1692722124781,"created":1692719560615,"url":"{{HOST}}:{{PORT}}/chat/completions","name":"chat/completions (long)","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\r\n \"model\": \"{{DEFAULT_MODEL}}\",\r\n \"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful, intelligent pirate assistant. Always answer as helpfully as possible, while being a pirate and using pirate language. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\"},\r\n {\"role\": \"user\", \"content\": \"How could one use electricity to cook an egg?\"},\r\n {\"role\": \"assistant\",\r\n \"content\": \"Shiver me timbers! Using electricity to cook an egg? Well, matey, I reckon that be a right curious idea! *adjusts spectacles* Now, I ain't sure if this be possible, but I'll do me best to help ye out.\\n\\nFirst things first, ye gotta understand that electricity be a powerful force, and it can't just cook an egg on its own. Ye see, electricity be like a mighty wind that can make things happen, but it needs somethin' to work with. So, if ye want to use electricity to cook an egg, ye gotta find a way to harness that power and make it do the cookin'.\\n\\nNow, I know what ye might be thinkin': \\\"How do I harness the power of electricity to cook an egg?\\\" Well, matey, there be a few ways to do it. One way be to use a special device called an \\\"electric frying pan.\\\" This be a pan that has a built-in heating element that gets hot when ye plug it into a wall socket. When the element gets hot, ye can crack an egg into the pan and watch as it cook\"\r\n },\r\n {\"role\": \"user\", \"content\": \"I don't have one of those, just a raw wire and plenty of power! How do we get it done?\"}],\r\n \"max_tokens\": 1024,\r\n \"temperature\": 0.5\r\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560561.5,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_0a66d6bfaff14b439ade641ce7469f16","parentId":"fld_2bc22ec3590240cd8d465fe084f5e14d","modified":1692722128583,"created":1692719560619,"url":"{{HOST}}:{{PORT}}/chat/completions","name":"chat/completions (stream)","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"model\": \"{{DEFAULT_MODEL}}\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Explain how I can set sail on the ocean using only power generated by seagulls?\"}],\n \"max_tokens\": 256,\n \"temperature\": 0.9,\n \"stream\": true\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560511.5,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_c1f5faeb3d8a4a0eb52c98d40c32f8f9","parentId":"fld_9584156ef4534e86b71735ed2b1e86e5","modified":1692722131493,"created":1692719560621,"url":"{{HOST}}:{{PORT}}/completions","name":"/completions","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\r\n \"model\": \"{{DEFAULT_MODEL}}\",\r\n \"prompt\": \"function downloadFile(string url, string outputPath) {\",\r\n \"max_tokens\": 256,\r\n \"temperature\": 0.5\r\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560621,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_f7ea027749b34f17a38f2bd549885f92","parentId":"fld_9584156ef4534e86b71735ed2b1e86e5","modified":1692722148262,"created":1692721748683,"url":"{{HOST}}:{{PORT}}/edits","name":"/edits","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"model\": \"{{DEFAULT_MODEL}}\",\n \"input\": \"What day of the wek is it?\",\n \"instruction\": \"Fix the spelling mistakes\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560616.25,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_fe1e62ed6c384eccb0a75fbaacfd8e92","parentId":"fld_9584156ef4534e86b71735ed2b1e86e5","modified":1692722327277,"created":1692722256486,"url":"{{HOST}}:{{PORT}}/embeddings","name":"/embeddings","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\n \"model\": \"{{DEFAULT_MODEL}}\",\n \"input\": \"A STRANGE GAME.\\nTHE ONLY WINNING MOVE IS NOT TO PLAY.\\n\\nHOW ABOUT A NICE GAME OF CHESS?\"\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560613.875,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"req_86b544c9e5324512ae2c830e8c2831ba","parentId":"fld_220b3e247cd940d0b615122f75b4da32","modified":1692722115897,"created":1692719560594,"url":"{{HOST}}:{{PORT}}/backend/shutdown","name":"backend/shutdown","description":"","method":"POST","body":{"mimeType":"application/json","text":"{\r\n \"model\": \"{{DEFAULT_MODEL}}\"\r\n}"},"parameters":[],"headers":[{"name":"Content-Type","value":"application/json"}],"authentication":{},"metaSortKey":-1692719560594,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"fld_220b3e247cd940d0b615122f75b4da32","parentId":"fld_911f4d2a05d84b59aff9d4924d1d3877","modified":1692719560584,"created":1692719560584,"name":"backend monitor","description":"","environment":{},"environmentPropertyOrder":null,"metaSortKey":-1692719560584,"_type":"request_group"},{"_id":"req_395dfb3a370d470d907532dc94f91d3f","parentId":"fld_220b3e247cd940d0b615122f75b4da32","modified":1692719560587,"created":1692719560587,"url":"{{HOST}}:{{PORT}}/backend/monitor","name":"backend monitor","description":"","method":"GET","body":{"mimeType":"","text":"{\r\n \"model\": \"{{DEFAULT_MODEL}}\"\r\n}"},"parameters":[],"headers":[],"authentication":{},"metaSortKey":-1692719560587,"isPrivate":false,"settingStoreCookies":true,"settingSendCookies":true,"settingDisableRenderRequestBody":false,"settingEncodeUrl":true,"settingRebuildPath":true,"settingFollowRedirects":"global","_type":"request"},{"_id":"env_b7b4937221a512490c06b95c4986a60800670c2f","parentId":"wrk_76923a29272642e49208d65ffe7e885a","modified":1692720330621,"created":1692719566107,"name":"Base Environment","data":{"PORT":8080,"DEFAULT_MODEL":"gpt-3.5-turbo"},"dataPropertyOrder":{"&":["PORT","DEFAULT_MODEL"]},"color":null,"isPrivate":false,"metaSortKey":1692719566107,"_type":"environment"},{"_id":"jar_b7b4937221a512490c06b95c4986a60800670c2f","parentId":"wrk_76923a29272642e49208d65ffe7e885a","modified":1692719566120,"created":1692719566120,"name":"Default Jar","cookies":[],"_type":"cookie_jar"},{"_id":"env_4a68ee3db2714cc69d81419acd6b2c31","parentId":"env_b7b4937221a512490c06b95c4986a60800670c2f","modified":1692719629896,"created":1692719607346,"name":"localhost","data":{"HOST":"localhost"},"dataPropertyOrder":{"&":["HOST"]},"color":null,"isPrivate":false,"metaSortKey":1692719607346,"_type":"environment"}]} -------------------------------------------------------------------------------- /realtime/realtime.py: -------------------------------------------------------------------------------- 1 | from RealtimeSTT import AudioToTextRecorder 2 | from openai import OpenAI 3 | import os 4 | import json 5 | import pygame 6 | import tempfile 7 | import threading 8 | import soundfile as sf 9 | import numpy as np 10 | import time 11 | 12 | # Configuration 13 | OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') 14 | OPENAI_MODEL = os.getenv('OPENAI_MODEL', 'qwen3-0.6b') 15 | OPENAI_BASE_URL = os.getenv('OPENAI_BASE_URL', 'http://localhost:8080') 16 | OPENAI_TTS_VOICE = os.getenv('OPENAI_TTS_VOICE', '') 17 | OPENAI_TTS_MODEL = os.getenv('OPENAI_TTS_MODEL', 'voice-en-us-amy-low') 18 | WHISPER_MODEL = os.getenv('WHISPER_MODEL', 'whisper-large-turbo-q5_0') 19 | WHISPER_LANGUAGE = os.getenv('WHISPER_LANGUAGE', 'it') 20 | SYSTEM_PROMPT = os.getenv('SYSTEM_PROMPT', 'You are a helpful assistant.') 21 | BACKGROUND_AUDIO = os.getenv('BACKGROUND_AUDIO', 'false').lower() in ['true', '1', 'yes', 'on'] 22 | DEBUG_PLAYBACK = os.getenv('DEBUG_PLAYBACK', 'false').lower() in ['true', '1', 'yes', 'on'] 23 | WAKE_WORD = os.getenv('WAKE_WORD', '') # Empty string means no wake word 24 | MCP_MODE = os.getenv('MCP_MODE', 'false').lower() in ['true', '1', 'yes', 'on'] 25 | 26 | # Initialize OpenAI client 27 | client = OpenAI( 28 | api_key=OPENAI_API_KEY, 29 | base_url=OPENAI_BASE_URL 30 | ) 31 | # Initialize OpenAI client 32 | mcpclient = OpenAI( 33 | api_key=OPENAI_API_KEY, 34 | base_url=OPENAI_BASE_URL+"/mcp" 35 | ) 36 | 37 | 38 | # Initialize pygame mixer for audio playback 39 | pygame.mixer.init() 40 | 41 | # Conversation history 42 | conversation_history = [] 43 | 44 | # Global variables for audio control 45 | audio_playing = False 46 | audio_thread = None 47 | 48 | def start_callback(): 49 | global audio_playing 50 | print("🎤 Recording started!") 51 | 52 | # Stop any playing audio when recording starts (only if background audio is enabled) 53 | if BACKGROUND_AUDIO and audio_playing: 54 | print("🔇 Stopping audio for recording...") 55 | pygame.mixer.music.stop() 56 | audio_playing = False 57 | 58 | def stop_callback(): 59 | print("⏹️ Recording stopped!") 60 | 61 | 62 | def play_audio_background(audio_file_path): 63 | """Play audio in background thread""" 64 | global audio_playing 65 | 66 | def audio_worker(): 67 | global audio_playing 68 | try: 69 | audio_playing = True 70 | pygame.mixer.music.load(audio_file_path) 71 | pygame.mixer.music.play() 72 | 73 | # Wait for playback to finish 74 | while pygame.mixer.music.get_busy() and audio_playing: 75 | pygame.time.wait(100) 76 | 77 | # Clean up 78 | if os.path.exists(audio_file_path): 79 | os.unlink(audio_file_path) 80 | 81 | audio_playing = False 82 | print("✅ Background audio playback completed") 83 | 84 | except Exception as e: 85 | print(f"❌ Background audio error: {str(e)}") 86 | audio_playing = False 87 | # Clean up file if it exists 88 | if os.path.exists(audio_file_path): 89 | os.unlink(audio_file_path) 90 | 91 | # Start audio in background thread 92 | audio_thread = threading.Thread(target=audio_worker, daemon=True) 93 | audio_thread.start() 94 | 95 | def play_audio_blocking(audio_file_path): 96 | """Play audio in foreground (blocking)""" 97 | global audio_playing 98 | try: 99 | audio_playing = True 100 | pygame.mixer.music.load(audio_file_path) 101 | pygame.mixer.music.play() 102 | 103 | # Wait for playback to finish 104 | while pygame.mixer.music.get_busy(): 105 | pygame.time.wait(100) 106 | 107 | # Clean up temporary file 108 | os.unlink(audio_file_path) 109 | 110 | audio_playing = False 111 | print("✅ Speech playback completed") 112 | 113 | except Exception as e: 114 | print(f"❌ Blocking audio error: {str(e)}") 115 | audio_playing = False 116 | # Clean up file if it exists 117 | if os.path.exists(audio_file_path): 118 | os.unlink(audio_file_path) 119 | 120 | def clean_text_for_tts(text): 121 | """Clean text for TTS by removing markdown, emojis and newlines""" 122 | import re 123 | 124 | # Remove emojis and special characters 125 | # Remove emoji ranges 126 | text = re.sub(r'[\U0001F600-\U0001F64F]', '', text) # Emoticons 127 | text = re.sub(r'[\U0001F300-\U0001F5FF]', '', text) # Misc Symbols and Pictographs 128 | text = re.sub(r'[\U0001F680-\U0001F6FF]', '', text) # Transport and Map 129 | text = re.sub(r'[\U0001F1E0-\U0001F1FF]', '', text) # Regional indicator symbols 130 | text = re.sub(r'[\U00002600-\U000026FF]', '', text) # Miscellaneous symbols 131 | text = re.sub(r'[\U00002700-\U000027BF]', '', text) # Dingbats 132 | text = re.sub(r'[\U0001F900-\U0001F9FF]', '', text) # Supplemental Symbols and Pictographs 133 | text = re.sub(r'[\U0001FA70-\U0001FAFF]', '', text) # Symbols and Pictographs Extended-A 134 | 135 | # Remove other common emoji patterns 136 | text = re.sub(r'[🔥🚀💡🎤⏹️🔊✅❌⚠️📝🎤🗣️🌐🔑🎯👤🤖]', '', text) 137 | 138 | # Remove markdown formatting 139 | # Remove bold/italic 140 | text = re.sub(r'\*\*(.*?)\*\*', r'\1', text) # **bold** 141 | text = re.sub(r'\*(.*?)\*', r'\1', text) # *italic* 142 | text = re.sub(r'__(.*?)__', r'\1', text) # __bold__ 143 | text = re.sub(r'_(.*?)_', r'\1', text) # _italic_ 144 | 145 | # Remove code blocks 146 | text = re.sub(r'```.*?```', '', text, flags=re.DOTALL) # ```code``` 147 | text = re.sub(r'`(.*?)`', r'\1', text) # `code` 148 | 149 | # Remove headers 150 | text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE) # # Header 151 | 152 | # Remove links but keep text 153 | text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text) # [text](url) 154 | 155 | # Remove newlines and extra whitespace 156 | text = re.sub(r'\n+', ' ', text) # Replace newlines with space 157 | text = re.sub(r'\s+', ' ', text) # Replace multiple spaces with single space 158 | 159 | # Strip leading/trailing whitespace 160 | text = text.strip() 161 | 162 | return text 163 | 164 | def text_to_speech(text): 165 | """Convert text to speech using OpenAI TTS API""" 166 | try: 167 | print("🔊 Generating speech...") 168 | 169 | # Clean text for TTS 170 | cleaned_text = clean_text_for_tts(text) 171 | 172 | # Call OpenAI TTS API 173 | response = client.audio.speech.create( 174 | model=OPENAI_TTS_MODEL, 175 | voice=OPENAI_TTS_VOICE, 176 | input=cleaned_text 177 | ) 178 | 179 | # Save audio to temporary file 180 | with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_file: 181 | temp_file.write(response.content) 182 | temp_file_path = temp_file.name 183 | 184 | # Play audio based on configuration 185 | if BACKGROUND_AUDIO: 186 | # Play audio in background (non-blocking) 187 | play_audio_background(temp_file_path) 188 | else: 189 | # Play audio in foreground (blocking) 190 | play_audio_blocking(temp_file_path) 191 | 192 | except Exception as e: 193 | print(f"❌ TTS Error: {str(e)}") 194 | 195 | def get_openai_response(user_text): 196 | """Get response from OpenAI API""" 197 | try: 198 | # Add user message to conversation history 199 | conversation_history.append({"role": "user", "content": user_text}) 200 | 201 | # Prepare messages for API call 202 | messages = [ 203 | {"role": "system", "content": SYSTEM_PROMPT} 204 | ] + conversation_history[-10:] # Keep last 10 messages for context 205 | 206 | if MCP_MODE: 207 | print("🤖 MCP Mode: ", MCP_MODE) 208 | response = mcpclient.chat.completions.create( 209 | model=OPENAI_MODEL, 210 | messages=messages, 211 | ) 212 | else: 213 | response = client.chat.completions.create( 214 | model=OPENAI_MODEL, 215 | messages=messages, 216 | ) 217 | 218 | assistant_response = response.choices[0].message.content 219 | 220 | # Add assistant response to conversation history 221 | conversation_history.append({"role": "assistant", "content": assistant_response}) 222 | 223 | return assistant_response 224 | 225 | except Exception as e: 226 | return f"Sorry, I encountered an error: {str(e)}" 227 | 228 | def validate_config(): 229 | """Validate configuration and exit if invalid""" 230 | if not OPENAI_API_KEY: 231 | print("❌ Error: OPENAI_API_KEY environment variable is not set!") 232 | print("💡 Please set it with: export OPENAI_API_KEY='your-api-key-here'") 233 | exit(1) 234 | 235 | def process_text(text): 236 | """Process the recorded text and get AI response""" 237 | if text and text.strip(): 238 | print(f"\n👤 You: {text}") 239 | 240 | # Get AI response 241 | ai_response = get_openai_response(text) 242 | print(f"🤖 Jarvis: {ai_response}") 243 | 244 | # Convert response to speech and play 245 | if ai_response: 246 | text_to_speech(ai_response) 247 | 248 | print() # Add spacing after response 249 | return ai_response 250 | return None 251 | 252 | def play_wav_file(wav_file_path): 253 | """Play a WAV file for debugging""" 254 | try: 255 | print(f"🔊 Playing WAV file: {wav_file_path}") 256 | 257 | # Check if file exists 258 | if not os.path.exists(wav_file_path): 259 | print(f"❌ WAV file not found: {wav_file_path}") 260 | return False 261 | 262 | # Get file size for debugging 263 | file_size = os.path.getsize(wav_file_path) 264 | print(f"📁 File size: {file_size} bytes") 265 | 266 | # Play the audio file 267 | pygame.mixer.music.load(wav_file_path) 268 | pygame.mixer.music.play() 269 | 270 | # Wait for playback to finish 271 | while pygame.mixer.music.get_busy(): 272 | pygame.time.wait(100) 273 | 274 | print("✅ WAV file playback completed") 275 | return True 276 | 277 | except Exception as e: 278 | print(f"❌ Error playing WAV file: {str(e)}") 279 | return False 280 | 281 | def save_audio_data_as_wav(audio_data, sample_rate=16000, filename="recorded.wav", play_for_debug=False): 282 | """Save audio data (numpy array) as a WAV file""" 283 | try: 284 | if audio_data is not None and len(audio_data) > 0: 285 | # Ensure audio data is in the correct format (float32, normalized) 286 | if audio_data.dtype != np.float32: 287 | audio_data = audio_data.astype(np.float32) 288 | 289 | # Save as WAV file 290 | sf.write(filename, audio_data, sample_rate) 291 | print(f"💾 Audio saved as {filename} (shape: {audio_data.shape})") 292 | 293 | # Play for debugging if requested 294 | if play_for_debug: 295 | play_wav_file(filename) 296 | 297 | return filename 298 | else: 299 | print("❌ No audio data to save") 300 | return None 301 | except Exception as e: 302 | print(f"❌ Error saving audio data: {str(e)}") 303 | return None 304 | 305 | def save_audio_frames_as_wav(frames, sample_rate=16000, filename="recorded.wav"): 306 | """Save audio frames as a WAV file""" 307 | try: 308 | # Convert frames to numpy array 309 | if frames: 310 | # Join all frames into a single byte string 311 | audio_bytes = b''.join(frames) 312 | # Convert to numpy array 313 | audio_array = np.frombuffer(audio_bytes, dtype=np.int16) 314 | # Convert to float32 and normalize 315 | audio_float = audio_array.astype(np.float32) / 32768.0 316 | 317 | # Save as WAV file 318 | sf.write(filename, audio_float, sample_rate) 319 | print(f"💾 Audio saved as {filename}") 320 | return filename 321 | else: 322 | print("❌ No audio frames to save") 323 | return None 324 | except Exception as e: 325 | print(f"❌ Error saving audio: {str(e)}") 326 | return None 327 | 328 | def transcribe_wav_with_openai(wav_file_path): 329 | """Transcribe WAV file using OpenAI Whisper API""" 330 | try: 331 | print("🎤 Transcribing with OpenAI Whisper...") 332 | with open(wav_file_path, 'rb') as audio_file: 333 | transcript = client.audio.transcriptions.create( 334 | model=WHISPER_MODEL, 335 | file=audio_file, 336 | language=WHISPER_LANGUAGE # Italian language 337 | ) 338 | 339 | return transcript.text 340 | except Exception as e: 341 | print(f"❌ OpenAI Whisper transcription error: {str(e)}") 342 | return None 343 | 344 | class CustomAudioRecorder(AudioToTextRecorder): 345 | """Custom AudioToTextRecorder that allows access to audio data""" 346 | 347 | def get_audio_data(self): 348 | """Get the processed audio data as numpy array""" 349 | return self.audio if hasattr(self, 'audio') and self.audio is not None else None 350 | 351 | def get_audio_frames(self): 352 | """Get the current audio frames (before processing)""" 353 | return self.frames if hasattr(self, 'frames') else [] 354 | 355 | def get_last_audio_frames(self): 356 | """Get the last recorded audio frames (before processing)""" 357 | return self.last_frames if hasattr(self, 'last_frames') else [] 358 | 359 | def clear_frames(self): 360 | """Clear the audio frames and data""" 361 | if hasattr(self, 'frames'): 362 | self.frames.clear() 363 | if hasattr(self, 'last_frames'): 364 | self.last_frames.clear() 365 | if hasattr(self, 'audio'): 366 | self.audio = None 367 | 368 | def process_audio_with_openai_whisper(recorder): 369 | """Process recorded audio using OpenAI Whisper API""" 370 | try: 371 | print("🔊 Processing audio with OpenAI Whisper...") 372 | 373 | # Get the processed audio data (numpy array) 374 | audio_data = recorder.get_audio_data() 375 | print(f"📊 Audio data available: {audio_data is not None}") 376 | 377 | if audio_data is not None: 378 | print(f"📊 Audio data shape: {audio_data.shape}, dtype: {audio_data.dtype}") 379 | 380 | # Save audio data as WAV file 381 | wav_filename = f"temp_audio_{int(time.time())}.wav" 382 | wav_path = save_audio_data_as_wav(audio_data, sample_rate=16000, filename=wav_filename, play_for_debug=DEBUG_PLAYBACK) 383 | 384 | if wav_path: 385 | # Transcribe using OpenAI Whisper 386 | transcription = transcribe_wav_with_openai(wav_path) 387 | 388 | # Clean up temporary file 389 | try: 390 | os.unlink(wav_path) 391 | except: 392 | pass 393 | 394 | return transcription 395 | else: 396 | return None 397 | else: 398 | print("❌ No audio data available") 399 | return None 400 | 401 | except Exception as e: 402 | print(f"❌ Error processing audio: {str(e)}") 403 | return None 404 | 405 | if __name__ == '__main__': 406 | # Validate configuration 407 | validate_config() 408 | 409 | print("🚀 Jarvis Voice Assistant Started!") 410 | print("💡 Configuration:") 411 | print(f" 📝 Chat Model: {OPENAI_MODEL}") 412 | print(f" 🎤 TTS Model: {OPENAI_TTS_MODEL}") 413 | print(f" 🗣️ TTS Voice: {OPENAI_TTS_VOICE}") 414 | print(f" 🌐 Base URL: {OPENAI_BASE_URL}") 415 | print(f" 🔊 Background Audio: {'✅ Enabled' if BACKGROUND_AUDIO else '❌ Disabled'}") 416 | print(f" 🎵 Debug Playback: {'✅ Enabled' if DEBUG_PLAYBACK else '❌ Disabled'}") 417 | print(f" 🎯 Wake Word: {'✅ ' + WAKE_WORD if WAKE_WORD else '❌ Disabled'}") 418 | print(f" 🔑 API Key: ✅ Set") 419 | print("🎯 Starting voice assistant...") 420 | if WAKE_WORD: 421 | print(f"💡 Say '{WAKE_WORD}' to activate the assistant!\n") 422 | else: 423 | print("💡 Just start speaking - no wake word needed!\n") 424 | 425 | # Configure wake word based on environment variable 426 | wake_word_config = {} 427 | if WAKE_WORD: 428 | wake_word_config["wake_words"] = WAKE_WORD 429 | wake_word_config["wakeword_backend"] = "pvporcupine" # Enable wake word backend 430 | else: 431 | wake_word_config["wakeword_backend"] = "" # Disable wake word backend 432 | 433 | recorder = CustomAudioRecorder(on_recording_start=start_callback, 434 | model="tiny", # Use smallest model since we're not using it for transcription 435 | on_recording_stop=stop_callback, 436 | **wake_word_config) 437 | 438 | # Start listening for voice activity 439 | recorder.listen() 440 | 441 | while True: 442 | try: 443 | print("🎧 Waiting for voice input...") 444 | 445 | # Use the built-in wait_audio method to handle voice activity detection 446 | recorder.wait_audio() 447 | 448 | print("✅ Audio recording completed, processing...") 449 | # Process the recorded audio with OpenAI Whisper 450 | recorded_text = process_audio_with_openai_whisper(recorder) 451 | 452 | if recorded_text: 453 | # Process the transcribed text 454 | process_text(recorded_text) 455 | else: 456 | print("⚠️ No audio was captured, please try again") 457 | 458 | # Clear frames for next recording 459 | recorder.clear_frames() 460 | 461 | # Start listening again for the next interaction 462 | recorder.listen() 463 | 464 | except KeyboardInterrupt: 465 | print("\n👋 Goodbye!") 466 | break 467 | except Exception as e: 468 | print(f"❌ Error in main loop: {str(e)}") 469 | continue 470 | --------------------------------------------------------------------------------