├── .devcontainer-scripts ├── postcreate.sh ├── poststart.sh └── utils.sh ├── .devcontainer ├── customization │ └── README.md ├── devcontainer.json ├── docker-compose-devcontainer.yml ├── grafana │ └── datasource.yml └── prometheus │ └── prometheus.yml ├── .dockerignore ├── .editorconfig ├── .env ├── .gitattributes ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── bump_deps.sh ├── bump_docs.sh ├── check_and_update.py ├── checksum_checker.sh ├── ci │ └── modelslist.go ├── dependabot.yml ├── labeler.yml ├── release.yml ├── stale.yml └── workflows │ ├── bump_deps.yaml │ ├── bump_docs.yaml │ ├── checksum_checker.yaml │ ├── dependabot_auto.yml │ ├── deploy-explorer.yaml │ ├── disabled │ ├── comment-pr.yaml │ └── test-gpu.yml │ ├── generate_grpc_cache.yaml │ ├── generate_intel_image.yaml │ ├── image-pr.yml │ ├── image.yml │ ├── image_build.yml │ ├── labeler.yml │ ├── localaibot_automerge.yml │ ├── notify-models.yaml │ ├── notify-releases.yaml │ ├── prlint.yaml │ ├── release.yaml │ ├── secscan.yaml │ ├── test-extra.yml │ ├── test.yml │ ├── update_swagger.yaml │ └── yaml-check.yml ├── .gitignore ├── .gitmodules ├── .vscode ├── extensions.json └── launch.json ├── .yamllint ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.aio ├── Earthfile ├── Entitlements.plist ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── aio ├── cpu │ ├── README.md │ ├── embeddings.yaml │ ├── image-gen.yaml │ ├── rerank.yaml │ ├── speech-to-text.yaml │ ├── text-to-speech.yaml │ ├── text-to-text.yaml │ ├── vad.yaml │ └── vision.yaml ├── entrypoint.sh ├── gpu-8g │ ├── embeddings.yaml │ ├── image-gen.yaml │ ├── rerank.yaml │ ├── speech-to-text.yaml │ ├── text-to-speech.yaml │ ├── text-to-text.yaml │ ├── vad.yaml │ └── vision.yaml └── intel │ ├── embeddings.yaml │ ├── image-gen.yaml │ ├── rerank.yaml │ ├── speech-to-text.yaml │ ├── text-to-speech.yaml │ ├── text-to-text.yaml │ ├── vad.yaml │ └── vision.yaml ├── assets.go ├── backend ├── backend.proto ├── cpp │ ├── grpc │ │ ├── .gitignore │ │ └── Makefile │ └── llama │ │ ├── CMakeLists.txt │ │ ├── Makefile │ │ ├── grpc-server.cpp │ │ ├── patches │ │ └── 01-llava.patch │ │ └── prepare.sh ├── go │ ├── bark │ │ ├── Makefile │ │ ├── gobark.cpp │ │ ├── gobark.go │ │ ├── gobark.h │ │ └── main.go │ ├── image │ │ └── stablediffusion-ggml │ │ │ ├── Makefile │ │ │ ├── gosd.cpp │ │ │ ├── gosd.go │ │ │ ├── gosd.h │ │ │ └── main.go │ ├── llm │ │ ├── langchain │ │ │ ├── langchain.go │ │ │ └── main.go │ │ └── llama │ │ │ ├── llama.go │ │ │ └── main.go │ ├── stores │ │ ├── debug.go │ │ ├── main.go │ │ ├── production.go │ │ └── store.go │ ├── transcribe │ │ └── whisper │ │ │ ├── main.go │ │ │ └── whisper.go │ ├── tts │ │ ├── main.go │ │ └── piper.go │ └── vad │ │ └── silero │ │ ├── main.go │ │ └── vad.go └── python │ ├── README.md │ ├── bark │ ├── Makefile │ ├── README.md │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh │ ├── chatterbox │ ├── Makefile │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh │ ├── common │ ├── libbackend.sh │ └── template │ │ ├── Makefile │ │ ├── backend.py │ │ ├── install.sh │ │ ├── protogen.sh │ │ ├── requirements-hipblas.txt │ │ ├── requirements-intel.txt │ │ ├── requirements.txt │ │ ├── run.sh │ │ └── test.sh │ ├── coqui │ ├── Makefile │ ├── README.md │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh │ ├── diffusers │ ├── Makefile │ ├── README.md │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh │ ├── exllama2 │ ├── .gitignore │ ├── Makefile │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-install.txt │ ├── requirements.txt │ ├── run.sh │ └── test.sh │ ├── faster-whisper │ ├── Makefile │ ├── backend.py │ ├── install.sh │ ├── protogen.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ └── test.sh │ ├── kokoro │ ├── Makefile │ ├── backend.py │ ├── install.sh │ ├── istftnet.py │ ├── kokoro.py │ ├── models.py │ ├── plbert.py │ ├── protogen.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ └── test.sh │ ├── rerankers │ ├── Makefile │ ├── README.md │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh │ ├── transformers │ ├── Makefile │ ├── README.md │ ├── backend.py │ ├── install.sh │ ├── requirements-cpu.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh │ └── vllm │ ├── Makefile │ ├── README.md │ ├── backend.py │ ├── install.sh │ ├── requirements-after.txt │ ├── requirements-cpu.txt │ ├── requirements-cublas11-after.txt │ ├── requirements-cublas11.txt │ ├── requirements-cublas12-after.txt │ ├── requirements-cublas12.txt │ ├── requirements-hipblas.txt │ ├── requirements-install.txt │ ├── requirements-intel.txt │ ├── requirements.txt │ ├── run.sh │ ├── test.py │ └── test.sh ├── configuration └── .keep ├── core ├── application │ ├── application.go │ ├── config_file_watcher.go │ └── startup.go ├── backend │ ├── backend_suite_test.go │ ├── embeddings.go │ ├── image.go │ ├── llm.go │ ├── llm_test.go │ ├── options.go │ ├── rerank.go │ ├── soundgeneration.go │ ├── stores.go │ ├── token_metrics.go │ ├── tokenize.go │ ├── transcript.go │ ├── tts.go │ ├── vad.go │ └── video.go ├── cli │ ├── api │ │ └── p2p.go │ ├── cli.go │ ├── context │ │ └── context.go │ ├── explorer.go │ ├── federated.go │ ├── models.go │ ├── run.go │ ├── soundgeneration.go │ ├── transcript.go │ ├── tts.go │ ├── util.go │ └── worker │ │ ├── worker.go │ │ ├── worker_llamacpp.go │ │ ├── worker_nop2p.go │ │ └── worker_p2p.go ├── clients │ └── store.go ├── config │ ├── application_config.go │ ├── backend_config.go │ ├── backend_config_filter.go │ ├── backend_config_loader.go │ ├── backend_config_test.go │ ├── config_suite_test.go │ ├── config_test.go │ ├── gallery.go │ ├── gguf.go │ └── guesser.go ├── dependencies_manager │ └── manager.go ├── explorer │ ├── database.go │ ├── database_test.go │ ├── discovery.go │ └── explorer_suite_test.go ├── gallery │ ├── gallery.go │ ├── gallery_suite_test.go │ ├── models.go │ ├── models_test.go │ ├── op.go │ ├── request.go │ └── request_test.go ├── http │ ├── app.go │ ├── app_test.go │ ├── elements │ │ ├── buttons.go │ │ ├── gallery.go │ │ ├── p2p.go │ │ └── progressbar.go │ ├── endpoints │ │ ├── elevenlabs │ │ │ ├── soundgeneration.go │ │ │ └── tts.go │ │ ├── explorer │ │ │ └── dashboard.go │ │ ├── jina │ │ │ └── rerank.go │ │ ├── localai │ │ │ ├── backend_monitor.go │ │ │ ├── gallery.go │ │ │ ├── get_token_metrics.go │ │ │ ├── metrics.go │ │ │ ├── p2p.go │ │ │ ├── stores.go │ │ │ ├── system.go │ │ │ ├── tokenize.go │ │ │ ├── tts.go │ │ │ ├── vad.go │ │ │ ├── video.go │ │ │ └── welcome.go │ │ └── openai │ │ │ ├── assistant.go │ │ │ ├── assistant_test.go │ │ │ ├── chat.go │ │ │ ├── completion.go │ │ │ ├── edit.go │ │ │ ├── embeddings.go │ │ │ ├── files.go │ │ │ ├── files_test.go │ │ │ ├── image.go │ │ │ ├── inference.go │ │ │ ├── list.go │ │ │ ├── realtime.go │ │ │ ├── realtime_model.go │ │ │ ├── transcription.go │ │ │ └── types │ │ │ └── realtime.go │ ├── explorer.go │ ├── http_suite_test.go │ ├── middleware │ │ ├── auth.go │ │ ├── request.go │ │ ├── strippathprefix.go │ │ └── strippathprefix_test.go │ ├── render.go │ ├── routes │ │ ├── elevenlabs.go │ │ ├── explorer.go │ │ ├── health.go │ │ ├── jina.go │ │ ├── localai.go │ │ ├── openai.go │ │ └── ui.go │ ├── static │ │ ├── assets │ │ │ ├── KFOlCnqEu92Fr1MmEU9fBBc9.ttf │ │ │ ├── KFOlCnqEu92Fr1MmEU9vAw.ttf │ │ │ ├── KFOlCnqEu92Fr1MmSU5fBBc9.ttf │ │ │ ├── KFOlCnqEu92Fr1MmWUlfBBc9.ttf │ │ │ ├── KFOlCnqEu92Fr1MmYUtfBBc9.ttf │ │ │ ├── KFOmCnqEu92Fr1Me5Q.ttf │ │ │ ├── KFOmCnqEu92Fr1Mu4mxP.ttf │ │ │ ├── UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuFuYMZg.ttf │ │ │ ├── UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuGKYMZg.ttf │ │ │ ├── UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuLyfMZg.ttf │ │ │ ├── alpine.js │ │ │ ├── flowbite.min.js │ │ │ ├── font1.css │ │ │ ├── font2.css │ │ │ ├── fontawesome.css │ │ │ ├── fontawesome │ │ │ │ ├── css │ │ │ │ │ ├── all.css │ │ │ │ │ ├── all.min.css │ │ │ │ │ ├── brands.css │ │ │ │ │ ├── brands.min.css │ │ │ │ │ ├── fontawesome.css │ │ │ │ │ ├── fontawesome.min.css │ │ │ │ │ ├── regular.css │ │ │ │ │ ├── regular.min.css │ │ │ │ │ ├── solid.css │ │ │ │ │ ├── solid.min.css │ │ │ │ │ ├── svg-with-js.css │ │ │ │ │ ├── svg-with-js.min.css │ │ │ │ │ ├── v4-font-face.css │ │ │ │ │ ├── v4-font-face.min.css │ │ │ │ │ ├── v4-shims.css │ │ │ │ │ ├── v4-shims.min.css │ │ │ │ │ ├── v5-font-face.css │ │ │ │ │ └── v5-font-face.min.css │ │ │ │ └── webfonts │ │ │ │ │ ├── fa-brands-400.ttf │ │ │ │ │ ├── fa-brands-400.woff2 │ │ │ │ │ ├── fa-regular-400.ttf │ │ │ │ │ ├── fa-regular-400.woff2 │ │ │ │ │ ├── fa-solid-900.ttf │ │ │ │ │ ├── fa-solid-900.woff2 │ │ │ │ │ ├── fa-v4compatibility.ttf │ │ │ │ │ └── fa-v4compatibility.woff2 │ │ │ ├── highlightjs.css │ │ │ ├── highlightjs.js │ │ │ ├── htmx.js │ │ │ ├── marked.js │ │ │ ├── purify.js │ │ │ ├── tailwindcss.js │ │ │ ├── tw-elements.css │ │ │ └── tw-elements.js │ │ ├── chat.js │ │ ├── favicon.svg │ │ ├── general.css │ │ ├── image.js │ │ ├── logo.png │ │ ├── logo_horizontal.png │ │ ├── p2panimation.js │ │ ├── talk.js │ │ └── tts.js │ ├── utils │ │ ├── baseurl.go │ │ └── baseurl_test.go │ └── views │ │ ├── 404.html │ │ ├── chat.html │ │ ├── error.html │ │ ├── explorer.html │ │ ├── index.html │ │ ├── login.html │ │ ├── models.html │ │ ├── p2p.html │ │ ├── partials │ │ ├── footer.html │ │ ├── head.html │ │ ├── inprogress.html │ │ ├── navbar.html │ │ └── navbar_explorer.html │ │ ├── talk.html │ │ ├── text2image.html │ │ └── tts.html ├── p2p │ ├── federated.go │ ├── federated_server.go │ ├── node.go │ ├── p2p.go │ ├── p2p_common.go │ └── p2p_disabled.go ├── schema │ ├── elevenlabs.go │ ├── jina.go │ ├── localai.go │ ├── openai.go │ ├── prediction.go │ ├── request.go │ ├── tokenize.go │ └── transcription.go └── services │ ├── backend_monitor.go │ ├── gallery.go │ ├── list_models.go │ └── metrics.go ├── custom-ca-certs └── .keep ├── docker-compose.yaml ├── docs ├── Dockerfile ├── README.md ├── assets │ ├── images │ │ ├── imagen.png │ │ ├── localai_screenshot.png │ │ ├── logos │ │ │ ├── logo.png │ │ │ └── logo.svg │ │ └── screenshots │ │ │ ├── screenshot_chat.png │ │ │ ├── screenshot_gallery.png │ │ │ ├── screenshot_home.png │ │ │ ├── screenshot_image.png │ │ │ ├── screenshot_login.png │ │ │ ├── screenshot_p2p.png │ │ │ ├── screenshot_talk.png │ │ │ └── screenshot_tts.png │ └── jsconfig.json ├── config.toml ├── content │ └── docs │ │ ├── advanced │ │ ├── _index.en.md │ │ ├── advanced-usage.md │ │ ├── fine-tuning.md │ │ └── installer.md │ │ ├── faq.md │ │ ├── features │ │ ├── GPU-acceleration.md │ │ ├── _index.en.md │ │ ├── audio-to-text.md │ │ ├── constrained_grammars.md │ │ ├── distributed_inferencing.md │ │ ├── embeddings.md │ │ ├── gpt-vision.md │ │ ├── image-generation.md │ │ ├── model-gallery.md │ │ ├── openai-functions.md │ │ ├── reranker.md │ │ ├── stores.md │ │ ├── text-generation.md │ │ └── text-to-audio.md │ │ ├── getting-started │ │ ├── _index.en.md │ │ ├── build.md │ │ ├── container-images.md │ │ ├── customize-model.md │ │ ├── kubernetes.md │ │ ├── models.md │ │ ├── quickstart.md │ │ └── try-it-out.md │ │ ├── integrations.md │ │ ├── overview.md │ │ ├── reference │ │ ├── _index.en.md │ │ ├── architecture.md │ │ ├── binaries.md │ │ ├── compatibility-table.md │ │ └── nvidia-l4t.md │ │ └── whats-new.md ├── data │ ├── landing.yaml │ └── version.json ├── docker-compose.yaml ├── go.mod ├── go.sum ├── layouts │ ├── 404.html │ ├── partials │ │ ├── docs │ │ │ ├── gitinfo.html │ │ │ ├── sidebar.html │ │ │ └── top-header.html │ │ ├── head.html │ │ ├── header.html │ │ └── logo.html │ └── shortcodes │ │ ├── github.html │ │ ├── pr.html │ │ └── version.html ├── netlify.toml ├── package.json ├── static │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── apple-touch-icon.png │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── favicon.ico │ ├── favicon.svg │ ├── install.sh │ └── site.webmanifest └── themes │ └── hugo-theme-relearn ├── entrypoint.sh ├── examples └── README.md ├── gallery ├── alpaca.yaml ├── arch-function.yaml ├── cerbero.yaml ├── chatml-hercules.yaml ├── chatml.yaml ├── codellama.yaml ├── command-r.yaml ├── deephermes.yaml ├── deepseek-r1.yaml ├── deepseek.yaml ├── dreamshaper.yaml ├── falcon3.yaml ├── flux-ggml.yaml ├── flux.yaml ├── gemma.yaml ├── granite.yaml ├── granite3-2.yaml ├── hermes-2-pro-mistral.yaml ├── hermes-vllm.yaml ├── index.yaml ├── llama3-instruct.yaml ├── llama3.1-instruct-grammar.yaml ├── llama3.1-instruct.yaml ├── llama3.1-reflective.yaml ├── llama3.2-fcall.yaml ├── llama3.2-quantized.yaml ├── llava.yaml ├── mathstral.yaml ├── mistral-0.3.yaml ├── moondream.yaml ├── mudler.yaml ├── noromaid.yaml ├── openvino.yaml ├── parler-tts.yaml ├── phi-2-chat.yaml ├── phi-2-orange.yaml ├── phi-3-chat.yaml ├── phi-3-vision.yaml ├── phi-4-chat-fcall.yaml ├── phi-4-chat.yaml ├── piper.yaml ├── qwen-fcall.yaml ├── qwen3.yaml ├── rerankers.yaml ├── rwkv.yaml ├── sd-ggml.yaml ├── sentencetransformers.yaml ├── smolvlm.yaml ├── stablediffusion3.yaml ├── tuluv2.yaml ├── vicuna-chat.yaml ├── virtual.yaml ├── vllm.yaml ├── whisper-base.yaml └── wizardlm2.yaml ├── go.mod ├── go.sum ├── internal └── version.go ├── main.go ├── models └── .keep ├── pkg ├── assets │ ├── extract.go │ └── list.go ├── audio │ └── audio.go ├── concurrency │ ├── concurrency_suite_test.go │ ├── jobresult.go │ └── jobresult_test.go ├── downloader │ ├── downloader_suite_test.go │ ├── huggingface.go │ ├── progress.go │ ├── uri.go │ └── uri_test.go ├── functions │ ├── function_structure.go │ ├── functions.go │ ├── functions_suite_test.go │ ├── functions_test.go │ ├── grammars │ │ ├── bnf_rules.go │ │ ├── grammars_suite_test.go │ │ ├── json_schema.go │ │ ├── json_schema_test.go │ │ ├── llama31_schema.go │ │ ├── llama31_schema_test.go │ │ ├── options.go │ │ ├── rules.go │ │ └── types.go │ ├── json_mode.go │ ├── parse.go │ └── parse_test.go ├── grpc │ ├── backend.go │ ├── base │ │ ├── base.go │ │ └── singlethread.go │ ├── client.go │ ├── embed.go │ ├── interface.go │ └── server.go ├── langchain │ ├── huggingface.go │ └── langchain.go ├── library │ └── dynaload.go ├── model │ ├── filters.go │ ├── initializers.go │ ├── loader.go │ ├── loader_options.go │ ├── loader_test.go │ ├── model.go │ ├── model_suite_test.go │ ├── process.go │ └── watchdog.go ├── oci │ ├── blob.go │ ├── blob_test.go │ ├── image.go │ ├── image_test.go │ ├── oci_suite_test.go │ ├── ollama.go │ └── ollama_test.go ├── sound │ ├── float32.go │ └── int16.go ├── startup │ ├── model_preload.go │ ├── model_preload_test.go │ └── startup_suite_test.go ├── store │ └── client.go ├── templates │ ├── cache.go │ ├── evaluator.go │ ├── evaluator_test.go │ ├── multimodal.go │ ├── multimodal_test.go │ └── templates_suite_test.go ├── utils │ ├── base64.go │ ├── base64_test.go │ ├── config.go │ ├── ffmpeg.go │ ├── hash.go │ ├── json.go │ ├── logging.go │ ├── path.go │ ├── strings.go │ ├── untar.go │ └── utils_suite_test.go ├── xsync │ ├── map.go │ ├── map_test.go │ └── sync_suite_test.go └── xsysinfo │ ├── cpu.go │ ├── gguf.go │ └── gpu.go ├── prompt-templates ├── alpaca.tmpl ├── getting_started.tmpl ├── ggml-gpt4all-j.tmpl ├── koala.tmpl ├── llama2-chat-message.tmpl ├── vicuna.tmpl └── wizardlm.tmpl ├── renovate.json ├── scripts ├── latest_hf.py ├── model_gallery_info.py └── prepare-libs.sh ├── swagger ├── docs.go ├── swagger.json └── swagger.yaml ├── tests ├── e2e-aio │ ├── e2e_suite_test.go │ ├── e2e_test.go │ └── sample_data_test.go ├── e2e-fixtures │ └── gpu.yaml ├── e2e │ ├── e2e_suite_test.go │ └── e2e_test.go ├── fixtures │ └── gallery_simple.yaml ├── integration │ ├── integration_suite_test.go │ └── stores_test.go └── models_fixtures │ ├── completion.tmpl │ ├── config.yaml │ ├── embeddings.yaml │ ├── ggml-gpt4all-j.tmpl │ ├── gpt4.yaml │ ├── gpt4_2.yaml │ ├── grpc.yaml │ ├── rwkv.yaml │ └── whisper.yaml └── webui_static.yaml /.devcontainer-scripts/postcreate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /workspace 4 | 5 | # Get the files into the volume without a bind mount 6 | if [ ! -d ".git" ]; then 7 | git clone https://github.com/mudler/LocalAI.git . 8 | else 9 | git fetch 10 | fi 11 | 12 | echo "Standard Post-Create script completed." 13 | 14 | if [ -f "/devcontainer-customization/postcreate.sh" ]; then 15 | echo "Launching customization postcreate.sh" 16 | bash "/devcontainer-customization/postcreate.sh" 17 | fi -------------------------------------------------------------------------------- /.devcontainer-scripts/poststart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /workspace 4 | 5 | # Grab the pre-stashed backend assets to avoid build issues 6 | cp -r /build/backend-assets /workspace/backend-assets 7 | 8 | # Ensures generated source files are present upon load 9 | make prepare 10 | 11 | echo "Standard Post-Start script completed." 12 | 13 | if [ -f "/devcontainer-customization/poststart.sh" ]; then 14 | echo "Launching customization poststart.sh" 15 | bash "/devcontainer-customization/poststart.sh" 16 | fi -------------------------------------------------------------------------------- /.devcontainer/customization/README.md: -------------------------------------------------------------------------------- 1 | Place any additional resources your environment requires in this directory 2 | 3 | Script hooks are currently called for: 4 | `postcreate.sh` and `poststart.sh` 5 | 6 | If files with those names exist here, they will be called at the end of the normal script. 7 | 8 | This is a good place to set things like `git config --global user.name` are set - and to handle any other files that are mounted via this directory. 9 | 10 | To assist in doing so, `source /.devcontainer-scripts/utils.sh` will provide utility functions that may be useful - for example: 11 | 12 | ``` 13 | #!/bin/bash 14 | 15 | source "/.devcontainer-scripts/utils.sh" 16 | 17 | sshfiles=("config", "key.pub") 18 | 19 | setup_ssh "${sshfiles[@]}" 20 | 21 | config_user "YOUR NAME" "YOUR EMAIL" 22 | 23 | config_remote "REMOTE NAME" "REMOTE URL" 24 | 25 | ``` -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/devcontainers/spec/main/schemas/devContainer.schema.json", 3 | "name": "LocalAI", 4 | "workspaceFolder": "/workspace", 5 | "dockerComposeFile": [ "./docker-compose-devcontainer.yml" ], 6 | "service": "api", 7 | "shutdownAction": "stopCompose", 8 | "customizations": { 9 | "vscode": { 10 | "extensions": [ 11 | "golang.go", 12 | "ms-vscode.makefile-tools", 13 | "ms-azuretools.vscode-docker", 14 | "ms-python.python", 15 | "ms-python.debugpy", 16 | "wayou.vscode-todo-highlight", 17 | "waderyan.gitblame" 18 | ] 19 | } 20 | }, 21 | "forwardPorts": [8080, 3000], 22 | "postCreateCommand": "bash /.devcontainer-scripts/postcreate.sh", 23 | "postStartCommand": "bash /.devcontainer-scripts/poststart.sh" 24 | } -------------------------------------------------------------------------------- /.devcontainer/grafana/datasource.yml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: 1 3 | 4 | datasources: 5 | - name: Prometheus 6 | type: prometheus 7 | url: http://prometheus:9090 8 | isDefault: true 9 | access: proxy 10 | editable: true 11 | -------------------------------------------------------------------------------- /.devcontainer/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | scrape_timeout: 10s 4 | evaluation_interval: 15s 5 | alerting: 6 | alertmanagers: 7 | - static_configs: 8 | - targets: [] 9 | scheme: http 10 | timeout: 10s 11 | api_version: v1 12 | scrape_configs: 13 | - job_name: prometheus 14 | honor_timestamps: true 15 | scrape_interval: 15s 16 | scrape_timeout: 10s 17 | metrics_path: /metrics 18 | scheme: http 19 | static_configs: 20 | - targets: 21 | - localhost:9090 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .github 3 | .vscode 4 | .devcontainer 5 | models 6 | examples/chatbot-ui/models 7 | examples/rwkv/models 8 | examples/**/models 9 | Dockerfile* 10 | __pycache__ 11 | 12 | # SonarQube 13 | .scannerwork 14 | 15 | # backend virtual environments 16 | **/venv 17 | backend/python/**/source -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.go] 13 | indent_style = tab 14 | 15 | [Makefile] 16 | indent_style = tab 17 | 18 | [*.proto] 19 | indent_size = 2 20 | 21 | [*.py] 22 | indent_size = 4 23 | 24 | [*.js] 25 | indent_size = 2 26 | 27 | [*.yaml] 28 | indent_size = 2 29 | 30 | [*.md] 31 | trim_trailing_whitespace = false 32 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | backend/cpp/llama/*.hpp linguist-vendored -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [mudler] 4 | custom: 5 | - https://www.buymeacoffee.com/mudler 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug, unconfirmed, up-for-grabs 6 | --- 7 | 8 | 9 | 10 | **LocalAI version:** 11 | 12 | 13 | **Environment, CPU architecture, OS, and Version:** 14 | 15 | 16 | **Describe the bug** 17 | 18 | 19 | **To Reproduce** 20 | 21 | 22 | **Expected behavior** 23 | 24 | 25 | **Logs** 26 | 27 | 28 | **Additional context** 29 | 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Community Support 4 | url: https://github.com/go-skynet/LocalAI/discussions 5 | about: Please ask and answer questions here. 6 | - name: Discord 7 | url: https://discord.gg/uJAeKSAGDy 8 | about: Join our community on Discord! 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement, up-for-grabs 6 | --- 7 | 8 | 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | 12 | 13 | **Describe the solution you'd like** 14 | 15 | 16 | **Describe alternatives you've considered** 17 | 18 | 19 | **Additional context** 20 | 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **Description** 2 | 3 | This PR fixes # 4 | 5 | **Notes for Reviewers** 6 | 7 | 8 | **[Signed commits](../CONTRIBUTING.md#signing-off-on-commits-developer-certificate-of-origin)** 9 | - [ ] Yes, I signed my commits. 10 | 11 | -------------------------------------------------------------------------------- /.github/bump_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | REPO=$1 4 | BRANCH=$2 5 | VAR=$3 6 | 7 | LAST_COMMIT=$(curl -s -H "Accept: application/vnd.github.VERSION.sha" "https://api.github.com/repos/$REPO/commits/$BRANCH") 8 | 9 | # Read $VAR from Makefile (only first match) 10 | set +e 11 | CURRENT_COMMIT="$(grep -m1 "^$VAR?=" Makefile | cut -d'=' -f2)" 12 | set -e 13 | 14 | sed -i Makefile -e "s/$VAR?=.*/$VAR?=$LAST_COMMIT/" 15 | 16 | if [ -z "$CURRENT_COMMIT" ]; then 17 | echo "Could not find $VAR in Makefile." 18 | exit 0 19 | fi 20 | 21 | echo "Changes: https://github.com/$REPO/compare/${CURRENT_COMMIT}..${LAST_COMMIT}" >> "${VAR}_message.txt" 22 | echo "${LAST_COMMIT}" >> "${VAR}_commit.txt" -------------------------------------------------------------------------------- /.github/bump_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | REPO=$1 4 | 5 | LATEST_TAG=$(curl -s "https://api.github.com/repos/$REPO/releases/latest" | jq -r '.tag_name') 6 | 7 | cat <<< $(jq ".version = \"$LATEST_TAG\"" docs/data/version.json) > docs/data/version.json 8 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | enhancement: 2 | - head-branch: ['^feature', 'feature'] 3 | 4 | dependencies: 5 | - any: 6 | - changed-files: 7 | - any-glob-to-any-file: 'Makefile' 8 | - changed-files: 9 | - any-glob-to-any-file: '*.mod' 10 | - changed-files: 11 | - any-glob-to-any-file: '*.sum' 12 | 13 | kind/documentation: 14 | - any: 15 | - changed-files: 16 | - any-glob-to-any-file: 'docs/*' 17 | - changed-files: 18 | - any-glob-to-any-file: '*.md' 19 | 20 | area/ai-model: 21 | - any: 22 | - changed-files: 23 | - any-glob-to-any-file: 'gallery/*' 24 | 25 | examples: 26 | - any: 27 | - changed-files: 28 | - any-glob-to-any-file: 'examples/*' 29 | 30 | ci: 31 | - any: 32 | - changed-files: 33 | - any-glob-to-any-file: '.github/*' 34 | -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | # .github/release.yml 2 | 3 | changelog: 4 | exclude: 5 | labels: 6 | - ignore-for-release 7 | categories: 8 | - title: Breaking Changes 🛠 9 | labels: 10 | - Semver-Major 11 | - breaking-change 12 | - title: "Bug fixes :bug:" 13 | labels: 14 | - bug 15 | - regression 16 | - title: "🖧 P2P area" 17 | labels: 18 | - area/p2p 19 | - title: Exciting New Features 🎉 20 | labels: 21 | - Semver-Minor 22 | - enhancement 23 | - ux 24 | - roadmap 25 | - title: 🧠 Models 26 | labels: 27 | - area/ai-model 28 | - title: 📖 Documentation and examples 29 | labels: 30 | - kind/documentation 31 | - examples 32 | - title: 👒 Dependencies 33 | labels: 34 | - dependencies 35 | - title: Other Changes 36 | labels: 37 | - "*" 38 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 45 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 10 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - issue/willfix 8 | # Label to use when marking an issue as stale 9 | staleLabel: issue/stale 10 | # Comment to post when marking an issue as stale. Set to `false` to disable 11 | markComment: > 12 | This issue has been automatically marked as stale because it has not had 13 | recent activity. It will be closed if no further activity occurs. Thank you 14 | for your contributions. 15 | # Comment to post when closing a stale issue. Set to `false` to disable 16 | closeComment: > 17 | This issue is being automatically closed due to inactivity. 18 | However, you may choose to reopen this issue. -------------------------------------------------------------------------------- /.github/workflows/bump_docs.yaml: -------------------------------------------------------------------------------- 1 | name: Bump dependencies 2 | on: 3 | schedule: 4 | - cron: 0 20 * * * 5 | workflow_dispatch: 6 | jobs: 7 | bump: 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | include: 12 | - repository: "mudler/LocalAI" 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Bump dependencies 🔧 17 | run: | 18 | bash .github/bump_docs.sh ${{ matrix.repository }} 19 | - name: Create Pull Request 20 | uses: peter-evans/create-pull-request@v7 21 | with: 22 | token: ${{ secrets.UPDATE_BOT_TOKEN }} 23 | push-to-fork: ci-forks/LocalAI 24 | commit-message: ':arrow_up: Update docs version ${{ matrix.repository }}' 25 | title: 'docs: :arrow_up: update docs version ${{ matrix.repository }}' 26 | branch: "update/docs" 27 | body: Bump of ${{ matrix.repository }} version inside docs 28 | signoff: true 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- 1 | name: "Pull Request Labeler" 2 | on: 3 | - pull_request_target 4 | 5 | jobs: 6 | labeler: 7 | permissions: 8 | contents: read 9 | pull-requests: write 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/labeler@v5 -------------------------------------------------------------------------------- /.github/workflows/localaibot_automerge.yml: -------------------------------------------------------------------------------- 1 | name: LocalAI-bot auto-merge 2 | on: 3 | - pull_request_target 4 | 5 | permissions: 6 | contents: write 7 | pull-requests: write 8 | packages: read 9 | 10 | jobs: 11 | dependabot: 12 | runs-on: ubuntu-latest 13 | if: ${{ github.actor == 'localai-bot' }} 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v4 17 | 18 | - name: Approve a PR if not already approved 19 | run: | 20 | gh pr checkout "$PR_URL" 21 | if [ "$(gh pr status --json reviewDecision -q .currentBranch.reviewDecision)" != "APPROVED" ]; 22 | then 23 | gh pr review --approve "$PR_URL" 24 | else 25 | echo "PR already approved."; 26 | fi 27 | env: 28 | PR_URL: ${{github.event.pull_request.html_url}} 29 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 30 | 31 | - name: Enable auto-merge for LocalAIBot PRs 32 | run: gh pr merge --auto --squash "$PR_URL" 33 | env: 34 | PR_URL: ${{github.event.pull_request.html_url}} 35 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 36 | -------------------------------------------------------------------------------- /.github/workflows/prlint.yaml: -------------------------------------------------------------------------------- 1 | name: Check PR style 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - reopened 8 | - edited 9 | - synchronize 10 | 11 | jobs: 12 | title-lint: 13 | runs-on: ubuntu-latest 14 | permissions: 15 | statuses: write 16 | steps: 17 | - uses: aslafy-z/conventional-pr-title-action@v3 18 | env: 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | # check-pr-description: 21 | # runs-on: ubuntu-latest 22 | # steps: 23 | # - uses: actions/checkout@v2 24 | # - uses: jadrol/pr-description-checker-action@v1.0.0 25 | # id: description-checker 26 | # with: 27 | # repo-token: ${{ secrets.GITHUB_TOKEN }} 28 | # exempt-labels: no qa 29 | -------------------------------------------------------------------------------- /.github/workflows/secscan.yaml: -------------------------------------------------------------------------------- 1 | name: "Security Scan" 2 | 3 | # Run workflow each time code is pushed to your repository and on a schedule. 4 | # The scheduled workflow runs every at 00:00 on Sunday UTC time. 5 | on: 6 | push: 7 | schedule: 8 | - cron: '0 0 * * 0' 9 | 10 | jobs: 11 | tests: 12 | runs-on: ubuntu-latest 13 | env: 14 | GO111MODULE: on 15 | steps: 16 | - name: Checkout Source 17 | uses: actions/checkout@v4 18 | if: ${{ github.actor != 'dependabot[bot]' }} 19 | - name: Run Gosec Security Scanner 20 | if: ${{ github.actor != 'dependabot[bot]' }} 21 | uses: securego/gosec@v2.22.4 22 | with: 23 | # we let the report trigger content trigger a failure using the GitHub Security features. 24 | args: '-no-fail -fmt sarif -out results.sarif ./...' 25 | - name: Upload SARIF file 26 | if: ${{ github.actor != 'dependabot[bot]' }} 27 | uses: github/codeql-action/upload-sarif@v3 28 | with: 29 | # Path to SARIF file relative to the root of the repository 30 | sarif_file: results.sarif 31 | -------------------------------------------------------------------------------- /.github/workflows/update_swagger.yaml: -------------------------------------------------------------------------------- 1 | name: Update swagger 2 | on: 3 | schedule: 4 | - cron: 0 20 * * * 5 | workflow_dispatch: 6 | jobs: 7 | swagger: 8 | strategy: 9 | fail-fast: false 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-go@v5 14 | with: 15 | go-version: 'stable' 16 | - name: Dependencies 17 | run: | 18 | sudo apt-get update 19 | sudo apt-get install protobuf-compiler 20 | - run: | 21 | go install github.com/swaggo/swag/cmd/swag@latest 22 | go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af 23 | go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 24 | - name: Bump swagger 🔧 25 | run: | 26 | make protogen-go swagger 27 | - name: Create Pull Request 28 | uses: peter-evans/create-pull-request@v7 29 | with: 30 | token: ${{ secrets.UPDATE_BOT_TOKEN }} 31 | push-to-fork: ci-forks/LocalAI 32 | commit-message: 'feat(swagger): update swagger' 33 | title: 'feat(swagger): update swagger' 34 | branch: "update/swagger" 35 | body: Update swagger 36 | signoff: true 37 | 38 | -------------------------------------------------------------------------------- /.github/workflows/yaml-check.yml: -------------------------------------------------------------------------------- 1 | name: 'Yamllint GitHub Actions' 2 | on: 3 | - pull_request 4 | jobs: 5 | yamllint: 6 | name: 'Yamllint' 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: 'Checkout' 10 | uses: actions/checkout@master 11 | - name: 'Yamllint' 12 | uses: karancode/yamllint-github-action@master 13 | with: 14 | yamllint_file_or_dir: 'gallery' 15 | yamllint_strict: false 16 | yamllint_comment: true 17 | env: 18 | GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # go-llama build artifacts 2 | /sources/ 3 | __pycache__/ 4 | *.a 5 | *.o 6 | get-sources 7 | prepare-sources 8 | /backend/cpp/llama/grpc-server 9 | /backend/cpp/llama/llama.cpp 10 | /backend/cpp/llama-* 11 | 12 | *.log 13 | 14 | go-ggml-transformers 15 | go-gpt2 16 | whisper.cpp 17 | /bloomz 18 | go-bert 19 | 20 | # LocalAI build binary 21 | LocalAI 22 | local-ai 23 | # prevent above rules from omitting the helm chart 24 | !charts/* 25 | # prevent above rules from omitting the api/localai folder 26 | !api/localai 27 | !core/**/localai 28 | 29 | # Ignore models 30 | models/* 31 | test-models/ 32 | test-dir/ 33 | 34 | release/ 35 | 36 | # just in case 37 | .DS_Store 38 | .idea 39 | 40 | # Generated during build 41 | backend-assets/* 42 | !backend-assets/.keep 43 | prepare 44 | /ggml-metal.metal 45 | docs/static/gallery.html 46 | 47 | # Protobuf generated files 48 | *.pb.go 49 | *pb2.py 50 | *pb2_grpc.py 51 | 52 | # SonarQube 53 | .scannerwork 54 | 55 | # backend virtual environments 56 | **/venv 57 | 58 | # per-developer customization files for the development container 59 | .devcontainer/customization/* -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "docs/themes/hugo-theme-relearn"] 2 | path = docs/themes/hugo-theme-relearn 3 | url = https://github.com/McShelby/hugo-theme-relearn.git 4 | [submodule "docs/themes/lotusdocs"] 5 | path = docs/themes/lotusdocs 6 | url = https://github.com/colinwilson/lotusdocs 7 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "golang.go" 4 | ] 5 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Python: Current File", 6 | "type": "debugpy", 7 | "request": "launch", 8 | "program": "${file}", 9 | "console": "integratedTerminal", 10 | "justMyCode": false, 11 | "cwd": "${fileDirname}", 12 | "env": { 13 | "OPENAI_API_BASE": "http://localhost:8080/v1", 14 | "OPENAI_API_KEY": "abc" 15 | } 16 | }, 17 | { 18 | "name": "Launch LocalAI API", 19 | "type": "go", 20 | "request": "launch", 21 | "mode": "debug", 22 | "program": "${workspaceRoot}", 23 | "args": [], 24 | "env": { 25 | "LOCALAI_LOG_LEVEL": "debug", 26 | "LOCALAI_P2P": "true", 27 | "LOCALAI_FEDERATED": "true" 28 | }, 29 | "buildFlags": ["-tags", "p2p tts", "-v"], 30 | "envFile": "${workspaceFolder}/.env", 31 | "cwd": "${workspaceRoot}" 32 | } 33 | ] 34 | } -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | extends: default 2 | 3 | rules: 4 | line-length: disable -------------------------------------------------------------------------------- /Dockerfile.aio: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=ubuntu:22.04 2 | 3 | FROM ${BASE_IMAGE} 4 | 5 | RUN apt-get update && apt-get install -y pciutils && apt-get clean 6 | 7 | COPY aio/ /aio 8 | ENTRYPOINT [ "/aio/entrypoint.sh" ] -------------------------------------------------------------------------------- /Earthfile: -------------------------------------------------------------------------------- 1 | VERSION 0.7 2 | 3 | build: 4 | FROM DOCKERFILE -f Dockerfile . 5 | SAVE ARTIFACT /usr/bin/local-ai AS LOCAL local-ai 6 | -------------------------------------------------------------------------------- /Entitlements.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.apple.security.network.client 6 | 7 | com.apple.security.network.server 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023-2025 Ettore Di Giacinto (mudler@localai.io) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /aio/cpu/README.md: -------------------------------------------------------------------------------- 1 | ## AIO CPU size 2 | 3 | Use this image with CPU-only. 4 | 5 | Please keep using only C++ backends so the base image is as small as possible (without CUDA, cuDNN, python, etc). -------------------------------------------------------------------------------- /aio/cpu/embeddings.yaml: -------------------------------------------------------------------------------- 1 | embeddings: true 2 | name: text-embedding-ada-002 3 | parameters: 4 | model: huggingface://bartowski/granite-embedding-107m-multilingual-GGUF/granite-embedding-107m-multilingual-f16.gguf 5 | 6 | usage: | 7 | You can test this model with curl like this: 8 | 9 | curl http://localhost:8080/embeddings -X POST -H "Content-Type: application/json" -d '{ 10 | "input": "Your text string goes here", 11 | "model": "text-embedding-ada-002" 12 | }' -------------------------------------------------------------------------------- /aio/cpu/image-gen.yaml: -------------------------------------------------------------------------------- 1 | name: stablediffusion 2 | backend: stablediffusion-ggml 3 | cfg_scale: 4.5 4 | 5 | options: 6 | - sampler:euler 7 | parameters: 8 | model: stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf 9 | step: 25 10 | 11 | download_files: 12 | - filename: "stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf" 13 | sha256: "b8944e9fe0b69b36ae1b5bb0185b3a7b8ef14347fe0fa9af6c64c4829022261f" 14 | uri: "huggingface://second-state/stable-diffusion-v1-5-GGUF/stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf" 15 | 16 | usage: | 17 | curl http://localhost:8080/v1/images/generations \ 18 | -H "Content-Type: application/json" \ 19 | -d '{ 20 | "prompt": "|", 21 | "step": 25, 22 | "size": "512x512" 23 | }' -------------------------------------------------------------------------------- /aio/cpu/rerank.yaml: -------------------------------------------------------------------------------- 1 | name: jina-reranker-v1-base-en 2 | backend: rerankers 3 | parameters: 4 | model: cross-encoder 5 | 6 | usage: | 7 | You can test this model with curl like this: 8 | 9 | curl http://localhost:8080/v1/rerank \ 10 | -H "Content-Type: application/json" \ 11 | -d '{ 12 | "model": "jina-reranker-v1-base-en", 13 | "query": "Organic skincare products for sensitive skin", 14 | "documents": [ 15 | "Eco-friendly kitchenware for modern homes", 16 | "Biodegradable cleaning supplies for eco-conscious consumers", 17 | "Organic cotton baby clothes for sensitive skin", 18 | "Natural organic skincare range for sensitive skin", 19 | "Tech gadgets for smart homes: 2024 edition", 20 | "Sustainable gardening tools and compost solutions", 21 | "Sensitive skin-friendly facial cleansers and toners", 22 | "Organic food wraps and storage solutions", 23 | "All-natural pet food for dogs with allergies", 24 | "Yoga mats made from recycled materials" 25 | ], 26 | "top_n": 3 27 | }' 28 | -------------------------------------------------------------------------------- /aio/cpu/speech-to-text.yaml: -------------------------------------------------------------------------------- 1 | name: whisper-1 2 | backend: whisper 3 | parameters: 4 | model: ggml-whisper-base.bin 5 | 6 | usage: | 7 | ## example audio file 8 | wget --quiet --show-progress -O gb1.ogg https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg 9 | 10 | ## Send the example audio file to the transcriptions endpoint 11 | curl http://localhost:8080/v1/audio/transcriptions \ 12 | -H "Content-Type: multipart/form-data" \ 13 | -F file="@$PWD/gb1.ogg" -F model="whisper-1" 14 | 15 | download_files: 16 | - filename: "ggml-whisper-base.bin" 17 | sha256: "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe" 18 | uri: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin" -------------------------------------------------------------------------------- /aio/cpu/text-to-speech.yaml: -------------------------------------------------------------------------------- 1 | name: tts-1 2 | download_files: 3 | - filename: voice-en-us-amy-low.tar.gz 4 | uri: https://github.com/rhasspy/piper/releases/download/v0.0.2/voice-en-us-amy-low.tar.gz 5 | 6 | parameters: 7 | model: en-us-amy-low.onnx 8 | 9 | usage: | 10 | To test if this model works as expected, you can use the following curl command: 11 | 12 | curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{ 13 | "model":"voice-en-us-amy-low", 14 | "input": "Hi, this is a test." 15 | }' -------------------------------------------------------------------------------- /aio/cpu/vad.yaml: -------------------------------------------------------------------------------- 1 | backend: silero-vad 2 | name: silero-vad 3 | parameters: 4 | model: silero-vad.onnx 5 | download_files: 6 | - filename: silero-vad.onnx 7 | uri: https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx 8 | sha256: a4a068cd6cf1ea8355b84327595838ca748ec29a25bc91fc82e6c299ccdc5808 -------------------------------------------------------------------------------- /aio/gpu-8g/embeddings.yaml: -------------------------------------------------------------------------------- 1 | embeddings: true 2 | name: text-embedding-ada-002 3 | parameters: 4 | model: huggingface://bartowski/granite-embedding-107m-multilingual-GGUF/granite-embedding-107m-multilingual-f16.gguf 5 | 6 | usage: | 7 | You can test this model with curl like this: 8 | 9 | curl http://localhost:8080/embeddings -X POST -H "Content-Type: application/json" -d '{ 10 | "input": "Your text string goes here", 11 | "model": "text-embedding-ada-002" 12 | }' -------------------------------------------------------------------------------- /aio/gpu-8g/image-gen.yaml: -------------------------------------------------------------------------------- 1 | name: stablediffusion 2 | parameters: 3 | model: DreamShaper_8_pruned.safetensors 4 | backend: diffusers 5 | step: 25 6 | f16: true 7 | 8 | diffusers: 9 | pipeline_type: StableDiffusionPipeline 10 | cuda: true 11 | enable_parameters: "negative_prompt,num_inference_steps" 12 | scheduler_type: "k_dpmpp_2m" 13 | 14 | download_files: 15 | - filename: DreamShaper_8_pruned.safetensors 16 | uri: huggingface://Lykon/DreamShaper/DreamShaper_8_pruned.safetensors 17 | 18 | usage: | 19 | curl http://localhost:8080/v1/images/generations \ 20 | -H "Content-Type: application/json" \ 21 | -d '{ 22 | "prompt": "|", 23 | "step": 25, 24 | "size": "512x512" 25 | }' -------------------------------------------------------------------------------- /aio/gpu-8g/rerank.yaml: -------------------------------------------------------------------------------- 1 | name: jina-reranker-v1-base-en 2 | backend: rerankers 3 | parameters: 4 | model: cross-encoder 5 | 6 | usage: | 7 | You can test this model with curl like this: 8 | 9 | curl http://localhost:8080/v1/rerank \ 10 | -H "Content-Type: application/json" \ 11 | -d '{ 12 | "model": "jina-reranker-v1-base-en", 13 | "query": "Organic skincare products for sensitive skin", 14 | "documents": [ 15 | "Eco-friendly kitchenware for modern homes", 16 | "Biodegradable cleaning supplies for eco-conscious consumers", 17 | "Organic cotton baby clothes for sensitive skin", 18 | "Natural organic skincare range for sensitive skin", 19 | "Tech gadgets for smart homes: 2024 edition", 20 | "Sustainable gardening tools and compost solutions", 21 | "Sensitive skin-friendly facial cleansers and toners", 22 | "Organic food wraps and storage solutions", 23 | "All-natural pet food for dogs with allergies", 24 | "Yoga mats made from recycled materials" 25 | ], 26 | "top_n": 3 27 | }' 28 | -------------------------------------------------------------------------------- /aio/gpu-8g/speech-to-text.yaml: -------------------------------------------------------------------------------- 1 | name: whisper-1 2 | backend: whisper 3 | parameters: 4 | model: ggml-whisper-base.bin 5 | 6 | usage: | 7 | ## example audio file 8 | wget --quiet --show-progress -O gb1.ogg https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg 9 | 10 | ## Send the example audio file to the transcriptions endpoint 11 | curl http://localhost:8080/v1/audio/transcriptions \ 12 | -H "Content-Type: multipart/form-data" \ 13 | -F file="@$PWD/gb1.ogg" -F model="whisper-1" 14 | 15 | download_files: 16 | - filename: "ggml-whisper-base.bin" 17 | sha256: "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe" 18 | uri: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin" -------------------------------------------------------------------------------- /aio/gpu-8g/text-to-speech.yaml: -------------------------------------------------------------------------------- 1 | name: tts-1 2 | download_files: 3 | - filename: voice-en-us-amy-low.tar.gz 4 | uri: https://github.com/rhasspy/piper/releases/download/v0.0.2/voice-en-us-amy-low.tar.gz 5 | 6 | parameters: 7 | model: en-us-amy-low.onnx 8 | 9 | usage: | 10 | To test if this model works as expected, you can use the following curl command: 11 | 12 | curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{ 13 | "model":"tts-1", 14 | "input": "Hi, this is a test." 15 | }' -------------------------------------------------------------------------------- /aio/gpu-8g/vad.yaml: -------------------------------------------------------------------------------- 1 | backend: silero-vad 2 | name: silero-vad 3 | parameters: 4 | model: silero-vad.onnx 5 | download_files: 6 | - filename: silero-vad.onnx 7 | uri: https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx 8 | sha256: a4a068cd6cf1ea8355b84327595838ca748ec29a25bc91fc82e6c299ccdc5808 -------------------------------------------------------------------------------- /aio/intel/embeddings.yaml: -------------------------------------------------------------------------------- 1 | embeddings: true 2 | name: text-embedding-ada-002 3 | parameters: 4 | model: huggingface://bartowski/granite-embedding-107m-multilingual-GGUF/granite-embedding-107m-multilingual-f16.gguf 5 | 6 | usage: | 7 | You can test this model with curl like this: 8 | 9 | curl http://localhost:8080/embeddings -X POST -H "Content-Type: application/json" -d '{ 10 | "input": "Your text string goes here", 11 | "model": "text-embedding-ada-002" 12 | }' -------------------------------------------------------------------------------- /aio/intel/image-gen.yaml: -------------------------------------------------------------------------------- 1 | name: stablediffusion 2 | parameters: 3 | model: Lykon/dreamshaper-8 4 | backend: diffusers 5 | step: 25 6 | f16: true 7 | diffusers: 8 | pipeline_type: StableDiffusionPipeline 9 | cuda: true 10 | enable_parameters: "negative_prompt,num_inference_steps" 11 | scheduler_type: "k_dpmpp_2m" 12 | 13 | usage: | 14 | curl http://localhost:8080/v1/images/generations \ 15 | -H "Content-Type: application/json" \ 16 | -d '{ 17 | "prompt": "|", 18 | "step": 25, 19 | "size": "512x512" 20 | }' -------------------------------------------------------------------------------- /aio/intel/rerank.yaml: -------------------------------------------------------------------------------- 1 | name: jina-reranker-v1-base-en 2 | backend: rerankers 3 | parameters: 4 | model: cross-encoder 5 | 6 | usage: | 7 | You can test this model with curl like this: 8 | 9 | curl http://localhost:8080/v1/rerank \ 10 | -H "Content-Type: application/json" \ 11 | -d '{ 12 | "model": "jina-reranker-v1-base-en", 13 | "query": "Organic skincare products for sensitive skin", 14 | "documents": [ 15 | "Eco-friendly kitchenware for modern homes", 16 | "Biodegradable cleaning supplies for eco-conscious consumers", 17 | "Organic cotton baby clothes for sensitive skin", 18 | "Natural organic skincare range for sensitive skin", 19 | "Tech gadgets for smart homes: 2024 edition", 20 | "Sustainable gardening tools and compost solutions", 21 | "Sensitive skin-friendly facial cleansers and toners", 22 | "Organic food wraps and storage solutions", 23 | "All-natural pet food for dogs with allergies", 24 | "Yoga mats made from recycled materials" 25 | ], 26 | "top_n": 3 27 | }' 28 | -------------------------------------------------------------------------------- /aio/intel/speech-to-text.yaml: -------------------------------------------------------------------------------- 1 | name: whisper-1 2 | backend: whisper 3 | parameters: 4 | model: ggml-whisper-base.bin 5 | 6 | usage: | 7 | ## example audio file 8 | wget --quiet --show-progress -O gb1.ogg https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg 9 | 10 | ## Send the example audio file to the transcriptions endpoint 11 | curl http://localhost:8080/v1/audio/transcriptions \ 12 | -H "Content-Type: multipart/form-data" \ 13 | -F file="@$PWD/gb1.ogg" -F model="whisper-1" 14 | 15 | download_files: 16 | - filename: "ggml-whisper-base.bin" 17 | sha256: "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe" 18 | uri: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin" -------------------------------------------------------------------------------- /aio/intel/text-to-speech.yaml: -------------------------------------------------------------------------------- 1 | name: tts-1 2 | download_files: 3 | - filename: voice-en-us-amy-low.tar.gz 4 | uri: https://github.com/rhasspy/piper/releases/download/v0.0.2/voice-en-us-amy-low.tar.gz 5 | 6 | parameters: 7 | model: en-us-amy-low.onnx 8 | 9 | usage: | 10 | To test if this model works as expected, you can use the following curl command: 11 | 12 | curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{ 13 | "model":"tts-1", 14 | "input": "Hi, this is a test." 15 | }' -------------------------------------------------------------------------------- /aio/intel/vad.yaml: -------------------------------------------------------------------------------- 1 | backend: silero-vad 2 | name: silero-vad 3 | parameters: 4 | model: silero-vad.onnx 5 | download_files: 6 | - filename: silero-vad.onnx 7 | uri: https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx 8 | sha256: a4a068cd6cf1ea8355b84327595838ca748ec29a25bc91fc82e6c299ccdc5808 -------------------------------------------------------------------------------- /assets.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | rice "github.com/GeertJohan/go.rice" 5 | ) 6 | 7 | var backendAssets *rice.Box 8 | 9 | func init() { 10 | var err error 11 | backendAssets, err = rice.FindBox("backend-assets") 12 | if err != nil { 13 | panic(err) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /backend/cpp/grpc/.gitignore: -------------------------------------------------------------------------------- 1 | installed_packages/ 2 | grpc_build/ 3 | grpc_repo/ 4 | -------------------------------------------------------------------------------- /backend/cpp/llama/patches/01-llava.patch: -------------------------------------------------------------------------------- 1 | diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp 2 | index 3cd0d2fa..6c5e811a 100644 3 | --- a/tools/mtmd/clip.cpp 4 | +++ b/tools/mtmd/clip.cpp 5 | @@ -2608,7 +2608,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima 6 | struct ggml_tensor * patches = ggml_graph_get_tensor(gf, "patches"); 7 | int* patches_data = (int*)malloc(ggml_nbytes(patches)); 8 | for (int i = 0; i < num_patches; i++) { 9 | - patches_data[i] = i + 1; 10 | + patches_data[i] = i; 11 | } 12 | ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches)); 13 | free(patches_data); -------------------------------------------------------------------------------- /backend/go/bark/Makefile: -------------------------------------------------------------------------------- 1 | INCLUDE_PATH := $(abspath ./) 2 | LIBRARY_PATH := $(abspath ./) 3 | 4 | AR?=ar 5 | 6 | BUILD_TYPE?= 7 | # keep standard at C11 and C++11 8 | CXXFLAGS = -I. -I$(INCLUDE_PATH)/../../../sources/bark.cpp/examples -I$(INCLUDE_PATH)/../../../sources/bark.cpp/spm-headers -I$(INCLUDE_PATH)/../../../sources/bark.cpp -O3 -DNDEBUG -std=c++17 -fPIC 9 | LDFLAGS = -L$(LIBRARY_PATH) -L$(LIBRARY_PATH)/../../../sources/bark.cpp/build/examples -lbark -lstdc++ -lm 10 | 11 | # warnings 12 | CXXFLAGS += -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function 13 | 14 | gobark.o: 15 | $(CXX) $(CXXFLAGS) gobark.cpp -o gobark.o -c $(LDFLAGS) 16 | 17 | libbark.a: gobark.o 18 | cp $(INCLUDE_PATH)/../../../sources/bark.cpp/build/libbark.a ./ 19 | $(AR) rcs libbark.a gobark.o 20 | $(AR) rcs libbark.a $(LIBRARY_PATH)/../../../sources/bark.cpp/build/encodec.cpp/ggml/src/CMakeFiles/ggml.dir/ggml.c.o 21 | $(AR) rcs libbark.a $(LIBRARY_PATH)/../../../sources/bark.cpp/build/encodec.cpp/ggml/src/CMakeFiles/ggml.dir/ggml-alloc.c.o 22 | $(AR) rcs libbark.a $(LIBRARY_PATH)/../../../sources/bark.cpp/build/encodec.cpp/ggml/src/CMakeFiles/ggml.dir/ggml-backend.c.o 23 | 24 | clean: 25 | rm -f gobark.o libbark.a -------------------------------------------------------------------------------- /backend/go/bark/gobark.h: -------------------------------------------------------------------------------- 1 | #ifdef __cplusplus 2 | extern "C" { 3 | #endif 4 | int load_model(char *model); 5 | int tts(char *text,int threads, char *dst ); 6 | #ifdef __cplusplus 7 | } 8 | #endif -------------------------------------------------------------------------------- /backend/go/bark/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each model 4 | import ( 5 | "flag" 6 | 7 | grpc "github.com/mudler/LocalAI/pkg/grpc" 8 | ) 9 | 10 | var ( 11 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 12 | ) 13 | 14 | func main() { 15 | flag.Parse() 16 | 17 | if err := grpc.StartServer(*addr, &Bark{}); err != nil { 18 | panic(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /backend/go/image/stablediffusion-ggml/gosd.h: -------------------------------------------------------------------------------- 1 | #ifdef __cplusplus 2 | extern "C" { 3 | #endif 4 | int load_model(char *model, char* options[], int threads, int diffusionModel); 5 | int gen_image(char *text, char *negativeText, int width, int height, int steps, int seed, char *dst, float cfg_scale); 6 | #ifdef __cplusplus 7 | } 8 | #endif -------------------------------------------------------------------------------- /backend/go/image/stablediffusion-ggml/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each model 4 | import ( 5 | "flag" 6 | 7 | grpc "github.com/mudler/LocalAI/pkg/grpc" 8 | ) 9 | 10 | var ( 11 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 12 | ) 13 | 14 | func main() { 15 | flag.Parse() 16 | 17 | if err := grpc.StartServer(*addr, &SDGGML{}); err != nil { 18 | panic(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /backend/go/llm/langchain/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each model 4 | 5 | import ( 6 | "flag" 7 | 8 | grpc "github.com/mudler/LocalAI/pkg/grpc" 9 | ) 10 | 11 | var ( 12 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | 18 | if err := grpc.StartServer(*addr, &LLM{}); err != nil { 19 | panic(err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /backend/go/llm/llama/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // GRPC Falcon server 4 | 5 | // Note: this is started internally by LocalAI and a server is allocated for each model 6 | 7 | import ( 8 | "flag" 9 | 10 | grpc "github.com/mudler/LocalAI/pkg/grpc" 11 | ) 12 | 13 | var ( 14 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 15 | ) 16 | 17 | func main() { 18 | flag.Parse() 19 | 20 | if err := grpc.StartServer(*addr, &LLM{}); err != nil { 21 | panic(err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /backend/go/stores/debug.go: -------------------------------------------------------------------------------- 1 | //go:build debug 2 | // +build debug 3 | 4 | package main 5 | 6 | import ( 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | func assert(cond bool, msg string) { 11 | if !cond { 12 | log.Fatal().Stack().Msg(msg) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /backend/go/stores/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each store 4 | 5 | import ( 6 | "flag" 7 | "os" 8 | 9 | grpc "github.com/mudler/LocalAI/pkg/grpc" 10 | "github.com/rs/zerolog" 11 | "github.com/rs/zerolog/log" 12 | ) 13 | 14 | var ( 15 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 16 | ) 17 | 18 | func main() { 19 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 20 | 21 | flag.Parse() 22 | 23 | if err := grpc.StartServer(*addr, NewStore()); err != nil { 24 | panic(err) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /backend/go/stores/production.go: -------------------------------------------------------------------------------- 1 | //go:build !debug 2 | // +build !debug 3 | 4 | package main 5 | 6 | func assert(cond bool, msg string) { 7 | } 8 | -------------------------------------------------------------------------------- /backend/go/transcribe/whisper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each model 4 | 5 | import ( 6 | "flag" 7 | 8 | grpc "github.com/mudler/LocalAI/pkg/grpc" 9 | ) 10 | 11 | var ( 12 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | 18 | if err := grpc.StartServer(*addr, &Whisper{}); err != nil { 19 | panic(err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /backend/go/tts/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each model 4 | 5 | import ( 6 | "flag" 7 | 8 | grpc "github.com/mudler/LocalAI/pkg/grpc" 9 | ) 10 | 11 | var ( 12 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | 18 | if err := grpc.StartServer(*addr, &Piper{}); err != nil { 19 | panic(err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /backend/go/vad/silero/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Note: this is started internally by LocalAI and a server is allocated for each model 4 | 5 | import ( 6 | "flag" 7 | 8 | grpc "github.com/mudler/LocalAI/pkg/grpc" 9 | ) 10 | 11 | var ( 12 | addr = flag.String("addr", "localhost:50051", "the address to connect to") 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | 18 | if err := grpc.StartServer(*addr, &VAD{}); err != nil { 19 | panic(err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /backend/python/README.md: -------------------------------------------------------------------------------- 1 | # Common commands about conda environment 2 | 3 | ## Create a new empty conda environment 4 | 5 | ``` 6 | conda create --name python= -y 7 | 8 | conda create --name autogptq python=3.11 -y 9 | ``` 10 | 11 | ## To activate the environment 12 | 13 | As of conda 4.4 14 | ``` 15 | conda activate autogptq 16 | ``` 17 | 18 | The conda version older than 4.4 19 | 20 | ``` 21 | source activate autogptq 22 | ``` 23 | 24 | ## Install the packages to your environment 25 | 26 | Sometimes you need to install the packages from the conda-forge channel 27 | 28 | By using `conda` 29 | ``` 30 | conda install 31 | 32 | conda install -c conda-forge 33 | ``` 34 | 35 | Or by using `pip` 36 | ``` 37 | pip install 38 | ``` 39 | -------------------------------------------------------------------------------- /backend/python/bark/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: ttsbark 2 | ttsbark: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running bark..." 8 | bash run.sh 9 | @echo "bark run." 10 | 11 | .PHONY: test 12 | test: protogen 13 | @echo "Testing bark..." 14 | bash test.sh 15 | @echo "bark tested." 16 | 17 | .PHONY: protogen 18 | protogen: backend_pb2_grpc.py backend_pb2.py 19 | 20 | .PHONY: protogen-clean 21 | protogen-clean: 22 | $(RM) backend_pb2_grpc.py backend_pb2.py 23 | 24 | backend_pb2_grpc.py backend_pb2.py: 25 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 26 | 27 | .PHONY: clean 28 | clean: protogen-clean 29 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/bark/README.md: -------------------------------------------------------------------------------- 1 | # Creating a separate environment for ttsbark project 2 | 3 | ``` 4 | make ttsbark 5 | ``` 6 | 7 | # Testing the gRPC server 8 | 9 | ``` 10 | -m unittest test_ttsbark.py 11 | ``` 12 | 13 | For example 14 | ``` 15 | /opt/conda/envs/bark/bin/python -m unittest extra/grpc/bark/test_ttsbark.py 16 | `````` -------------------------------------------------------------------------------- /backend/python/bark/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/bark/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | accelerate 3 | torch==2.4.1 4 | torchaudio==2.4.1 -------------------------------------------------------------------------------- /backend/python/bark/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | torchaudio==2.4.1+cu118 4 | transformers 5 | accelerate -------------------------------------------------------------------------------- /backend/python/bark/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | torchaudio==2.4.1 3 | transformers 4 | accelerate -------------------------------------------------------------------------------- /backend/python/bark/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch==2.4.1+rocm6.0 3 | torchaudio==2.4.1+rocm6.0 4 | transformers 5 | accelerate -------------------------------------------------------------------------------- /backend/python/bark/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | torchaudio==2.3.1+cxx11.abi 5 | oneccl_bind_pt==2.3.100+xpu 6 | optimum[openvino] 7 | setuptools 8 | transformers 9 | accelerate -------------------------------------------------------------------------------- /backend/python/bark/requirements.txt: -------------------------------------------------------------------------------- 1 | bark==0.1.5 2 | grpcio==1.72.0 3 | protobuf 4 | certifi -------------------------------------------------------------------------------- /backend/python/bark/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/bark/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/chatterbox/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: coqui 2 | coqui: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running coqui..." 8 | bash run.sh 9 | @echo "coqui run." 10 | 11 | .PHONY: test 12 | test: protogen 13 | @echo "Testing coqui..." 14 | bash test.sh 15 | @echo "coqui tested." 16 | 17 | .PHONY: protogen 18 | protogen: backend_pb2_grpc.py backend_pb2.py 19 | 20 | .PHONY: protogen-clean 21 | protogen-clean: 22 | $(RM) backend_pb2_grpc.py backend_pb2.py 23 | 24 | backend_pb2_grpc.py backend_pb2.py: 25 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 26 | 27 | .PHONY: clean 28 | clean: protogen-clean 29 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/chatterbox/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/chatterbox/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | torch==2.6.0 3 | torchaudio==2.6.0 4 | transformers==4.46.3 5 | chatterbox-tts 6 | -------------------------------------------------------------------------------- /backend/python/chatterbox/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.6.0+cu118 3 | torchaudio==2.6.0+cu118 4 | transformers==4.46.3 5 | chatterbox-tts 6 | accelerate -------------------------------------------------------------------------------- /backend/python/chatterbox/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.6.0 2 | torchaudio==2.6.0 3 | transformers==4.46.3 4 | chatterbox-tts 5 | accelerate 6 | -------------------------------------------------------------------------------- /backend/python/chatterbox/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch==2.6.0+rocm6.0 3 | torchaudio==2.6.0+rocm6.0 4 | transformers==4.46.3 5 | chatterbox-tts 6 | accelerate 7 | -------------------------------------------------------------------------------- /backend/python/chatterbox/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | torchaudio==2.3.1+cxx11.abi 5 | transformers==4.46.3 6 | chatterbox-tts 7 | accelerate 8 | oneccl_bind_pt==2.3.100+xpu 9 | optimum[openvino] 10 | setuptools 11 | transformers==4.48.3 12 | accelerate -------------------------------------------------------------------------------- /backend/python/chatterbox/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | certifi 4 | packaging 5 | setuptools -------------------------------------------------------------------------------- /backend/python/chatterbox/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/chatterbox/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/common/template/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := install 2 | 3 | .PHONY: install 4 | install: 5 | bash install.sh 6 | $(MAKE) protogen 7 | 8 | .PHONY: protogen 9 | protogen: backend_pb2_grpc.py backend_pb2.py 10 | 11 | .PHONY: protogen-clean 12 | protogen-clean: 13 | $(RM) backend_pb2_grpc.py backend_pb2.py 14 | 15 | backend_pb2_grpc.py backend_pb2.py: 16 | bash protogen.sh 17 | 18 | .PHONY: clean 19 | clean: protogen-clean 20 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/common/template/backend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import grpc 3 | import backend_pb2 4 | import backend_pb2_grpc 5 | -------------------------------------------------------------------------------- /backend/python/common/template/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/common/template/protogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto -------------------------------------------------------------------------------- /backend/python/common/template/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch -------------------------------------------------------------------------------- /backend/python/common/template/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | oneccl_bind_pt==2.3.100+xpu 5 | optimum[openvino] -------------------------------------------------------------------------------- /backend/python/common/template/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | grpcio-tools -------------------------------------------------------------------------------- /backend/python/common/template/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/common/template/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/coqui/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: coqui 2 | coqui: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running coqui..." 8 | bash run.sh 9 | @echo "coqui run." 10 | 11 | .PHONY: test 12 | test: protogen 13 | @echo "Testing coqui..." 14 | bash test.sh 15 | @echo "coqui tested." 16 | 17 | .PHONY: protogen 18 | protogen: backend_pb2_grpc.py backend_pb2.py 19 | 20 | .PHONY: protogen-clean 21 | protogen-clean: 22 | $(RM) backend_pb2_grpc.py backend_pb2.py 23 | 24 | backend_pb2_grpc.py backend_pb2.py: 25 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 26 | 27 | .PHONY: clean 28 | clean: protogen-clean 29 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/coqui/README.md: -------------------------------------------------------------------------------- 1 | # Creating a separate environment for ttsbark project 2 | 3 | ``` 4 | make coqui 5 | ``` 6 | 7 | # Testing the gRPC server 8 | 9 | ``` 10 | make test 11 | ``` -------------------------------------------------------------------------------- /backend/python/coqui/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/coqui/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | transformers==4.48.3 2 | accelerate 3 | torch==2.4.1 4 | coqui-tts -------------------------------------------------------------------------------- /backend/python/coqui/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | torchaudio==2.4.1+cu118 4 | transformers==4.48.3 5 | accelerate 6 | coqui-tts -------------------------------------------------------------------------------- /backend/python/coqui/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | torchaudio==2.4.1 3 | transformers==4.48.3 4 | accelerate 5 | coqui-tts -------------------------------------------------------------------------------- /backend/python/coqui/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch==2.4.1+rocm6.0 3 | torchaudio==2.4.1+rocm6.0 4 | transformers==4.48.3 5 | accelerate 6 | coqui-tts -------------------------------------------------------------------------------- /backend/python/coqui/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | torchaudio==2.3.1+cxx11.abi 5 | oneccl_bind_pt==2.3.100+xpu 6 | optimum[openvino] 7 | setuptools 8 | transformers==4.48.3 9 | accelerate 10 | coqui-tts -------------------------------------------------------------------------------- /backend/python/coqui/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | certifi 4 | packaging==24.1 -------------------------------------------------------------------------------- /backend/python/coqui/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/coqui/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/diffusers/Makefile: -------------------------------------------------------------------------------- 1 | export CONDA_ENV_PATH = "diffusers.yml" 2 | 3 | ifeq ($(BUILD_TYPE), hipblas) 4 | export CONDA_ENV_PATH = "diffusers-rocm.yml" 5 | endif 6 | 7 | # Intel GPU are supposed to have dependencies installed in the main python 8 | # environment, so we skip conda installation for SYCL builds. 9 | # https://github.com/intel/intel-extension-for-pytorch/issues/538 10 | ifneq (,$(findstring sycl,$(BUILD_TYPE))) 11 | export SKIP_CONDA=1 12 | endif 13 | 14 | .PHONY: diffusers 15 | diffusers: protogen 16 | bash install.sh 17 | 18 | .PHONY: run 19 | run: protogen 20 | @echo "Running diffusers..." 21 | bash run.sh 22 | @echo "Diffusers run." 23 | 24 | test: protogen 25 | bash test.sh 26 | 27 | .PHONY: protogen 28 | protogen: backend_pb2_grpc.py backend_pb2.py 29 | 30 | .PHONY: protogen-clean 31 | protogen-clean: 32 | $(RM) backend_pb2_grpc.py backend_pb2.py 33 | 34 | backend_pb2_grpc.py backend_pb2.py: 35 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 36 | 37 | .PHONY: clean 38 | clean: protogen-clean 39 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/diffusers/README.md: -------------------------------------------------------------------------------- 1 | # Creating a separate environment for the diffusers project 2 | 3 | ``` 4 | make diffusers 5 | ``` -------------------------------------------------------------------------------- /backend/python/diffusers/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/diffusers/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | diffusers 2 | opencv-python 3 | transformers 4 | accelerate 5 | compel 6 | peft 7 | sentencepiece 8 | torch==2.4.1 9 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/diffusers/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | diffusers 4 | opencv-python 5 | transformers 6 | accelerate 7 | compel 8 | peft 9 | sentencepiece 10 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/diffusers/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | diffusers 3 | opencv-python 4 | transformers 5 | accelerate 6 | compel 7 | peft 8 | sentencepiece 9 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/diffusers/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch==2.3.1+rocm6.0 3 | torchvision==0.18.1+rocm6.0 4 | diffusers 5 | opencv-python 6 | transformers 7 | accelerate 8 | compel 9 | peft 10 | sentencepiece 11 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/diffusers/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | torchvision==0.18.1+cxx11.abi 5 | oneccl_bind_pt==2.3.100+xpu 6 | optimum[openvino] 7 | setuptools 8 | diffusers 9 | opencv-python 10 | transformers 11 | accelerate 12 | compel 13 | peft 14 | sentencepiece 15 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/diffusers/requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools 2 | grpcio==1.72.0 3 | pillow 4 | protobuf 5 | certifi 6 | -------------------------------------------------------------------------------- /backend/python/diffusers/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/diffusers/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/exllama2/.gitignore: -------------------------------------------------------------------------------- 1 | source -------------------------------------------------------------------------------- /backend/python/exllama2/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: exllama2 2 | exllama2: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running exllama2..." 8 | bash run.sh 9 | @echo "exllama2 run." 10 | 11 | .PHONY: protogen 12 | protogen: backend_pb2_grpc.py backend_pb2.py 13 | 14 | .PHONY: protogen-clean 15 | protogen-clean: 16 | $(RM) backend_pb2_grpc.py backend_pb2.py 17 | 18 | backend_pb2_grpc.py backend_pb2.py: 19 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 20 | 21 | .PHONY: clean 22 | clean: protogen-clean 23 | $(RM) -r venv source __pycache__ -------------------------------------------------------------------------------- /backend/python/exllama2/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | LIMIT_TARGETS="cublas" 5 | EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation" 6 | EXLLAMA2_VERSION=c0ddebaaaf8ffd1b3529c2bb654e650bce2f790f 7 | 8 | source $(dirname $0)/../common/libbackend.sh 9 | 10 | installRequirements 11 | 12 | git clone https://github.com/turboderp/exllamav2 $MY_DIR/source 13 | pushd ${MY_DIR}/source && git checkout -b build ${EXLLAMA2_VERSION} && popd 14 | 15 | # This installs exllamav2 in JIT mode so it will compile the appropriate torch extension at runtime 16 | EXLLAMA_NOCOMPILE= uv pip install ${EXTRA_PIP_INSTALL_FLAGS} ${MY_DIR}/source/ 17 | -------------------------------------------------------------------------------- /backend/python/exllama2/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | accelerate 3 | torch==2.4.1 -------------------------------------------------------------------------------- /backend/python/exllama2/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | transformers 4 | accelerate -------------------------------------------------------------------------------- /backend/python/exllama2/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | transformers 3 | accelerate -------------------------------------------------------------------------------- /backend/python/exllama2/requirements-install.txt: -------------------------------------------------------------------------------- 1 | # This is here to trigger the install script to add --no-build-isolation to the uv pip install commands 2 | # exllama2 does not specify it's build requirements per PEP517, so we need to provide some things ourselves 3 | wheel 4 | setuptools -------------------------------------------------------------------------------- /backend/python/exllama2/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | certifi 4 | wheel 5 | setuptools -------------------------------------------------------------------------------- /backend/python/exllama2/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | LIMIT_TARGETS="cublas" 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/exllama2/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/faster-whisper/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := install 2 | 3 | .PHONY: install 4 | install: 5 | bash install.sh 6 | $(MAKE) protogen 7 | 8 | .PHONY: protogen 9 | protogen: backend_pb2_grpc.py backend_pb2.py 10 | 11 | .PHONY: protogen-clean 12 | protogen-clean: 13 | $(RM) backend_pb2_grpc.py backend_pb2.py 14 | 15 | backend_pb2_grpc.py backend_pb2.py: 16 | bash protogen.sh 17 | 18 | .PHONY: clean 19 | clean: protogen-clean 20 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/faster-whisper/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/faster-whisper/protogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto -------------------------------------------------------------------------------- /backend/python/faster-whisper/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | faster-whisper 2 | opencv-python 3 | accelerate 4 | compel 5 | peft 6 | sentencepiece 7 | torch==2.4.1 8 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/faster-whisper/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | faster-whisper 4 | opencv-python 5 | accelerate 6 | compel 7 | peft 8 | sentencepiece 9 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/faster-whisper/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | faster-whisper 3 | opencv-python 4 | accelerate 5 | compel 6 | peft 7 | sentencepiece 8 | optimum-quanto -------------------------------------------------------------------------------- /backend/python/faster-whisper/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch 3 | faster-whisper -------------------------------------------------------------------------------- /backend/python/faster-whisper/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | oneccl_bind_pt==2.3.100+xpu 5 | optimum[openvino] 6 | faster-whisper -------------------------------------------------------------------------------- /backend/python/faster-whisper/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | grpcio-tools -------------------------------------------------------------------------------- /backend/python/faster-whisper/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/faster-whisper/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/kokoro/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := install 2 | 3 | .PHONY: install 4 | install: 5 | bash install.sh 6 | $(MAKE) protogen 7 | 8 | .PHONY: protogen 9 | protogen: backend_pb2_grpc.py backend_pb2.py 10 | 11 | .PHONY: protogen-clean 12 | protogen-clean: 13 | $(RM) backend_pb2_grpc.py backend_pb2.py 14 | 15 | backend_pb2_grpc.py backend_pb2.py: 16 | bash protogen.sh 17 | 18 | .PHONY: clean 19 | clean: protogen-clean 20 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/kokoro/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/kokoro/plbert.py: -------------------------------------------------------------------------------- 1 | # https://huggingface.co/hexgrad/Kokoro-82M/blob/main/plbert.py 2 | # https://github.com/yl4579/StyleTTS2/blob/main/Utils/PLBERT/util.py 3 | from transformers import AlbertConfig, AlbertModel 4 | 5 | class CustomAlbert(AlbertModel): 6 | def forward(self, *args, **kwargs): 7 | # Call the original forward method 8 | outputs = super().forward(*args, **kwargs) 9 | # Only return the last_hidden_state 10 | return outputs.last_hidden_state 11 | 12 | def load_plbert(): 13 | plbert_config = {'vocab_size': 178, 'hidden_size': 768, 'num_attention_heads': 12, 'intermediate_size': 2048, 'max_position_embeddings': 512, 'num_hidden_layers': 12, 'dropout': 0.1} 14 | albert_base_configuration = AlbertConfig(**plbert_config) 15 | bert = CustomAlbert(albert_base_configuration) 16 | return bert 17 | -------------------------------------------------------------------------------- /backend/python/kokoro/protogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto -------------------------------------------------------------------------------- /backend/python/kokoro/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | transformers -------------------------------------------------------------------------------- /backend/python/kokoro/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | transformers -------------------------------------------------------------------------------- /backend/python/kokoro/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | transformers -------------------------------------------------------------------------------- /backend/python/kokoro/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch==2.4.1+rocm6.0 3 | transformers -------------------------------------------------------------------------------- /backend/python/kokoro/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | oneccl_bind_pt==2.3.100+xpu 5 | transformers -------------------------------------------------------------------------------- /backend/python/kokoro/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | phonemizer 4 | scipy 5 | munch 6 | setuptools 7 | soundfile -------------------------------------------------------------------------------- /backend/python/kokoro/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/kokoro/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/rerankers/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: rerankers 2 | rerankers: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running rerankers..." 8 | bash run.sh 9 | @echo "rerankers run." 10 | 11 | # It is not working well by using command line. It only6 works with IDE like VSCode. 12 | .PHONY: test 13 | test: protogen 14 | @echo "Testing rerankers..." 15 | bash test.sh 16 | @echo "rerankers tested." 17 | 18 | .PHONY: protogen 19 | protogen: backend_pb2_grpc.py backend_pb2.py 20 | 21 | .PHONY: protogen-clean 22 | protogen-clean: 23 | $(RM) backend_pb2_grpc.py backend_pb2.py 24 | 25 | backend_pb2_grpc.py backend_pb2.py: 26 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 27 | 28 | .PHONY: clean 29 | clean: protogen-clean 30 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/rerankers/README.md: -------------------------------------------------------------------------------- 1 | # Creating a separate environment for the reranker project 2 | 3 | ``` 4 | make reranker 5 | ``` -------------------------------------------------------------------------------- /backend/python/rerankers/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/rerankers/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | accelerate 3 | torch==2.4.1 4 | rerankers[transformers] -------------------------------------------------------------------------------- /backend/python/rerankers/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | transformers 3 | accelerate 4 | torch==2.4.1+cu118 5 | rerankers[transformers] -------------------------------------------------------------------------------- /backend/python/rerankers/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | accelerate 3 | torch==2.4.1 4 | rerankers[transformers] -------------------------------------------------------------------------------- /backend/python/rerankers/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | transformers 3 | accelerate 4 | torch==2.4.1+rocm6.0 5 | rerankers[transformers] -------------------------------------------------------------------------------- /backend/python/rerankers/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | transformers 4 | accelerate 5 | torch==2.3.1+cxx11.abi 6 | oneccl_bind_pt==2.3.100+xpu 7 | rerankers[transformers] 8 | optimum[openvino] 9 | setuptools -------------------------------------------------------------------------------- /backend/python/rerankers/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | certifi -------------------------------------------------------------------------------- /backend/python/rerankers/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/rerankers/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/transformers/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: transformers 2 | transformers: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running transformers..." 8 | bash run.sh 9 | @echo "transformers run." 10 | 11 | # It is not working well by using command line. It only6 works with IDE like VSCode. 12 | .PHONY: test 13 | test: protogen 14 | @echo "Testing transformers..." 15 | bash test.sh 16 | @echo "transformers tested." 17 | 18 | .PHONY: protogen 19 | protogen: backend_pb2_grpc.py backend_pb2.py 20 | 21 | .PHONY: protogen-clean 22 | protogen-clean: 23 | $(RM) backend_pb2_grpc.py backend_pb2.py 24 | 25 | backend_pb2_grpc.py backend_pb2.py: 26 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 27 | 28 | .PHONY: clean 29 | clean: protogen-clean 30 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/transformers/README.md: -------------------------------------------------------------------------------- 1 | # Creating a separate environment for the transformers project 2 | 3 | ``` 4 | make transformers 5 | ``` -------------------------------------------------------------------------------- /backend/python/transformers/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | # This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. 7 | # This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. 8 | # We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index 9 | # the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index 10 | if [ "x${BUILD_PROFILE}" == "xintel" ]; then 11 | EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" 12 | fi 13 | 14 | installRequirements 15 | -------------------------------------------------------------------------------- /backend/python/transformers/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | llvmlite==0.43.0 3 | numba==0.60.0 4 | accelerate 5 | transformers 6 | bitsandbytes 7 | outetts 8 | sentence-transformers==3.4.1 9 | protobuf==6.31.0 -------------------------------------------------------------------------------- /backend/python/transformers/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | torch==2.4.1+cu118 3 | llvmlite==0.43.0 4 | numba==0.60.0 5 | accelerate 6 | transformers 7 | bitsandbytes 8 | outetts 9 | sentence-transformers==4.1.0 10 | protobuf==6.31.0 -------------------------------------------------------------------------------- /backend/python/transformers/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | torch==2.4.1 2 | accelerate 3 | llvmlite==0.43.0 4 | numba==0.60.0 5 | transformers 6 | bitsandbytes 7 | outetts 8 | sentence-transformers==4.1.0 9 | protobuf==6.31.0 -------------------------------------------------------------------------------- /backend/python/transformers/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | torch==2.4.1+rocm6.0 3 | accelerate 4 | transformers 5 | llvmlite==0.43.0 6 | numba==0.60.0 7 | bitsandbytes 8 | outetts 9 | bitsandbytes 10 | sentence-transformers==4.1.0 11 | protobuf==6.31.0 -------------------------------------------------------------------------------- /backend/python/transformers/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | torch==2.3.1+cxx11.abi 4 | oneccl_bind_pt==2.3.100+xpu 5 | optimum[openvino] 6 | llvmlite==0.43.0 7 | numba==0.60.0 8 | transformers 9 | intel-extension-for-transformers 10 | bitsandbytes 11 | outetts 12 | sentence-transformers==4.1.0 13 | protobuf==6.31.0 -------------------------------------------------------------------------------- /backend/python/transformers/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf==6.31.0 3 | certifi 4 | setuptools 5 | scipy==1.15.1 6 | numpy>=2.0.0 -------------------------------------------------------------------------------- /backend/python/transformers/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | if [ -d "/opt/intel" ]; then 5 | # Assumes we are using the Intel oneAPI container image 6 | # https://github.com/intel/intel-extension-for-pytorch/issues/538 7 | export XPU=1 8 | fi 9 | 10 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/transformers/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /backend/python/vllm/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: vllm 2 | vllm: protogen 3 | bash install.sh 4 | 5 | .PHONY: run 6 | run: protogen 7 | @echo "Running vllm..." 8 | bash run.sh 9 | @echo "vllm run." 10 | 11 | .PHONY: test 12 | test: protogen 13 | @echo "Testing vllm..." 14 | bash test.sh 15 | @echo "vllm tested." 16 | 17 | .PHONY: protogen 18 | protogen: backend_pb2_grpc.py backend_pb2.py 19 | 20 | .PHONY: protogen-clean 21 | protogen-clean: 22 | $(RM) backend_pb2_grpc.py backend_pb2.py 23 | 24 | backend_pb2_grpc.py backend_pb2.py: 25 | python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto 26 | 27 | .PHONY: clean 28 | clean: protogen-clean 29 | rm -rf venv __pycache__ -------------------------------------------------------------------------------- /backend/python/vllm/README.md: -------------------------------------------------------------------------------- 1 | # Creating a separate environment for the vllm project 2 | 3 | ``` 4 | make vllm 5 | ``` -------------------------------------------------------------------------------- /backend/python/vllm/requirements-after.txt: -------------------------------------------------------------------------------- 1 | vllm -------------------------------------------------------------------------------- /backend/python/vllm/requirements-cpu.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | torch==2.4.1 3 | transformers -------------------------------------------------------------------------------- /backend/python/vllm/requirements-cublas11-after.txt: -------------------------------------------------------------------------------- 1 | flash-attn -------------------------------------------------------------------------------- /backend/python/vllm/requirements-cublas11.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cu118 2 | accelerate 3 | torch==2.4.1+cu118 4 | transformers 5 | bitsandbytes -------------------------------------------------------------------------------- /backend/python/vllm/requirements-cublas12-after.txt: -------------------------------------------------------------------------------- 1 | flash-attn -------------------------------------------------------------------------------- /backend/python/vllm/requirements-cublas12.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | torch==2.4.1 3 | transformers 4 | bitsandbytes -------------------------------------------------------------------------------- /backend/python/vllm/requirements-hipblas.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/rocm6.0 2 | accelerate 3 | torch==2.4.1+rocm6.0 4 | transformers 5 | bitsandbytes -------------------------------------------------------------------------------- /backend/python/vllm/requirements-install.txt: -------------------------------------------------------------------------------- 1 | # mabma does not specify it's build dependencies per PEP517, so we need to disable build isolation 2 | # this also means that we need to install the basic build dependencies into the venv ourselves 3 | # https://github.com/Dao-AILab/causal-conv1d/issues/24 4 | packaging 5 | setuptools 6 | wheel -------------------------------------------------------------------------------- /backend/python/vllm/requirements-intel.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 2 | intel-extension-for-pytorch==2.3.110+xpu 3 | accelerate 4 | torch==2.3.1+cxx11.abi 5 | transformers 6 | optimum[openvino] 7 | setuptools 8 | bitsandbytes 9 | oneccl_bind_pt==2.3.100+xpu -------------------------------------------------------------------------------- /backend/python/vllm/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio==1.72.0 2 | protobuf 3 | certifi 4 | setuptools -------------------------------------------------------------------------------- /backend/python/vllm/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source $(dirname $0)/../common/libbackend.sh 3 | 4 | startBackend $@ -------------------------------------------------------------------------------- /backend/python/vllm/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source $(dirname $0)/../common/libbackend.sh 5 | 6 | runUnittests 7 | -------------------------------------------------------------------------------- /configuration/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/configuration/.keep -------------------------------------------------------------------------------- /core/application/application.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "github.com/mudler/LocalAI/core/config" 5 | "github.com/mudler/LocalAI/pkg/model" 6 | "github.com/mudler/LocalAI/pkg/templates" 7 | ) 8 | 9 | type Application struct { 10 | backendLoader *config.BackendConfigLoader 11 | modelLoader *model.ModelLoader 12 | applicationConfig *config.ApplicationConfig 13 | templatesEvaluator *templates.Evaluator 14 | } 15 | 16 | func newApplication(appConfig *config.ApplicationConfig) *Application { 17 | return &Application{ 18 | backendLoader: config.NewBackendConfigLoader(appConfig.ModelPath), 19 | modelLoader: model.NewModelLoader(appConfig.ModelPath, appConfig.SingleBackend), 20 | applicationConfig: appConfig, 21 | templatesEvaluator: templates.NewEvaluator(appConfig.ModelPath), 22 | } 23 | } 24 | 25 | func (a *Application) BackendLoader() *config.BackendConfigLoader { 26 | return a.backendLoader 27 | } 28 | 29 | func (a *Application) ModelLoader() *model.ModelLoader { 30 | return a.modelLoader 31 | } 32 | 33 | func (a *Application) ApplicationConfig() *config.ApplicationConfig { 34 | return a.applicationConfig 35 | } 36 | 37 | func (a *Application) TemplatesEvaluator() *templates.Evaluator { 38 | return a.templatesEvaluator 39 | } 40 | -------------------------------------------------------------------------------- /core/backend/backend_suite_test.go: -------------------------------------------------------------------------------- 1 | package backend_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestBackend(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Backend test suite") 13 | } 14 | -------------------------------------------------------------------------------- /core/backend/image.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "github.com/mudler/LocalAI/core/config" 5 | 6 | "github.com/mudler/LocalAI/pkg/grpc/proto" 7 | model "github.com/mudler/LocalAI/pkg/model" 8 | ) 9 | 10 | func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error) { 11 | 12 | opts := ModelOptions(backendConfig, appConfig) 13 | inferenceModel, err := loader.Load( 14 | opts..., 15 | ) 16 | if err != nil { 17 | return nil, err 18 | } 19 | defer loader.Close() 20 | 21 | fn := func() error { 22 | _, err := inferenceModel.GenerateImage( 23 | appConfig.Context, 24 | &proto.GenerateImageRequest{ 25 | Height: int32(height), 26 | Width: int32(width), 27 | Mode: int32(mode), 28 | Step: int32(step), 29 | Seed: int32(seed), 30 | CLIPSkip: int32(backendConfig.Diffusers.ClipSkip), 31 | PositivePrompt: positive_prompt, 32 | NegativePrompt: negative_prompt, 33 | Dst: dst, 34 | Src: src, 35 | EnableParameters: backendConfig.Diffusers.EnableParameters, 36 | }) 37 | return err 38 | } 39 | 40 | return fn, nil 41 | } 42 | -------------------------------------------------------------------------------- /core/backend/rerank.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/mudler/LocalAI/core/config" 8 | "github.com/mudler/LocalAI/pkg/grpc/proto" 9 | model "github.com/mudler/LocalAI/pkg/model" 10 | ) 11 | 12 | func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, backendConfig config.BackendConfig) (*proto.RerankResult, error) { 13 | opts := ModelOptions(backendConfig, appConfig) 14 | rerankModel, err := loader.Load(opts...) 15 | if err != nil { 16 | return nil, err 17 | } 18 | defer loader.Close() 19 | 20 | if rerankModel == nil { 21 | return nil, fmt.Errorf("could not load rerank model") 22 | } 23 | 24 | res, err := rerankModel.Rerank(context.Background(), request) 25 | 26 | return res, err 27 | } 28 | -------------------------------------------------------------------------------- /core/backend/stores.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "github.com/mudler/LocalAI/core/config" 5 | 6 | "github.com/mudler/LocalAI/pkg/grpc" 7 | "github.com/mudler/LocalAI/pkg/model" 8 | ) 9 | 10 | func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string) (grpc.Backend, error) { 11 | if storeName == "" { 12 | storeName = "default" 13 | } 14 | 15 | sc := []model.Option{ 16 | model.WithBackendString(model.LocalStoreBackend), 17 | model.WithAssetDir(appConfig.AssetsDestination), 18 | model.WithModel(storeName), 19 | } 20 | 21 | return sl.Load(sc...) 22 | } 23 | -------------------------------------------------------------------------------- /core/backend/token_metrics.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/mudler/LocalAI/core/config" 8 | "github.com/mudler/LocalAI/pkg/grpc/proto" 9 | model "github.com/mudler/LocalAI/pkg/model" 10 | ) 11 | 12 | func TokenMetrics( 13 | modelFile string, 14 | loader *model.ModelLoader, 15 | appConfig *config.ApplicationConfig, 16 | backendConfig config.BackendConfig) (*proto.MetricsResponse, error) { 17 | 18 | opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile)) 19 | model, err := loader.Load(opts...) 20 | if err != nil { 21 | return nil, err 22 | } 23 | defer loader.Close() 24 | 25 | if model == nil { 26 | return nil, fmt.Errorf("could not loadmodel model") 27 | } 28 | 29 | res, err := model.GetTokenMetrics(context.Background(), &proto.MetricsRequest{}) 30 | 31 | return res, err 32 | } 33 | -------------------------------------------------------------------------------- /core/backend/tokenize.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "github.com/mudler/LocalAI/core/config" 5 | "github.com/mudler/LocalAI/core/schema" 6 | "github.com/mudler/LocalAI/pkg/grpc" 7 | "github.com/mudler/LocalAI/pkg/model" 8 | ) 9 | 10 | func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (schema.TokenizeResponse, error) { 11 | 12 | var inferenceModel grpc.Backend 13 | var err error 14 | 15 | opts := ModelOptions(backendConfig, appConfig) 16 | inferenceModel, err = loader.Load(opts...) 17 | if err != nil { 18 | return schema.TokenizeResponse{}, err 19 | } 20 | defer loader.Close() 21 | 22 | predictOptions := gRPCPredictOpts(backendConfig, loader.ModelPath) 23 | predictOptions.Prompt = s 24 | 25 | // tokenize the string 26 | resp, err := inferenceModel.TokenizeString(appConfig.Context, predictOptions) 27 | if err != nil { 28 | return schema.TokenizeResponse{}, err 29 | } 30 | 31 | if resp.Tokens == nil { 32 | resp.Tokens = make([]int32, 0) 33 | } 34 | 35 | return schema.TokenizeResponse{ 36 | Tokens: resp.Tokens, 37 | }, nil 38 | 39 | } 40 | -------------------------------------------------------------------------------- /core/backend/vad.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/mudler/LocalAI/core/config" 7 | "github.com/mudler/LocalAI/core/schema" 8 | "github.com/mudler/LocalAI/pkg/grpc/proto" 9 | "github.com/mudler/LocalAI/pkg/model" 10 | ) 11 | 12 | func VAD(request *schema.VADRequest, 13 | ctx context.Context, 14 | ml *model.ModelLoader, 15 | appConfig *config.ApplicationConfig, 16 | backendConfig config.BackendConfig) (*schema.VADResponse, error) { 17 | opts := ModelOptions(backendConfig, appConfig) 18 | vadModel, err := ml.Load(opts...) 19 | if err != nil { 20 | return nil, err 21 | } 22 | defer ml.Close() 23 | 24 | req := proto.VADRequest{ 25 | Audio: request.Audio, 26 | } 27 | resp, err := vadModel.VAD(ctx, &req) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | segments := []schema.VADSegment{} 33 | for _, s := range resp.Segments { 34 | segments = append(segments, schema.VADSegment{Start: s.Start, End: s.End}) 35 | } 36 | 37 | return &schema.VADResponse{ 38 | Segments: segments, 39 | }, nil 40 | } 41 | -------------------------------------------------------------------------------- /core/backend/video.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "github.com/mudler/LocalAI/core/config" 5 | 6 | "github.com/mudler/LocalAI/pkg/grpc/proto" 7 | model "github.com/mudler/LocalAI/pkg/model" 8 | ) 9 | 10 | func VideoGeneration(height, width int32, prompt, startImage, endImage, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error) { 11 | 12 | opts := ModelOptions(backendConfig, appConfig) 13 | inferenceModel, err := loader.Load( 14 | opts..., 15 | ) 16 | if err != nil { 17 | return nil, err 18 | } 19 | defer loader.Close() 20 | 21 | fn := func() error { 22 | _, err := inferenceModel.GenerateVideo( 23 | appConfig.Context, 24 | &proto.GenerateVideoRequest{ 25 | Height: height, 26 | Width: width, 27 | Prompt: prompt, 28 | StartImage: startImage, 29 | EndImage: endImage, 30 | Dst: dst, 31 | }) 32 | return err 33 | } 34 | 35 | return fn, nil 36 | } 37 | -------------------------------------------------------------------------------- /core/cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | cliContext "github.com/mudler/LocalAI/core/cli/context" 5 | "github.com/mudler/LocalAI/core/cli/worker" 6 | ) 7 | 8 | var CLI struct { 9 | cliContext.Context `embed:""` 10 | 11 | Run RunCMD `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"` 12 | Federated FederatedCLI `cmd:"" help:"Run LocalAI in federated mode"` 13 | Models ModelsCMD `cmd:"" help:"Manage LocalAI models and definitions"` 14 | TTS TTSCMD `cmd:"" help:"Convert text to speech"` 15 | SoundGeneration SoundGenerationCMD `cmd:"" help:"Generates audio files from text or audio"` 16 | Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"` 17 | Worker worker.Worker `cmd:"" help:"Run workers to distribute workload (llama.cpp-only)"` 18 | Util UtilCMD `cmd:"" help:"Utility commands"` 19 | Explorer ExplorerCMD `cmd:"" help:"Run p2p explorer"` 20 | } 21 | -------------------------------------------------------------------------------- /core/cli/context/context.go: -------------------------------------------------------------------------------- 1 | package cliContext 2 | 3 | import ( 4 | rice "github.com/GeertJohan/go.rice" 5 | ) 6 | 7 | type Context struct { 8 | Debug bool `env:"LOCALAI_DEBUG,DEBUG" default:"false" hidden:"" help:"DEPRECATED, use --log-level=debug instead. Enable debug logging"` 9 | LogLevel *string `env:"LOCALAI_LOG_LEVEL" enum:"error,warn,info,debug,trace" help:"Set the level of logs to output [${enum}]"` 10 | 11 | // This field is not a command line argument/flag, the struct tag excludes it from the parsed CLI 12 | BackendAssets *rice.Box `kong:"-"` 13 | } 14 | -------------------------------------------------------------------------------- /core/cli/federated.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | 6 | cliContext "github.com/mudler/LocalAI/core/cli/context" 7 | "github.com/mudler/LocalAI/core/p2p" 8 | ) 9 | 10 | type FederatedCLI struct { 11 | Address string `env:"LOCALAI_ADDRESS,ADDRESS" default:":8080" help:"Bind address for the API server" group:"api"` 12 | Peer2PeerToken string `env:"LOCALAI_P2P_TOKEN,P2P_TOKEN,TOKEN" name:"p2ptoken" help:"Token for P2P mode (optional)" group:"p2p"` 13 | RandomWorker bool `env:"LOCALAI_RANDOM_WORKER,RANDOM_WORKER" default:"false" help:"Select a random worker from the pool" group:"p2p"` 14 | Peer2PeerNetworkID string `env:"LOCALAI_P2P_NETWORK_ID,P2P_NETWORK_ID" help:"Network ID for P2P mode, can be set arbitrarly by the user for grouping a set of instances." group:"p2p"` 15 | TargetWorker string `env:"LOCALAI_TARGET_WORKER,TARGET_WORKER" help:"Target worker to run the federated server on" group:"p2p"` 16 | } 17 | 18 | func (f *FederatedCLI) Run(ctx *cliContext.Context) error { 19 | 20 | fs := p2p.NewFederatedServer(f.Address, p2p.NetworkID(f.Peer2PeerNetworkID, p2p.FederatedID), f.Peer2PeerToken, !f.RandomWorker, f.TargetWorker) 21 | 22 | return fs.Start(context.Background()) 23 | } 24 | -------------------------------------------------------------------------------- /core/cli/worker/worker.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | type WorkerFlags struct { 4 | BackendAssetsPath string `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"` 5 | ExtraLLamaCPPArgs string `name:"llama-cpp-args" env:"LOCALAI_EXTRA_LLAMA_CPP_ARGS,EXTRA_LLAMA_CPP_ARGS" help:"Extra arguments to pass to llama-cpp-rpc-server"` 6 | } 7 | 8 | type Worker struct { 9 | P2P P2P `cmd:"" name:"p2p-llama-cpp-rpc" help:"Starts a LocalAI llama.cpp worker in P2P mode (requires a token)"` 10 | LLamaCPP LLamaCPP `cmd:"" name:"llama-cpp-rpc" help:"Starts a llama.cpp worker in standalone mode"` 11 | } 12 | -------------------------------------------------------------------------------- /core/cli/worker/worker_llamacpp.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "syscall" 8 | 9 | cliContext "github.com/mudler/LocalAI/core/cli/context" 10 | "github.com/mudler/LocalAI/pkg/assets" 11 | "github.com/mudler/LocalAI/pkg/library" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | type LLamaCPP struct { 16 | WorkerFlags `embed:""` 17 | } 18 | 19 | func (r *LLamaCPP) Run(ctx *cliContext.Context) error { 20 | // Extract files from the embedded FS 21 | err := assets.ExtractFiles(ctx.BackendAssets, r.BackendAssetsPath) 22 | log.Debug().Msgf("Extracting backend assets files to %s", r.BackendAssetsPath) 23 | if err != nil { 24 | log.Warn().Msgf("Failed extracting backend assets files: %s (might be required for some backends to work properly)", err) 25 | } 26 | 27 | if len(os.Args) < 4 { 28 | return fmt.Errorf("usage: local-ai worker llama-cpp-rpc -- ") 29 | } 30 | 31 | grpcProcess := assets.ResolvePath( 32 | r.BackendAssetsPath, 33 | "util", 34 | "llama-cpp-rpc-server", 35 | ) 36 | 37 | args := strings.Split(r.ExtraLLamaCPPArgs, " ") 38 | args, grpcProcess = library.LoadLDSO(r.BackendAssetsPath, args, grpcProcess) 39 | args = append([]string{grpcProcess}, args...) 40 | return syscall.Exec( 41 | grpcProcess, 42 | args, 43 | os.Environ()) 44 | } 45 | -------------------------------------------------------------------------------- /core/cli/worker/worker_nop2p.go: -------------------------------------------------------------------------------- 1 | //go:build !p2p 2 | // +build !p2p 3 | 4 | package worker 5 | 6 | import ( 7 | "fmt" 8 | 9 | cliContext "github.com/mudler/LocalAI/core/cli/context" 10 | ) 11 | 12 | type P2P struct{} 13 | 14 | func (r *P2P) Run(ctx *cliContext.Context) error { 15 | return fmt.Errorf("p2p mode is not enabled in this build") 16 | } 17 | -------------------------------------------------------------------------------- /core/config/backend_config_filter.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "regexp" 4 | 5 | type BackendConfigFilterFn func(string, *BackendConfig) bool 6 | 7 | func NoFilterFn(_ string, _ *BackendConfig) bool { return true } 8 | 9 | func BuildNameFilterFn(filter string) (BackendConfigFilterFn, error) { 10 | if filter == "" { 11 | return NoFilterFn, nil 12 | } 13 | rxp, err := regexp.Compile(filter) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return func(name string, config *BackendConfig) bool { 18 | if config != nil { 19 | return rxp.MatchString(config.Name) 20 | } 21 | return rxp.MatchString(name) 22 | }, nil 23 | } 24 | 25 | func BuildUsecaseFilterFn(usecases BackendConfigUsecases) BackendConfigFilterFn { 26 | if usecases == FLAG_ANY { 27 | return NoFilterFn 28 | } 29 | return func(name string, config *BackendConfig) bool { 30 | if config == nil { 31 | return false // TODO: Potentially make this a param, for now, no known usecase to include 32 | } 33 | return config.HasUsecases(usecases) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /core/config/config_suite_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestConfig(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Config test suite") 13 | } 14 | -------------------------------------------------------------------------------- /core/config/gallery.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type Gallery struct { 4 | URL string `json:"url" yaml:"url"` 5 | Name string `json:"name" yaml:"name"` 6 | } 7 | -------------------------------------------------------------------------------- /core/config/guesser.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | gguf "github.com/gpustack/gguf-parser-go" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func guessDefaultsFromFile(cfg *BackendConfig, modelPath string, defaultCtx int) { 12 | if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" { 13 | log.Debug().Msgf("guessDefaultsFromFile: %s", "guessing disabled with LOCALAI_DISABLE_GUESSING") 14 | return 15 | } 16 | 17 | if modelPath == "" { 18 | log.Debug().Msgf("guessDefaultsFromFile: %s", "modelPath is empty") 19 | return 20 | } 21 | 22 | // We try to guess only if we don't have a template defined already 23 | guessPath := filepath.Join(modelPath, cfg.ModelFileName()) 24 | 25 | // try to parse the gguf file 26 | f, err := gguf.ParseGGUFFile(guessPath) 27 | if err == nil { 28 | guessGGUFFromFile(cfg, f, defaultCtx) 29 | return 30 | } 31 | 32 | if cfg.ContextSize == nil { 33 | if defaultCtx == 0 { 34 | defaultCtx = defaultContextSize 35 | } 36 | cfg.ContextSize = &defaultCtx 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/dependencies_manager/manager.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/mudler/LocalAI/pkg/downloader" 9 | "github.com/mudler/LocalAI/pkg/utils" 10 | "gopkg.in/yaml.v3" 11 | ) 12 | 13 | type Asset struct { 14 | FileName string `yaml:"filename"` 15 | URL string `yaml:"url"` 16 | SHA string `yaml:"sha"` 17 | } 18 | 19 | func main() { 20 | 21 | // read the YAML file which contains a list of assets 22 | // and download them in the asset path 23 | assets := []Asset{} 24 | 25 | assetFile := os.Args[1] 26 | destPath := os.Args[2] 27 | 28 | // read the YAML file 29 | f, err := os.ReadFile(assetFile) 30 | if err != nil { 31 | panic(err) 32 | } 33 | // unmarshal the YAML data into a struct 34 | if err := yaml.Unmarshal(f, &assets); err != nil { 35 | panic(err) 36 | } 37 | 38 | // download the assets 39 | for _, asset := range assets { 40 | uri := downloader.URI(asset.URL) 41 | if err := uri.DownloadFile(filepath.Join(destPath, asset.FileName), asset.SHA, 1, 1, utils.DisplayDownloadFunction); err != nil { 42 | panic(err) 43 | } 44 | } 45 | 46 | fmt.Println("Finished downloading assets") 47 | } 48 | -------------------------------------------------------------------------------- /core/explorer/explorer_suite_test.go: -------------------------------------------------------------------------------- 1 | package explorer_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestExplorer(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Explorer test suite") 13 | } 14 | -------------------------------------------------------------------------------- /core/gallery/gallery_suite_test.go: -------------------------------------------------------------------------------- 1 | package gallery_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | func TestGallery(t *testing.T) { 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Gallery test suite") 14 | } 15 | 16 | var _ = BeforeSuite(func() { 17 | if os.Getenv("FIXTURES") == "" { 18 | Fail("FIXTURES env var not set") 19 | } 20 | }) 21 | -------------------------------------------------------------------------------- /core/gallery/op.go: -------------------------------------------------------------------------------- 1 | package gallery 2 | 3 | import "github.com/mudler/LocalAI/core/config" 4 | 5 | type GalleryOp struct { 6 | Id string 7 | GalleryModelName string 8 | ConfigURL string 9 | Delete bool 10 | 11 | Req GalleryModel 12 | Galleries []config.Gallery 13 | } 14 | 15 | type GalleryOpStatus struct { 16 | Deletion bool `json:"deletion"` // Deletion is true if the operation is a deletion 17 | FileName string `json:"file_name"` 18 | Error error `json:"error"` 19 | Processed bool `json:"processed"` 20 | Message string `json:"message"` 21 | Progress float64 `json:"progress"` 22 | TotalFileSize string `json:"file_size"` 23 | DownloadedFileSize string `json:"downloaded_size"` 24 | GalleryModelName string `json:"gallery_model_name"` 25 | } 26 | -------------------------------------------------------------------------------- /core/gallery/request_test.go: -------------------------------------------------------------------------------- 1 | package gallery_test 2 | 3 | import ( 4 | . "github.com/mudler/LocalAI/core/gallery" 5 | . "github.com/onsi/ginkgo/v2" 6 | . "github.com/onsi/gomega" 7 | ) 8 | 9 | var _ = Describe("Gallery API tests", func() { 10 | Context("requests", func() { 11 | It("parses github with a branch", func() { 12 | req := GalleryModel{ 13 | Metadata: Metadata{ 14 | URL: "github:go-skynet/model-gallery/gpt4all-j.yaml@main", 15 | }, 16 | } 17 | e, err := GetGalleryConfigFromURL(req.URL, "") 18 | Expect(err).ToNot(HaveOccurred()) 19 | Expect(e.Name).To(Equal("gpt4all-j")) 20 | }) 21 | }) 22 | }) 23 | -------------------------------------------------------------------------------- /core/http/endpoints/localai/p2p.go: -------------------------------------------------------------------------------- 1 | package localai 2 | 3 | import ( 4 | "github.com/gofiber/fiber/v2" 5 | "github.com/mudler/LocalAI/core/config" 6 | "github.com/mudler/LocalAI/core/p2p" 7 | "github.com/mudler/LocalAI/core/schema" 8 | ) 9 | 10 | // ShowP2PNodes returns the P2P Nodes 11 | // @Summary Returns available P2P nodes 12 | // @Success 200 {object} []schema.P2PNodesResponse "Response" 13 | // @Router /api/p2p [get] 14 | func ShowP2PNodes(appConfig *config.ApplicationConfig) func(*fiber.Ctx) error { 15 | // Render index 16 | return func(c *fiber.Ctx) error { 17 | return c.JSON(schema.P2PNodesResponse{ 18 | Nodes: p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.WorkerID)), 19 | FederatedNodes: p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.FederatedID)), 20 | }) 21 | } 22 | } 23 | 24 | // ShowP2PToken returns the P2P token 25 | // @Summary Show the P2P token 26 | // @Success 200 {string} string "Response" 27 | // @Router /api/p2p/token [get] 28 | func ShowP2PToken(appConfig *config.ApplicationConfig) func(*fiber.Ctx) error { 29 | return func(c *fiber.Ctx) error { return c.Send([]byte(appConfig.P2PToken)) } 30 | } 31 | -------------------------------------------------------------------------------- /core/http/endpoints/localai/system.go: -------------------------------------------------------------------------------- 1 | package localai 2 | 3 | import ( 4 | "github.com/gofiber/fiber/v2" 5 | "github.com/mudler/LocalAI/core/config" 6 | "github.com/mudler/LocalAI/core/schema" 7 | "github.com/mudler/LocalAI/pkg/model" 8 | ) 9 | 10 | // SystemInformations returns the system informations 11 | // @Summary Show the LocalAI instance information 12 | // @Success 200 {object} schema.SystemInformationResponse "Response" 13 | // @Router /system [get] 14 | func SystemInformations(ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(*fiber.Ctx) error { 15 | return func(c *fiber.Ctx) error { 16 | availableBackends, err := ml.ListAvailableBackends(appConfig.AssetsDestination) 17 | if err != nil { 18 | return err 19 | } 20 | loadedModels := ml.ListModels() 21 | for b := range appConfig.ExternalGRPCBackends { 22 | availableBackends = append(availableBackends, b) 23 | } 24 | 25 | sysmodels := []schema.SysInfoModel{} 26 | for _, m := range loadedModels { 27 | sysmodels = append(sysmodels, schema.SysInfoModel{ID: m.ID}) 28 | } 29 | return c.JSON( 30 | schema.SystemInformationResponse{ 31 | Backends: availableBackends, 32 | Models: sysmodels, 33 | }, 34 | ) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /core/http/endpoints/localai/tokenize.go: -------------------------------------------------------------------------------- 1 | package localai 2 | 3 | import ( 4 | "github.com/gofiber/fiber/v2" 5 | "github.com/mudler/LocalAI/core/backend" 6 | "github.com/mudler/LocalAI/core/config" 7 | "github.com/mudler/LocalAI/core/http/middleware" 8 | "github.com/mudler/LocalAI/core/schema" 9 | "github.com/mudler/LocalAI/pkg/model" 10 | ) 11 | 12 | // TokenizeEndpoint exposes a REST API to tokenize the content 13 | // @Summary Tokenize the input. 14 | // @Param request body schema.TokenizeRequest true "Request" 15 | // @Success 200 {object} schema.TokenizeResponse "Response" 16 | // @Router /v1/tokenize [post] 17 | func TokenizeEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error { 18 | return func(ctx *fiber.Ctx) error { 19 | input, ok := ctx.Locals(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.TokenizeRequest) 20 | if !ok || input.Model == "" { 21 | return fiber.ErrBadRequest 22 | } 23 | 24 | cfg, ok := ctx.Locals(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.BackendConfig) 25 | if !ok || cfg == nil { 26 | return fiber.ErrBadRequest 27 | } 28 | 29 | tokenResponse, err := backend.ModelTokenize(input.Content, ml, *cfg, appConfig) 30 | if err != nil { 31 | return err 32 | } 33 | return ctx.JSON(tokenResponse) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /core/http/http_suite_test.go: -------------------------------------------------------------------------------- 1 | package http_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestLocalAI(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "LocalAI test suite") 13 | } 14 | -------------------------------------------------------------------------------- /core/http/middleware/strippathprefix.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/gofiber/fiber/v2" 7 | ) 8 | 9 | // StripPathPrefix returns a middleware that strips a path prefix from the request path. 10 | // The path prefix is obtained from the X-Forwarded-Prefix HTTP request header. 11 | func StripPathPrefix() fiber.Handler { 12 | return func(c *fiber.Ctx) error { 13 | for _, prefix := range c.GetReqHeaders()["X-Forwarded-Prefix"] { 14 | if prefix != "" { 15 | path := c.Path() 16 | pos := len(prefix) 17 | 18 | if prefix[pos-1] == '/' { 19 | pos-- 20 | } else { 21 | prefix += "/" 22 | } 23 | 24 | if strings.HasPrefix(path, prefix) { 25 | c.Path(path[pos:]) 26 | break 27 | } else if prefix[:pos] == path { 28 | c.Redirect(prefix) 29 | return nil 30 | } 31 | } 32 | } 33 | 34 | return c.Next() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /core/http/routes/elevenlabs.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "github.com/gofiber/fiber/v2" 5 | "github.com/mudler/LocalAI/core/config" 6 | "github.com/mudler/LocalAI/core/http/endpoints/elevenlabs" 7 | "github.com/mudler/LocalAI/core/http/middleware" 8 | "github.com/mudler/LocalAI/core/schema" 9 | "github.com/mudler/LocalAI/pkg/model" 10 | ) 11 | 12 | func RegisterElevenLabsRoutes(app *fiber.App, 13 | re *middleware.RequestExtractor, 14 | cl *config.BackendConfigLoader, 15 | ml *model.ModelLoader, 16 | appConfig *config.ApplicationConfig) { 17 | 18 | // Elevenlabs 19 | app.Post("/v1/text-to-speech/:voice-id", 20 | re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TTS)), 21 | re.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.ElevenLabsTTSRequest) }), 22 | elevenlabs.TTSEndpoint(cl, ml, appConfig)) 23 | 24 | app.Post("/v1/sound-generation", 25 | re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_SOUND_GENERATION)), 26 | re.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.ElevenLabsSoundGenerationRequest) }), 27 | elevenlabs.SoundGenerationEndpoint(cl, ml, appConfig)) 28 | 29 | } 30 | -------------------------------------------------------------------------------- /core/http/routes/explorer.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "github.com/gofiber/fiber/v2" 5 | coreExplorer "github.com/mudler/LocalAI/core/explorer" 6 | "github.com/mudler/LocalAI/core/http/endpoints/explorer" 7 | ) 8 | 9 | func RegisterExplorerRoutes(app *fiber.App, db *coreExplorer.Database) { 10 | app.Get("/", explorer.Dashboard()) 11 | app.Post("/network/add", explorer.AddNetwork(db)) 12 | app.Get("/networks", explorer.ShowNetworks(db)) 13 | } 14 | -------------------------------------------------------------------------------- /core/http/routes/health.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import "github.com/gofiber/fiber/v2" 4 | 5 | func HealthRoutes(app *fiber.App) { 6 | // Service health checks 7 | ok := func(c *fiber.Ctx) error { 8 | return c.SendStatus(200) 9 | } 10 | 11 | app.Get("/healthz", ok) 12 | app.Get("/readyz", ok) 13 | } 14 | -------------------------------------------------------------------------------- /core/http/routes/jina.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "github.com/mudler/LocalAI/core/config" 5 | "github.com/mudler/LocalAI/core/http/endpoints/jina" 6 | "github.com/mudler/LocalAI/core/http/middleware" 7 | "github.com/mudler/LocalAI/core/schema" 8 | 9 | "github.com/gofiber/fiber/v2" 10 | "github.com/mudler/LocalAI/pkg/model" 11 | ) 12 | 13 | func RegisterJINARoutes(app *fiber.App, 14 | re *middleware.RequestExtractor, 15 | cl *config.BackendConfigLoader, 16 | ml *model.ModelLoader, 17 | appConfig *config.ApplicationConfig) { 18 | 19 | // POST endpoint to mimic the reranking 20 | app.Post("/v1/rerank", 21 | re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_RERANK)), 22 | re.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.JINARerankRequest) }), 23 | jina.JINARerankEndpoint(cl, ml, appConfig)) 24 | } 25 | -------------------------------------------------------------------------------- /core/http/static/assets/KFOlCnqEu92Fr1MmEU9fBBc9.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOlCnqEu92Fr1MmEU9fBBc9.ttf -------------------------------------------------------------------------------- /core/http/static/assets/KFOlCnqEu92Fr1MmEU9vAw.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOlCnqEu92Fr1MmEU9vAw.ttf -------------------------------------------------------------------------------- /core/http/static/assets/KFOlCnqEu92Fr1MmSU5fBBc9.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOlCnqEu92Fr1MmSU5fBBc9.ttf -------------------------------------------------------------------------------- /core/http/static/assets/KFOlCnqEu92Fr1MmWUlfBBc9.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOlCnqEu92Fr1MmWUlfBBc9.ttf -------------------------------------------------------------------------------- /core/http/static/assets/KFOlCnqEu92Fr1MmYUtfBBc9.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOlCnqEu92Fr1MmYUtfBBc9.ttf -------------------------------------------------------------------------------- /core/http/static/assets/KFOmCnqEu92Fr1Me5Q.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOmCnqEu92Fr1Me5Q.ttf -------------------------------------------------------------------------------- /core/http/static/assets/KFOmCnqEu92Fr1Mu4mxP.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/KFOmCnqEu92Fr1Mu4mxP.ttf -------------------------------------------------------------------------------- /core/http/static/assets/UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuFuYMZg.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuFuYMZg.ttf -------------------------------------------------------------------------------- /core/http/static/assets/UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuGKYMZg.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuGKYMZg.ttf -------------------------------------------------------------------------------- /core/http/static/assets/UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuLyfMZg.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuLyfMZg.ttf -------------------------------------------------------------------------------- /core/http/static/assets/font1.css: -------------------------------------------------------------------------------- 1 | /* 2 | https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&family=Roboto:wght@400;500&display=swap 3 | */ 4 | 5 | @font-face { 6 | font-family: 'Inter'; 7 | font-style: normal; 8 | font-weight: 400; 9 | font-display: swap; 10 | src: url(./UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuLyfMZg.ttf) format('truetype'); 11 | } 12 | @font-face { 13 | font-family: 'Inter'; 14 | font-style: normal; 15 | font-weight: 600; 16 | font-display: swap; 17 | src: url(./UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuGKYMZg.ttf) format('truetype'); 18 | } 19 | @font-face { 20 | font-family: 'Inter'; 21 | font-style: normal; 22 | font-weight: 700; 23 | font-display: swap; 24 | src: url(./UcCO3FwrK3iLTeHuS_fvQtMwCp50KnMw2boKoduKmMEVuFuYMZg.ttf) format('truetype'); 25 | } 26 | @font-face { 27 | font-family: 'Roboto'; 28 | font-style: normal; 29 | font-weight: 400; 30 | font-display: swap; 31 | src: url(./KFOmCnqEu92Fr1Me5Q.ttf) format('truetype'); 32 | } 33 | @font-face { 34 | font-family: 'Roboto'; 35 | font-style: normal; 36 | font-weight: 500; 37 | font-display: swap; 38 | src: url(./KFOlCnqEu92Fr1MmEU9vAw.ttf) format('truetype'); 39 | } 40 | -------------------------------------------------------------------------------- /core/http/static/assets/font2.css: -------------------------------------------------------------------------------- 1 | /* 2 | https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,900&display=swap 3 | */ 4 | 5 | @font-face { 6 | font-family: 'Roboto'; 7 | font-style: normal; 8 | font-weight: 300; 9 | font-display: swap; 10 | src: url(./KFOlCnqEu92Fr1MmSU5fBBc9.ttf) format('truetype'); 11 | } 12 | @font-face { 13 | font-family: 'Roboto'; 14 | font-style: normal; 15 | font-weight: 400; 16 | font-display: swap; 17 | src: url(./KFOmCnqEu92Fr1Mu4mxP.ttf) format('truetype'); 18 | } 19 | @font-face { 20 | font-family: 'Roboto'; 21 | font-style: normal; 22 | font-weight: 500; 23 | font-display: swap; 24 | src: url(./KFOlCnqEu92Fr1MmEU9fBBc9.ttf) format('truetype'); 25 | } 26 | @font-face { 27 | font-family: 'Roboto'; 28 | font-style: normal; 29 | font-weight: 700; 30 | font-display: swap; 31 | src: url(./KFOlCnqEu92Fr1MmWUlfBBc9.ttf) format('truetype'); 32 | } 33 | @font-face { 34 | font-family: 'Roboto'; 35 | font-style: normal; 36 | font-weight: 900; 37 | font-display: swap; 38 | src: url(./KFOlCnqEu92Fr1MmYUtfBBc9.ttf) format('truetype'); 39 | } 40 | -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/css/regular.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | * Copyright 2024 Fonticons, Inc. 5 | */ 6 | :root, :host { 7 | --fa-style-family-classic: 'Font Awesome 6 Free'; 8 | --fa-font-regular: normal 400 1em/1 'Font Awesome 6 Free'; } 9 | 10 | @font-face { 11 | font-family: 'Font Awesome 6 Free'; 12 | font-style: normal; 13 | font-weight: 400; 14 | font-display: block; 15 | src: url("../webfonts/fa-regular-400.woff2") format("woff2"), url("../webfonts/fa-regular-400.ttf") format("truetype"); } 16 | 17 | .far, 18 | .fa-regular { 19 | font-weight: 400; } 20 | -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/css/regular.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | * Copyright 2024 Fonticons, Inc. 5 | */ 6 | :host,:root{--fa-style-family-classic:"Font Awesome 6 Free";--fa-font-regular:normal 400 1em/1 "Font Awesome 6 Free"}@font-face{font-family:"Font Awesome 6 Free";font-style:normal;font-weight:400;font-display:block;src:url(../webfonts/fa-regular-400.woff2) format("woff2"),url(../webfonts/fa-regular-400.ttf) format("truetype")}.fa-regular,.far{font-weight:400} -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/css/solid.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | * Copyright 2024 Fonticons, Inc. 5 | */ 6 | :root, :host { 7 | --fa-style-family-classic: 'Font Awesome 6 Free'; 8 | --fa-font-solid: normal 900 1em/1 'Font Awesome 6 Free'; } 9 | 10 | @font-face { 11 | font-family: 'Font Awesome 6 Free'; 12 | font-style: normal; 13 | font-weight: 900; 14 | font-display: block; 15 | src: url("../webfonts/fa-solid-900.woff2") format("woff2"), url("../webfonts/fa-solid-900.ttf") format("truetype"); } 16 | 17 | .fas, 18 | .fa-solid { 19 | font-weight: 900; } 20 | -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/css/solid.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | * Copyright 2024 Fonticons, Inc. 5 | */ 6 | :host,:root{--fa-style-family-classic:"Font Awesome 6 Free";--fa-font-solid:normal 900 1em/1 "Font Awesome 6 Free"}@font-face{font-family:"Font Awesome 6 Free";font-style:normal;font-weight:900;font-display:block;src:url(../webfonts/fa-solid-900.woff2) format("woff2"),url(../webfonts/fa-solid-900.ttf) format("truetype")}.fa-solid,.fas{font-weight:900} -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/css/v5-font-face.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | * Copyright 2024 Fonticons, Inc. 5 | */ 6 | @font-face { 7 | font-family: 'Font Awesome 5 Brands'; 8 | font-display: block; 9 | font-weight: 400; 10 | src: url("../webfonts/fa-brands-400.woff2") format("woff2"), url("../webfonts/fa-brands-400.ttf") format("truetype"); } 11 | 12 | @font-face { 13 | font-family: 'Font Awesome 5 Free'; 14 | font-display: block; 15 | font-weight: 900; 16 | src: url("../webfonts/fa-solid-900.woff2") format("woff2"), url("../webfonts/fa-solid-900.ttf") format("truetype"); } 17 | 18 | @font-face { 19 | font-family: 'Font Awesome 5 Free'; 20 | font-display: block; 21 | font-weight: 400; 22 | src: url("../webfonts/fa-regular-400.woff2") format("woff2"), url("../webfonts/fa-regular-400.ttf") format("truetype"); } 23 | -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/css/v5-font-face.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | * Copyright 2024 Fonticons, Inc. 5 | */ 6 | @font-face{font-family:"Font Awesome 5 Brands";font-display:block;font-weight:400;src:url(../webfonts/fa-brands-400.woff2) format("woff2"),url(../webfonts/fa-brands-400.ttf) format("truetype")}@font-face{font-family:"Font Awesome 5 Free";font-display:block;font-weight:900;src:url(../webfonts/fa-solid-900.woff2) format("woff2"),url(../webfonts/fa-solid-900.ttf) format("truetype")}@font-face{font-family:"Font Awesome 5 Free";font-display:block;font-weight:400;src:url(../webfonts/fa-regular-400.woff2) format("woff2"),url(../webfonts/fa-regular-400.ttf) format("truetype")} -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-v4compatibility.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-v4compatibility.ttf -------------------------------------------------------------------------------- /core/http/static/assets/fontawesome/webfonts/fa-v4compatibility.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/assets/fontawesome/webfonts/fa-v4compatibility.woff2 -------------------------------------------------------------------------------- /core/http/static/assets/highlightjs.css: -------------------------------------------------------------------------------- 1 | /*! 2 | Theme: Default 3 | Description: Original highlight.js style 4 | Author: (c) Ivan Sagalaev 5 | Maintainer: @highlightjs/core-team 6 | Website: https://highlightjs.org/ 7 | License: see project LICENSE 8 | Touched: 2021 9 | */pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}.hljs{background:#f3f3f3;color:#444}.hljs-comment{color:#697070}.hljs-punctuation,.hljs-tag{color:#444a}.hljs-tag .hljs-attr,.hljs-tag .hljs-name{color:#444}.hljs-attribute,.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-name,.hljs-selector-tag{font-weight:700}.hljs-deletion,.hljs-number,.hljs-quote,.hljs-selector-class,.hljs-selector-id,.hljs-string,.hljs-template-tag,.hljs-type{color:#800}.hljs-section,.hljs-title{color:#800;font-weight:700}.hljs-link,.hljs-operator,.hljs-regexp,.hljs-selector-attr,.hljs-selector-pseudo,.hljs-symbol,.hljs-template-variable,.hljs-variable{color:#ab5656}.hljs-literal{color:#695}.hljs-addition,.hljs-built_in,.hljs-bullet,.hljs-code{color:#397300}.hljs-meta{color:#1f7199}.hljs-meta .hljs-string{color:#38a}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:700} -------------------------------------------------------------------------------- /core/http/static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/logo.png -------------------------------------------------------------------------------- /core/http/static/logo_horizontal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/core/http/static/logo_horizontal.png -------------------------------------------------------------------------------- /core/http/utils/baseurl.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/gofiber/fiber/v2" 7 | ) 8 | 9 | // BaseURL returns the base URL for the given HTTP request context. 10 | // It takes into account that the app may be exposed by a reverse-proxy under a different protocol, host and path. 11 | // The returned URL is guaranteed to end with `/`. 12 | // The method should be used in conjunction with the StripPathPrefix middleware. 13 | func BaseURL(c *fiber.Ctx) string { 14 | path := c.Path() 15 | origPath := c.OriginalURL() 16 | 17 | if path != origPath && strings.HasSuffix(origPath, path) { 18 | pathPrefix := origPath[:len(origPath)-len(path)+1] 19 | 20 | return c.BaseURL() + pathPrefix 21 | } 22 | 23 | return c.BaseURL() + "/" 24 | } 25 | -------------------------------------------------------------------------------- /core/http/utils/baseurl_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net/http/httptest" 5 | "testing" 6 | 7 | "github.com/gofiber/fiber/v2" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestBaseURL(t *testing.T) { 12 | for _, tc := range []struct { 13 | name string 14 | prefix string 15 | expectURL string 16 | }{ 17 | { 18 | name: "without prefix", 19 | prefix: "/", 20 | expectURL: "http://example.com/", 21 | }, 22 | { 23 | name: "with prefix", 24 | prefix: "/myprefix/", 25 | expectURL: "http://example.com/myprefix/", 26 | }, 27 | } { 28 | t.Run(tc.name, func(t *testing.T) { 29 | app := fiber.New() 30 | actualURL := "" 31 | 32 | app.Get(tc.prefix+"hello/world", func(c *fiber.Ctx) error { 33 | if tc.prefix != "/" { 34 | c.Path("/hello/world") 35 | } 36 | actualURL = BaseURL(c) 37 | return nil 38 | }) 39 | 40 | req := httptest.NewRequest("GET", tc.prefix+"hello/world", nil) 41 | resp, err := app.Test(req, -1) 42 | 43 | require.NoError(t, err) 44 | require.Equal(t, 200, resp.StatusCode, "response status code") 45 | require.Equal(t, tc.expectURL, actualURL, "base URL") 46 | }) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /core/p2p/p2p_common.go: -------------------------------------------------------------------------------- 1 | package p2p 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | var logLevel = strings.ToLower(os.Getenv("LOCALAI_P2P_LOGLEVEL")) 9 | 10 | const ( 11 | logLevelDebug = "debug" 12 | logLevelInfo = "info" 13 | ) 14 | 15 | func init() { 16 | if logLevel == "" { 17 | logLevel = logLevelInfo 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /core/p2p/p2p_disabled.go: -------------------------------------------------------------------------------- 1 | //go:build !p2p 2 | // +build !p2p 3 | 4 | package p2p 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | 10 | "github.com/mudler/edgevpn/pkg/node" 11 | ) 12 | 13 | func GenerateToken(DHTInterval, OTPInterval int) string { 14 | return "not implemented" 15 | } 16 | 17 | func (f *FederatedServer) Start(ctx context.Context) error { 18 | return fmt.Errorf("not implemented") 19 | } 20 | 21 | func ServiceDiscoverer(ctx context.Context, node *node.Node, token, servicesID string, fn func(string, NodeData), allocate bool) error { 22 | return fmt.Errorf("not implemented") 23 | } 24 | 25 | func ExposeService(ctx context.Context, host, port, token, servicesID string) (*node.Node, error) { 26 | return nil, fmt.Errorf("not implemented") 27 | } 28 | 29 | func IsP2PEnabled() bool { 30 | return false 31 | } 32 | 33 | func NewNode(token string) (*node.Node, error) { 34 | return nil, fmt.Errorf("not implemented") 35 | } 36 | -------------------------------------------------------------------------------- /core/schema/elevenlabs.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type ElevenLabsTTSRequest struct { 4 | Text string `json:"text" yaml:"text"` 5 | ModelID string `json:"model_id" yaml:"model_id"` 6 | LanguageCode string `json:"language_code" yaml:"language_code"` 7 | } 8 | 9 | type ElevenLabsSoundGenerationRequest struct { 10 | Text string `json:"text" yaml:"text"` 11 | ModelID string `json:"model_id" yaml:"model_id"` 12 | Duration *float32 `json:"duration_seconds,omitempty" yaml:"duration_seconds,omitempty"` 13 | Temperature *float32 `json:"prompt_influence,omitempty" yaml:"prompt_influence,omitempty"` 14 | DoSample *bool `json:"do_sample,omitempty" yaml:"do_sample,omitempty"` 15 | } 16 | 17 | func (elttsr *ElevenLabsTTSRequest) ModelName(s *string) string { 18 | if s != nil { 19 | elttsr.ModelID = *s 20 | } 21 | return elttsr.ModelID 22 | } 23 | 24 | func (elsgr *ElevenLabsSoundGenerationRequest) ModelName(s *string) string { 25 | if s != nil { 26 | elsgr.ModelID = *s 27 | } 28 | return elsgr.ModelID 29 | } 30 | -------------------------------------------------------------------------------- /core/schema/jina.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | // RerankRequest defines the structure of the request payload 4 | type JINARerankRequest struct { 5 | BasicModelRequest 6 | Query string `json:"query"` 7 | Documents []string `json:"documents"` 8 | TopN int `json:"top_n"` 9 | Backend string `json:"backend"` 10 | } 11 | 12 | // DocumentResult represents a single document result 13 | type JINADocumentResult struct { 14 | Index int `json:"index"` 15 | Document JINAText `json:"document"` 16 | RelevanceScore float64 `json:"relevance_score"` 17 | } 18 | 19 | // Text holds the text of the document 20 | type JINAText struct { 21 | Text string `json:"text"` 22 | } 23 | 24 | // RerankResponse defines the structure of the response payload 25 | type JINARerankResponse struct { 26 | Model string `json:"model"` 27 | Usage JINAUsageInfo `json:"usage"` 28 | Results []JINADocumentResult `json:"results"` 29 | } 30 | 31 | // UsageInfo holds information about usage of tokens 32 | type JINAUsageInfo struct { 33 | TotalTokens int `json:"total_tokens"` 34 | PromptTokens int `json:"prompt_tokens"` 35 | } 36 | -------------------------------------------------------------------------------- /core/schema/request.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | // This file and type represent a generic request to LocalAI - as opposed to requests to LocalAI-specific endpoints, which live in localai.go 4 | type LocalAIRequest interface { 5 | ModelName(*string) string 6 | } 7 | 8 | type BasicModelRequest struct { 9 | Model string `json:"model" yaml:"model"` 10 | // TODO: Should this also include the following fields from the OpenAI side of the world? 11 | // If so, changes should be made to core/http/middleware/request.go to match 12 | 13 | // Context context.Context `json:"-"` 14 | // Cancel context.CancelFunc `json:"-"` 15 | } 16 | 17 | func (bmr *BasicModelRequest) ModelName(s *string) string { 18 | if s != nil { 19 | bmr.Model = *s 20 | } 21 | return bmr.Model 22 | } 23 | -------------------------------------------------------------------------------- /core/schema/tokenize.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | type TokenizeRequest struct { 4 | BasicModelRequest 5 | Content string `json:"content"` 6 | } 7 | 8 | type TokenizeResponse struct { 9 | Tokens []int32 `json:"tokens"` 10 | } 11 | -------------------------------------------------------------------------------- /core/schema/transcription.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import "time" 4 | 5 | type TranscriptionSegment struct { 6 | Id int `json:"id"` 7 | Start time.Duration `json:"start"` 8 | End time.Duration `json:"end"` 9 | Text string `json:"text"` 10 | Tokens []int `json:"tokens"` 11 | } 12 | 13 | type TranscriptionResult struct { 14 | Segments []TranscriptionSegment `json:"segments"` 15 | Text string `json:"text"` 16 | } 17 | -------------------------------------------------------------------------------- /custom-ca-certs/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/custom-ca-certs/.keep -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | api: 3 | # See https://localai.io/basics/getting_started/#container-images for 4 | # a list of available container images (or build your own with the provided Dockerfile) 5 | # Available images with CUDA, ROCm, SYCL 6 | # Image list (quay.io): https://quay.io/repository/go-skynet/local-ai?tab=tags 7 | # Image list (dockerhub): https://hub.docker.com/r/localai/localai 8 | image: quay.io/go-skynet/local-ai:master-ffmpeg-core 9 | build: 10 | context: . 11 | dockerfile: Dockerfile 12 | args: 13 | - IMAGE_TYPE=core 14 | - BASE_IMAGE=ubuntu:22.04 15 | ports: 16 | - 8080:8080 17 | env_file: 18 | - .env 19 | environment: 20 | - MODELS_PATH=/models 21 | # - DEBUG=true 22 | volumes: 23 | - ./models:/models:cached 24 | - ./images/:/tmp/generated/images/ 25 | command: 26 | # Here we can specify a list of models to run (see quickstart https://localai.io/basics/getting_started/#running-models ) 27 | # or an URL pointing to a YAML configuration file, for example: 28 | # - https://gist.githubusercontent.com/mudler/ad601a0488b497b69ec549150d9edd18/raw/a8a8869ef1bb7e3830bf5c0bae29a0cce991ff8d/phi-2.yaml 29 | - phi-2 30 | -------------------------------------------------------------------------------- /docs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM klakegg/hugo:ext-alpine 2 | 3 | RUN apk add git && \ 4 | git config --global --add safe.directory /src 5 | -------------------------------------------------------------------------------- /docs/assets/images/imagen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/imagen.png -------------------------------------------------------------------------------- /docs/assets/images/localai_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/localai_screenshot.png -------------------------------------------------------------------------------- /docs/assets/images/logos/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/logos/logo.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_chat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_chat.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_gallery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_gallery.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_home.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_image.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_login.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_p2p.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_p2p.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_talk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_talk.png -------------------------------------------------------------------------------- /docs/assets/images/screenshots/screenshot_tts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/assets/images/screenshots/screenshot_tts.png -------------------------------------------------------------------------------- /docs/assets/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "paths": { 5 | "*": [ 6 | "../../../../.cache/hugo_cache/modules/filecache/modules/pkg/mod/github.com/gohugoio/hugo-mod-jslibs-dist/popperjs/v2@v2.21100.20000/package/dist/cjs/*", 7 | "../../../../.cache/hugo_cache/modules/filecache/modules/pkg/mod/github.com/twbs/bootstrap@v5.3.2+incompatible/js/*" 8 | ] 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /docs/content/docs/advanced/_index.en.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 20 3 | title: "Advanced" 4 | description: "Advanced usage" 5 | icon: settings 6 | lead: "" 7 | date: 2020-10-06T08:49:15+00:00 8 | lastmod: 2020-10-06T08:49:15+00:00 9 | draft: false 10 | images: [] 11 | --- -------------------------------------------------------------------------------- /docs/content/docs/features/_index.en.md: -------------------------------------------------------------------------------- 1 | 2 | +++ 3 | disableToc = false 4 | title = "Features" 5 | weight = 8 6 | icon = "feature_search" 7 | url = "/features/" 8 | +++ 9 | -------------------------------------------------------------------------------- /docs/content/docs/getting-started/_index.en.md: -------------------------------------------------------------------------------- 1 | 2 | +++ 3 | disableToc = false 4 | title = "Getting started" 5 | weight = 2 6 | icon = "rocket_launch" 7 | +++ 8 | -------------------------------------------------------------------------------- /docs/content/docs/getting-started/kubernetes.md: -------------------------------------------------------------------------------- 1 | +++ 2 | disableToc = false 3 | title = "Run with Kubernetes" 4 | weight = 6 5 | url = '/basics/kubernetes/' 6 | ico = "rocket_launch" 7 | +++ 8 | 9 | 10 | For installing LocalAI in Kubernetes, the deployment file from the `examples` can be used and customized as preferred: 11 | 12 | ``` 13 | kubectl apply -f https://raw.githubusercontent.com/mudler/LocalAI-examples/refs/heads/main/kubernetes/deployment.yaml 14 | ``` 15 | 16 | For Nvidia GPUs: 17 | 18 | ``` 19 | kubectl apply -f https://raw.githubusercontent.com/mudler/LocalAI-examples/refs/heads/main/kubernetes/deployment-nvidia.yaml 20 | ``` 21 | 22 | Alternatively, the [helm chart](https://github.com/go-skynet/helm-charts) can be used as well: 23 | 24 | ```bash 25 | # Install the helm repository 26 | helm repo add go-skynet https://go-skynet.github.io/helm-charts/ 27 | # Update the repositories 28 | helm repo update 29 | # Get the values 30 | helm show values go-skynet/local-ai > values.yaml 31 | 32 | # Edit the values if needed 33 | # vim values.yaml ... 34 | 35 | # Install the helm chart 36 | helm install local-ai go-skynet/local-ai -f values.yaml 37 | ``` 38 | -------------------------------------------------------------------------------- /docs/content/docs/reference/_index.en.md: -------------------------------------------------------------------------------- 1 | --- 2 | weight: 23 3 | title: "References" 4 | description: "Reference" 5 | icon: menu_book 6 | lead: "" 7 | date: 2020-10-06T08:49:15+00:00 8 | lastmod: 2020-10-06T08:49:15+00:00 9 | draft: false 10 | images: [] 11 | --- -------------------------------------------------------------------------------- /docs/data/version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "v2.29.0" 3 | } 4 | -------------------------------------------------------------------------------- /docs/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | 3 | services: 4 | 5 | site: 6 | image: docsy/docsy-example 7 | build: 8 | context: . 9 | command: server 10 | ports: 11 | - "1313:1313" 12 | volumes: 13 | - .:/src 14 | -------------------------------------------------------------------------------- /docs/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/McShelby/hugo-theme-relearn.git 2 | 3 | go 1.19 4 | 5 | require github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20200 // indirect 6 | -------------------------------------------------------------------------------- /docs/go.sum: -------------------------------------------------------------------------------- 1 | github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20200 h1:SmpwwN3DNzJWbV+IT8gaFu07ENUFpCvKou5BHYUKuVs= 2 | github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20200/go.mod h1:kx8MBj9T7SFR8ZClWvKZPmmUxBaltkoXvnWlZZcSnYA= 3 | github.com/gohugoio/hugo-mod-jslibs-dist/popperjs/v2 v2.21100.20000/go.mod h1:mFberT6ZtcchrsDtfvJM7aAH2bDKLdOnruUHl0hlapI= 4 | github.com/twbs/bootstrap v5.3.2+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0= 5 | -------------------------------------------------------------------------------- /docs/layouts/404.html: -------------------------------------------------------------------------------- 1 | {{ define "main"}} 2 |
3 |
4 |

Not found

5 |

Oops! This page doesn't exist. Try going back to our home page.

6 | 7 |

You can learn how to make a 404 page like this in Custom 404 Pages.

8 |
9 |
10 | {{ end }} 11 | -------------------------------------------------------------------------------- /docs/layouts/partials/logo.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/github.html: -------------------------------------------------------------------------------- 1 | {{ if .IsNamedParams }} 2 | {{ .Get "user" }} 3 | {{ else }} 4 | {{ .Get 0 }} 5 | {{ end }} 6 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/pr.html: -------------------------------------------------------------------------------- 1 | {{ if .IsNamedParams }} 2 | {{ .Get "number" }} 3 | {{ else }} 4 | {{ .Get 0 }} 5 | {{ end }} 6 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/version.html: -------------------------------------------------------------------------------- 1 | {{ $.Site.Data.version.version }} -------------------------------------------------------------------------------- /docs/netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | [build.environment] 3 | HUGO_VERSION = "0.146.3" 4 | GO_VERSION = "1.22.2" 5 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tech-doc-hugo", 3 | "version": "0.0.1", 4 | "description": "Hugo theme for technical documentation.", 5 | "main": "none.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/google/docsy-example.git" 12 | }, 13 | "author": "", 14 | "license": "ISC", 15 | "bugs": { 16 | "url": "https://github.com/google/docsy-example/issues" 17 | }, 18 | "homepage": "https://github.com/google/docsy-example#readme", 19 | "devDependencies": { 20 | "autoprefixer": "^10.4.0", 21 | "postcss": "^8.3.7", 22 | "postcss-cli": "^10.0.0" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /docs/static/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/static/android-chrome-192x192.png -------------------------------------------------------------------------------- /docs/static/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/static/android-chrome-512x512.png -------------------------------------------------------------------------------- /docs/static/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/static/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/static/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/static/favicon-16x16.png -------------------------------------------------------------------------------- /docs/static/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/static/favicon-32x32.png -------------------------------------------------------------------------------- /docs/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/docs/static/favicon.ico -------------------------------------------------------------------------------- /docs/static/site.webmanifest: -------------------------------------------------------------------------------- 1 | {"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} -------------------------------------------------------------------------------- /docs/themes/hugo-theme-relearn: -------------------------------------------------------------------------------- 1 | 9a020e7eadb7d8203f5b01b18756c72d94773ec9 -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # LocalAI Examples 2 | 3 | LocalAI examples were moved to a dedicated repository: https://github.com/mudler/LocalAI-examples 4 | -------------------------------------------------------------------------------- /gallery/alpaca.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "alpaca" 3 | 4 | config_file: | 5 | context_size: 4096 6 | f16: true 7 | mmap: true 8 | template: 9 | chat: | 10 | Below is an instruction that describes a task. Write a response that appropriately completes the request. 11 | 12 | ### Instruction: 13 | {{.Input}} 14 | 15 | ### Response: 16 | completion: | 17 | {{.Input}} 18 | -------------------------------------------------------------------------------- /gallery/cerbero.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | config_file: | 3 | backend: llama-cpp 4 | context_size: 8192 5 | f16: false 6 | name: cerbero 7 | 8 | template: 9 | completion: "{{.Input}}" 10 | chat: "Questa è una conversazione tra un umano ed un assistente AI.\n{{.Input}}\n[|Assistente|] " 11 | roles: 12 | user: "[|Umano|] " 13 | system: "[|Umano|] " 14 | assistant: "[|Assistente|] " 15 | 16 | stopwords: 17 | - "[|Umano|]" 18 | 19 | trimsuffix: 20 | - "\n" 21 | -------------------------------------------------------------------------------- /gallery/codellama.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "codellama" 3 | 4 | config_file: | 5 | backend: llama-cpp 6 | context_size: 4096 7 | f16: true 8 | mmap: true 9 | -------------------------------------------------------------------------------- /gallery/deepseek-r1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "deepseek-r1" 3 | 4 | config_file: | 5 | context_size: 131072 6 | mmap: true 7 | f16: true 8 | stopwords: 9 | - <｜begin▁of▁sentence｜> 10 | - <｜end▁of▁sentence｜> 11 | - <｜User｜> 12 | - <｜Assistant｜> 13 | template: 14 | chat_message: | 15 | {{if eq .RoleName "system" -}}{{.Content }} 16 | {{ end -}} 17 | {{if eq .RoleName "user" -}}<｜User｜>{{.Content}} 18 | {{end -}} 19 | {{if eq .RoleName "assistant" -}}<｜Assistant｜>{{.Content}}<｜end▁of▁sentence｜>{{end}} 20 | completion: | 21 | {{.Input}} 22 | chat: | 23 | {{.Input -}}<｜Assistant｜> 24 | -------------------------------------------------------------------------------- /gallery/deepseek.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "deepseek" 3 | 4 | config_file: | 5 | mmap: true 6 | context_size: 8192 7 | template: 8 | chat_message: |- 9 | {{if eq .RoleName "user" -}}User: {{.Content }} 10 | {{ end -}} 11 | {{if eq .RoleName "assistant" -}}Assistant: {{.Content}}<｜end▁of▁sentence｜>{{end}} 12 | {{if eq .RoleName "system" -}}{{.Content}} 13 | {{end -}} 14 | chat: | 15 | {{.Input -}} 16 | Assistant: # Space is preserved for templating reasons, but line does not end with one for the linter. 17 | completion: | 18 | {{.Input}} 19 | stopwords: 20 | - '<｜end▁of▁sentence｜>' 21 | -------------------------------------------------------------------------------- /gallery/dreamshaper.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "dreamshaper" 3 | 4 | config_file: | 5 | backend: diffusers 6 | step: 25 7 | f16: true 8 | 9 | diffusers: 10 | pipeline_type: StableDiffusionPipeline 11 | cuda: true 12 | enable_parameters: "negative_prompt,num_inference_steps" 13 | scheduler_type: "k_dpmpp_2m" 14 | -------------------------------------------------------------------------------- /gallery/flux-ggml.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "flux-ggml" 3 | 4 | config_file: | 5 | backend: stablediffusion-ggml 6 | step: 25 7 | options: 8 | - "diffusion_model" 9 | - "clip_l_path:clip_l.safetensors" 10 | - "t5xxl_path:t5xxl_fp16.safetensors" 11 | - "vae_path:ae.safetensors" 12 | - "sampler:euler" 13 | 14 | cfg_scale: 1 15 | -------------------------------------------------------------------------------- /gallery/flux.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "flux" 3 | 4 | config_file: | 5 | backend: diffusers 6 | f16: true 7 | low_vram: true 8 | step: 25 9 | 10 | diffusers: 11 | cuda: true 12 | enable_parameters: num_inference_steps 13 | pipeline_type: FluxPipeline 14 | 15 | cfg_scale: 1 16 | -------------------------------------------------------------------------------- /gallery/granite.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "granite" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|{{ .RoleName }}|> 9 | {{ if .FunctionCall -}} 10 | Function call: 11 | {{ else if eq .RoleName "tool" -}} 12 | Function response: 13 | {{ end -}} 14 | {{ if .Content -}} 15 | {{.Content }} 16 | {{ end -}} 17 | {{ if .FunctionCall -}} 18 | {{toJson .FunctionCall}} 19 | {{ end -}} 20 | function: | 21 | <|system|> 22 | You are a function calling AI model. You are provided with functions to execute. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: 23 | {{range .Functions}} 24 | {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }} 25 | {{end}} 26 | For each function call return a json object with function name and arguments 27 | {{.Input -}} 28 | <|assistant|> 29 | chat: | 30 | {{.Input -}} 31 | <|assistant|> 32 | completion: | 33 | {{.Input}} 34 | context_size: 4096 35 | f16: true 36 | stopwords: 37 | - '<|im_end|>' 38 | - '' 39 | - '' 40 | - '<|' 41 | -------------------------------------------------------------------------------- /gallery/llava.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "llava" 3 | 4 | config_file: | 5 | backend: llama-cpp 6 | context_size: 4096 7 | f16: true 8 | 9 | mmap: true 10 | roles: 11 | user: "USER:" 12 | assistant: "ASSISTANT:" 13 | system: "SYSTEM:" 14 | 15 | template: 16 | chat: | 17 | A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. 18 | {{.Input}} 19 | ASSISTANT: 20 | -------------------------------------------------------------------------------- /gallery/moondream.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "moondream2" 3 | 4 | 5 | config_file: | 6 | context_size: 2046 7 | roles: 8 | user: "\nQuestion: " 9 | system: "\nSystem: " 10 | assistant: "\nAnswer: " 11 | stopwords: 12 | - "Question:" 13 | - "<|endoftext|>" 14 | f16: true 15 | template: 16 | completion: | 17 | Complete the following sentence: {{.Input}} 18 | chat: "{{.Input}}\nAnswer:\n" 19 | -------------------------------------------------------------------------------- /gallery/openvino.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: openvino 3 | 4 | config_file: | 5 | backend: transformers 6 | context_size: 8192 7 | type: OVModelForCausalLM 8 | template: 9 | use_tokenizer_template: true 10 | -------------------------------------------------------------------------------- /gallery/parler-tts.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | config_file: | 3 | backend: parler-tts 4 | -------------------------------------------------------------------------------- /gallery/phi-2-chat.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "phi-2-chatml" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|im_start|>{{ .RoleName }} 9 | {{.Content}}<|im_end|> 10 | chat: | 11 | {{.Input}} 12 | <|im_start|>assistant 13 | completion: | 14 | {{.Input}} 15 | context_size: 4096 16 | f16: true 17 | stopwords: 18 | - <|im_end|> 19 | -------------------------------------------------------------------------------- /gallery/phi-2-orange.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "phi-2-orange" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}} 9 | {{if .Content}}{{.Content}}{{end}}<|im_end|> 10 | chat: | 11 | {{.Input}} 12 | <|im_start|>assistant 13 | completion: | 14 | {{.Input}} 15 | context_size: 4096 16 | f16: true 17 | stopwords: 18 | - <|im_end|> 19 | - 20 | -------------------------------------------------------------------------------- /gallery/phi-3-chat.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "phi-3-chat" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|{{ .RoleName }}|> 9 | {{.Content}}<|end|> 10 | chat: | 11 | {{.Input}} 12 | <|assistant|> 13 | completion: | 14 | {{.Input}} 15 | context_size: 4096 16 | f16: true 17 | stopwords: 18 | - <|end|> 19 | - <|endoftext|> 20 | -------------------------------------------------------------------------------- /gallery/phi-3-vision.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "phi3-vision" 3 | 4 | config_file: | 5 | name: phi3-vision 6 | backend: vllm 7 | parameters: 8 | model: microsoft/Phi-3-vision-128k-instruct 9 | trust_remote_code: true 10 | max_model_len: 32768 11 | template: 12 | chat_message: |- 13 | <|{{ .RoleName }}|> 14 | {{.Content}}<|end|> 15 | chat: >- 16 | {{.Input}} 17 | 18 | <|assistant|> 19 | 20 | completion: | 21 | {{.Input}} 22 | use_tokenizer_template: false 23 | multimodal: "{{ range .Images }}<|image_{{ add1 .ID}}|>{{end}}\n{{.Text}}" 24 | # XXX: The one below can be dropped after a new release is out 25 | image: "<|image_{{ add1 .ID }}|>\n{{.Text}}" 26 | -------------------------------------------------------------------------------- /gallery/phi-4-chat-fcall.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "phi-4-chat" 3 | 4 | config_file: | 5 | mmap: true 6 | function: 7 | json_regex_match: 8 | - "(?s)(.*?)" 9 | capture_llm_results: 10 | - (?s)(.*?) 11 | replace_llm_results: 12 | - key: (?s)(.*?) 13 | value: "" 14 | grammar: 15 | properties_order: "name,arguments" 16 | template: 17 | chat_message: | 18 | <|im_start|>{{ .RoleName }}<|im_sep|> 19 | {{.Content}}<|im_end|> 20 | chat: | 21 | {{.Input}} 22 | <|im_start|>assistant<|im_sep|> 23 | completion: | 24 | {{.Input}} 25 | function: | 26 | <|im_start|>system<|im_sep|> 27 | You are an AI assistant that executes function calls, and these are the tools at your disposal: 28 | {{range .Functions}} 29 | {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }} 30 | {{end}} 31 | {{.Input}}<|im_end|> 32 | context_size: 4096 33 | f16: true 34 | stopwords: 35 | - <|end|> 36 | - <|endoftext|> 37 | - <|im_end|> 38 | -------------------------------------------------------------------------------- /gallery/phi-4-chat.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "phi-4-chat" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|im_start|>{{ .RoleName }}<|im_sep|> 9 | {{.Content}}<|im_end|> 10 | chat: | 11 | {{.Input}} 12 | <|im_start|>assistant<|im_sep|> 13 | completion: | 14 | {{.Input}} 15 | context_size: 4096 16 | f16: true 17 | stopwords: 18 | - <|end|> 19 | - <|endoftext|> 20 | - <|im_end|> 21 | -------------------------------------------------------------------------------- /gallery/piper.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | config_file: | 3 | backend: piper 4 | -------------------------------------------------------------------------------- /gallery/qwen3.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "qwen3" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|im_start|>{{ .RoleName }} 9 | {{ if .FunctionCall -}} 10 | {{ else if eq .RoleName "tool" -}} 11 | {{ end -}} 12 | {{ if .Content -}} 13 | {{.Content }} 14 | {{ end -}} 15 | {{ if .FunctionCall -}} 16 | {{toJson .FunctionCall}} 17 | {{ end -}}<|im_end|> 18 | function: | 19 | <|im_start|>system 20 | You are a function calling AI model. You are provided with functions to execute. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: 21 | {{range .Functions}} 22 | {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }} 23 | {{end}} 24 | For each function call return a json object with function name and arguments 25 | <|im_end|> 26 | {{.Input -}} 27 | <|im_start|>assistant 28 | chat: | 29 | {{.Input -}} 30 | <|im_start|>assistant 31 | completion: | 32 | {{.Input}} 33 | context_size: 8192 34 | f16: true 35 | stopwords: 36 | - '<|im_end|>' 37 | - '' 38 | - '' 39 | - '<|endoftext|>' 40 | -------------------------------------------------------------------------------- /gallery/rerankers.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | config_file: | 3 | backend: rerankers 4 | -------------------------------------------------------------------------------- /gallery/rwkv.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "rwkv" 3 | 4 | config_file: | 5 | parameters: 6 | top_k: 80 7 | temperature: 0.9 8 | max_tokens: 4098 9 | top_p: 0.8 10 | context_size: 4098 11 | 12 | roles: 13 | user: "User: " 14 | system: "System: " 15 | assistant: "Assistant: " 16 | 17 | stopwords: 18 | - 'Assistant:' 19 | - '' 20 | 21 | template: 22 | chat: "{{.Input}}\nAssistant: " 23 | completion: | 24 | {{.Input}} 25 | -------------------------------------------------------------------------------- /gallery/sd-ggml.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "sd-ggml" 3 | 4 | config_file: | 5 | backend: stablediffusion-ggml 6 | step: 25 7 | cfg_scale: 4.5 8 | options: 9 | - "clip_l_path:clip_l.safetensors" 10 | - "clip_g_path:clip_g.safetensors" 11 | - "t5xxl_path:t5xxl-Q5_0.gguf" 12 | - "sampler:euler" 13 | -------------------------------------------------------------------------------- /gallery/sentencetransformers.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "sentencetransformers" 3 | 4 | config_file: | 5 | backend: sentencetransformers 6 | -------------------------------------------------------------------------------- /gallery/smolvlm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: smolvlm 3 | # yamllint disable-line rule:trailing-spaces 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | {{if eq .RoleName "assistant"}}Assistant{{else if eq .RoleName "system"}}System{{else if eq .RoleName "user"}}User{{end}}: {{.Content }} 9 | chat: "<|im_start|>\n{{.Input -}}\nAssistant: " 10 | completion: | 11 | {{-.Input}} 12 | f16: true 13 | stopwords: 14 | - '<|im_end|>' 15 | - '' 16 | - '' 17 | - '<|' 18 | - '' 19 | - '<|endoftext|>' 20 | -------------------------------------------------------------------------------- /gallery/stablediffusion3.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "stable-diffusion-3-medium" 3 | 4 | config_file: | 5 | backend: diffusers 6 | diffusers: 7 | cuda: true 8 | enable_parameters: negative_prompt,num_inference_steps 9 | pipeline_type: StableDiffusion3Pipeline 10 | f16: false 11 | name: sd3 12 | parameters: 13 | model: v2ray/stable-diffusion-3-medium-diffusers 14 | step: 25 15 | -------------------------------------------------------------------------------- /gallery/tuluv2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "tuluv2" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: | 8 | <|{{ .RoleName }}|> 9 | {{ if .FunctionCall -}} 10 | Function call: 11 | {{ else if eq .RoleName "tool" -}} 12 | Function response: 13 | {{ end -}} 14 | {{ if .Content -}} 15 | {{.Content }} 16 | {{ end -}} 17 | {{ if .FunctionCall -}} 18 | {{toJson .FunctionCall}} 19 | {{ end -}} 20 | function: | 21 | <|{{ .RoleName }}|> 22 | {{ if .FunctionCall -}} 23 | Function call: 24 | {{ else if eq .RoleName "tool" -}} 25 | Function response: 26 | {{ end -}} 27 | {{ if .Content -}} 28 | {{.Content }} 29 | {{ end -}} 30 | {{ if .FunctionCall -}} 31 | {{toJson .FunctionCall}} 32 | {{ end -}} 33 | chat: | 34 | {{.Input -}} 35 | <|assistant|> 36 | completion: | 37 | {{.Input}} 38 | context_size: 4096 39 | f16: true 40 | stopwords: 41 | - '<|im_end|>' 42 | - '' 43 | - '<|endoftext|>' 44 | -------------------------------------------------------------------------------- /gallery/vicuna-chat.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "vicuna-chat" 3 | 4 | description: | 5 | Vicuna chat 6 | 7 | license: "LLaMA" 8 | 9 | config_file: | 10 | backend: llama-cpp 11 | context_size: 4096 12 | roles: 13 | user: "User: " 14 | system: "System: " 15 | assistant: "Assistant: " 16 | f16: true 17 | stopwords: 18 | - <|end|> 19 | - <|endoftext|> 20 | - 21 | template: 22 | completion: | 23 | Complete the following sentence: {{.Input}} 24 | chat: | 25 | {{.Input}} 26 | ASSISTANT: 27 | -------------------------------------------------------------------------------- /gallery/virtual.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "virtual" 3 | 4 | description: | 5 | A Base model definition 6 | 7 | license: "N/A" 8 | -------------------------------------------------------------------------------- /gallery/whisper-base.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "whisper-base" 3 | 4 | config_file: | 5 | backend: whisper 6 | -------------------------------------------------------------------------------- /gallery/wizardlm2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "wizardlm2" 3 | 4 | config_file: | 5 | mmap: true 6 | template: 7 | chat_message: |- 8 | {{if eq .RoleName "assistant"}}ASSISTANT: {{.Content}}{{else if eq .RoleName "system"}}{{.Content}}{{else if eq .RoleName "user"}}USER: {{.Content}}{{end}} 9 | chat: "{{.Input}}ASSISTANT: " 10 | completion: |- 11 | {{.Input}} 12 | context_size: 32768 13 | f16: true 14 | stopwords: 15 | - 16 | -------------------------------------------------------------------------------- /internal/version.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "fmt" 4 | 5 | var Version = "" 6 | var Commit = "" 7 | 8 | func PrintableVersion() string { 9 | return fmt.Sprintf("%s (%s)", Version, Commit) 10 | } 11 | -------------------------------------------------------------------------------- /models/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mudler/LocalAI/489c28991609d6a3aac84733727d054afdfd0f84/models/.keep -------------------------------------------------------------------------------- /pkg/assets/list.go: -------------------------------------------------------------------------------- 1 | package assets 2 | 3 | import ( 4 | "os" 5 | 6 | rice "github.com/GeertJohan/go.rice" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | func ListFiles(content *rice.Box) (files []string) { 11 | err := content.Walk("", func(path string, info os.FileInfo, err error) error { 12 | if err != nil { 13 | return err 14 | } 15 | 16 | if info.IsDir() { 17 | return nil 18 | } 19 | 20 | files = append(files, path) 21 | return nil 22 | }) 23 | if err != nil { 24 | log.Error().Err(err).Msg("error walking the rice box") 25 | } 26 | return 27 | } 28 | -------------------------------------------------------------------------------- /pkg/concurrency/concurrency_suite_test.go: -------------------------------------------------------------------------------- 1 | package concurrency 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestConcurrency(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Concurrency test suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/downloader/downloader_suite_test.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestDownloader(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Downloader test suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/functions/function_structure.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/mudler/LocalAI/pkg/functions/grammars" 7 | ) 8 | 9 | type Item struct { 10 | Type string `json:"type"` 11 | Properties map[string]interface{} `json:"properties"` 12 | } 13 | 14 | type JSONFunctionStructure struct { 15 | OneOf []Item `json:"oneOf,omitempty"` 16 | AnyOf []Item `json:"anyOf,omitempty"` 17 | Defs map[string]interface{} `json:"$defs,omitempty"` 18 | } 19 | 20 | func (j JSONFunctionStructure) Grammar(options ...func(*grammars.GrammarOption)) (string, error) { 21 | grammarOpts := &grammars.GrammarOption{} 22 | grammarOpts.Apply(options...) 23 | 24 | dat, err := json.Marshal(j) 25 | if err != nil { 26 | return "", err 27 | } 28 | 29 | converter := NewSchemaConverter(*grammarOpts) 30 | return converter.GrammarFromBytes(dat, options...) 31 | } 32 | 33 | type SchemaConverter interface { 34 | GrammarFromBytes([]byte, ...func(*grammars.GrammarOption)) (string, error) 35 | } 36 | 37 | func NewSchemaConverter(opt grammars.GrammarOption) SchemaConverter { 38 | switch { 39 | case opt.SchemaType == grammars.LLama31Schema: 40 | return grammars.NewLLama31SchemaConverter(opt.FunctionName) 41 | } 42 | return grammars.NewJSONSchemaConverter(opt.PropOrder) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/functions/functions_suite_test.go: -------------------------------------------------------------------------------- 1 | package functions_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestFunctions(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Functions test suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/functions/grammars/grammars_suite_test.go: -------------------------------------------------------------------------------- 1 | package grammars_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/mudler/LocalAI/pkg/functions" 7 | 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | func TestGrammar(t *testing.T) { 13 | RegisterFailHandler(Fail) 14 | RunSpecs(t, "Grammar test suite") 15 | } 16 | 17 | func createFunction(field1 string, field2 string, name string, properties map[string]interface{}) map[string]interface{} { 18 | property := map[string]interface{}{} 19 | property[field1] = FunctionName{Const: name} 20 | property[field2] = Argument{ 21 | Type: "object", 22 | Properties: properties, 23 | } 24 | return property 25 | } 26 | -------------------------------------------------------------------------------- /pkg/functions/grammars/types.go: -------------------------------------------------------------------------------- 1 | package grammars 2 | 3 | type SchemaConverterType int 4 | 5 | const ( 6 | JSONSchema SchemaConverterType = iota 7 | LLama31Schema 8 | ) 9 | 10 | const ( 11 | LlamaType string = "llama3.1" 12 | JSONType string = "json" 13 | ) 14 | 15 | func (s SchemaConverterType) String() string { 16 | switch s { 17 | case JSONSchema: 18 | return JSONType 19 | case LLama31Schema: 20 | return LlamaType 21 | } 22 | return "unknown" 23 | } 24 | 25 | func NewType(t string) SchemaConverterType { 26 | switch t { 27 | case JSONType: 28 | return JSONSchema 29 | case LlamaType: 30 | return LLama31Schema 31 | } 32 | return JSONSchema 33 | } 34 | -------------------------------------------------------------------------------- /pkg/functions/json_mode.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | const ( 4 | JSONBNF = `root ::= object 5 | value ::= object | array | string | number | ("true" | "false" | "null") ws 6 | 7 | object ::= 8 | "{" ws ( 9 | string ":" ws value 10 | ("," ws string ":" ws value)* 11 | )? "}" ws 12 | 13 | array ::= 14 | "[" ws ( 15 | value 16 | ("," ws value)* 17 | )? "]" ws 18 | 19 | string ::= 20 | "\"" ( 21 | [^"\\] | 22 | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes 23 | )* "\"" ws 24 | 25 | number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws 26 | 27 | ws ::= ([ \t\n] ws)?` 28 | ) 29 | -------------------------------------------------------------------------------- /pkg/grpc/interface.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | pb "github.com/mudler/LocalAI/pkg/grpc/proto" 5 | ) 6 | 7 | type LLM interface { 8 | Busy() bool 9 | Lock() 10 | Unlock() 11 | Locking() bool 12 | Predict(*pb.PredictOptions) (string, error) 13 | PredictStream(*pb.PredictOptions, chan string) error 14 | Load(*pb.ModelOptions) error 15 | Embeddings(*pb.PredictOptions) ([]float32, error) 16 | GenerateImage(*pb.GenerateImageRequest) error 17 | GenerateVideo(*pb.GenerateVideoRequest) error 18 | AudioTranscription(*pb.TranscriptRequest) (pb.TranscriptResult, error) 19 | TTS(*pb.TTSRequest) error 20 | SoundGeneration(*pb.SoundGenerationRequest) error 21 | TokenizeString(*pb.PredictOptions) (pb.TokenizationResponse, error) 22 | Status() (pb.StatusResponse, error) 23 | 24 | StoresSet(*pb.StoresSetOptions) error 25 | StoresDelete(*pb.StoresDeleteOptions) error 26 | StoresGet(*pb.StoresGetOptions) (pb.StoresGetResult, error) 27 | StoresFind(*pb.StoresFindOptions) (pb.StoresFindResult, error) 28 | 29 | VAD(*pb.VADRequest) (pb.VADResponse, error) 30 | } 31 | 32 | func newReply(s string) *pb.Reply { 33 | return &pb.Reply{Message: []byte(s)} 34 | } 35 | -------------------------------------------------------------------------------- /pkg/langchain/huggingface.go: -------------------------------------------------------------------------------- 1 | package langchain 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/tmc/langchaingo/llms" 8 | "github.com/tmc/langchaingo/llms/huggingface" 9 | ) 10 | 11 | type HuggingFace struct { 12 | modelPath string 13 | token string 14 | } 15 | 16 | func NewHuggingFace(repoId, token string) (*HuggingFace, error) { 17 | if token == "" { 18 | return nil, fmt.Errorf("no huggingface token provided") 19 | } 20 | return &HuggingFace{ 21 | modelPath: repoId, 22 | token: token, 23 | }, nil 24 | } 25 | 26 | func (s *HuggingFace) PredictHuggingFace(text string, opts ...PredictOption) (*Predict, error) { 27 | po := NewPredictOptions(opts...) 28 | 29 | // Init client 30 | llm, err := huggingface.New(huggingface.WithToken(s.token)) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | // Convert from LocalAI to LangChainGo format of options 36 | co := []llms.CallOption{ 37 | llms.WithModel(po.Model), 38 | llms.WithMaxTokens(po.MaxTokens), 39 | llms.WithTemperature(po.Temperature), 40 | llms.WithStopWords(po.StopWords), 41 | } 42 | 43 | // Call Inference API 44 | ctx := context.Background() 45 | completion, err := llm.Call(ctx, text, co...) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | return &Predict{ 51 | Completion: completion, 52 | }, nil 53 | } 54 | -------------------------------------------------------------------------------- /pkg/model/filters.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | process "github.com/mudler/go-processmanager" 5 | ) 6 | 7 | type GRPCProcessFilter = func(id string, p *process.Process) bool 8 | 9 | func all(_ string, _ *process.Process) bool { 10 | return true 11 | } 12 | 13 | func allExcept(s string) GRPCProcessFilter { 14 | return func(id string, p *process.Process) bool { 15 | return id != s 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /pkg/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "sync" 5 | 6 | grpc "github.com/mudler/LocalAI/pkg/grpc" 7 | process "github.com/mudler/go-processmanager" 8 | ) 9 | 10 | type Model struct { 11 | ID string `json:"id"` 12 | address string 13 | client grpc.Backend 14 | process *process.Process 15 | sync.Mutex 16 | } 17 | 18 | func NewModel(ID, address string, process *process.Process) *Model { 19 | return &Model{ 20 | ID: ID, 21 | address: address, 22 | process: process, 23 | } 24 | } 25 | 26 | func (m *Model) Process() *process.Process { 27 | return m.process 28 | } 29 | 30 | func (m *Model) GRPC(parallel bool, wd *WatchDog) grpc.Backend { 31 | if m.client != nil { 32 | return m.client 33 | } 34 | 35 | enableWD := false 36 | if wd != nil { 37 | enableWD = true 38 | } 39 | 40 | m.Lock() 41 | defer m.Unlock() 42 | m.client = grpc.NewClient(m.address, parallel, wd, enableWD) 43 | return m.client 44 | } 45 | -------------------------------------------------------------------------------- /pkg/model/model_suite_test.go: -------------------------------------------------------------------------------- 1 | package model_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestModel(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "LocalAI model test") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/oci/blob_test.go: -------------------------------------------------------------------------------- 1 | package oci_test 2 | 3 | import ( 4 | "os" 5 | 6 | . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | var _ = Describe("OCI", func() { 12 | Context("pulling images", func() { 13 | It("should fetch blobs correctly", func() { 14 | f, err := os.CreateTemp("", "ollama") 15 | Expect(err).NotTo(HaveOccurred()) 16 | defer os.RemoveAll(f.Name()) 17 | err = FetchImageBlob("registry.ollama.ai/library/gemma", "sha256:c1864a5eb19305c40519da12cc543519e48a0697ecd30e15d5ac228644957d12", f.Name(), nil) 18 | Expect(err).NotTo(HaveOccurred()) 19 | }) 20 | }) 21 | }) 22 | -------------------------------------------------------------------------------- /pkg/oci/image_test.go: -------------------------------------------------------------------------------- 1 | package oci_test 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | 7 | . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var _ = Describe("OCI", func() { 13 | 14 | Context("when template is loaded successfully", func() { 15 | It("should evaluate the template correctly", func() { 16 | if runtime.GOOS == "darwin" { 17 | Skip("Skipping test on darwin") 18 | } 19 | imageName := "alpine" 20 | img, err := GetImage(imageName, "", nil, nil) 21 | Expect(err).NotTo(HaveOccurred()) 22 | 23 | size, err := GetOCIImageSize(imageName, "", nil, nil) 24 | Expect(err).NotTo(HaveOccurred()) 25 | 26 | Expect(size).ToNot(Equal(int64(0))) 27 | 28 | // Create tempdir 29 | dir, err := os.MkdirTemp("", "example") 30 | Expect(err).NotTo(HaveOccurred()) 31 | defer os.RemoveAll(dir) 32 | 33 | err = ExtractOCIImage(img, dir) 34 | Expect(err).NotTo(HaveOccurred()) 35 | }) 36 | }) 37 | }) 38 | -------------------------------------------------------------------------------- /pkg/oci/oci_suite_test.go: -------------------------------------------------------------------------------- 1 | package oci_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestOCI(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "OCI test suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/oci/ollama_test.go: -------------------------------------------------------------------------------- 1 | package oci_test 2 | 3 | import ( 4 | "os" 5 | 6 | . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | var _ = Describe("OCI", func() { 12 | Context("ollama", func() { 13 | It("pulls model files", func() { 14 | f, err := os.CreateTemp("", "ollama") 15 | Expect(err).NotTo(HaveOccurred()) 16 | defer os.RemoveAll(f.Name()) 17 | err = OllamaFetchModel("gemma:2b", f.Name(), nil) 18 | Expect(err).NotTo(HaveOccurred()) 19 | }) 20 | }) 21 | }) 22 | -------------------------------------------------------------------------------- /pkg/sound/float32.go: -------------------------------------------------------------------------------- 1 | package sound 2 | 3 | import ( 4 | "encoding/binary" 5 | "math" 6 | ) 7 | 8 | func BytesFloat32(bytes []byte) float32 { 9 | bits := binary.LittleEndian.Uint32(bytes) 10 | float := math.Float32frombits(bits) 11 | return float 12 | } 13 | -------------------------------------------------------------------------------- /pkg/startup/startup_suite_test.go: -------------------------------------------------------------------------------- 1 | package startup_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestStartup(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "LocalAI startup test") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/templates/templates_suite_test.go: -------------------------------------------------------------------------------- 1 | package templates_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestTemplates(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Templates test suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/utils/config.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func SaveConfig(filePath, fileName string, obj any) { 12 | file, err := json.MarshalIndent(obj, "", " ") 13 | if err != nil { 14 | log.Error().Err(err).Msg("failed to JSON marshal the uploadedFiles") 15 | } 16 | 17 | absolutePath := filepath.Join(filePath, fileName) 18 | err = os.WriteFile(absolutePath, file, 0600) 19 | if err != nil { 20 | log.Error().Err(err).Str("filepath", absolutePath).Msg("failed to save configuration file") 21 | } 22 | } 23 | 24 | func LoadConfig(filePath, fileName string, obj interface{}) { 25 | uploadFilePath := filepath.Join(filePath, fileName) 26 | 27 | _, err := os.Stat(uploadFilePath) 28 | if os.IsNotExist(err) { 29 | log.Debug().Msgf("No configuration file found at %s", uploadFilePath) 30 | return 31 | } 32 | 33 | file, err := os.ReadFile(uploadFilePath) 34 | if err != nil { 35 | log.Error().Err(err).Str("filepath", uploadFilePath).Msg("failed to read file") 36 | } else { 37 | err = json.Unmarshal(file, &obj) 38 | if err != nil { 39 | log.Error().Err(err).Str("filepath", uploadFilePath).Msg("failed to parse file as JSON") 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pkg/utils/hash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | ) 7 | 8 | func MD5(s string) string { 9 | return fmt.Sprintf("%x", md5.Sum([]byte(s))) 10 | } 11 | -------------------------------------------------------------------------------- /pkg/utils/json.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "regexp" 4 | 5 | var matchNewlines = regexp.MustCompile(`[\r\n]`) 6 | 7 | const doubleQuote = `"[^"\\]*(?:\\[\s\S][^"\\]*)*"` 8 | 9 | func EscapeNewLines(s string) string { 10 | return regexp.MustCompile(doubleQuote).ReplaceAllStringFunc(s, func(s string) string { 11 | return matchNewlines.ReplaceAllString(s, "\\n") 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/utils/logging.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/rs/zerolog/log" 7 | ) 8 | 9 | var lastProgress time.Time = time.Now() 10 | var startTime time.Time = time.Now() 11 | 12 | func ResetDownloadTimers() { 13 | lastProgress = time.Now() 14 | startTime = time.Now() 15 | } 16 | 17 | func DisplayDownloadFunction(fileName string, current string, total string, percentage float64) { 18 | currentTime := time.Now() 19 | 20 | if currentTime.Sub(lastProgress) >= 5*time.Second { 21 | 22 | lastProgress = currentTime 23 | 24 | // calculate ETA based on percentage and elapsed time 25 | var eta time.Duration 26 | if percentage > 0 { 27 | elapsed := currentTime.Sub(startTime) 28 | eta = time.Duration(float64(elapsed)*(100/percentage) - float64(elapsed)) 29 | } 30 | 31 | if total != "" { 32 | log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta) 33 | } else { 34 | log.Info().Msgf("Downloading: %s", current) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pkg/utils/strings.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 9 | 10 | func init() { 11 | rand.Seed(time.Now().UnixNano()) 12 | } 13 | 14 | func RandString(n int) string { 15 | b := make([]rune, n) 16 | for i := range b { 17 | b[i] = letterRunes[rand.Intn(len(letterRunes))] 18 | } 19 | return string(b) 20 | } 21 | 22 | func Unique(arr []string) []string { 23 | unique := make(map[string]bool) 24 | var result []string 25 | for _, item := range arr { 26 | if _, ok := unique[item]; !ok { 27 | unique[item] = true 28 | result = append(result, item) 29 | } 30 | } 31 | return result 32 | } 33 | -------------------------------------------------------------------------------- /pkg/utils/utils_suite_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestUtils(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Utils test suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/xsync/map_test.go: -------------------------------------------------------------------------------- 1 | package xsync_test 2 | 3 | import ( 4 | . "github.com/mudler/LocalAI/pkg/xsync" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var _ = Describe("SyncMap", func() { 11 | 12 | Context("Syncmap", func() { 13 | It("sets and gets", func() { 14 | m := NewSyncedMap[string, string]() 15 | m.Set("foo", "bar") 16 | Expect(m.Get("foo")).To(Equal("bar")) 17 | }) 18 | It("deletes", func() { 19 | m := NewSyncedMap[string, string]() 20 | m.Set("foo", "bar") 21 | m.Delete("foo") 22 | Expect(m.Get("foo")).To(Equal("")) 23 | Expect(m.Exists("foo")).To(Equal(false)) 24 | }) 25 | }) 26 | }) 27 | -------------------------------------------------------------------------------- /pkg/xsync/sync_suite_test.go: -------------------------------------------------------------------------------- 1 | package xsync_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestSync(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "LocalAI sync test") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/xsysinfo/cpu.go: -------------------------------------------------------------------------------- 1 | package xsysinfo 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/jaypipes/ghw" 7 | "github.com/klauspost/cpuid/v2" 8 | ) 9 | 10 | func CPUCapabilities() ([]string, error) { 11 | cpu, err := ghw.CPU() 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | caps := map[string]struct{}{} 17 | 18 | for _, proc := range cpu.Processors { 19 | for _, c := range proc.Capabilities { 20 | 21 | caps[c] = struct{}{} 22 | } 23 | 24 | } 25 | 26 | ret := []string{} 27 | for c := range caps { 28 | ret = append(ret, c) 29 | } 30 | 31 | // order 32 | sort.Strings(ret) 33 | return ret, nil 34 | } 35 | 36 | func HasCPUCaps(ids ...cpuid.FeatureID) bool { 37 | return cpuid.CPU.Supports(ids...) 38 | } 39 | 40 | func CPUPhysicalCores() int { 41 | if cpuid.CPU.PhysicalCores == 0 { 42 | return 1 43 | } 44 | return cpuid.CPU.PhysicalCores 45 | } 46 | -------------------------------------------------------------------------------- /pkg/xsysinfo/gpu.go: -------------------------------------------------------------------------------- 1 | package xsysinfo 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | 7 | "github.com/jaypipes/ghw" 8 | "github.com/jaypipes/ghw/pkg/gpu" 9 | ) 10 | 11 | var ( 12 | gpuCache []*gpu.GraphicsCard 13 | gpuCacheOnce sync.Once 14 | gpuCacheErr error 15 | ) 16 | 17 | func GPUs() ([]*gpu.GraphicsCard, error) { 18 | gpuCacheOnce.Do(func() { 19 | gpu, err := ghw.GPU() 20 | if err != nil { 21 | gpuCacheErr = err 22 | return 23 | } 24 | gpuCache = gpu.GraphicsCards 25 | }) 26 | 27 | return gpuCache, gpuCacheErr 28 | } 29 | 30 | func TotalAvailableVRAM() (uint64, error) { 31 | gpus, err := GPUs() 32 | if err != nil { 33 | return 0, err 34 | } 35 | 36 | var totalVRAM uint64 37 | for _, gpu := range gpus { 38 | if gpu != nil && gpu.Node != nil && gpu.Node.Memory != nil { 39 | if gpu.Node.Memory.TotalUsableBytes > 0 { 40 | totalVRAM += uint64(gpu.Node.Memory.TotalUsableBytes) 41 | } 42 | } 43 | } 44 | 45 | return totalVRAM, nil 46 | } 47 | 48 | func HasGPU(vendor string) bool { 49 | gpus, err := GPUs() 50 | if err != nil { 51 | return false 52 | } 53 | if vendor == "" { 54 | return len(gpus) > 0 55 | } 56 | for _, gpu := range gpus { 57 | if strings.Contains(gpu.String(), vendor) { 58 | return true 59 | } 60 | } 61 | return false 62 | } 63 | -------------------------------------------------------------------------------- /prompt-templates/alpaca.tmpl: -------------------------------------------------------------------------------- 1 | Below is an instruction that describes a task. Write a response that appropriately completes the request. 2 | 3 | ### Instruction: 4 | {{.Input}} 5 | 6 | ### Response: -------------------------------------------------------------------------------- /prompt-templates/getting_started.tmpl: -------------------------------------------------------------------------------- 1 | {{.Input}} 2 | ### Response: 3 | -------------------------------------------------------------------------------- /prompt-templates/ggml-gpt4all-j.tmpl: -------------------------------------------------------------------------------- 1 | The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response. 2 | ### Prompt: 3 | {{.Input}} 4 | ### Response: 5 | -------------------------------------------------------------------------------- /prompt-templates/koala.tmpl: -------------------------------------------------------------------------------- 1 | BEGINNING OF CONVERSATION: USER: {{.Input}} GPT: -------------------------------------------------------------------------------- /prompt-templates/llama2-chat-message.tmpl: -------------------------------------------------------------------------------- 1 | {{if eq .RoleName "assistant"}}{{.Content}}{{else}} 2 | [INST] 3 | {{if .SystemPrompt}}{{.SystemPrompt}}{{else if eq .RoleName "system"}}<>{{.Content}}<> 4 | 5 | {{else if .Content}}{{.Content}}{{end}} 6 | [/INST] 7 | {{end}} -------------------------------------------------------------------------------- /prompt-templates/vicuna.tmpl: -------------------------------------------------------------------------------- 1 | Below is an instruction that describes a task. Write a response that appropriately completes the request. 2 | 3 | ### Instruction: 4 | {{.Input}} 5 | 6 | ### Response: -------------------------------------------------------------------------------- /prompt-templates/wizardlm.tmpl: -------------------------------------------------------------------------------- 1 | {{.Input}} 2 | 3 | ### Response: -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": ["config:base"] 4 | } 5 | -------------------------------------------------------------------------------- /scripts/latest_hf.py: -------------------------------------------------------------------------------- 1 | ## Gets latest GGUF models from HF: 2 | ## Example: 3 | ## local-ai run hermes-2-theta-llama-3-8b 4 | ## OPENAI_BASE_URL="http://192.168.xx.xx:8080" python scripts/latest_hf.py 5 | 6 | import requests 7 | import subprocess 8 | import os 9 | import sys 10 | # get current directory where the script is 11 | current_dir = os.path.dirname(os.path.realpath(__file__)) 12 | 13 | def get_latest_model(): 14 | search_term = "GGUF" 15 | if len(sys.argv) > 2 and sys.argv[1]: 16 | search_term = sys.argv[1] 17 | url = "https://huggingface.co/api/models" 18 | params = {"sort": "lastModified", "direction": -1, "limit": 30, "search": search_term} 19 | response = requests.get(url, params=params) 20 | 21 | if response.status_code == 200: 22 | models = response.json() 23 | if models: 24 | for model in models: 25 | print(f"Model: {model['modelId']}") 26 | subprocess.run(["python", current_dir+"/model_gallery_info.py", model['modelId']]) 27 | 28 | else: 29 | print("No models found.") 30 | else: 31 | print(f"Failed to fetch models. Status code: {response.status_code}") 32 | 33 | 34 | if __name__ == "__main__": 35 | get_latest_model() 36 | -------------------------------------------------------------------------------- /scripts/prepare-libs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p backend-assets/lib 4 | 5 | OS="$(uname)" 6 | 7 | if [ "$OS" == "Darwin" ]; then 8 | LIBS="$(otool -L $1 | awk 'NR > 1 { system("echo " $1) } ' | xargs echo)" 9 | elif [ "$OS" == "Linux" ]; then 10 | LIBS="$(ldd $1 | awk 'NF == 4 { system("echo " $3) } ' | xargs echo)" 11 | else 12 | echo "Unsupported OS" 13 | exit 1 14 | fi 15 | 16 | for lib in $LIBS; do 17 | cp -f $lib backend-assets/lib 18 | done 19 | 20 | echo "===============================" 21 | echo "Copied libraries to backend-assets/lib" 22 | echo "$LIBS" 23 | echo "===============================" -------------------------------------------------------------------------------- /tests/e2e-fixtures/gpu.yaml: -------------------------------------------------------------------------------- 1 | context_size: 2048 2 | mirostat: 2 3 | mirostat_tau: 5.0 4 | mirostat_eta: 0.1 5 | f16: true 6 | threads: 1 7 | gpu_layers: 90 8 | name: gpt-4 9 | mmap: true 10 | parameters: 11 | model: ggllm-test-model.bin 12 | rope_freq_base: 10000 13 | max_tokens: 20 14 | rope_freq_scale: 1 15 | temperature: 0.2 16 | top_k: 40 17 | top_p: 0.95 18 | -------------------------------------------------------------------------------- /tests/e2e/e2e_suite_test.go: -------------------------------------------------------------------------------- 1 | package e2e_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | var ( 12 | localAIURL = os.Getenv("LOCALAI_API") 13 | ) 14 | 15 | func TestLocalAI(t *testing.T) { 16 | RegisterFailHandler(Fail) 17 | RunSpecs(t, "LocalAI E2E test suite") 18 | } 19 | -------------------------------------------------------------------------------- /tests/fixtures/gallery_simple.yaml: -------------------------------------------------------------------------------- 1 | name: "cerebras" 2 | description: | 3 | cerebras 4 | license: "Apache 2.0" 5 | 6 | config_file: | 7 | parameters: 8 | model: cerebras 9 | top_k: 80 10 | temperature: 0.2 11 | top_p: 0.7 12 | context_size: 1024 13 | stopwords: 14 | - "HUMAN:" 15 | - "GPT:" 16 | roles: 17 | user: "" 18 | system: "" 19 | template: 20 | completion: "cerebras-completion" 21 | chat: cerebras-chat 22 | 23 | files: 24 | - filename: "cerebras" 25 | sha256: "c947051ae4dba9530ca55d923a7a484acd65664c8633462c8ccd4bb7848f2c65" 26 | uri: "https://huggingface.co/concedo/cerebras-111M-ggml/resolve/main/cerebras-111m-q4_2.bin" 27 | 28 | prompt_templates: 29 | - name: "cerebras-completion" 30 | content: | 31 | Complete the prompt 32 | ### Prompt: 33 | {{.Input}} 34 | ### Response: 35 | - name: "cerebras-chat" 36 | content: | 37 | The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response. 38 | ### Prompt: 39 | {{.Input}} 40 | ### Response: -------------------------------------------------------------------------------- /tests/integration/integration_suite_test.go: -------------------------------------------------------------------------------- 1 | package integration_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | func TestLocalAI(t *testing.T) { 14 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 15 | RegisterFailHandler(Fail) 16 | RunSpecs(t, "LocalAI test suite") 17 | } 18 | -------------------------------------------------------------------------------- /tests/models_fixtures/completion.tmpl: -------------------------------------------------------------------------------- 1 | {{.Input}} -------------------------------------------------------------------------------- /tests/models_fixtures/config.yaml: -------------------------------------------------------------------------------- 1 | - name: list1 2 | parameters: 3 | model: testmodel.ggml 4 | top_p: 80 5 | top_k: 0.9 6 | temperature: 0.1 7 | context_size: 200 8 | stopwords: 9 | - "HUMAN:" 10 | - "### Response:" 11 | roles: 12 | user: "HUMAN:" 13 | system: "GPT:" 14 | template: 15 | completion: completion 16 | chat: ggml-gpt4all-j 17 | - name: list2 18 | parameters: 19 | top_p: 80 20 | top_k: 0.9 21 | temperature: 0.1 22 | model: testmodel.ggml 23 | context_size: 200 24 | stopwords: 25 | - "HUMAN:" 26 | - "### Response:" 27 | roles: 28 | user: "HUMAN:" 29 | system: "GPT:" 30 | template: 31 | completion: completion 32 | chat: ggml-gpt4all-j -------------------------------------------------------------------------------- /tests/models_fixtures/embeddings.yaml: -------------------------------------------------------------------------------- 1 | name: text-embedding-ada-002 2 | embeddings: true 3 | parameters: 4 | model: huggingface://hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF/llama-3.2-1b-instruct-q4_k_m.gguf -------------------------------------------------------------------------------- /tests/models_fixtures/ggml-gpt4all-j.tmpl: -------------------------------------------------------------------------------- 1 | The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response. 2 | ### Prompt: 3 | {{.Input}} 4 | ### Response: 5 | -------------------------------------------------------------------------------- /tests/models_fixtures/gpt4.yaml: -------------------------------------------------------------------------------- 1 | name: gpt4all 2 | parameters: 3 | model: testmodel.ggml 4 | top_p: 80 5 | top_k: 0.9 6 | temperature: 0.1 7 | context_size: 200 8 | stopwords: 9 | - "HUMAN:" 10 | - "### Response:" 11 | roles: 12 | user: "HUMAN:" 13 | system: "GPT:" 14 | template: 15 | completion: completion 16 | chat: ggml-gpt4all-j -------------------------------------------------------------------------------- /tests/models_fixtures/gpt4_2.yaml: -------------------------------------------------------------------------------- 1 | name: gpt4all-2 2 | parameters: 3 | model: testmodel.ggml 4 | top_p: 80 5 | top_k: 0.9 6 | temperature: 0.1 7 | context_size: 200 8 | stopwords: 9 | - "HUMAN:" 10 | - "### Response:" 11 | roles: 12 | user: "HUMAN:" 13 | system: "GPT:" 14 | template: 15 | completion: completion 16 | chat: ggml-gpt4all-j -------------------------------------------------------------------------------- /tests/models_fixtures/grpc.yaml: -------------------------------------------------------------------------------- 1 | name: code-search-ada-code-001 2 | backend: sentencetransformers 3 | embeddings: true 4 | parameters: 5 | model: all-MiniLM-L6-v2 -------------------------------------------------------------------------------- /tests/models_fixtures/rwkv.yaml: -------------------------------------------------------------------------------- 1 | name: rwkv_test 2 | parameters: 3 | model: huggingface://bartowski/rwkv-6-world-7b-GGUF/rwkv-6-world-7b-Q4_K_M.gguf 4 | top_k: 80 5 | temperature: 0.9 6 | max_tokens: 4098 7 | top_p: 0.8 8 | context_size: 4098 9 | 10 | roles: 11 | user: "User: " 12 | system: "System: " 13 | assistant: "Assistant: " 14 | 15 | stopwords: 16 | - 'Assistant:' 17 | - '' 18 | 19 | template: 20 | chat: | 21 | {{.Input}} 22 | Assistant: 23 | completion: | 24 | {{.Input}} -------------------------------------------------------------------------------- /tests/models_fixtures/whisper.yaml: -------------------------------------------------------------------------------- 1 | name: whisper-1 2 | backend: whisper 3 | parameters: 4 | model: whisper-en --------------------------------------------------------------------------------