├── .env.example ├── .flake8 ├── .github └── ISSUE_TEMPLATE │ ├── 01-🐛-bug-report.md │ ├── 02-🚀-feature-request.md │ ├── 03-📚-documentation.md │ └── config.yaml ├── .gitignore ├── .prettierignore ├── .prettierrc ├── CHANGELOG.md ├── README.md ├── asset ├── ellm_chatbot_vid.webp └── ellm_modelui.png ├── docs └── model │ ├── ipex_models.md │ ├── onnxruntime_cpu_models.md │ ├── onnxruntime_directml_models.md │ └── openvino_models.md ├── ellm_api_server.spec ├── pyproject.toml ├── requirements-build.txt ├── requirements-common.txt ├── requirements-cpu.txt ├── requirements-cuda.txt ├── requirements-directml.txt ├── requirements-ipex.txt ├── requirements-lint.txt ├── requirements-openvino.txt ├── requirements-webui.txt ├── scripts ├── images │ └── catdog.png └── python │ ├── get_model.py │ ├── httpx_client.py │ ├── httpx_client_stream.py │ ├── httpx_client_vision.py │ ├── httpx_client_vision_stream.py │ ├── litellm_client.py │ ├── litellm_vision_client.py │ ├── openai_client.py │ └── openai_vision_client.py ├── setup.py └── src └── embeddedllm ├── backend ├── __init__.py ├── base_engine.py ├── ipex_engine.py ├── onnxruntime_engine.py └── openvino_engine.py ├── engine.py ├── entrypoints ├── __init__.py ├── api_server.py ├── chat_server.py ├── modelui.py └── webui.py ├── inputs.py ├── multimodal_models └── openvino │ └── ov_phi3_vision.py ├── protocol.py ├── sampling_params.py ├── sequence.py ├── utils.py └── version.py /.env.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.env.example -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.flake8 -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-🐛-bug-report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.github/ISSUE_TEMPLATE/01-🐛-bug-report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02-🚀-feature-request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.github/ISSUE_TEMPLATE/02-🚀-feature-request.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/03-📚-documentation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.github/ISSUE_TEMPLATE/03-📚-documentation.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.github/ISSUE_TEMPLATE/config.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.gitignore -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.prettierignore -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/.prettierrc -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/README.md -------------------------------------------------------------------------------- /asset/ellm_chatbot_vid.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/asset/ellm_chatbot_vid.webp -------------------------------------------------------------------------------- /asset/ellm_modelui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/asset/ellm_modelui.png -------------------------------------------------------------------------------- /docs/model/ipex_models.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/docs/model/ipex_models.md -------------------------------------------------------------------------------- /docs/model/onnxruntime_cpu_models.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/docs/model/onnxruntime_cpu_models.md -------------------------------------------------------------------------------- /docs/model/onnxruntime_directml_models.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/docs/model/onnxruntime_directml_models.md -------------------------------------------------------------------------------- /docs/model/openvino_models.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/docs/model/openvino_models.md -------------------------------------------------------------------------------- /ellm_api_server.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/ellm_api_server.spec -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements-build.txt: -------------------------------------------------------------------------------- 1 | # Should be mirrored in pyproject.toml 2 | packaging 3 | setuptools>=49.4.0 4 | wheel 5 | -------------------------------------------------------------------------------- /requirements-common.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-common.txt -------------------------------------------------------------------------------- /requirements-cpu.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-cpu.txt -------------------------------------------------------------------------------- /requirements-cuda.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-cuda.txt -------------------------------------------------------------------------------- /requirements-directml.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-directml.txt -------------------------------------------------------------------------------- /requirements-ipex.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-ipex.txt -------------------------------------------------------------------------------- /requirements-lint.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-lint.txt -------------------------------------------------------------------------------- /requirements-openvino.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/requirements-openvino.txt -------------------------------------------------------------------------------- /requirements-webui.txt: -------------------------------------------------------------------------------- 1 | gradio~=4.43.0 2 | -------------------------------------------------------------------------------- /scripts/images/catdog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/images/catdog.png -------------------------------------------------------------------------------- /scripts/python/get_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/get_model.py -------------------------------------------------------------------------------- /scripts/python/httpx_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/httpx_client.py -------------------------------------------------------------------------------- /scripts/python/httpx_client_stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/httpx_client_stream.py -------------------------------------------------------------------------------- /scripts/python/httpx_client_vision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/httpx_client_vision.py -------------------------------------------------------------------------------- /scripts/python/httpx_client_vision_stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/httpx_client_vision_stream.py -------------------------------------------------------------------------------- /scripts/python/litellm_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/litellm_client.py -------------------------------------------------------------------------------- /scripts/python/litellm_vision_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/litellm_vision_client.py -------------------------------------------------------------------------------- /scripts/python/openai_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/openai_client.py -------------------------------------------------------------------------------- /scripts/python/openai_vision_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/scripts/python/openai_vision_client.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/setup.py -------------------------------------------------------------------------------- /src/embeddedllm/backend/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/embeddedllm/backend/base_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/backend/base_engine.py -------------------------------------------------------------------------------- /src/embeddedllm/backend/ipex_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/backend/ipex_engine.py -------------------------------------------------------------------------------- /src/embeddedllm/backend/onnxruntime_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/backend/onnxruntime_engine.py -------------------------------------------------------------------------------- /src/embeddedllm/backend/openvino_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/backend/openvino_engine.py -------------------------------------------------------------------------------- /src/embeddedllm/engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/engine.py -------------------------------------------------------------------------------- /src/embeddedllm/entrypoints/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/embeddedllm/entrypoints/api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/entrypoints/api_server.py -------------------------------------------------------------------------------- /src/embeddedllm/entrypoints/chat_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/entrypoints/chat_server.py -------------------------------------------------------------------------------- /src/embeddedllm/entrypoints/modelui.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/entrypoints/modelui.py -------------------------------------------------------------------------------- /src/embeddedllm/entrypoints/webui.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/entrypoints/webui.py -------------------------------------------------------------------------------- /src/embeddedllm/inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/inputs.py -------------------------------------------------------------------------------- /src/embeddedllm/multimodal_models/openvino/ov_phi3_vision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/multimodal_models/openvino/ov_phi3_vision.py -------------------------------------------------------------------------------- /src/embeddedllm/protocol.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/protocol.py -------------------------------------------------------------------------------- /src/embeddedllm/sampling_params.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/sampling_params.py -------------------------------------------------------------------------------- /src/embeddedllm/sequence.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/embeddedllm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EmbeddedLLM/embeddedllm/HEAD/src/embeddedllm/utils.py -------------------------------------------------------------------------------- /src/embeddedllm/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.3.1" 2 | --------------------------------------------------------------------------------