├── .github ├── release.yml └── workflows │ ├── codeql.yml │ ├── docker_release.yml │ ├── e2e_test.yml │ ├── gh_project.yml │ ├── lint.yml │ ├── publish_release.yml │ └── unit_test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .taplo.toml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── docker ├── Dockerfile.base ├── README.md ├── docker-bake.hcl ├── run_apiserver.py └── run_autodeploy.py ├── docs ├── README.md ├── docs │ ├── _static │ │ ├── assets │ │ │ ├── LlamaLogoBrowserTab.png │ │ │ └── LlamaSquareBlack.svg │ │ ├── css │ │ │ ├── algolia.css │ │ │ └── custom.css │ │ └── js │ │ │ ├── algolia.js │ │ │ ├── leadfeeder.js │ │ │ └── mendablesearch.js │ ├── api_reference │ │ └── llama_deploy │ │ │ ├── apiserver.md │ │ │ ├── control_plane.md │ │ │ ├── message_queues │ │ │ ├── index.md │ │ │ ├── kafka.md │ │ │ ├── rabbitmq.md │ │ │ ├── redis.md │ │ │ └── simple.md │ │ │ ├── python_sdk.md │ │ │ ├── services.md │ │ │ └── types.md │ ├── css │ │ ├── algolia.css │ │ ├── custom.css │ │ └── style.css │ ├── index.md │ ├── javascript │ │ ├── algolia.js │ │ ├── llms_example.js │ │ └── mendablesearch.js │ └── module_guides │ │ └── llama_deploy │ │ ├── 10_getting_started.md │ │ ├── 20_core_components.md │ │ ├── 30_python_sdk.md │ │ ├── 40_llamactl.md │ │ ├── 50_observability.md │ │ ├── apiserver.json │ │ └── index.md ├── mkdocs.yml ├── overrides │ ├── main.html │ └── partials │ │ ├── copyright.html │ │ └── search.html ├── pyproject.toml └── uv.lock ├── e2e_tests ├── README.md ├── __init__.py ├── apiserver │ ├── __init__.py │ ├── conftest.py │ ├── deployments │ │ ├── deployment1.yml │ │ ├── deployment2.yml │ │ ├── deployment_env_git.yml │ │ ├── deployment_env_local.yml │ │ ├── deployment_hitl.yml │ │ ├── deployment_reload1.yml │ │ ├── deployment_reload2.yml │ │ ├── deployment_streaming.yml │ │ └── src │ │ │ ├── .env │ │ │ ├── __init__.py │ │ │ ├── workflow.py │ │ │ ├── workflow_env.py │ │ │ ├── workflow_hitl.py │ │ │ └── workflow_reload.py │ ├── rc │ │ ├── deployment.yml │ │ └── src │ │ │ ├── __init__.py │ │ │ └── workflow.py │ ├── test_autodeploy.py │ ├── test_deploy.py │ ├── test_env_vars_git.py │ ├── test_env_vars_local.py │ ├── test_hitl.py │ ├── test_reload.py │ ├── test_service_entrypoint.py │ ├── test_status.py │ └── test_streaming.py ├── basic_hitl │ ├── __init__.py │ ├── conftest.py │ ├── test_run_client.py │ └── workflow.py ├── basic_session │ ├── __init__.py │ ├── conftest.py │ ├── test_run_client.py │ └── workflow.py ├── basic_streaming │ ├── __init__.py │ ├── conftest.py │ ├── test_run_client.py │ └── workflow.py ├── basic_workflow │ ├── __init__.py │ ├── conftest.py │ ├── test_run_client.py │ └── workflow.py ├── conftest.py ├── core │ ├── __init__.py │ ├── conftest.py │ ├── test_services.py │ └── workflow.py ├── message_queues │ ├── __init__.py │ ├── kafka │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── docker-compose.yml │ │ ├── test_message_queue.py │ │ └── workflow.py │ ├── rabbitmq │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── docker-compose.yml │ │ ├── test_message_queue.py │ │ └── workflow.py │ ├── redis │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── docker-compose.yml │ │ ├── test_message_queue.py │ │ └── workflow.py │ └── simple │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_message_queue.py │ │ └── test_server.py └── utils.py ├── examples ├── google_cloud_run │ ├── Dockerfile │ ├── README.md │ ├── deployment.yml │ └── src │ │ └── workflow.py ├── llamacloud │ └── google_drive │ │ ├── README.md │ │ ├── deployment.yml │ │ └── src │ │ ├── config.yml │ │ ├── setup_pipeline.py │ │ └── workflow.py ├── python_fullstack │ ├── README.md │ ├── docker-compose.yml │ ├── frontend │ │ ├── .gitignore │ │ ├── assets │ │ │ └── favicon.ico │ │ ├── dockerfile │ │ ├── frontend │ │ │ ├── __init__.py │ │ │ ├── frontend.py │ │ │ ├── session_list │ │ │ │ ├── __init__.py │ │ │ │ ├── component.py │ │ │ │ └── state.py │ │ │ ├── state.py │ │ │ └── style.py │ │ ├── requirements.txt │ │ └── rxconfig.py │ ├── llama_deploy_frontend.png │ ├── python_fullstack.yaml │ └── workflows │ │ ├── __init__.py │ │ ├── agent_workflow.py │ │ ├── data │ │ └── attention.pdf │ │ ├── dockerfile │ │ ├── rag_workflow.py │ │ └── requirements.txt ├── quick_start │ ├── README.md │ ├── quick_start.yml │ ├── src │ │ └── workflow.py │ └── ui │ │ ├── .gitignore │ │ ├── app │ │ ├── confetti │ │ │ └── page.tsx │ │ ├── favicon.ico │ │ ├── globals.css │ │ ├── layout.tsx │ │ └── page.tsx │ │ ├── eslint.config.mjs │ │ ├── next.config.ts │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── pnpm-lock.yaml │ │ ├── postcss.config.mjs │ │ ├── public │ │ ├── file.svg │ │ └── logo-dark-light.svg │ │ └── tsconfig.json ├── redis_message_queue │ ├── README.md │ ├── docker-compose.yml │ └── src │ │ ├── deployment.yml │ │ └── workflow.py └── redis_state_store │ ├── README.md │ ├── docker-compose.yml │ ├── redis_store.yml │ ├── requirements.txt │ └── src │ └── workflow.py ├── llama_deploy ├── __init__.py ├── apiserver │ ├── __init__.py │ ├── __main__.py │ ├── app.py │ ├── deployment.py │ ├── deployment_config_parser.py │ ├── routers │ │ ├── __init__.py │ │ ├── deployments.py │ │ └── status.py │ ├── server.py │ ├── settings.py │ ├── source_managers │ │ ├── __init__.py │ │ ├── base.py │ │ ├── git.py │ │ └── local.py │ ├── stats.py │ └── tracing.py ├── cli │ ├── __init__.py │ ├── __main__.py │ ├── config.py │ ├── deploy.py │ ├── init.py │ ├── internal │ │ ├── config.py │ │ └── utils.py │ ├── run.py │ ├── serve.py │ ├── sessions.py │ └── status.py ├── client │ ├── __init__.py │ ├── base.py │ ├── client.py │ └── models │ │ ├── __init__.py │ │ ├── apiserver.py │ │ ├── core.py │ │ └── model.py ├── control_plane │ ├── __init__.py │ ├── config.py │ ├── server.py │ └── utils.py ├── message_queues │ ├── __init__.py │ ├── apache_kafka.py │ ├── base.py │ ├── rabbitmq.py │ ├── redis.py │ └── simple │ │ ├── __init__.py │ │ ├── client.py │ │ ├── config.py │ │ └── server.py ├── services │ ├── __init__.py │ ├── network_service_manager.py │ └── workflow.py └── types │ ├── __init__.py │ ├── apiserver.py │ └── core.py ├── pyproject.toml ├── system_diagram.png ├── templates └── basic │ ├── src │ ├── __init__.py │ └── workflow.py │ └── ui │ ├── .gitignore │ ├── app │ ├── favicon.ico │ ├── globals.css │ ├── layout.tsx │ └── page.tsx │ ├── eslint.config.mjs │ ├── next.config.ts │ ├── package-lock.json │ ├── package.json │ ├── postcss.config.mjs │ ├── public │ ├── file.svg │ └── logo-dark-light.svg │ └── tsconfig.json ├── tests ├── __init__.py ├── apiserver │ ├── conftest.py │ ├── data │ │ ├── .env │ │ ├── env_variables.yaml │ │ ├── example.yaml │ │ ├── git_service.yaml │ │ ├── local.yaml │ │ ├── python_dependencies.yaml │ │ ├── service_ports.yaml │ │ ├── with_ui.yaml │ │ └── workflow │ │ │ ├── __init__.py │ │ │ └── workflow_test.py │ ├── routers │ │ ├── __init__.py │ │ ├── test_deployments.py │ │ └── test_status.py │ ├── source_managers │ │ ├── __init__.py │ │ ├── test_git.py │ │ └── test_local.py │ ├── test_app.py │ ├── test_config_parser.py │ ├── test_deployment.py │ ├── test_server.py │ └── test_settings.py ├── cli │ ├── __init__.py │ ├── conftest.py │ ├── data │ │ ├── config.yaml │ │ └── deployment.yaml │ ├── internal │ │ ├── __init__.py │ │ └── test_config.py │ ├── test_cli.py │ ├── test_config.py │ ├── test_deploy.py │ ├── test_init.py │ ├── test_run.py │ ├── test_serve.py │ ├── test_sessions.py │ └── test_status.py ├── client │ ├── models │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_apiserver.py │ │ ├── test_core.py │ │ └── test_model.py │ └── test_client.py ├── conftest.py ├── control_plane │ ├── __init__.py │ ├── conftest.py │ ├── test_config.py │ ├── test_server.py │ └── test_utils.py ├── message_queues │ ├── simple │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_client.py │ │ ├── test_config.py │ │ └── test_server.py │ ├── test_apache_kafka.py │ ├── test_rabbitmq.py │ └── test_redis.py └── services │ ├── __init__.py │ ├── test_network_workflow.py │ ├── test_workflow_service.py │ ├── test_workflow_service_config.py │ └── test_workflow_state.py └── uv.lock /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | categories: 3 | - title: Breaking Changes ⚠️ 4 | labels: 5 | - breaking-change 6 | - title: New Features 🎉 7 | labels: 8 | - '*' 9 | - title: Bug Fixes 🐛 10 | labels: 11 | - bug 12 | - title: Documentation 📚 13 | labels: 14 | - documentation 15 | - example 16 | -------------------------------------------------------------------------------- /.github/workflows/docker_release.yml: -------------------------------------------------------------------------------- 1 | name: Docker image release 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - "v[0-9]+.[0-9]+.[0-9]+*" 10 | 11 | env: 12 | DOCKER_REPO_NAME: llamaindex/llama-deploy 13 | 14 | jobs: 15 | build-and-push: 16 | name: Build base image 17 | runs-on: ubuntu-latest 18 | # don't run from forks 19 | if: github.repository_owner == 'run-llama' 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v3 27 | 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | 31 | - name: Login to DockerHub 32 | uses: docker/login-action@v3 33 | with: 34 | username: ${{ secrets.DOCKER_HUB_USER }} 35 | password: ${{ secrets.DOCKER_HUB_TOKEN }} 36 | 37 | - name: Docker meta 38 | id: meta 39 | uses: docker/metadata-action@v5 40 | with: 41 | images: $DOCKER_REPO_NAME 42 | 43 | - name: Build images 44 | uses: docker/bake-action@v5 45 | env: 46 | IMAGE_TAG_SUFFIX: ${{ steps.meta.outputs.version }} 47 | LLAMA_DEPLOY_VERSION: ${{ steps.meta.outputs.version }} 48 | with: 49 | workdir: docker 50 | targets: all 51 | push: true 52 | -------------------------------------------------------------------------------- /.github/workflows/e2e_test.yml: -------------------------------------------------------------------------------- 1 | name: E2E Testing 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | e2e-tests: 11 | runs-on: ubuntu-latest 12 | # E2E tests might get stuck, timeout aggressively for faster feedback 13 | timeout-minutes: 10 14 | strategy: 15 | # Let the matrix finish to see if the failure was transient 16 | fail-fast: false 17 | matrix: 18 | python-version: ["3.10", "3.11", "3.12"] 19 | test-package: 20 | [ 21 | "basic_hitl", 22 | "basic_streaming", 23 | "apiserver", 24 | "basic_session", 25 | "basic_workflow", 26 | "core", 27 | ] 28 | steps: 29 | - uses: actions/checkout@v3 30 | 31 | - name: Install uv and set the python version 32 | uses: astral-sh/setup-uv@v5 33 | with: 34 | python-version: ${{ matrix.python-version }} 35 | 36 | - name: Run All E2E Tests 37 | run: uv run -- pytest e2e_tests/${{ matrix.test-package }} -s 38 | 39 | e2e-message-queues: 40 | runs-on: ubuntu-latest 41 | # E2E tests might get stuck, timeout aggressively for faster feedback 42 | timeout-minutes: 10 43 | strategy: 44 | # Let the matrix finish to see if the failure was transient 45 | fail-fast: false 46 | matrix: 47 | test-package: ["kafka", "rabbitmq", "redis", "simple"] 48 | steps: 49 | - uses: actions/checkout@v3 50 | 51 | - name: Install uv and set the python version 52 | uses: astral-sh/setup-uv@v5 53 | 54 | - name: Run E2E Tests for message queues 55 | run: uv run -- pytest e2e_tests/message_queues/${{ matrix.test-package }} -s 56 | -------------------------------------------------------------------------------- /.github/workflows/gh_project.yml: -------------------------------------------------------------------------------- 1 | name: Add issues to GitHub project 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | 8 | jobs: 9 | add-to-project: 10 | name: Add new issues to project for triage 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/add-to-project@v1.0.2 14 | with: 15 | project-url: https://github.com/orgs/run-llama/projects/8 16 | github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} 17 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Linting 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | 15 | - name: Install uv 16 | uses: astral-sh/setup-uv@v5 17 | 18 | - name: Set up Python 19 | run: uv python install 20 | 21 | - name: Run linter 22 | shell: bash 23 | run: uv run -- pre-commit run -a 24 | -------------------------------------------------------------------------------- /.github/workflows/publish_release.yml: -------------------------------------------------------------------------------- 1 | name: Publish llama-index to PyPI / GitHub 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | tags: 7 | - "v*" 8 | 9 | jobs: 10 | build-n-publish: 11 | name: Build and publish to PyPI 12 | if: github.repository == 'run-llama/llama_deploy' 13 | runs-on: ubuntu-latest 14 | permissions: 15 | contents: write 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | - name: Install uv 21 | uses: astral-sh/setup-uv@v5 22 | 23 | - name: Build and publish 24 | env: 25 | UV_PUBLISH_TOKEN: ${{ secrets.LLAMA_AGENTS_PYPI_TOKEN }} 26 | run: | 27 | uv build 28 | uv publish 29 | 30 | - name: Create GitHub Release 31 | uses: ncipollo/release-action@v1 32 | with: 33 | artifacts: "dist/*" 34 | generateReleaseNotes: true 35 | -------------------------------------------------------------------------------- /.github/workflows/unit_test.yml: -------------------------------------------------------------------------------- 1 | name: Unit Testing 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | unit-tests: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | # You can use PyPy versions in python-version. 14 | # For example, pypy-2.7 and pypy-3.8 15 | matrix: 16 | python-version: ["3.10", "3.11", "3.12"] 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Install uv and set the python version 21 | uses: astral-sh/setup-uv@v5 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | 25 | - name: Run testing 26 | shell: bash 27 | run: uv run -- pytest --cov --cov-report=xml tests 28 | 29 | - if: matrix.python-version == '3.12' 30 | name: Report Coveralls 31 | uses: coverallsapp/github-action@v2 32 | env: 33 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # Unit test / coverage reports 6 | .coverage 7 | .coverage.* 8 | coverage.* 9 | .pytest_cache/ 10 | 11 | # Build artifacts 12 | dist/ 13 | 14 | # Project related 15 | .tool-versions 16 | 17 | # IDEs 18 | .idea 19 | .DS_Store 20 | .vscode 21 | .zed 22 | .claude 23 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | default_language_version: 3 | python: python3 4 | 5 | repos: 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v5.0.0 8 | hooks: 9 | - id: check-byte-order-marker 10 | - id: check-merge-conflict 11 | - id: check-symlinks 12 | - id: check-toml 13 | - id: check-yaml 14 | args: [--allow-multiple-documents] 15 | - id: detect-private-key 16 | - id: end-of-file-fixer 17 | - id: mixed-line-ending 18 | - id: trailing-whitespace 19 | 20 | - repo: https://github.com/charliermarsh/ruff-pre-commit 21 | rev: v0.7.3 22 | 23 | hooks: 24 | - id: ruff 25 | args: [--fix, --exit-non-zero-on-fix] 26 | - id: ruff-format 27 | 28 | - repo: https://github.com/pre-commit/mirrors-mypy 29 | rev: v1.15.0 30 | hooks: 31 | - id: mypy 32 | additional_dependencies: 33 | [ 34 | "llama_deploy[kafka,rabbitmq,redis,awssqs]", 35 | "llama_index_core", 36 | "types-Deprecated", 37 | "types-PyYAML", 38 | "types-botocore", 39 | "types-aiobotocore", 40 | "types-protobuf==4.24.0.4", 41 | "types-redis", 42 | "types-requests", 43 | "types-setuptools", 44 | "types-click", 45 | ] 46 | args: 47 | [ 48 | --disallow-untyped-defs, 49 | --ignore-missing-imports, 50 | --python-version=3.11, 51 | ] 52 | exclude: ^(examples/|e2e_tests/|tests/message_queues/test_aws.py) 53 | 54 | - repo: https://github.com/adamchainz/blacken-docs 55 | rev: 1.16.0 56 | hooks: 57 | - id: blacken-docs 58 | name: black-docs-text 59 | alias: black 60 | types_or: [rst, markdown, tex] 61 | additional_dependencies: [black==23.10.1] 62 | # Using PEP 8's line length in docs prevents excess left/right scrolling 63 | args: [--line-length=79] 64 | 65 | - repo: https://github.com/codespell-project/codespell 66 | rev: v2.3.0 67 | hooks: 68 | - id: codespell 69 | additional_dependencies: [tomli] 70 | 71 | - repo: https://github.com/pappasam/toml-sort 72 | rev: v0.23.1 73 | hooks: 74 | - id: toml-sort-fix 75 | -------------------------------------------------------------------------------- /.taplo.toml: -------------------------------------------------------------------------------- 1 | [formatting] 2 | align_comments = false 3 | reorder_keys = false 4 | # Following are to be consistent with toml-sort 5 | indent_string = " " 6 | array_trailing_comma = false 7 | compact_arrays = true 8 | compact_inline_tables = true 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/run-llama/llama_deploy/8c5430d57910cb90a0cf82fb1e0ff68f6299f692/CHANGELOG.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to llama_deploy 2 | 3 | Thank you for your interest in contributing to llama_deploy! This document provides guidelines and 4 | instructions for contributing to the project. 5 | 6 | ## Getting Started 7 | 8 | 1. Fork the repository 9 | 2. Clone your fork: `git clone https://github.com/yourusername/llama_deploy.git` 10 | 3. Set up your development environment (see below) 11 | 4. Create a feature branch: `git checkout -b feature/my-feature` 12 | 13 | ## Development Environment 14 | 15 | ### Installing uv 16 | 17 | We use `uv` for package management. If you don't have it installed: 18 | 19 | ``` 20 | curl -LsSf https://astral.sh/uv/install.sh | sh 21 | ``` 22 | 23 | Or on macOS with Homebrew: 24 | ``` 25 | brew install uv 26 | ``` 27 | 28 | For more installation options, visit the [uv documentation](https://docs.astral.sh/uv/getting-started/installation/). 29 | 30 | ### Setting up the project 31 | 32 | Run the unit tests to verify your setup: 33 | ``` 34 | uv run -- pytest tests 35 | ``` 36 | 37 | You can also run the end-to-end tests: 38 | ``` 39 | uv run -- pytest e2e_tests 40 | ``` 41 | 42 | ## Making Changes 43 | 44 | 1. Make your changes 45 | 2. Add or update tests as needed 46 | 3. Run tests to make sure everything passes 47 | 4. Update documentation if necessary 48 | 5. Commit your changes with a descriptive message 49 | 50 | ## Pull Request Process 51 | 52 | 1. Push changes to your fork 53 | 2. Submit a pull request to the main repository 54 | 3. Add a clear description of the changes 55 | 4. Address any review comments 56 | 57 | ## Code Style 58 | 59 | Follow the existing code style in the project. We use: 60 | 61 | - Black for code formatting 62 | - Ruff for linting 63 | 64 | ## Testing 65 | 66 | - Write tests for any new functionality 67 | - Ensure all tests pass before submitting a PR 68 | - Run tests with: `uv run -- pytest` 69 | 70 | ## Documentation 71 | 72 | - Update documentation for any changed functionality 73 | - Document new features thoroughly 74 | 75 | ## Questions? 76 | 77 | If you have questions about contributing, please open an issue in the repository. 78 | 79 | Thank you for contributing to llama_deploy! 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 LlamaIndex 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GIT_ROOT ?= $(shell git rev-parse --show-toplevel) 2 | 3 | help: ## Show all Makefile targets. 4 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' 5 | 6 | format: ## Run code autoformatters (black). 7 | pre-commit install 8 | git ls-files | xargs pre-commit run black --files 9 | 10 | lint: ## Run linters: pre-commit (black, ruff, codespell) and mypy 11 | pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files 12 | 13 | test: ## Run tests via pytest 14 | pytest tests 15 | -------------------------------------------------------------------------------- /docker/Dockerfile.base: -------------------------------------------------------------------------------- 1 | ARG build_image 2 | ARG dist_image 3 | 4 | FROM $build_image AS build-image 5 | 6 | ARG llama_deploy_version 7 | ARG llama_deploy_version_sha 8 | ARG llama_deploy_extras="" 9 | ARG git_clone_options 10 | 11 | RUN apt-get update && \ 12 | apt-get install -y --no-install-recommends \ 13 | build-essential \ 14 | git 15 | 16 | # Shallow clone the repo, we install from git 17 | RUN git clone ${git_clone_options} --branch=${llama_deploy_version} https://github.com/run-llama/llama_deploy.git /opt/llama_deploy 18 | WORKDIR /opt/llama_deploy 19 | RUN git checkout ${llama_deploy_version_sha} 20 | 21 | # Use a virtualenv we can copy over the next build stage 22 | RUN python3 -m venv --system-site-packages /opt/venv 23 | ENV PATH="/opt/venv/bin:$PATH" 24 | 25 | # Install llama_deploy 26 | RUN pip install --upgrade pip && \ 27 | pip install --no-cache-dir ".$llama_deploy_extras" 28 | 29 | FROM $dist_image AS final 30 | 31 | ENV PATH="/opt/venv/bin:$PATH" 32 | RUN apt-get update && \ 33 | apt-get install -y --no-install-recommends git nodejs npm && \ 34 | npm i pnpm --global 35 | 36 | ARG apiserver_port 37 | ARG rc_path 38 | ARG prometheus_enabled 39 | ARG prometheus_port 40 | 41 | EXPOSE ${apiserver_port} 42 | EXPOSE ${prometheus_port} 43 | 44 | COPY --from=build-image /opt/venv /opt/venv 45 | COPY ./run_apiserver.py /opt 46 | COPY ./run_autodeploy.py /opt 47 | 48 | # Default configuration, override with "docker run -e NAME=value" 49 | ENV LLAMA_DEPLOY_APISERVER_RC_PATH=${rc_path} 50 | ENV LLAMA_DEPLOY_APISERVER_HOST=0.0.0.0 51 | ENV LLAMA_DEPLOY_APISERVER_PORT=${apiserver_port} 52 | ENV LLAMA_DEPLOY_APISERVER_PROMETHEUS_ENABLED=${prometheus_enabled} 53 | ENV LLAMA_DEPLOY_APISERVER_PROMETHEUS_PORT=${prometheus_port} 54 | 55 | FROM final AS base 56 | 57 | ENTRYPOINT [ "python", "/opt/run_apiserver.py" ] 58 | 59 | FROM final AS autodeploy 60 | 61 | ENTRYPOINT [ "python", "/opt/run_autodeploy.py" ] 62 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker build system 2 | 3 | This folder contains the files needed to build the base `llama_deploy` images that 4 | can be used to simplify deployments by reducing boiler plate code. 5 | 6 | ## Image Development 7 | 8 | Images are built with [BuildKit](https://docs.docker.com/build/buildkit/) and we use 9 | `bake` to orchestrate the process. To build all the available images run: 10 | 11 | ```sh 12 | docker buildx bake all 13 | ``` 14 | 15 | You can override any `variable` defined in the `docker-bake.hcl` file and build custom 16 | images, for example if you want to use a branch from the llama_deploy repo instead of 17 | an official release, run: 18 | 19 | ```sh 20 | LLAMA_DEPLOY_VERSION=mybranch_or_tag docker buildx bake 21 | ``` 22 | 23 | ### Multi-Platform Builds 24 | 25 | `llama_deploy` images support multiple architectures. Depending on your operating 26 | system and Docker environment, you might not be able to build all of them locally. 27 | 28 | This is the error you might encounter: 29 | 30 | ``` 31 | multiple platforms feature is currently not supported for docker driver. Please switch to a different driver 32 | (eg. “docker buildx create --use”) 33 | ``` 34 | 35 | Make sure `containerd` image store is enabled, following the instruction in the [Docker documentation](https://docs.docker.com/build/building/multi-platform/#enable-the-containerd-image-store). 36 | 37 | If the problem persists, one solution is to override the `platform` option and 38 | limit local builds to the same architecture as your computer's. For example, on an Apple M1 you can limit the builds 39 | to ARM only by invoking `bake` like this: 40 | 41 | ```sh 42 | docker buildx bake control_plane --set "*.platform=linux/arm64" 43 | ``` 44 | -------------------------------------------------------------------------------- /docker/docker-bake.hcl: -------------------------------------------------------------------------------- 1 | variable "IMAGE_NAME" { 2 | default = "llamaindex/llama-deploy" 3 | } 4 | 5 | variable "IMAGE_TAG_SUFFIX" { 6 | default = "local" 7 | } 8 | 9 | variable "LLAMA_DEPLOY_VERSION" { 10 | default = "main" 11 | } 12 | 13 | variable "LLAMA_DEPLOY_VERSION_SHA" { 14 | default = "" 15 | } 16 | 17 | variable "GIT_CLONE_OPTIONS" { 18 | default = "--depth=1" 19 | } 20 | 21 | variable "BUILD_IMAGE" { 22 | default = "python:3.12-slim" 23 | } 24 | 25 | variable "DIST_IMAGE" { 26 | default = "python:3.12-slim" 27 | } 28 | 29 | variable "APISERVER_PORT" { 30 | default = 4501 31 | } 32 | 33 | variable "PROMETHEUS_PORT" { 34 | default = 9000 35 | } 36 | 37 | variable "PROMETHEUS_ENABLED" { 38 | default = true 39 | } 40 | 41 | variable "ENTRYPOINT_SCRIPT" { 42 | default = "run_apiserver.py" 43 | } 44 | 45 | variable "RC_PATH" { 46 | default = "/data" 47 | } 48 | 49 | target "default" { 50 | dockerfile = "Dockerfile.base" 51 | tags = ["${IMAGE_NAME}:${IMAGE_TAG_SUFFIX}"] 52 | target = "base" 53 | args = { 54 | build_image = "${BUILD_IMAGE}" 55 | dist_image = "${DIST_IMAGE}" 56 | llama_deploy_version = "${LLAMA_DEPLOY_VERSION}" 57 | llama_deploy_version_sha = "${LLAMA_DEPLOY_VERSION_SHA}" 58 | llama_deploy_extras = "[awssqs, rabbitmq, kafka, redis]" 59 | git_clone_options = "${GIT_CLONE_OPTIONS}" 60 | apiserver_port = "${APISERVER_PORT}" 61 | prometheus_port = "${PROMETHEUS_PORT}" 62 | prometheus_enabled = "${PROMETHEUS_ENABLED}" 63 | entrypoint_script = "${ENTRYPOINT_SCRIPT}" 64 | rc_path = "${RC_PATH}" 65 | } 66 | platforms = ["linux/amd64", "linux/arm64"] 67 | } 68 | 69 | target "autodeploy" { 70 | inherits = ["default"] 71 | tags = ["${IMAGE_NAME}:${IMAGE_TAG_SUFFIX}-autodeploy"] 72 | target = "autodeploy" 73 | args = { 74 | entrypoint_script = "run_autodeploy.py", 75 | apiserver_port = 8080, 76 | } 77 | } 78 | 79 | group "all" { 80 | targets = ["default", "autodeploy"] 81 | } 82 | -------------------------------------------------------------------------------- /docker/run_apiserver.py: -------------------------------------------------------------------------------- 1 | import uvicorn 2 | from prometheus_client import start_http_server 3 | 4 | from llama_deploy.apiserver.settings import settings 5 | 6 | if __name__ == "__main__": 7 | if settings.prometheus_enabled: 8 | start_http_server(settings.prometheus_port) 9 | 10 | uvicorn.run( 11 | "llama_deploy.apiserver.app:app", 12 | host=settings.host, 13 | port=settings.port, 14 | ) 15 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # LlamaDeploy Documentation 2 | 3 | This repository contains the documentation for LlamaDeploy, built using MkDocs with Material theme. 4 | 5 | ## Setup 6 | 7 | ### Prerequisites 8 | - Python 3.10 or higher 9 | - uv (for dependency management) 10 | 11 | ### Installation 12 | 13 | 1. Clone the repository 14 | 2. Install dependencies using uv: 15 | ```bash 16 | uv sync 17 | ``` 18 | 19 | ## Development 20 | 21 | To start the documentation server locally: 22 | ```bash 23 | uv run mkdocs serve 24 | ``` 25 | 26 | This will start a development server at `http://127.0.0.1:8000`. 27 | 28 | ## Building 29 | 30 | LlamaDeploy is part of LlamaIndex [documentation portal](https://docs.llamaindex.ai/) 31 | so the build is performed from the [main repository](https://github.com/run-llama/llama_index). 32 | 33 | > [!WARNING] 34 | > When a documentation change is merged here, the change won't be visible until a new 35 | > build is triggered from the LlamaIndex repository. 36 | 37 | 38 | ## Contributing 39 | 40 | Contributions are very welcome! 41 | 42 | 1. Create a new branch for your changes 43 | 2. Make your changes to the documentation 44 | 3. Test locally using `uv run mkdocs serve` 45 | 4. Submit a pull request 46 | -------------------------------------------------------------------------------- /docs/docs/_static/assets/LlamaLogoBrowserTab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/run-llama/llama_deploy/8c5430d57910cb90a0cf82fb1e0ff68f6299f692/docs/docs/_static/assets/LlamaLogoBrowserTab.png -------------------------------------------------------------------------------- /docs/docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | #my-component-root *, 2 | #headlessui-portal-root * { 3 | z-index: 1000000000000; 4 | font-size: 100%; 5 | } 6 | 7 | textarea { 8 | border: 0; 9 | padding: 0; 10 | } 11 | 12 | article p { 13 | margin-bottom: 10px !important; 14 | } 15 | -------------------------------------------------------------------------------- /docs/docs/_static/js/leadfeeder.js: -------------------------------------------------------------------------------- 1 | (function (ss, ex) { 2 | window.ldfdr = 3 | window.ldfdr || 4 | function () { 5 | (ldfdr._q = ldfdr._q || []).push([].slice.call(arguments)); 6 | }; 7 | (function (d, s) { 8 | fs = d.getElementsByTagName(s)[0]; 9 | function ce(src) { 10 | var cs = d.createElement(s); 11 | cs.src = src; 12 | cs.async = 1; 13 | fs.parentNode.insertBefore(cs, fs); 14 | } 15 | ce( 16 | "https://sc.lfeeder.com/lftracker_v1_" + 17 | ss + 18 | (ex ? "_" + ex : "") + 19 | ".js", 20 | ); 21 | })(document, "script"); 22 | })("Xbp1oaEnqwn8EdVj"); 23 | -------------------------------------------------------------------------------- /docs/docs/_static/js/mendablesearch.js: -------------------------------------------------------------------------------- 1 | document.addEventListener("DOMContentLoaded", () => { 2 | // Load the external dependencies 3 | function loadScript(src, onLoadCallback) { 4 | const script = document.createElement("script"); 5 | script.src = src; 6 | script.onload = onLoadCallback; 7 | document.head.appendChild(script); 8 | } 9 | 10 | function createRootElement() { 11 | const rootElement = document.createElement("div"); 12 | rootElement.id = "my-component-root"; 13 | document.body.appendChild(rootElement); 14 | return rootElement; 15 | } 16 | 17 | function initializeMendable() { 18 | const rootElement = createRootElement(); 19 | const { MendableFloatingButton } = Mendable; 20 | 21 | const icon = React.createElement( 22 | "p", 23 | { 24 | style: { 25 | color: "#ffffff", 26 | fontSize: "40px", 27 | width: "48px", 28 | height: "48px", 29 | margin: "0px", 30 | padding: "0px", 31 | display: "flex", 32 | alignItems: "center", 33 | justifyContent: "center", 34 | }, 35 | }, 36 | "🦙", 37 | ); 38 | 39 | const mendableFloatingButton = React.createElement(MendableFloatingButton, { 40 | style: { darkMode: false, accentColor: "#010810" }, 41 | floatingButtonStyle: { color: "#ffffff", backgroundColor: "#010810" }, 42 | anon_key: "d0fb5ab6-ae6c-49dc-8d38-5115fe8e4755", // Public ANON key, ok to be public 43 | messageSettings: { 44 | openSourcesInNewTab: false, 45 | prettySources: true, 46 | }, 47 | showSimpleSearch: true, 48 | icon: icon, 49 | }); 50 | 51 | ReactDOM.render(mendableFloatingButton, rootElement); 52 | } 53 | 54 | loadScript("https://unpkg.com/react@17/umd/react.production.min.js", () => { 55 | loadScript( 56 | "https://unpkg.com/react-dom@17/umd/react-dom.production.min.js", 57 | () => { 58 | loadScript( 59 | "https://unpkg.com/@mendable/search@0.0.150/dist/umd/mendable.min.js", 60 | initializeMendable, 61 | ); 62 | }, 63 | ); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/apiserver.md: -------------------------------------------------------------------------------- 1 | # `apiserver` 2 | 3 | ::: llama_deploy.apiserver.deployment 4 | 5 | ::: llama_deploy.apiserver.deployment_config_parser 6 | options: 7 | members: 8 | - DeploymentConfig 9 | 10 | ::: llama_deploy.apiserver.source_managers 11 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/control_plane.md: -------------------------------------------------------------------------------- 1 | # `control_plane` 2 | 3 | ::: llama_deploy.control_plane 4 | options: 5 | show_docstring_parameters: true 6 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/message_queues/index.md: -------------------------------------------------------------------------------- 1 | # message_queues 2 | 3 | ::: llama_deploy.message_queues.base 4 | options: 5 | members: 6 | - AbstractMessageQueue 7 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/message_queues/kafka.md: -------------------------------------------------------------------------------- 1 | # apache_kafka 2 | 3 | ::: llama_deploy.message_queues.apache_kafka 4 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/message_queues/rabbitmq.md: -------------------------------------------------------------------------------- 1 | # rabbitmq 2 | 3 | ::: llama_deploy.message_queues.rabbitmq 4 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/message_queues/redis.md: -------------------------------------------------------------------------------- 1 | # redis 2 | 3 | ::: llama_deploy.message_queues.redis 4 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/message_queues/simple.md: -------------------------------------------------------------------------------- 1 | # simple 2 | 3 | ::: llama_deploy.message_queues.simple 4 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/python_sdk.md: -------------------------------------------------------------------------------- 1 | # Python SDK 2 | 3 | ## Client 4 | 5 | ::: llama_deploy.client.Client 6 | options: 7 | show_bases: false 8 | 9 | 10 | ## API Server functionalities 11 | 12 | ::: llama_deploy.client.models.apiserver 13 | 14 | ## Control Plane functionalities 15 | 16 | ::: llama_deploy.client.models.core 17 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/services.md: -------------------------------------------------------------------------------- 1 | # `services` 2 | 3 | ::: llama_deploy.services 4 | -------------------------------------------------------------------------------- /docs/docs/api_reference/llama_deploy/types.md: -------------------------------------------------------------------------------- 1 | # `types` 2 | 3 | ::: llama_deploy.types 4 | -------------------------------------------------------------------------------- /docs/docs/css/custom.css: -------------------------------------------------------------------------------- 1 | #my-component-root *, 2 | #headlessui-portal-root * { 3 | z-index: 1000000000000; 4 | font-size: 100%; 5 | } 6 | 7 | textarea { 8 | border: 0; 9 | padding: 0; 10 | } 11 | 12 | article p { 13 | margin-bottom: 10px !important; 14 | } 15 | -------------------------------------------------------------------------------- /docs/docs/css/style.css: -------------------------------------------------------------------------------- 1 | .md-container .jp-Cell-outputWrapper .jp-OutputPrompt.jp-OutputArea-prompt, 2 | .md-container .jp-Cell-inputWrapper .jp-InputPrompt.jp-InputArea-prompt { 3 | display: none !important; 4 | } 5 | 6 | /* CSS styles for side-by-side layout */ 7 | .container { 8 | display: flex-col; 9 | justify-content: space-between; 10 | margin-bottom: 20px; /* Adjust spacing between sections */ 11 | position: sticky; 12 | top: 2.4rem; 13 | z-index: 1000; /* Ensure it's above other content */ 14 | background-color: white; /* Match your page background */ 15 | padding: 0.2rem; 16 | } 17 | 18 | .example-heading { 19 | margin: 0.2rem !important; 20 | } 21 | 22 | .usage-examples { 23 | width: 100%; /* Adjust the width as needed */ 24 | border: 1px solid var(--md-default-fg-color--light); 25 | border-radius: 2px; 26 | padding: 0.2rem; 27 | } 28 | 29 | /* Additional styling for the toggle */ 30 | .toggle-example { 31 | cursor: pointer; 32 | color: white; 33 | text-decoration: underline; 34 | background-color: var(--md-primary-fg-color); 35 | padding: 0.2rem; 36 | border-radius: 2px; 37 | } 38 | 39 | .hidden { 40 | display: none; 41 | } 42 | 43 | /* mendable search styling */ 44 | #my-component-root > div { 45 | bottom: 100px; 46 | } 47 | -------------------------------------------------------------------------------- /docs/docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/run-llama/llama_deploy/8c5430d57910cb90a0cf82fb1e0ff68f6299f692/docs/docs/index.md -------------------------------------------------------------------------------- /docs/docs/javascript/llms_example.js: -------------------------------------------------------------------------------- 1 | var exampleTemplate = `

Framework Usage

2 | `; 6 | 7 | var exampleMarkdown = `\`\`\`python 8 | from llama_index.core import Settings 9 | 10 | # global default 11 | Settings.llm = llm 12 | 13 | # per-component 14 | # objects that use an LLM accept it as a kwarg 15 | index.as_query_engine(llm=llm) 16 | 17 | index.as_chat_engine(llm=llm) 18 | \`\`\``; 19 | 20 | function addToggleToExample() { 21 | const toggleExample = document.querySelector(".toggle-example"); 22 | const usageExamples = document.querySelector(".usage-examples"); 23 | 24 | toggleExample.addEventListener("click", function () { 25 | console.log("clicked!"); 26 | console.log(usageExamples); 27 | usageExamples.classList.toggle("hidden"); 28 | }); 29 | } 30 | 31 | // Add marked package as