├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── config.yml │ └── feature_request.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── ci.yaml │ ├── code-check.yaml │ ├── dependency-manager.yml │ ├── docs.yaml │ └── publish.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── README.pypi.md ├── assets ├── icon.png ├── logo-dark-mode.png └── logo-light-mode.png ├── docs ├── CNAME ├── assets │ ├── favicon.png │ ├── icon.svg │ ├── logo-dark-mode.png │ └── logo-light-mode.png ├── getting-started.md ├── index.md ├── learn │ ├── adapters │ │ ├── index.md │ │ ├── langchain │ │ │ ├── callbacks.md │ │ │ ├── dependency.md │ │ │ ├── fastapi.md │ │ │ ├── index.md │ │ │ └── router.md │ │ └── openai │ │ │ ├── dependency.md │ │ │ ├── fastapi.md │ │ │ ├── index.md │ │ │ └── router.md │ ├── index.md │ ├── streaming.md │ └── websockets.md ├── reference │ ├── adapters │ │ ├── langchain.md │ │ └── openai.md │ ├── index.md │ ├── lanarky.md │ ├── misc.md │ ├── streaming.md │ └── websockets.md └── stylesheets │ └── extra.css ├── examples ├── chatgpt-clone │ ├── README.md │ └── app.py └── paulGPT │ ├── README.md │ ├── app.py │ └── db │ ├── index.faiss │ └── index.pkl ├── lanarky ├── __init__.py ├── adapters │ ├── __init__.py │ ├── langchain │ │ ├── __init__.py │ │ ├── callbacks.py │ │ ├── dependencies.py │ │ ├── responses.py │ │ ├── routing.py │ │ └── utils.py │ └── openai │ │ ├── __init__.py │ │ ├── dependencies.py │ │ ├── resources.py │ │ ├── responses.py │ │ ├── routing.py │ │ └── utils.py ├── applications.py ├── clients.py ├── events.py ├── logging.py ├── py.typed ├── responses.py ├── utils.py └── websockets.py ├── mkdocs.yml ├── poetry.lock ├── pyproject.toml └── tests ├── adapters ├── langchain │ ├── test_langchain_callbacks.py │ ├── test_langchain_dependencies.py │ ├── test_langchain_responses.py │ └── test_langchain_routing.py └── openai │ ├── test_openai_dependencies.py │ ├── test_openai_resources.py │ ├── test_openai_responses.py │ └── test_openai_routing.py ├── conftest.py ├── test_applications.py ├── test_logging.py ├── test_responses.py └── test_websockets.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | lanarky/clients.py 4 | lanarky/adapters/*/__init__.py 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug report 2 | description: Create a report to help us improve 3 | title: "bug: " 4 | labels: ["bug"] 5 | body: 6 | - type: textarea 7 | id: scenario 8 | attributes: 9 | label: Scenario 10 | description: Describe the scenario in which you encountered the bug 11 | validations: 12 | required: true 13 | - type: textarea 14 | id: actual 15 | attributes: 16 | label: Actual result 17 | description: Describe what actually happened 18 | validations: 19 | required: true 20 | - type: textarea 21 | id: expected 22 | attributes: 23 | label: Expected result 24 | description: Describe what you expected to happen 25 | validations: 26 | required: true 27 | - type: textarea 28 | id: criteria 29 | attributes: 30 | label: Acceptance criteria 31 | description: Describe the acceptance criteria(s) for this bug 32 | value: | 33 | - [ ] 34 | validations: 35 | required: false 36 | - type: checkboxes 37 | id: contribute 38 | attributes: 39 | label: Contribute 40 | description: Would you like to fix this bug? 41 | options: 42 | - label: Yes, I can fix this bug and open a PR 43 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Ask a question 4 | url: https://github.com/ajndkr/lanarky/discussions/new 5 | about: Ask a question or request support for using Lanarky 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Suggest a new idea for this project 3 | title: "feat: " 4 | labels: ["feature"] 5 | body: 6 | - type: textarea 7 | id: description 8 | attributes: 9 | label: Description 10 | description: Describe the feature you are requesting 11 | placeholder: "As **[the actor]**, I want **[the something]** so I can **[the goal]**" 12 | validations: 13 | required: true 14 | - type: textarea 15 | id: criteria 16 | attributes: 17 | label: Acceptance criteria 18 | description: Describe the acceptance criteria(s) for this bug. 19 | value: | 20 | - [ ] 21 | validations: 22 | required: false 23 | - type: checkboxes 24 | id: contribute 25 | attributes: 26 | label: Contribute 27 | description: Would you like to contribute to this feature? 28 | options: 29 | - label: Yes, I can implement this feature and open a PR 30 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 10 | 11 | 12 | 13 | Fixes # (issue) 14 | 15 | ### Changelog: 16 | 17 | 19 | 20 | - 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: # Allow running on-demand 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - .github/workflows/ci.yaml 10 | - poetry.lock 11 | - pyproject.toml 12 | - lanarky/** 13 | - tests/** 14 | - "!**.md" 15 | pull_request: 16 | types: [opened, synchronize, reopened, ready_for_review] 17 | paths: 18 | - .github/workflows/ci.yaml 19 | - poetry.lock 20 | - pyproject.toml 21 | - lanarky/** 22 | - tests/** 23 | - "!**.md" 24 | 25 | env: 26 | PYTHON_VERSION: 3.9 27 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 29 | 30 | jobs: 31 | build-and-test: 32 | if: github.event.pull_request.draft == false 33 | runs-on: ubuntu-latest 34 | steps: 35 | - name: Code checkout 36 | uses: actions/checkout@v3 37 | 38 | - name: Setup python 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: ${{ env.PYTHON_VERSION }} 42 | 43 | - name: Install dependencies 44 | run: | 45 | python -m pip install --upgrade pip poetry 46 | poetry install --all-extras 47 | 48 | - name: Run unit tests 49 | run: make tests 50 | 51 | - name: Upload coverage 52 | run: make coverage 53 | -------------------------------------------------------------------------------- /.github/workflows/code-check.yaml: -------------------------------------------------------------------------------- 1 | name: Code check 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | types: [opened, synchronize, reopened, ready_for_review] 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 12 | cancel-in-progress: true 13 | 14 | env: 15 | PYTHON_VERSION: 3.9 16 | 17 | jobs: 18 | code-check: 19 | if: github.event.pull_request.draft == false 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Code checkout 23 | uses: actions/checkout@v3 24 | 25 | - name: Setup python 26 | uses: actions/setup-python@v4 27 | with: 28 | python-version: ${{ env.PYTHON_VERSION }} 29 | 30 | - uses: actions/cache@v3 31 | with: 32 | path: ~/.cache/pre-commit 33 | key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} 34 | 35 | - name: Download pre-commit 36 | run: pip install pre-commit 37 | 38 | - name: Run pre-commit hooks 39 | run: pre-commit run --all --show-diff-on-failure 40 | -------------------------------------------------------------------------------- /.github/workflows/dependency-manager.yml: -------------------------------------------------------------------------------- 1 | name: Dependency Manager 2 | 3 | on: 4 | workflow_dispatch: # Allow running on-demand 5 | schedule: 6 | # Every first day of the month at 1PM UTC 7 | - cron: 0 13 1 * * 8 | 9 | env: 10 | PYTHON_VERSION: 3.9 11 | 12 | jobs: 13 | upgrade: 14 | name: Weekly dependency upgrade 15 | runs-on: ubuntu-latest 16 | env: 17 | BRANCH_NAME: deps/weekly-upgrade 18 | steps: 19 | - name: Code checkout 20 | uses: actions/checkout@v4 21 | 22 | - name: Setup python 23 | uses: actions/setup-python@v4 24 | with: 25 | python-version: ${{ env.PYTHON_VERSION }} 26 | 27 | - name: Upgrade base dependencies 28 | run: | 29 | pip install -U poetry 30 | poetry update 31 | 32 | - name: Upgrade pre-commit hooks 33 | run: | 34 | pip install pre-commit 35 | pre-commit autoupdate 36 | 37 | - name: Detect changes 38 | id: changes 39 | run: echo "count=$(git status --porcelain=v1 2>/dev/null | wc -l)" >> $GITHUB_OUTPUT 40 | 41 | - name: Commit & push changes 42 | if: steps.changes.outputs.count > 0 43 | run: | 44 | git config user.name "github-actions" 45 | git config user.email "github-actions@github.com" 46 | git add . 47 | git commit -m "build(deps): upgrade dependencies" 48 | git push -f origin ${{ github.ref_name }}:$BRANCH_NAME 49 | 50 | - name: Open pull request 51 | if: steps.changes.outputs.count > 0 52 | env: 53 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | run: | 55 | PR=$(gh pr list --head $BRANCH_NAME --json number -q '.[0].number') 56 | if [ -z $PR ]; then 57 | gh pr create \ 58 | --draft \ 59 | --head $BRANCH_NAME \ 60 | --title "build(deps): upgrade dependencies" \ 61 | --body "Full log: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ 62 | --label "dependencies" 63 | else 64 | echo "Pull request already exists, won't create a new one." 65 | fi 66 | -------------------------------------------------------------------------------- /.github/workflows/docs.yaml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | 11 | env: 12 | PYTHON_VERSION: 3.9 13 | 14 | jobs: 15 | publish: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | 21 | - name: Configure Git Credentials 22 | run: | 23 | git config user.name github-actions[bot] 24 | git config user.email 41898282+github-actions[bot]@users.noreply.github.com 25 | 26 | - name: Setup Python 27 | uses: actions/setup-python@v4 28 | with: 29 | python-version: ${{ env.PYTHON_VERSION }} 30 | 31 | - name: Update Environment Variables 32 | run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 33 | 34 | - name: Setup Cache 35 | uses: actions/cache@v3 36 | with: 37 | key: mkdocs-material-${{ env.cache_id }} 38 | path: .cache 39 | restore-keys: | 40 | mkdocs-material- 41 | 42 | - name: Deploy GitHub Pages 43 | run: | 44 | pip install 'mkdocs-material[imaging]' mdx-include termynal 'mkdocstrings[python]' 45 | mkdocs gh-deploy --force 46 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | workflow_dispatch: # Allow running on-demand 5 | push: 6 | tags: 7 | - "*" 8 | 9 | concurrency: 10 | group: ${{ github.workflow }}-${{ github.ref }} 11 | cancel-in-progress: true 12 | 13 | env: 14 | PYTHON_VERSION: 3.9 15 | 16 | jobs: 17 | build-and-publish: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Code checkout 21 | uses: actions/checkout@v3 22 | 23 | - name: Setup python 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ env.PYTHON_VERSION }} 27 | 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip poetry 31 | poetry install 32 | 33 | - name: Set Poetry config 34 | run: poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} 35 | 36 | - name: Publish package 37 | run: poetry publish --build 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | untracked/ 2 | 3 | # Created by https://www.toptal.com/developers/gitignore/api/macos,python,visualstudiocode 4 | # Edit at https://www.toptal.com/developers/gitignore?templates=macos,python,visualstudiocode 5 | 6 | ### macOS ### 7 | # General 8 | .DS_Store 9 | .AppleDouble 10 | .LSOverride 11 | 12 | # Icon must end with two \r 13 | Icon 14 | 15 | 16 | # Thumbnails 17 | ._* 18 | 19 | # Files that might appear in the root of a volume 20 | .DocumentRevisions-V100 21 | .fseventsd 22 | .Spotlight-V100 23 | .TemporaryItems 24 | .Trashes 25 | .VolumeIcon.icns 26 | .com.apple.timemachine.donotpresent 27 | 28 | # Directories potentially created on remote AFP share 29 | .AppleDB 30 | .AppleDesktop 31 | Network Trash Folder 32 | Temporary Items 33 | .apdisk 34 | 35 | ### macOS Patch ### 36 | # iCloud generated files 37 | *.icloud 38 | 39 | ### Python ### 40 | # Byte-compiled / optimized / DLL files 41 | __pycache__/ 42 | *.py[cod] 43 | *$py.class 44 | 45 | # C extensions 46 | *.so 47 | 48 | # Distribution / packaging 49 | .Python 50 | build/ 51 | develop-eggs/ 52 | dist/ 53 | downloads/ 54 | eggs/ 55 | .eggs/ 56 | lib/ 57 | lib64/ 58 | parts/ 59 | sdist/ 60 | var/ 61 | wheels/ 62 | share/python-wheels/ 63 | *.egg-info/ 64 | .installed.cfg 65 | *.egg 66 | MANIFEST 67 | 68 | # PyInstaller 69 | # Usually these files are written by a python script from a template 70 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 71 | *.manifest 72 | *.spec 73 | 74 | # Installer logs 75 | pip-log.txt 76 | pip-delete-this-directory.txt 77 | 78 | # Unit test / coverage reports 79 | htmlcov/ 80 | .tox/ 81 | .nox/ 82 | .coverage 83 | .coverage.* 84 | .cache 85 | nosetests.xml 86 | coverage.xml 87 | *.cover 88 | *.py,cover 89 | .hypothesis/ 90 | .pytest_cache/ 91 | cover/ 92 | 93 | # Translations 94 | *.mo 95 | *.pot 96 | 97 | # Django stuff: 98 | *.log 99 | local_settings.py 100 | db.sqlite3 101 | db.sqlite3-journal 102 | 103 | # Flask stuff: 104 | instance/ 105 | .webassets-cache 106 | 107 | # Scrapy stuff: 108 | .scrapy 109 | 110 | # Sphinx documentation 111 | docs/_build/ 112 | 113 | # PyBuilder 114 | .pybuilder/ 115 | target/ 116 | 117 | # Jupyter Notebook 118 | .ipynb_checkpoints 119 | 120 | # IPython 121 | profile_default/ 122 | ipython_config.py 123 | 124 | # pyenv 125 | # For a library or package, you might want to ignore these files since the code is 126 | # intended to run in multiple environments; otherwise, check them in: 127 | # .python-version 128 | 129 | # pipenv 130 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 131 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 132 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 133 | # install all needed dependencies. 134 | #Pipfile.lock 135 | 136 | # poetry 137 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 138 | # This is especially recommended for binary packages to ensure reproducibility, and is more 139 | # commonly ignored for libraries. 140 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 141 | #poetry.lock 142 | 143 | # pdm 144 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 145 | #pdm.lock 146 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 147 | # in version control. 148 | # https://pdm.fming.dev/#use-with-ide 149 | .pdm.toml 150 | 151 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 152 | __pypackages__/ 153 | 154 | # Celery stuff 155 | celerybeat-schedule 156 | celerybeat.pid 157 | 158 | # SageMath parsed files 159 | *.sage.py 160 | 161 | # Environments 162 | .env 163 | .venv 164 | env/ 165 | venv/ 166 | ENV/ 167 | env.bak/ 168 | venv.bak/ 169 | 170 | # Spyder project settings 171 | .spyderproject 172 | .spyproject 173 | 174 | # Rope project settings 175 | .ropeproject 176 | 177 | # mkdocs documentation 178 | /site 179 | 180 | # mypy 181 | .mypy_cache/ 182 | .dmypy.json 183 | dmypy.json 184 | 185 | # Pyre type checker 186 | .pyre/ 187 | 188 | # pytype static type analyzer 189 | .pytype/ 190 | 191 | # Cython debug symbols 192 | cython_debug/ 193 | 194 | # PyCharm 195 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 196 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 197 | # and can be added to the global gitignore or merged into this file. For a more nuclear 198 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 199 | #.idea/ 200 | 201 | ### Python Patch ### 202 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 203 | poetry.toml 204 | 205 | # ruff 206 | .ruff_cache/ 207 | 208 | # LSP config files 209 | pyrightconfig.json 210 | 211 | ### VisualStudioCode ### 212 | .vscode/* 213 | !.vscode/settings.json 214 | !.vscode/tasks.json 215 | !.vscode/launch.json 216 | !.vscode/extensions.json 217 | !.vscode/*.code-snippets 218 | 219 | # Local History for Visual Studio Code 220 | .history/ 221 | 222 | # Built Visual Studio Code Extensions 223 | *.vsix 224 | 225 | ### VisualStudioCode Patch ### 226 | # Ignore all local history of files 227 | .history 228 | .ionide 229 | 230 | # End of https://www.toptal.com/developers/gitignore/api/macos,python,visualstudiocode 231 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.9 3 | 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v4.6.0 7 | hooks: 8 | - id: check-ast 9 | - id: check-builtin-literals 10 | - id: check-docstring-first 11 | - id: check-json 12 | - id: check-merge-conflict 13 | - id: check-toml 14 | - id: check-xml 15 | - id: check-yaml 16 | args: [--allow-multiple-documents] 17 | - id: debug-statements 18 | - id: detect-private-key 19 | - id: end-of-file-fixer 20 | exclude: \.(txt|json)$ 21 | - id: fix-byte-order-marker 22 | - id: fix-encoding-pragma 23 | args: [--remove] 24 | - id: mixed-line-ending 25 | - id: trailing-whitespace 26 | args: [--markdown-linebreak-ext=md] 27 | 28 | - repo: https://github.com/psf/black 29 | rev: 24.4.2 30 | hooks: 31 | - id: black 32 | 33 | - repo: https://github.com/charliermarsh/ruff-pre-commit 34 | rev: v0.4.3 35 | hooks: 36 | - id: ruff 37 | args: [--fix, --exit-non-zero-on-fix] 38 | 39 | - repo: https://github.com/PyCQA/isort 40 | rev: 5.13.2 41 | hooks: 42 | - id: isort 43 | args: [--profile, black] 44 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | This document contains information about contributing to this repository. Please read it before contributing. 2 | 3 | ## Setup Instructions 4 | 5 | `lanarky` is built with Python 3.9 and managed by Poetry. 6 | Clone this repository and follow the steps below to get started. 7 | 8 | ### Create conda environment: 9 | 10 | ```bash 11 | conda create -n lanarky python=3.9 -y 12 | conda activate lanarky 13 | ``` 14 | 15 | You can choose any other environment manager of your choice. 16 | 17 | ### Install Poetry: 18 | 19 | ```bash 20 | pip install poetry 21 | ``` 22 | 23 | ### Install Lanarky: 24 | 25 | ```bash 26 | poetry install --all-extras 27 | ``` 28 | 29 | ## Pre-Commit 30 | 31 | `lanarky` uses `pre-commit` to run code checks and tests before every commit. 32 | 33 | To install the pre-commit hooks, run the following commands: 34 | 35 | ```bash 36 | poetry run pre-commit install 37 | ``` 38 | 39 | To run the pre-commit hooks on all files, run the following command: 40 | 41 | ```bash 42 | make pre-commit 43 | ``` 44 | 45 | ## Bump Version 46 | 47 | Lanarky uses Makefile to bump versions: 48 | 49 | ```bash 50 | make bump 51 | ``` 52 | 53 | Note: The make recipe bumps version and auto-commits the changes. 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Ajinkya Indulkar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help tests coverage pre-commit 2 | help: ## shows this help message 3 | @echo "Usage:\n\tmake " 4 | @echo "\nAvailable targets:" 5 | @awk 'BEGIN {FS = ":.*##"; } /^[$$()% a-zA-Z_-]+:.*?##/ \ 6 | { printf "\t\033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ \ 7 | { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 8 | 9 | tests: ## run unit tests with coverage 10 | poetry run pytest \ 11 | --cov=lanarky --cov-report=term-missing:skip-covered \ 12 | -p pytest_asyncio -v 13 | find . -type d -name '__pycache__' -exec rm -r {} + 14 | 15 | coverage: ## run unit tests with coverage 16 | poetry run coveralls 17 | 18 | pre-commit: ## run pre-commit hooks 19 | poetry run pre-commit run --all-files 20 | 21 | bump: ## bump version 22 | @read -p "Enter version bump (patch|minor|major): " arg; \ 23 | if [ "$$arg" != "patch" ] && [ "$$arg" != "minor" ] && [ "$$arg" != "major" ]; then \ 24 | echo "Usage: make bump (patch|minor|major)"; \ 25 | exit 1; \ 26 | fi; \ 27 | current_version=$$(poetry version -s); \ 28 | poetry version $$arg; \ 29 | new_version=$$(poetry version -s); \ 30 | git add pyproject.toml; \ 31 | git commit -m "bump(ver): $$current_version → $$new_version"; 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | lanarky-logo-light-mode 4 | lanarky-logo-dark-mode 5 | 6 |

The web framework for building LLM microservices.

7 | 8 | [![Stars](https://img.shields.io/github/stars/ajndkr/lanarky)](https://github.com/ajndkr/lanarky/stargazers) 9 | [![License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/ajndkr/lanarky/blob/main/LICENSE) 10 | [![Twitter](https://img.shields.io/twitter/follow/LanarkyAPI?style=social)](https://twitter.com/intent/follow?screen_name=LanarkyAPI) 11 | 12 | [![Python](https://img.shields.io/pypi/pyversions/lanarky.svg)](https://pypi.org/project/lanarky/) 13 | [![Coverage](https://coveralls.io/repos/github/ajndkr/lanarky/badge.svg?branch=main)](https://coveralls.io/github/ajndkr/lanarky?branch=main) 14 | [![Version](https://badge.fury.io/py/lanarky.svg)](https://pypi.org/project/lanarky/) 15 | [![Stats](https://img.shields.io/pypi/dm/lanarky.svg)](https://pypistats.org/packages/lanarky) 16 | 17 |
18 | 19 | > ⚠️ **Disclaimer**: This project is now in maintenance mode. I won't be adding new features or actively maintaining the project as I have moved on to other projects and priorities. While I will address critical bugs and security issues as needed, active development has ceased from my end. I do encourage the community to continue to contribute to the project if they find it useful. Thank you for using lanarky! 20 | 21 | Lanarky is a **python (3.9+)** web framework for developers who want to build microservices using LLMs. 22 | Here are some of its key features: 23 | 24 | - **LLM-first**: Unlike other web frameworks, lanarky is built specifically for LLM developers. 25 | It's unopinionated in terms of how you build your microservices and guarantees zero vendor lock-in 26 | with any LLM tooling frameworks or cloud providers 27 | - **Fast & Modern**: Built on top of FastAPI, lanarky offers all the FastAPI features you know and love. 28 | If you are new to FastAPI, visit [fastapi.tiangolo.com](https://fastapi.tiangolo.com) to learn more 29 | - **Streaming**: Streaming is essential for many real-time LLM applications, like chatbots. Lanarky has 30 | got you covered with built-in streaming support over **HTTP** and **WebSockets**. 31 | - **Open-source**: Lanarky is open-source and free to use. Forever. 32 | 33 | To learn more about lanarky and get started, you can find the full documentation on [lanarky.ajndkr.com](https://lanarky.ajndkr.com) 34 | 35 | ## Installation 36 | 37 | The library is available on PyPI and can be installed via `pip`: 38 | 39 | ```bash 40 | pip install lanarky 41 | ``` 42 | 43 | ## Getting Started 44 | 45 | Lanarky provides a powerful abstraction layer to allow developers to build 46 | simple LLM microservices in just a few lines of code. 47 | 48 | Here's an example to build a simple microservice that uses OpenAI's `ChatCompletion` service: 49 | 50 | ```python 51 | from lanarky import Lanarky 52 | from lanarky.adapters.openai.resources import ChatCompletionResource 53 | from lanarky.adapters.openai.routing import OpenAIAPIRouter 54 | 55 | app = Lanarky() 56 | router = OpenAIAPIRouter() 57 | 58 | 59 | @router.post("/chat") 60 | def chat(stream: bool = True) -> ChatCompletionResource: 61 | system = "You are a sassy assistant" 62 | return ChatCompletionResource(stream=stream, system=system) 63 | 64 | 65 | app.include_router(router) 66 | ``` 67 | 68 | Visit [Getting Started](https://lanarky.ajndkr.com/getting-started) for the full tutorial on building 69 | and testing your first LLM microservice with Lanarky. 70 | 71 | ## Contributing 72 | 73 | [![Code check](https://github.com/ajndkr/lanarky/actions/workflows/code-check.yaml/badge.svg)](https://github.com/ajndkr/lanarky/actions/workflows/code-check.yaml) 74 | [![Publish](https://github.com/ajndkr/lanarky/actions/workflows/publish.yaml/badge.svg)](https://github.com/ajndkr/lanarky/actions/workflows/publish.yaml) 75 | 76 | Contributions are more than welcome! If you have an idea for a new feature or want to help improve lanarky, 77 | please create an issue or submit a pull request on [GitHub](https://github.com/ajndkr/lanarky). 78 | 79 | See [CONTRIBUTING.md](https://github.com/ajndkr/lanarky/blob/main/CONTRIBUTING.md) for more information. 80 | 81 | See [Lanarky Roadmap](https://github.com/users/ajndkr/projects/6) for the list of new features and future milestones. 82 | 83 | ## License 84 | 85 | The library is released under the [MIT License](https://github.com/ajndkr/lanarky/blob/main/LICENSE). 86 | -------------------------------------------------------------------------------- /README.pypi.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | lanarky-logo-light-mode 4 | 5 |

The web framework for building LLM microservices.

6 | 7 | [![Stars](https://img.shields.io/github/stars/ajndkr/lanarky)](https://github.com/ajndkr/lanarky/stargazers) 8 | [![License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/ajndkr/lanarky/blob/main/LICENSE) 9 | [![Twitter](https://img.shields.io/twitter/follow/LanarkyAPI?style=social)](https://twitter.com/intent/follow?screen_name=LanarkyAPI) 10 | 11 | [![Python](https://img.shields.io/pypi/pyversions/lanarky.svg)](https://pypi.org/project/lanarky/) 12 | [![Coverage](https://coveralls.io/repos/github/ajndkr/lanarky/badge.svg?branch=main)](https://coveralls.io/github/ajndkr/lanarky?branch=main) 13 | [![Version](https://badge.fury.io/py/lanarky.svg)](https://pypi.org/project/lanarky/) 14 | [![Stats](https://img.shields.io/pypi/dm/lanarky.svg)](https://pypistats.org/packages/lanarky) 15 | 16 |
17 | 18 | > ⚠️ **Disclaimer**: This project is now in maintenance mode. I won't be adding new features or actively maintaining the project as I have moved on to other projects and priorities. While I will address critical bugs and security issues as needed, active development has ceased from my end. I do encourage the community to continue to contribute to the project if they find it useful. Thank you for using lanarky! 19 | 20 | Lanarky is a **python (3.9+)** web framework for developers who want to build microservices using LLMs. 21 | Here are some of its key features: 22 | 23 | - **LLM-first**: Unlike other web frameworks, lanarky is built specifically for LLM developers. 24 | It's unopinionated in terms of how you build your microservices and guarantees zero vendor lock-in 25 | with any LLM tooling frameworks or cloud providers 26 | - **Fast & Modern**: Built on top of FastAPI, lanarky offers all the FastAPI features you know and love. 27 | If you are new to FastAPI, visit [fastapi.tiangolo.com](https://fastapi.tiangolo.com) to learn more 28 | - **Streaming**: Streaming is essential for many real-time LLM applications, like chatbots. Lanarky has 29 | got you covered with built-in streaming support over **HTTP** and **WebSockets**. 30 | - **Open-source**: Lanarky is open-source and free to use. Forever. 31 | 32 | To learn more about lanarky and get started, you can find the full documentation on [lanarky.ajndkr.com](https://lanarky.ajndkr.com) 33 | 34 | ## Installation 35 | 36 | The library is available on PyPI and can be installed via `pip`: 37 | 38 | ```bash 39 | pip install lanarky 40 | ``` 41 | 42 | ## Getting Started 43 | 44 | Lanarky provides a powerful abstraction layer to allow developers to build 45 | simple LLM microservices in just a few lines of code. 46 | 47 | Here's an example to build a simple microservice that uses OpenAI's `ChatCompletion` service: 48 | 49 | ```python 50 | from lanarky import Lanarky 51 | from lanarky.adapters.openai.resources import ChatCompletionResource 52 | from lanarky.adapters.openai.routing import OpenAIAPIRouter 53 | 54 | app = Lanarky() 55 | router = OpenAIAPIRouter() 56 | 57 | 58 | @router.post("/chat") 59 | def chat(stream: bool = True) -> ChatCompletionResource: 60 | system = "You are a sassy assistant" 61 | return ChatCompletionResource(stream=stream, system=system) 62 | 63 | 64 | app.include_router(router) 65 | ``` 66 | 67 | Visit [Getting Started](https://lanarky.ajndkr.com/getting-started) for the full tutorial on building 68 | and testing your first LLM microservice with Lanarky. 69 | 70 | ## Contributing 71 | 72 | [![Code check](https://github.com/ajndkr/lanarky/actions/workflows/code-check.yaml/badge.svg)](https://github.com/ajndkr/lanarky/actions/workflows/code-check.yaml) 73 | [![Publish](https://github.com/ajndkr/lanarky/actions/workflows/publish.yaml/badge.svg)](https://github.com/ajndkr/lanarky/actions/workflows/publish.yaml) 74 | 75 | Contributions are more than welcome! If you have an idea for a new feature or want to help improve lanarky, 76 | please create an issue or submit a pull request on [GitHub](https://github.com/ajndkr/lanarky). 77 | 78 | See [CONTRIBUTING.md](https://github.com/ajndkr/lanarky/blob/main/CONTRIBUTING.md) for more information. 79 | 80 | ## License 81 | 82 | The library is released under the [MIT License](https://github.com/ajndkr/lanarky/blob/main/LICENSE). 83 | -------------------------------------------------------------------------------- /assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/assets/icon.png -------------------------------------------------------------------------------- /assets/logo-dark-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/assets/logo-dark-mode.png -------------------------------------------------------------------------------- /assets/logo-light-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/assets/logo-light-mode.png -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | lanarky.ajndkr.com 2 | -------------------------------------------------------------------------------- /docs/assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/docs/assets/favicon.png -------------------------------------------------------------------------------- /docs/assets/icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/assets/logo-dark-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/docs/assets/logo-dark-mode.png -------------------------------------------------------------------------------- /docs/assets/logo-light-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/docs/assets/logo-light-mode.png -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - navigation 4 | - footer 5 | --- 6 | 7 | Let's build our first LLM microservice with Lanarky! 8 | 9 | ## Dependencies 10 | 11 | First, we will install Lanarky with the OpenAI adapter: 12 | 13 | 14 | 15 | ``` 16 | $ pip install lanarky[openai] 17 | ``` 18 | 19 | ## Application 20 | 21 | !!! info 22 | 23 | You need to set the `OPENAI_API_KEY` environment variable to use OpenAI. 24 | Visit [openai.com](https://openai.com) to get your API key. 25 | 26 | ```python 27 | import os 28 | 29 | from lanarky import Lanarky 30 | from lanarky.adapters.openai.resources import ChatCompletionResource 31 | from lanarky.adapters.openai.routing import OpenAIAPIRouter 32 | 33 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 34 | 35 | app = Lanarky() 36 | router = OpenAIAPIRouter() 37 | 38 | 39 | @router.post("/chat") 40 | def chat(stream: bool = True) -> ChatCompletionResource: 41 | system = "You are a sassy assistant" 42 | return ChatCompletionResource(stream=stream, system=system) 43 | 44 | 45 | app.include_router(router) 46 | ``` 47 | 48 | Run application: 49 | 50 | 51 | 52 | ``` 53 | $ pip install uvicorn 54 | $ uvicorn app:app --reload 55 | ``` 56 | 57 | !!! tip 58 | 59 | Swagger docs will be available at [http://localhost:8000/docs](http://localhost:8000/docs). 60 | 61 | ## Client 62 | 63 | Now that the application script is running, we will setup a client script for testing. 64 | 65 | Create `client.py`: 66 | 67 | ```python 68 | import click 69 | 70 | from lanarky.clients import StreamingClient 71 | 72 | 73 | @click.command() 74 | @click.option("--input", required=True) 75 | @click.option("--stream", is_flag=True) 76 | def main(input: str, stream: bool): 77 | client = StreamingClient() 78 | for event in client.stream_response( 79 | "POST", 80 | "/chat", 81 | params={"stream": str(stream).lower()}, 82 | json={"messages": [dict(role="user", content=input)]}, 83 | ): 84 | print(f"{event.event}: {event.data}") 85 | 86 | 87 | if __name__ == "__main__": 88 | main() 89 | ``` 90 | 91 | Since we have exposed only `stream` as the query parameter, we can test 2 scenarios: 92 | 93 | 1. Recieve output as it is generated: 94 | 95 | 96 | 97 | ``` 98 | $ python client.py --input "hi" --stream 99 | completion: 100 | completion: Well 101 | completion: , 102 | completion: hello 103 | completion: there 104 | completion: ! 105 | completion: How 106 | completion: can 107 | completion: I 108 | completion: sass 109 | completion: ... 110 | completion: I 111 | completion: mean 112 | completion: assist 113 | completion: you 114 | completion: today 115 | completion: ? 116 | ``` 117 | 118 | 2. Recieve all output at once: 119 | 120 | 121 | 122 | ``` 123 | $ python client.py --input "hi" 124 | completion: Oh, hello there! What can I sass...I mean assist you with today? 125 | ``` 126 | 127 | ## Next Steps 128 | 129 | Congrats on building your first LLM microservice with Lanarky! 130 | 131 | Now that you have a basic understanding of how Lanarky works, let's learn more about 132 | the core concepts of Lanarky. 133 | 134 | [Let's Learn!](./learn/index.md){ .md-button .md-button--primary } 135 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - navigation 4 | - toc 5 | - footer 6 | --- 7 | 8 | 11 | 12 |
13 | 14 | lanarky-logo-light-mode 15 | lanarky-logo-dark-mode 16 | 17 |

The web framework for building LLM microservices.

18 | 19 | 20 | License 21 | 22 | 23 | Coverage 24 | 25 | 26 | Stats 27 | 28 | 29 |
30 | 31 | > ⚠️ **Disclaimer**: This project is now in maintenance mode. I won't be adding new features or actively maintaining 32 | the project as I have moved on to other projects and priorities. While I will address critical bugs and security 33 | issues as needed, active development has ceased from my end. I do encourage the community to continue to contribute 34 | to the project if they find it useful. Thank you for using lanarky! 35 | 36 | Lanarky is a **python (3.9+)** web framework for developers who want to build microservices using LLMs. 37 | Here are some of its key features: 38 | 39 | - **LLM-first**: Unlike other web frameworks, lanarky is built specifically for LLM developers. 40 | It's unopinionated in terms of how you build your microservices and guarantees zero vendor lock-in 41 | with any LLM tooling frameworks or cloud providers 42 | - **Fast & Modern**: Built on top of FastAPI, lanarky offers all the FastAPI features you know and love. 43 | If you are new to FastAPI, visit [fastapi.tiangolo.com](https://fastapi.tiangolo.com) to learn more 44 | - **Streaming**: Streaming is essential for many real-time LLM applications, like chatbots. Lanarky has 45 | got you covered with built-in streaming support over **HTTP** and **WebSockets**. 46 | - **Open-source**: Lanarky is open-source and free to use. Forever. 47 | 48 | 49 | 50 | ``` 51 | $ pip install lanarky 52 | ``` 53 | -------------------------------------------------------------------------------- /docs/learn/adapters/index.md: -------------------------------------------------------------------------------- 1 | # Adapters 2 | 3 | The **Adapters API** allows Lanarky users to build microservices using popular LLM frameworks. 4 | 5 | We will cover the following adapters in depth: 6 | 7 | - [OpenAI](./openai/index.md): build microservices using the 8 | [OpenAI Python SDK](https://platform.openai.com/docs/api-reference?lang=python) 9 | - [LangChain](./langchain/index.md): build microservices using the 10 | [LangChain](https://www.langchain.com/) 11 | 12 | !!! note "Note from Author" 13 | 14 | The **Adapters API** is still in active development. I will add more adapters in the future. 15 | -------------------------------------------------------------------------------- /docs/learn/adapters/langchain/callbacks.md: -------------------------------------------------------------------------------- 1 | Lanarky offers a collection of callback handlers for LangChain. These callback 2 | handlers are useful in executing intermediate callback events related to your LangChain 3 | microservice. 4 | 5 | Lanarky offers callback handlers for both streaming and WebSockets. We will take a look at 6 | both of them in this guide. 7 | 8 | !!! note 9 | 10 | All callback handlers can be imported from the `lanarky.adapters.langchain.callbacks` 11 | module. 12 | 13 | ## Supported Callbacks 14 | 15 | Lanarky offers callback handlers for the following events: 16 | 17 | ### Tokens 18 | 19 | - `TokenStreamingCallbackHandler`: handles streaming of the intermediate tokens over HTTP 20 | - `TokenWebSocketCallbackHandler`: handles streaming of the intermediate tokens over WebSockets 21 | 22 | Both callback handlers offer token streaming in two modes: `text` and `json`. In `text` mode, 23 | the callback handlers will use raw token string as event data. In `json` mode, the callback 24 | handlers will use a JSON object containing the token string as event data. 25 | 26 | These callback handlers are useful for all chains where the `llm` component supports streaming. 27 | 28 | ### Source Documents 29 | 30 | - `SourceDocumentStreamingCallbackHandler`: handles streaming of the source documents 31 | over HTTP 32 | - `SourceDocumentWebSocketCallbackHandler`: handles streaming of the source documents 33 | over WebSockets 34 | 35 | The source documents are sent at the end of a chain execution as a `source_documents` event. 36 | 37 | These callback handlers are useful for retrieval-based chains like `RetrievalQA`. 38 | 39 | ### Agents 40 | 41 | - `FinalTokenStreamingCallbackHandler`: handles streaming of the final answer tokens over HTTP 42 | - `FinalTokenWebSocketCallbackHandler`: handles streaming of the final answer tokens over WebSockets 43 | 44 | Both callback handlers are extension of the token streaming callback handlers where the tokens are 45 | streamed only when the LLM agent has reached the final step of its execution. 46 | 47 | These callback handlers are useful for all agent types like `ZeroShotAgent`. 48 | 49 | !!! note 50 | 51 | The callback handlers also inherit some functionality of the `FinalStreamingStdOutCallbackHandler` 52 | callback handler. Check out [LangChain Docs](https://api.python.langchain.com/en/latest/callbacks/langchain.callbacks.streaming_stdout_final_only.FinalStreamingStdOutCallbackHandler.html) to know more. 53 | 54 | ## Create custom lanarky callback handlers 55 | 56 | You can create your own lanarky callback handler by inheriting from: 57 | 58 | - `StreamingCallbackHandler`: useful for building microservices using server-sent events 59 | - `WebSocketCallbackHandler`: useful for building microservices using WebSockets 60 | 61 | For example, let's say you want to create a callback handler for streaming a message at the start of chain: 62 | 63 | ```python 64 | from lanarky.adapters.langchain.callbacks import StreamingCallbackHandler 65 | 66 | class ChainStartStreamingCallbackHandler(StreamingCallbackHandler): 67 | async def on_chain_start(self, *args: Any, **kwargs: dict[str, Any]) -> None: 68 | """Run when chain starts running.""" 69 | message = self._construct_message( 70 | data="Chain started", event="start" 71 | ) 72 | await self.send(message) 73 | ``` 74 | 75 | When the above callback handler is passed to the input list of callbacks, it will stream the following event: 76 | 77 | ``` 78 | event: start 79 | data: Chain started 80 | ``` 81 | 82 | !!! note 83 | 84 | You can learn more about the specific callback events in the 85 | [LangChain Docs](https://python.langchain.com/docs/modules/callbacks/) 86 | -------------------------------------------------------------------------------- /docs/learn/adapters/langchain/dependency.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | --- 5 | 6 | FastAPI offers a powerful [Dependency Injection](https://fastapi.tiangolo.com/tutorial/dependencies/) 7 | system that allows you to inject dependencies into your API endpoints. Lanarky extends this functionality 8 | by offering LangChain as a dependency. 9 | 10 | !!! example "Experimental" 11 | 12 | LLM-based dependency injection is an experimental feature. We will add more functionality 13 | based on community feedback and viable use cases. If you have ideas or suggestions, we 14 | would love to hear from you. Feel free to open an issue on 15 | [GitHub](https://github.com/ajndkr/lanarky/issues/new/choose). 16 | 17 | Let's take a look at how we can use LangChain as a dependency. 18 | 19 | ```python 20 | import os 21 | 22 | from langchain.chains import LLMChain 23 | from langchain.chat_models import ChatOpenAI 24 | from langchain.prompts import ( 25 | ChatPromptTemplate, 26 | HumanMessagePromptTemplate, 27 | PromptTemplate, 28 | ) 29 | 30 | from lanarky import Lanarky 31 | from lanarky.adapters.langchain.dependencies import Depends 32 | 33 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 34 | 35 | 36 | app = Lanarky() 37 | 38 | 39 | def chain_factory(temperature: float = 0.0, verbose: bool = False) -> LLMChain: 40 | return LLMChain( 41 | llm=ChatOpenAI(temperature=temperature), 42 | prompt=ChatPromptTemplate.from_messages( 43 | [ 44 | HumanMessagePromptTemplate( 45 | prompt=PromptTemplate.from_template("Respond in JSON: {input}") 46 | ), 47 | ] 48 | ), 49 | verbose=verbose, 50 | ) 51 | 52 | 53 | @app.post("/") 54 | async def endpoint(outputs: dict = Depends(chain_factory)): 55 | return outputs["text"] 56 | ``` 57 | 58 | In the above example, we pass `chain_factory` as a dependency to the endpoint. The endpoint 59 | exposes the dependency function arguments as query parameters. This allows us to configure 60 | the dependency at runtime. 61 | 62 | To test the above endpoint, let's create a client script: 63 | 64 | ```python 65 | import click 66 | import httpx 67 | 68 | 69 | @click.command() 70 | @click.option("--input", required=True) 71 | def main(input: str): 72 | url = "http://localhost:8000/" 73 | 74 | with httpx.Client() as client: 75 | response = client.post(url, json={"input": input}) 76 | if response.status_code == 200: 77 | data = response.json() 78 | print(f"Received: {data}") 79 | else: 80 | print(response.text) 81 | 82 | 83 | if __name__ == "__main__": 84 | main() 85 | ``` 86 | 87 | First, start the server: 88 | 89 | ```bash 90 | uvicorn app:app 91 | ``` 92 | 93 | Then, run the client script: 94 | 95 | 96 | 97 | ``` 98 | $ python client.py --input "Who won the world series in 2020?" 99 | Received: { 100 | "team": "Los Angeles Dodgers", 101 | "year": 2020 102 | } 103 | ``` 104 | -------------------------------------------------------------------------------- /docs/learn/adapters/langchain/fastapi.md: -------------------------------------------------------------------------------- 1 | Lanarky is built on top of FastAPI and offers backwards compatibility with all FastAPI features. 2 | Nonetheless, if your project uses FastAPI and Lanarky is not a drop-in replacement, you can still 3 | the low-level Lanarky modules to build your microservice. 4 | 5 | We will use the examples from the [LangChain API Router](./router.md) guide to demonstrate how to 6 | use the low-level modules as well as understand how the router works under the hood. 7 | 8 | ## Streaming 9 | 10 | LangChain adapter extends the `StreamingResponse` class to support streaming for LangChain microservices. 11 | 12 | !!! note 13 | 14 | Before you start, make sure you have read the [Streaming](../../streaming.md) and 15 | [LangChain API Router](./router.md) guides. 16 | 17 | ```python 18 | import os 19 | 20 | from fastapi import Depends 21 | from langchain.chains import ConversationChain 22 | from langchain.chat_models import ChatOpenAI 23 | from pydantic import BaseModel 24 | 25 | from lanarky import Lanarky 26 | from lanarky.adapters.langchain.callbacks import TokenStreamingCallbackHandler 27 | from lanarky.adapters.langchain.responses import StreamingResponse 28 | 29 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 30 | 31 | 32 | app = Lanarky() 33 | 34 | 35 | class ChatInput(BaseModel): 36 | input: str 37 | 38 | 39 | def chain_factory( 40 | temperature: float = 0.0, verbose: bool = False, streaming: bool = True 41 | ) -> ConversationChain: 42 | return ConversationChain( 43 | llm=ChatOpenAI(temperature=temperature, streaming=streaming), 44 | verbose=verbose, 45 | ) 46 | 47 | 48 | @app.post("/chat") 49 | async def chat( 50 | request: ChatInput, 51 | chain: ConversationChain = Depends(chain_factory) 52 | ): 53 | return StreamingResponse( 54 | chain=chain, 55 | config={ 56 | "inputs": request.model_dump(), 57 | "callbacks": [ 58 | TokenStreamingCallbackHandler(output_key=chain.output_key), 59 | ], 60 | }, 61 | ) 62 | ``` 63 | 64 | The `/chat` endpoint is similar to the one we created using `LangChainAPIRouter` in the 65 | [LangChain API Router](./router.md) guide. Besides the `StreamingResponse` class, we also use 66 | the `TokenStreamingCallbackHandler` callback handler to stream the intermediate tokens back to 67 | the client. Check out [Callbacks](./callbacks.md) to learn more about the lanarky callback 68 | handlers. 69 | 70 | !!! tip 71 | 72 | You can use the same client script from the [LangChain API Router](./router.md) guide to test 73 | the above example. 74 | 75 | ## Websockets 76 | 77 | In addition to streaming, LangChain adapter also supports websockets. Let's take a look at how we can 78 | build an LangChain microservice using websockets. 79 | 80 | ```python 81 | 82 | import os 83 | 84 | from fastapi import Depends 85 | from langchain.chains import ConversationChain 86 | from langchain.chat_models import ChatOpenAI 87 | from pydantic import BaseModel 88 | 89 | from lanarky import Lanarky 90 | from lanarky.adapters.langchain.callbacks import TokenWebSocketCallbackHandler 91 | from lanarky.events import Events 92 | from lanarky.websockets import WebSocket, WebsocketSession 93 | 94 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 95 | 96 | 97 | app = Lanarky() 98 | 99 | 100 | class ChatInput(BaseModel): 101 | input: str 102 | 103 | 104 | def chain_factory() -> ConversationChain: 105 | return ConversationChain(llm=ChatOpenAI(streaming=True)) 106 | 107 | 108 | @app.websocket("/ws") 109 | async def ws( 110 | websocket: WebSocket, 111 | chain: ConversationChain = Depends(chain_factory) 112 | ): 113 | async with WebsocketSession().connect(websocket) as session: 114 | async for data in session: 115 | await chain.acall( 116 | inputs=ChatInput(**data).model_dump(), 117 | callbacks=[ 118 | TokenWebSocketCallbackHandler( 119 | websocket=websocket, output_key=chain.output_key 120 | ) 121 | ], 122 | ) 123 | await websocket.send_json(dict(data="", event=Events.END)) 124 | ``` 125 | 126 | In this example, we use the `WebsocketSession` context manager to connect to the websocket 127 | and communicate with the client. We pass the client data to the `ConversationChain` and stream 128 | the response back to the client. 129 | 130 | !!! tip 131 | 132 | Similar to the streaming example, you can use the same client script from the 133 | [LangChain API Router](./router.md) guide to test the websocket example. 134 | -------------------------------------------------------------------------------- /docs/learn/adapters/langchain/index.md: -------------------------------------------------------------------------------- 1 | # LangChain Adapter 2 | 3 | The **LangChain Adapter** allows Lanarky users to build microservices using the 4 | [LangChain](https://www.langchain.com/) framework. 5 | 6 | To enable this adapter, install lanarky with extra dependencies: 7 | 8 | 9 | 10 | ``` 11 | $ pip install lanarky[langchain] 12 | ``` 13 | 14 | !!! tip 15 | 16 | LangChain is an LLM tooling framework to construct LLM chains and agents using 17 | LLM providers such OpenAI, Anthropic, etc. Visit their [Python SDK](https://python.langchain.com/docs/) 18 | documentation for more information. 19 | 20 | Here's an overview of the supported features: 21 | 22 | - [Langchain API Router](./router.md): Lanarky router for LangChain 23 | - [Callbacks](./callbacks.md): collection of Lanarky callbacks for LangChain 24 | 25 | Additionally, we will cover some advanced topics: 26 | 27 | - [Dependency Injection](./dependency.md): use LangChain as a dependency in your microservice 28 | - [FastAPI Backport](./fastapi.md): low-level modules for FastAPI users 29 | -------------------------------------------------------------------------------- /docs/learn/adapters/langchain/router.md: -------------------------------------------------------------------------------- 1 | # LangChain API Router 2 | 3 | The `LangChainAPIRouter` class is an abstraction layer which provides a quick and easy 4 | way to build microservices using LangChain. 5 | 6 | Let's understand how to use `LangChainAPIRouter` to build streaming and websocket 7 | endpoints. 8 | 9 | ## Streaming 10 | 11 | ```python 12 | import os 13 | 14 | from langchain.chains import ConversationChain 15 | from langchain.chat_models import ChatOpenAI 16 | 17 | from lanarky import Lanarky 18 | from lanarky.adapters.langchain.routing import LangchainAPIRouter 19 | 20 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 21 | 22 | 23 | app = Lanarky() 24 | router = LangchainAPIRouter() 25 | 26 | 27 | @router.post("/chat") 28 | def chat( 29 | temperature: float = 0.0, verbose: bool = False, streaming: bool = True 30 | ) -> ConversationChain: 31 | return ConversationChain( 32 | llm=ChatOpenAI(temperature=temperature, streaming=streaming), 33 | verbose=verbose, 34 | ) 35 | 36 | 37 | app.include_router(router) 38 | ``` 39 | 40 | In this example, we use `chat` as a `ConversationChain` factory function and send it 41 | to the router to build a streaming endpoint. The additional parameters such as 42 | `temperature`, `verbose`, and `streaming` are exposed as query parameters. 43 | 44 | To receive the events, we will use the following client script: 45 | 46 | ```python 47 | import json 48 | 49 | import click 50 | 51 | from lanarky.clients import StreamingClient 52 | 53 | 54 | @click.command() 55 | @click.option("--input", required=True) 56 | @click.option("--stream", is_flag=True) 57 | def main(input: str, stream: bool): 58 | client = StreamingClient() 59 | for event in client.stream_response( 60 | "POST", 61 | "/chat", 62 | params={"streaming": str(stream).lower()}, 63 | json={"input": input}, 64 | ): 65 | print(f"{event.event}: {json.loads(event.data)['token']}", end="", flush=True) 66 | 67 | 68 | if __name__ == "__main__": 69 | main() 70 | ``` 71 | 72 | First run the application server: 73 | 74 | 75 | 76 | ``` 77 | $ uvicorn app:app 78 | ``` 79 | 80 | Then run the client script: 81 | 82 | 83 | 84 | ``` 85 | $ python client.py --input "hi" 86 | completion: Hello! How can I assist you today? 87 | ``` 88 | 89 | ## Websocket 90 | 91 | ```python 92 | import os 93 | 94 | from langchain.chains import ConversationChain 95 | from langchain.chat_models import ChatOpenAI 96 | 97 | from lanarky import Lanarky 98 | from lanarky.adapters.langchain.routing import LangchainAPIRouter 99 | 100 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 101 | 102 | 103 | app = Lanarky() 104 | router = LangchainAPIRouter() 105 | 106 | 107 | @router.websocket("/ws") 108 | def chat() -> ConversationChain: 109 | return ConversationChain(llm=ChatOpenAI(streaming=True), verbose=True) 110 | 111 | 112 | app.include_router(router) 113 | ``` 114 | 115 | Similar to the streaming example, we use `chat` as a `ConversationChain` factory 116 | function and send it to the router to build a websocket endpoint. 117 | 118 | To communicate with the server, we will use the following client script: 119 | 120 | ```python 121 | import json 122 | from lanarky.clients import WebSocketClient 123 | 124 | 125 | def main(): 126 | client = WebSocketClient() 127 | with client.connect() as session: 128 | while True: 129 | user_input = input("\nEnter a message: ") 130 | session.send(dict(input=user_input)) 131 | print("Received: ", end="") 132 | for chunk in session.stream_response(): 133 | print(json.loads(chunk["data"])["token"], end="", flush=True) 134 | 135 | 136 | if __name__ == "__main__": 137 | main() 138 | ``` 139 | 140 | First run the application server: 141 | 142 | 143 | 144 | ``` 145 | $ uvicorn app:app 146 | ``` 147 | 148 | Then run the client script: 149 | 150 | 151 | 152 | ``` 153 | $ python client.py 154 | Enter a message: hi 155 | Received: Hello! How can I assist you today? 156 | Enter a message: i am lanarky 157 | Received: Hello Lanarky! It's nice to meet you. How can I assist 158 | you today? 159 | Enter a message: who am i? 160 | Received: You are Lanarky, as you mentioned earlier. Is there anything 161 | specific you would like to know about yourself? 162 | ``` 163 | 164 | !!! note "Note from Author" 165 | 166 | If you want to build more complex logic, I recommend using the low-level modules 167 | to define the endpoint from scratch: [Learn more](./fastapi.md) 168 | -------------------------------------------------------------------------------- /docs/learn/adapters/openai/dependency.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | --- 5 | 6 | FastAPI offers a powerful [Dependency Injection](https://fastapi.tiangolo.com/tutorial/dependencies/) 7 | system that allows you to inject dependencies into your API endpoints. Lanarky extends this functionality 8 | by offering OpenAI as a dependency. 9 | 10 | !!! example "Experimental" 11 | 12 | LLM-based dependency injection is an experimental feature. We will add more functionality 13 | based on community feedback and viable use cases. If you have ideas or suggestions, we 14 | would love to hear from you. Feel free to open an issue on 15 | [GitHub](https://github.com/ajndkr/lanarky/issues/new/choose). 16 | 17 | Let's take a look at how we can use OpenAI as a dependency. 18 | 19 | ```python 20 | import os 21 | 22 | from lanarky import Lanarky 23 | from lanarky.adapters.openai.dependencies import Depends 24 | from lanarky.adapters.openai.resources import ChatCompletion, ChatCompletionResource 25 | 26 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 27 | 28 | app = Lanarky() 29 | 30 | 31 | def chat_completion_factory() -> ChatCompletionResource: 32 | return ChatCompletionResource( 33 | system="You are a helpful assistant designed to output JSON.", 34 | model="gpt-3.5-turbo-1106", 35 | response_format={"type": "json_object"}, 36 | ) 37 | 38 | 39 | @app.post("/") 40 | async def endpoint(outputs: ChatCompletion = Depends(chat_completion_factory)): 41 | return outputs.choices[0].message.content 42 | ``` 43 | 44 | In the above example, we pass `chat_completion_factory` as a dependency to the `POST /` endpoint. 45 | Similar to how FasAPI handles dependencies, you can expose additional parameters by defining arguments 46 | in the `chat_completion_factory` function. For example, if you want to expose the `temperature` parameter, 47 | you can do so by adding a `temperature` argument to the `chat_completion_factory` function. 48 | 49 | ```python 50 | def chat_completion_factory(temperature: float = 0.5) -> ChatCompletionResource: 51 | return ChatCompletionResource( 52 | system="You are a helpful assistant designed to output JSON.", 53 | model="gpt-3.5-turbo-1106", 54 | response_format={"type": "json_object"}, 55 | temperature=temperature, 56 | ) 57 | ``` 58 | 59 | To test the above endpoint, let's create a client script: 60 | 61 | ```python 62 | import click 63 | import httpx 64 | 65 | 66 | @click.command() 67 | @click.option("--input", required=True) 68 | def main(input: str): 69 | url = "http://localhost:8000/" 70 | 71 | with httpx.Client() as client: 72 | response = client.post( 73 | url, json={"messages": [dict(role="user", content=input)]} 74 | ) 75 | if response.status_code == 200: 76 | data = response.json() 77 | print(f"Received: {data}") 78 | 79 | 80 | if __name__ == "__main__": 81 | main() 82 | ``` 83 | 84 | First, start the server: 85 | 86 | ```bash 87 | uvicorn app:app 88 | ``` 89 | 90 | Then, run the client script: 91 | 92 | 93 | 94 | ``` 95 | $ python client.py --input "Who won the world series in 2020?" 96 | Received: { 97 | "result": "The Los Angeles Dodgers won the World Series in 2020." 98 | } 99 | ``` 100 | -------------------------------------------------------------------------------- /docs/learn/adapters/openai/fastapi.md: -------------------------------------------------------------------------------- 1 | Lanarky is built on top of FastAPI and offers backwards compatibility with all FastAPI features. 2 | Nonetheless, if your project uses FastAPI and Lanarky is not a drop-in replacement, you can still 3 | the low-level Lanarky modules to build your microservice. 4 | 5 | We will use the examples from the [OpenAI API Router](./router.md) guide to demonstrate how to 6 | use the low-level modules as well as understand how the router works under the hood. 7 | 8 | ## Streaming 9 | 10 | OpenAI adapter extends the `StreamingResponse` class to support streaming for OpenAI microservices. 11 | 12 | !!! note 13 | 14 | Before you start, make sure you have read the [Streaming](../../streaming.md) and 15 | [OpenAI API Router](./router.md) guides. 16 | 17 | ```python 18 | import os 19 | 20 | from fastapi import Depends 21 | from pydantic import BaseModel 22 | 23 | from lanarky import Lanarky 24 | from lanarky.adapters.openai.resources import ChatCompletionResource, Message 25 | from lanarky.adapters.openai.responses import StreamingResponse 26 | 27 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 28 | 29 | app = Lanarky() 30 | 31 | 32 | class ChatInput(BaseModel): 33 | messages: list[Message] 34 | 35 | 36 | def chat_completion_factory(stream: bool = True) -> ChatCompletionResource: 37 | system = "You are a sassy assistant" 38 | return ChatCompletionResource(system=system, stream=stream) 39 | 40 | 41 | @app.post("/chat") 42 | async def chat( 43 | request: ChatInput, 44 | resource: ChatCompletionResource = Depends(chat_completion_factory), 45 | ): 46 | return StreamingResponse(resource=resource, **request.model_dump()) 47 | ``` 48 | 49 | The `/chat` endpoint is similar to the one we created using `OpenAIAPIRouter` in the 50 | [OpenAI API Router](./router.md) guide. 51 | 52 | !!! tip 53 | 54 | You can use the same client script from the [OpenAI API Router](./router.md) guide to test 55 | the above example. 56 | 57 | ## Websockets 58 | 59 | In addition to streaming, OpenAI adapter also supports websockets. Let's take a look at how we can 60 | build an OpenAI microservice using websockets. 61 | 62 | ```python 63 | import os 64 | 65 | from fastapi import Depends 66 | from pydantic import BaseModel 67 | 68 | from lanarky import Lanarky 69 | from lanarky.adapters.openai.resources import ChatCompletionResource, Message 70 | from lanarky.events import Events 71 | from lanarky.websockets import WebSocket, WebsocketSession 72 | 73 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 74 | 75 | app = Lanarky() 76 | 77 | 78 | class ChatInput(BaseModel): 79 | messages: list[Message] 80 | 81 | 82 | def chat_completion_factory() -> ChatCompletionResource: 83 | system = "You are a sassy assistant" 84 | return ChatCompletionResource(system=system, stream=True) 85 | 86 | 87 | @app.websocket("/ws") 88 | async def chat( 89 | websocket: WebSocket, 90 | resource: ChatCompletionResource = Depends(chat_completion_factory), 91 | ): 92 | async with WebsocketSession().connect(websocket) as session: 93 | async for data in session: 94 | async for chunk in resource.stream_response( 95 | **ChatInput(**data).model_dump() 96 | ): 97 | await websocket.send_json( 98 | dict(data=chunk, event=Events.COMPLETION) 99 | ) 100 | await websocket.send_json(dict(data="", event=Events.END)) 101 | ``` 102 | 103 | In this example, we use the `WebsocketSession` context manager to connect to the websocket 104 | and communicate with the client. We pass the client data to the OpenAI resource and stream 105 | the response back to the client. 106 | 107 | !!! tip 108 | 109 | Similar to the streaming example, you can use the same client script from the 110 | [OpenAI API Router](./router.md) guide to test the websocket example. 111 | -------------------------------------------------------------------------------- /docs/learn/adapters/openai/index.md: -------------------------------------------------------------------------------- 1 | # OpenAI Adapter 2 | 3 | The **OpenAI Adapter** allows Lanarky users to build microservices using the 4 | [OpenAI Python SDK](https://platform.openai.com/docs/api-reference?lang=python). 5 | 6 | To enable this adapter, install lanarky with extra dependencies: 7 | 8 | 9 | 10 | ``` 11 | $ pip install lanarky[openai] 12 | ``` 13 | 14 | !!! tip 15 | 16 | To use OpenAI, you need to create an openai account and generate an API key. 17 | Visit [openai.com](https://openai.com) for more information. 18 | 19 | To use the generated API key, you need to set the `OPENAI_API_KEY` environment 20 | variable. 21 | 22 | Here's an overview of the supported features: 23 | 24 | - [OpenAI API Router](./router.md): Lanarky router for OpenAI 25 | 26 | Additionally, we will cover some advanced topics: 27 | 28 | - [Dependency Injection](./dependency.md): use OpenAI as a dependency in your microservice 29 | - [FastAPI Backport](./fastapi.md): low-level modules for FastAPI users 30 | -------------------------------------------------------------------------------- /docs/learn/adapters/openai/router.md: -------------------------------------------------------------------------------- 1 | # OpenAI API Router 2 | 3 | The `OpenAIAPIRouter` class is an abstraction layer which provides a quick and easy 4 | way to build microservices using supported OpenAI models. 5 | 6 | !!! warning 7 | 8 | OpenAI SDK support is currently limited to chat models only (i.e. 9 | GPT-3.5-Turbo, GPT-4 and GPT-4-Turbo). Other models/services will be 10 | added in the future. 11 | 12 | Let's understand how to use `OpenAIAPIRouter` to build streaming and websocket 13 | endpoints. 14 | 15 | ## Streaming 16 | 17 | !!! note 18 | 19 | We are using the example from the [Getting Started](../../../getting-started.md) guide. 20 | 21 | ```python 22 | import os 23 | 24 | from lanarky import Lanarky 25 | from lanarky.adapters.openai.resources import ChatCompletionResource 26 | from lanarky.adapters.openai.routing import OpenAIAPIRouter 27 | 28 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 29 | 30 | 31 | app = Lanarky() 32 | router = OpenAIAPIRouter() 33 | 34 | 35 | @router.post("/chat") 36 | def chat(stream: bool = True) -> ChatCompletionResource: 37 | system = "You are a sassy assistant" 38 | return ChatCompletionResource(stream=stream, system=system) 39 | 40 | 41 | app.include_router(router) 42 | ``` 43 | 44 | In this example, `ChatCompletionResource` is a wrapper class to use the OpenAI 45 | Python SDK. `chat` acts as a factory function where we define the parameters of 46 | `ChatCompletionResource` and send it to the router to build the endpoint for us. 47 | The factory function arguments can be used to define query or header parameters 48 | which are exposed to the client. 49 | 50 | To receive the events, we will use the following client script: 51 | 52 | ```python 53 | import click 54 | 55 | from lanarky.clients import StreamingClient 56 | 57 | 58 | @click.command() 59 | @click.option("--input", required=True) 60 | @click.option("--stream", is_flag=True) 61 | def main(input: str, stream: bool): 62 | client = StreamingClient() 63 | for event in client.stream_response( 64 | "POST", 65 | "/chat", 66 | params={"stream": str(stream).lower()}, 67 | json={"messages": [dict(role="user", content=input)]}, 68 | ): 69 | print(f"{event.event}: {event.data}") 70 | 71 | 72 | if __name__ == "__main__": 73 | main() 74 | ``` 75 | 76 | First run the application server: 77 | 78 | 79 | 80 | ``` 81 | $ uvicorn app:app 82 | ``` 83 | 84 | Then run the client script: 85 | 86 | 87 | 88 | ``` 89 | $ python client.py --input "hi" 90 | completion: Oh, hello there! What can I sass...I mean assist 91 | you with today? 92 | ``` 93 | 94 | ## Websocket 95 | 96 | ```python 97 | import os 98 | 99 | from lanarky import Lanarky 100 | from lanarky.adapters.openai.resources import ChatCompletionResource 101 | from lanarky.adapters.openai.routing import OpenAIAPIRouter 102 | 103 | os.environ["OPENAI_API_KEY"] = "add-your-openai-api-key-here" 104 | 105 | 106 | app = Lanarky() 107 | router = OpenAIAPIRouter() 108 | 109 | 110 | @router.websocket("/ws") 111 | def chat() -> ChatCompletionResource: 112 | system = "You are a sassy assistant" 113 | return ChatCompletionResource(stream=True, system=system) 114 | 115 | 116 | app.include_router(router) 117 | ``` 118 | 119 | Similar to the streaming example, we use `chat` as a `ChatCompletionResource` 120 | factory function and send it to the router to build a websocket endpoint. 121 | 122 | To communicate with the server, we will use the following client script: 123 | 124 | ```python 125 | from lanarky.clients import WebSocketClient 126 | 127 | 128 | def main(): 129 | client = WebSocketClient() 130 | with client.connect() as session: 131 | messages = [] 132 | while True: 133 | user_input = input("\nEnter a message: ") 134 | messages.append(dict(role="user", content=user_input)) 135 | session.send(dict(messages=messages)) 136 | print("Received: ", end="") 137 | assistant_message = dict(role="assistant", content="") 138 | for chunk in session.stream_response(): 139 | print(chunk["data"], end="", flush=True) 140 | assistant_message["content"] += chunk["data"] 141 | messages.append(assistant_message) 142 | 143 | 144 | if __name__ == "__main__": 145 | main() 146 | ``` 147 | 148 | First run the application server: 149 | 150 | 151 | 152 | ``` 153 | $ uvicorn app:app 154 | ``` 155 | 156 | Then run the client script: 157 | 158 | 159 | 160 | ``` 161 | $ python client.py 162 | Enter a message: hi 163 | Received: Well, hello there! How can I assist you today? 164 | Enter a message: i am lanarky 165 | Received: Oh, aren't you just full of mischief, Lanarky? 166 | What trouble can I help you stir up today? 167 | Enter a message: who am i? 168 | Received: Well, that's a question only you can answer, Lanarky. 169 | But if I had to guess, based on your sassy spirit, I would say you're 170 | someone who loves to dance to the beat of your own drum and has a 171 | mischievous sense of humor. Am I close? 172 | ``` 173 | 174 | !!! note "Note from Author" 175 | 176 | If you want to build more complex logic, I recommend using the low-level modules 177 | to define the endpoint from scratch: [Learn more](./fastapi.md) 178 | -------------------------------------------------------------------------------- /docs/learn/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | - footer 5 | --- 6 | 7 | # Learn 8 | 9 | Hello user! Here we will learn about the core concepts of Lanarky. 10 | 11 | Here's a quick overview of what we will cover: 12 | 13 | - [Streaming](./streaming.md): build streaming microservices using FastAPI routers 14 | - [Websockets](./websockets.md): build websocket microservices using FastAPI routers 15 | - [Adapters](./adapters/index.md): build microservices using popular LLM frameworks 16 | 17 | ## Examples 18 | 19 | Lanarky comes with a set of examples to demonstrate its features: 20 | 21 | - [ChatGPT-clone](https://github.com/ajndkr/lanarky/tree/main/examples/chatgpt-clone): a chatbot 22 | application like [ChatGPT](https://chat.openai.com/) 23 | - [PaulGPT](https://github.com/ajndkr/lanarky/tree/main/examples/paulGPT): a chatbot application 24 | to answer questions about Paul Graham essay, "[What I worked on](http://www.paulgraham.com/worked.html)" 25 | -------------------------------------------------------------------------------- /docs/learn/streaming.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | --- 5 | 6 | Lanarky uses [Server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events) 7 | to implement streaming support over HTTP. 8 | 9 | ## `StreamingResponse` 10 | 11 | The `StreamingResponse` class follows the [`EventSource`](https://developer.mozilla.org/en-US/docs/Web/API/EventSource) 12 | protocol to send events to the client. 13 | 14 | ### Example 15 | 16 | To understand how to use `StreamingResponse`, let's look at an example. 17 | 18 | ```python 19 | from lanarky import Lanarky 20 | from lanarky.responses import StreamingResponse 21 | 22 | app = Lanarky() 23 | 24 | 25 | @app.get("/") 26 | def index(): 27 | def stream(): 28 | for word in ["Hello", "World!"]: 29 | yield word 30 | 31 | return StreamingResponse(content=stream()) 32 | ``` 33 | 34 | Here, we have a simple endpoint streams the message "Hello World!" to the client. 35 | `StreamingResponse` takes a generator function as its content, iterates over it and 36 | sends each item as an event to the client. 37 | 38 | To receive the events, let's build a simple client script. 39 | 40 | ```python 41 | from lanarky.clients import StreamingClient 42 | 43 | 44 | def main(): 45 | client = StreamingClient() 46 | for event in client.stream_response("GET", "/"): 47 | print(f"{event.event}: {event.data}") 48 | 49 | 50 | if __name__ == "__main__": 51 | main() 52 | ``` 53 | 54 | First run the application server. 55 | 56 | 57 | 58 | ``` 59 | $ uvicorn app:app 60 | ``` 61 | 62 | Then run the client script. 63 | 64 | 65 | 66 | ``` 67 | $ python client.py 68 | message: Hello 69 | message: World! 70 | ``` 71 | 72 | !!! warning 73 | 74 | The `StreamingResponse` classes inside the **Adapters API** behave differently from the 75 | above example. To learn more, see [Adapters API](./adapters/index.md) documentation. 76 | -------------------------------------------------------------------------------- /docs/learn/websockets.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | --- 5 | 6 | [WebSockets](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API) are useful for 7 | LLM microservices which require a bi-directional connection between the client and server. 8 | For example, a chat application would require a WebSocket to hold a persistent connection 9 | between the client and server during an active user session. 10 | 11 | Lanarky builds on top of FastAPI to support LLM microservices over WebSockets. 12 | 13 | ## `WebsocketSession` 14 | 15 | The `WebsocketSession` class establishes a WebSocket session inside an endpoint logic to define 16 | the interaction between the client and server. This is particularly useful for building chatbot 17 | applications. 18 | 19 | ### Example 20 | 21 | To understand how to use `WebsocketSession`, let's look at an example. 22 | 23 | ```python 24 | from lanarky import Lanarky 25 | from lanarky.websockets import WebSocket, WebsocketSession 26 | 27 | app = Lanarky() 28 | 29 | 30 | @app.websocket("/ws") 31 | async def endpoint(websocket: WebSocket): 32 | async with WebsocketSession().connect(websocket) as session: 33 | async for message in session: 34 | await websocket.send_json({"data": message["data"].capitalize()}) 35 | ``` 36 | 37 | Here, we have a simple websocket endpoint which capitalizes the message sent by the client. 38 | We use the `WebsocketSession` class to establish a session with the client. The session 39 | allows us to send and receive messages from the client. 40 | 41 | To receive the events, let's build a simple client script. 42 | 43 | ```python 44 | from lanarky.clients import WebSocketClient 45 | 46 | 47 | def main(): 48 | client = WebSocketClient(uri="ws://localhost:8001/ws") 49 | with client.connect() as session: 50 | while True: 51 | user_input = input("Enter a message: ") 52 | session.send(dict(data=user_input)) 53 | response = session.receive() 54 | print(f"Received: {response}") 55 | 56 | 57 | if __name__ == "__main__": 58 | main() 59 | ``` 60 | 61 | First run the application server. 62 | 63 | 64 | 65 | ``` 66 | $ uvicorn app:app 67 | ``` 68 | 69 | Then run the client script. 70 | 71 | 72 | 73 | ``` 74 | $ python client.py 75 | Enter a message: hi 76 | Received: {'data': 'Hi'} 77 | Enter a message: hola 78 | Received: {'data': 'Hola'} 79 | ``` 80 | -------------------------------------------------------------------------------- /docs/reference/adapters/langchain.md: -------------------------------------------------------------------------------- 1 | ::: lanarky.adapters.langchain.routing 2 | 3 | ::: lanarky.adapters.langchain.responses 4 | 5 | ::: lanarky.adapters.langchain.callbacks 6 | 7 | ::: lanarky.adapters.langchain.dependencies 8 | 9 | ::: lanarky.adapters.langchain.utils 10 | -------------------------------------------------------------------------------- /docs/reference/adapters/openai.md: -------------------------------------------------------------------------------- 1 | ::: lanarky.adapters.openai.resources 2 | 3 | ::: lanarky.adapters.openai.routing 4 | 5 | ::: lanarky.adapters.openai.responses 6 | 7 | ::: lanarky.adapters.openai.dependencies 8 | 9 | ::: lanarky.adapters.openai.utils 10 | -------------------------------------------------------------------------------- /docs/reference/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | - footer 5 | --- 6 | 7 | # API Reference 8 | 9 | This is Lanarky's API reference documentation. 10 | 11 | The API reference in split into multiple sections. First, we will cover the 12 | Core API: 13 | 14 | - [`Lanarky`](./lanarky.md) - The main application module 15 | - [`StreamingResponse`](./streaming.md) - `Response` class for streaming 16 | - [`WebSocketSession`](./websockets.md) - class for managing websocket sessions 17 | 18 | !!! note 19 | 20 | Lanarky also provides a collection of web clients for testing purposes. 21 | See [Miscellaneous](./misc.md) for more information. 22 | 23 | Next, we will cover the Adapter API: 24 | 25 | - [OpenAI](./adapters/openai.md): Adapter module for 26 | [OpenAI Python SDK](https://platform.openai.com/docs/api-reference?lang=python) 27 | - [LangChain](./adapters/langchain.md): Adapter module for 28 | [LangChain](https://www.langchain.com/) 29 | 30 | You can find all other utility functions/classes in the [Miscellaneous](./misc.md) section. 31 | -------------------------------------------------------------------------------- /docs/reference/lanarky.md: -------------------------------------------------------------------------------- 1 | # `Lanarky` class 2 | 3 | ::: lanarky.Lanarky 4 | options: 5 | members: [] 6 | -------------------------------------------------------------------------------- /docs/reference/misc.md: -------------------------------------------------------------------------------- 1 | Lanarky offers client classes to test streaming and websocket endpoints. 2 | 3 | ::: lanarky.clients 4 | 5 | ::: lanarky.logging 6 | -------------------------------------------------------------------------------- /docs/reference/streaming.md: -------------------------------------------------------------------------------- 1 | # `StreamingResponse` class 2 | 3 | ::: lanarky.responses.StreamingResponse 4 | -------------------------------------------------------------------------------- /docs/reference/websockets.md: -------------------------------------------------------------------------------- 1 | # `WebsocketSession` class 2 | 3 | ::: lanarky.websockets.WebsocketSession 4 | -------------------------------------------------------------------------------- /docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | [data-md-color-scheme="lanarky"] { 2 | --md-primary-fg-color: #4ce1b5; 3 | --md-primary-fg-color--light: #28f2b6; 4 | --md-primary-fg-color--dark: #0c9a78; 5 | 6 | --md-accent-fg-color: #f25353; 7 | --md-accent-fg-color--transparent: rgba(242, 83, 83, 0.1); 8 | } 9 | 10 | [data-md-color-scheme="slate"] { 11 | --md-primary-fg-color: #4ce1b5; 12 | --md-primary-fg-color--light: #28f2b6; 13 | --md-primary-fg-color--dark: #0c9a78; 14 | 15 | --md-accent-fg-color: #f25353; 16 | --md-accent-fg-color--transparent: rgba(242, 83, 83, 0.1); 17 | 18 | --md-hue: 210; 19 | } 20 | 21 | .md-header { 22 | color: #303841; 23 | } 24 | 25 | .md-tabs { 26 | color: #303841; 27 | } 28 | 29 | .md-typeset a { 30 | color: #0c9a78; 31 | } 32 | 33 | .md-header__topic:first-child { 34 | font-weight: 400; 35 | } 36 | 37 | .md-nav .md-nav__link { 38 | color: #0c9a78; 39 | } 40 | 41 | [data-md-color-scheme="lanarky"] img[src$="#only-dark"], 42 | [data-md-color-scheme="lanarky"] img[src$="#gh-dark-mode-only"] { 43 | display: none; 44 | } 45 | 46 | [data-md-color-scheme="slate"] img[src$="#only-light"], 47 | [data-md-color-scheme="slate"] img[src$="#gh-light-mode-only"] { 48 | display: none; 49 | } 50 | 51 | .md-typeset .md-button--primary { 52 | color: #303841; 53 | } 54 | 55 | .md-nav--primary .md-nav__item--active > .md-nav__link { 56 | color: #f25353; 57 | } 58 | -------------------------------------------------------------------------------- /examples/chatgpt-clone/README.md: -------------------------------------------------------------------------------- 1 | # ChatGPT-clone 2 | 3 | A chatbot application like [ChatGPT](https://chat.openai.com/), built with Lanarky. 4 | 5 | This example covers the following Lanarky features: 6 | 7 | - OpenAI Adapter 8 | - Streaming tokens via server-sent events 9 | 10 | To learn more about Lanarky, check out Lanarky's [full documentation](https://lanarky.ajndkr.com/learn/). 11 | 12 | ## Setup 13 | 14 | Install dependencies: 15 | 16 | ``` 17 | pip install 'lanarky[openai]' gradio 18 | ``` 19 | 20 | ## Run 21 | 22 | First we set the OpenAI API key: 23 | 24 | ```sh 25 | export OPENAI_API_KEY= 26 | ``` 27 | 28 | Then we run the server: 29 | 30 | ```sh 31 | python app.py 32 | ``` 33 | 34 | Once the server is running, open http://localhost:8000/ in your browser. 35 | -------------------------------------------------------------------------------- /examples/chatgpt-clone/app.py: -------------------------------------------------------------------------------- 1 | import gradio as gr 2 | 3 | from lanarky import Lanarky 4 | from lanarky.adapters.openai.resources import ChatCompletionResource 5 | from lanarky.adapters.openai.routing import OpenAIAPIRouter 6 | from lanarky.clients import StreamingClient 7 | 8 | app = Lanarky() 9 | router = OpenAIAPIRouter() 10 | 11 | 12 | @router.post("/chat") 13 | def chat(system: str = "You are a sassy assistant") -> ChatCompletionResource: 14 | return ChatCompletionResource(system=system, stream=True) 15 | 16 | 17 | app.include_router(router) 18 | 19 | 20 | def mount_playground(app: Lanarky) -> Lanarky: 21 | blocks = gr.Blocks( 22 | title="ChatGPT-clone", 23 | theme=gr.themes.Default( 24 | primary_hue=gr.themes.colors.teal, 25 | secondary_hue=gr.themes.colors.teal, 26 | text_size=gr.themes.sizes.text_lg, 27 | ), 28 | css="footer {visibility: hidden}", 29 | ) 30 | 31 | with blocks: 32 | blocks.load( 33 | None, 34 | None, 35 | js=""" 36 | () => { 37 | document.body.className = "white"; 38 | }""", 39 | ) 40 | gr.HTML( 41 | """
""" 42 | ) 43 | system_message = gr.Textbox( 44 | value="You are a sassy assistant", label="System Prompt" 45 | ) 46 | chatbot = gr.Chatbot(height=500, show_label=False) 47 | with gr.Row(): 48 | user_input = gr.Textbox( 49 | show_label=False, placeholder="Type a message...", scale=5 50 | ) 51 | clear_btn = gr.Button("Clear") 52 | 53 | def chat(history, system): 54 | messages = [] 55 | for human, assistant in history: 56 | if human: 57 | messages.append({"role": "user", "content": human}) 58 | if assistant: 59 | messages.append({"role": "assistant", "content": assistant}) 60 | 61 | history[-1][1] = "" 62 | for event in StreamingClient().stream_response( 63 | "POST", "/chat", json={"messages": messages}, params={"system": system} 64 | ): 65 | history[-1][1] += event.data 66 | yield history 67 | 68 | user_input.submit( 69 | lambda user_input, chatbot: ("", chatbot + [[user_input, None]]), 70 | [user_input, chatbot], 71 | [user_input, chatbot], 72 | queue=False, 73 | ).then(chat, [chatbot, system_message], chatbot) 74 | clear_btn.click(lambda: None, None, chatbot, queue=False) 75 | 76 | return gr.mount_gradio_app(app, blocks.queue(), "/") 77 | 78 | 79 | app = mount_playground(app) 80 | 81 | 82 | if __name__ == "__main__": 83 | import uvicorn 84 | 85 | uvicorn.run(app) 86 | -------------------------------------------------------------------------------- /examples/paulGPT/README.md: -------------------------------------------------------------------------------- 1 | # PaulGPT 2 | 3 | A chatbot application to answer questions about Paul Graham essay, "[What I worked on](http://www.paulgraham.com/worked.html)", 4 | built with Lanarky. 5 | 6 | This example covers the following Lanarky features: 7 | 8 | - LangChain Adapter 9 | - Streaming source documents via server-sent events 10 | 11 | To learn more about Lanarky, check out Lanarky's [full documentation](https://lanarky.ajndkr.com/learn/). 12 | 13 | ## Setup 14 | 15 | Install dependencies: 16 | 17 | ``` 18 | pip install 'lanarky[openai]' gradio faiss-cpu 19 | ``` 20 | 21 | ## Run 22 | 23 | First we set the OpenAI API key: 24 | 25 | ```sh 26 | export OPENAI_API_KEY= 27 | ``` 28 | 29 | Then we run the server: 30 | 31 | ```sh 32 | python app.py 33 | ``` 34 | 35 | Once the server is running, open http://localhost:8000/ in your browser. 36 | -------------------------------------------------------------------------------- /examples/paulGPT/app.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import gradio as gr 4 | from langchain.chains import RetrievalQA 5 | from langchain.chat_models import ChatOpenAI 6 | from langchain.embeddings.openai import OpenAIEmbeddings 7 | from langchain.vectorstores.faiss import FAISS 8 | 9 | from lanarky import Lanarky 10 | from lanarky.adapters.langchain.routing import LangchainAPIRouter 11 | from lanarky.clients import StreamingClient 12 | 13 | app = Lanarky() 14 | router = LangchainAPIRouter() 15 | 16 | 17 | @router.post("/chat") 18 | def chat() -> RetrievalQA: 19 | db = FAISS.load_local("db/", OpenAIEmbeddings()) 20 | return RetrievalQA.from_chain_type( 21 | ChatOpenAI(streaming=True), 22 | retriever=db.as_retriever(search_kwargs={"k": 2}), 23 | return_source_documents=True, 24 | ) 25 | 26 | 27 | app.include_router(router) 28 | 29 | 30 | SOURCE_DOCUMENT_TEMPLATE = """ 31 |
Source {idx}{page_content}
32 | """ 33 | 34 | 35 | def mount_playground(app: Lanarky) -> Lanarky: 36 | blocks = gr.Blocks( 37 | title="paulGPT", 38 | theme=gr.themes.Default( 39 | primary_hue=gr.themes.colors.teal, secondary_hue=gr.themes.colors.teal 40 | ), 41 | css="footer {visibility: hidden}", 42 | ) 43 | 44 | with blocks: 45 | blocks.load( 46 | None, 47 | None, 48 | js=""" 49 | () => { 50 | document.body.className = "white"; 51 | }""", 52 | ) 53 | gr.HTML( 54 | """
""" 55 | ) 56 | chatbot = gr.Chatbot(height=500, show_label=False) 57 | with gr.Row(): 58 | user_input = gr.Textbox( 59 | show_label=False, placeholder="Type a message...", scale=5 60 | ) 61 | clear_btn = gr.Button("Clear") 62 | 63 | def chat(history): 64 | history[-1][1] = "" 65 | for event in StreamingClient().stream_response( 66 | "POST", "/chat", json={"query": history[-1][0]} 67 | ): 68 | if event.event == "completion": 69 | history[-1][1] += json.loads(event.data)["token"] 70 | yield history 71 | elif event.event == "source_documents": 72 | for idx, document in enumerate( 73 | json.loads(event.data)["source_documents"] 74 | ): 75 | history[-1][1] += SOURCE_DOCUMENT_TEMPLATE.format( 76 | idx=idx, 77 | page_content=document["page_content"], 78 | ) 79 | yield history 80 | elif event.event == "error": 81 | raise gr.Error(event.data) 82 | 83 | user_input.submit( 84 | lambda user_input, chatbot: ("", chatbot + [[user_input, None]]), 85 | [user_input, chatbot], 86 | [user_input, chatbot], 87 | queue=False, 88 | ).then(chat, chatbot, chatbot) 89 | clear_btn.click(lambda: None, None, chatbot, queue=False) 90 | 91 | return gr.mount_gradio_app(app, blocks.queue(), "/") 92 | 93 | 94 | app = mount_playground(app) 95 | 96 | 97 | if __name__ == "__main__": 98 | import uvicorn 99 | 100 | uvicorn.run(app) 101 | -------------------------------------------------------------------------------- /examples/paulGPT/db/index.faiss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/examples/paulGPT/db/index.faiss -------------------------------------------------------------------------------- /examples/paulGPT/db/index.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/examples/paulGPT/db/index.pkl -------------------------------------------------------------------------------- /lanarky/__init__.py: -------------------------------------------------------------------------------- 1 | from .applications import Lanarky as Lanarky 2 | -------------------------------------------------------------------------------- /lanarky/adapters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/lanarky/adapters/__init__.py -------------------------------------------------------------------------------- /lanarky/adapters/langchain/__init__.py: -------------------------------------------------------------------------------- 1 | from importlib.util import find_spec 2 | 3 | if not find_spec("langchain"): 4 | raise ImportError( 5 | "run `pip install lanarky[langchain]` to use the langchain adapter" 6 | ) 7 | -------------------------------------------------------------------------------- /lanarky/adapters/langchain/callbacks.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional 2 | 3 | from fastapi.websockets import WebSocket 4 | from langchain.callbacks.base import AsyncCallbackHandler 5 | from langchain.callbacks.streaming_stdout_final_only import ( 6 | FinalStreamingStdOutCallbackHandler, 7 | ) 8 | from langchain.globals import get_llm_cache 9 | from langchain.schema.document import Document 10 | from pydantic import BaseModel 11 | from starlette.types import Message, Send 12 | 13 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 14 | from lanarky.utils import StrEnum, model_dump_json 15 | 16 | 17 | class LangchainEvents(StrEnum): 18 | SOURCE_DOCUMENTS = "source_documents" 19 | 20 | 21 | class LanarkyCallbackHandler(AsyncCallbackHandler): 22 | """Base callback handler for Lanarky applications.""" 23 | 24 | def __init__(self, **kwargs: dict[str, Any]) -> None: 25 | super().__init__(**kwargs) 26 | self.llm_cache_used = get_llm_cache() is not None 27 | 28 | @property 29 | def always_verbose(self) -> bool: 30 | """Verbose mode is always enabled for Lanarky applications.""" 31 | return True 32 | 33 | async def on_chat_model_start(self, *args: Any, **kwargs: Any) -> Any: ... 34 | 35 | 36 | class StreamingCallbackHandler(LanarkyCallbackHandler): 37 | """Callback handler for streaming responses.""" 38 | 39 | def __init__( 40 | self, 41 | *, 42 | send: Send = None, 43 | **kwargs: dict[str, Any], 44 | ) -> None: 45 | """Constructor method. 46 | 47 | Args: 48 | send: The ASGI send callable. 49 | **kwargs: Keyword arguments to pass to the parent constructor. 50 | """ 51 | super().__init__(**kwargs) 52 | 53 | self._send = send 54 | self.streaming = None 55 | 56 | @property 57 | def send(self) -> Send: 58 | return self._send 59 | 60 | @send.setter 61 | def send(self, value: Send) -> None: 62 | """Setter method for send property.""" 63 | if not callable(value): 64 | raise ValueError("value must be a Callable") 65 | self._send = value 66 | 67 | def _construct_message(self, data: str, event: Optional[str] = None) -> Message: 68 | """Constructs message payload. 69 | 70 | Args: 71 | data: The data payload. 72 | event: The event name. 73 | """ 74 | chunk = ServerSentEvent(data=data, event=event) 75 | return { 76 | "type": "http.response.body", 77 | "body": ensure_bytes(chunk, None), 78 | "more_body": True, 79 | } 80 | 81 | 82 | class TokenStreamMode(StrEnum): 83 | TEXT = "text" 84 | JSON = "json" 85 | 86 | 87 | class TokenEventData(BaseModel): 88 | """Event data payload for tokens.""" 89 | 90 | token: str = "" 91 | 92 | 93 | def get_token_data(token: str, mode: TokenStreamMode) -> str: 94 | """Get token data based on mode. 95 | 96 | Args: 97 | token: The token to use. 98 | mode: The stream mode. 99 | """ 100 | if mode not in list(TokenStreamMode): 101 | raise ValueError(f"Invalid stream mode: {mode}") 102 | 103 | if mode == TokenStreamMode.TEXT: 104 | return token 105 | else: 106 | return model_dump_json(TokenEventData(token=token)) 107 | 108 | 109 | class TokenStreamingCallbackHandler(StreamingCallbackHandler): 110 | """Callback handler for streaming tokens.""" 111 | 112 | def __init__( 113 | self, 114 | *, 115 | output_key: str, 116 | mode: TokenStreamMode = TokenStreamMode.JSON, 117 | **kwargs: dict[str, Any], 118 | ) -> None: 119 | """Constructor method. 120 | 121 | Args: 122 | output_key: chain output key. 123 | mode: The stream mode. 124 | **kwargs: Keyword arguments to pass to the parent constructor. 125 | """ 126 | super().__init__(**kwargs) 127 | 128 | self.output_key = output_key 129 | 130 | if mode not in list(TokenStreamMode): 131 | raise ValueError(f"Invalid stream mode: {mode}") 132 | self.mode = mode 133 | 134 | async def on_chain_start(self, *args: Any, **kwargs: dict[str, Any]) -> None: 135 | """Run when chain starts running.""" 136 | self.streaming = False 137 | 138 | async def on_llm_new_token(self, token: str, **kwargs: dict[str, Any]) -> None: 139 | """Run on new LLM token. Only available when streaming is enabled.""" 140 | if not self.streaming: 141 | self.streaming = True 142 | 143 | if self.llm_cache_used: # cache missed (or was never enabled) if we are here 144 | self.llm_cache_used = False 145 | 146 | message = self._construct_message( 147 | data=get_token_data(token, self.mode), event=Events.COMPLETION 148 | ) 149 | await self.send(message) 150 | 151 | async def on_chain_end( 152 | self, outputs: dict[str, Any], **kwargs: dict[str, Any] 153 | ) -> None: 154 | """Run when chain ends running. 155 | 156 | Final output is streamed only if LLM cache is enabled. 157 | """ 158 | if self.llm_cache_used or not self.streaming: 159 | if self.output_key in outputs: 160 | message = self._construct_message( 161 | data=get_token_data(outputs[self.output_key], self.mode), 162 | event=Events.COMPLETION, 163 | ) 164 | await self.send(message) 165 | else: 166 | raise KeyError(f"missing outputs key: {self.output_key}") 167 | 168 | 169 | class SourceDocumentsEventData(BaseModel): 170 | """Event data payload for source documents.""" 171 | 172 | source_documents: list[dict[str, Any]] 173 | 174 | 175 | class SourceDocumentsStreamingCallbackHandler(StreamingCallbackHandler): 176 | """Callback handler for streaming source documents.""" 177 | 178 | async def on_chain_end( 179 | self, outputs: dict[str, Any], **kwargs: dict[str, Any] 180 | ) -> None: 181 | """Run when chain ends running.""" 182 | if "source_documents" in outputs: 183 | if not isinstance(outputs["source_documents"], list): 184 | raise ValueError("source_documents must be a list") 185 | if not isinstance(outputs["source_documents"][0], Document): 186 | raise ValueError("source_documents must be a list of Document") 187 | 188 | # NOTE: langchain is using pydantic_v1 for `Document` 189 | source_documents: list[dict] = [ 190 | document.dict() for document in outputs["source_documents"] 191 | ] 192 | message = self._construct_message( 193 | data=model_dump_json( 194 | SourceDocumentsEventData(source_documents=source_documents) 195 | ), 196 | event=LangchainEvents.SOURCE_DOCUMENTS, 197 | ) 198 | await self.send(message) 199 | 200 | 201 | class FinalTokenStreamingCallbackHandler( 202 | TokenStreamingCallbackHandler, FinalStreamingStdOutCallbackHandler 203 | ): 204 | """Callback handler for streaming final answer tokens. 205 | 206 | Useful for streaming responses from Langchain Agents. 207 | """ 208 | 209 | def __init__( 210 | self, 211 | *, 212 | answer_prefix_tokens: Optional[list[str]] = None, 213 | strip_tokens: bool = True, 214 | stream_prefix: bool = False, 215 | **kwargs: dict[str, Any], 216 | ) -> None: 217 | """Constructor method. 218 | 219 | Args: 220 | answer_prefix_tokens: The answer prefix tokens to use. 221 | strip_tokens: Whether to strip tokens. 222 | stream_prefix: Whether to stream the answer prefix. 223 | **kwargs: Keyword arguments to pass to the parent constructor. 224 | """ 225 | super().__init__(output_key=None, **kwargs) 226 | 227 | FinalStreamingStdOutCallbackHandler.__init__( 228 | self, 229 | answer_prefix_tokens=answer_prefix_tokens, 230 | strip_tokens=strip_tokens, 231 | stream_prefix=stream_prefix, 232 | ) 233 | 234 | async def on_llm_start(self, *args: Any, **kwargs: dict[str, Any]) -> None: 235 | """Run when LLM starts running.""" 236 | self.answer_reached = False 237 | self.streaming = False 238 | 239 | async def on_llm_new_token(self, token: str, **kwargs: dict[str, Any]) -> None: 240 | """Run on new LLM token. Only available when streaming is enabled.""" 241 | if not self.streaming: 242 | self.streaming = True 243 | 244 | # Remember the last n tokens, where n = len(answer_prefix_tokens) 245 | self.append_to_last_tokens(token) 246 | 247 | # Check if the last n tokens match the answer_prefix_tokens list ... 248 | if self.check_if_answer_reached(): 249 | self.answer_reached = True 250 | if self.stream_prefix: 251 | message = self._construct_message( 252 | data=get_token_data("".join(self.last_tokens), self.mode), 253 | event=Events.COMPLETION, 254 | ) 255 | await self.send(message) 256 | 257 | # ... if yes, then print tokens from now on 258 | if self.answer_reached: 259 | message = self._construct_message( 260 | data=get_token_data(token, self.mode), event=Events.COMPLETION 261 | ) 262 | await self.send(message) 263 | 264 | 265 | class WebSocketCallbackHandler(LanarkyCallbackHandler): 266 | """Callback handler for websocket sessions.""" 267 | 268 | def __init__( 269 | self, 270 | *, 271 | mode: TokenStreamMode = TokenStreamMode.JSON, 272 | websocket: WebSocket = None, 273 | **kwargs: dict[str, Any], 274 | ) -> None: 275 | """Constructor method. 276 | 277 | Args: 278 | mode: The stream mode. 279 | websocket: The websocket to use. 280 | **kwargs: Keyword arguments to pass to the parent constructor. 281 | """ 282 | super().__init__(**kwargs) 283 | 284 | if mode not in list(TokenStreamMode): 285 | raise ValueError(f"Invalid stream mode: {mode}") 286 | self.mode = mode 287 | 288 | self._websocket = websocket 289 | self.streaming = None 290 | 291 | @property 292 | def websocket(self) -> WebSocket: 293 | return self._websocket 294 | 295 | @websocket.setter 296 | def websocket(self, value: WebSocket) -> None: 297 | """Setter method for send property.""" 298 | if not isinstance(value, WebSocket): 299 | raise ValueError("value must be a WebSocket") 300 | self._websocket = value 301 | 302 | def _construct_message(self, data: str, event: Optional[str] = None) -> Message: 303 | """Constructs message payload. 304 | 305 | Args: 306 | data: The data payload. 307 | event: The event name. 308 | """ 309 | return dict(data=data, event=event) 310 | 311 | 312 | class TokenWebSocketCallbackHandler(WebSocketCallbackHandler): 313 | """Callback handler for sending tokens in websocket sessions.""" 314 | 315 | def __init__(self, *, output_key: str, **kwargs: dict[str, Any]) -> None: 316 | """Constructor method. 317 | 318 | Args: 319 | output_key: chain output key. 320 | **kwargs: Keyword arguments to pass to the parent constructor. 321 | """ 322 | super().__init__(**kwargs) 323 | 324 | self.output_key = output_key 325 | 326 | async def on_chain_start(self, *args: Any, **kwargs: dict[str, Any]) -> None: 327 | """Run when chain starts running.""" 328 | self.streaming = False 329 | 330 | async def on_llm_new_token(self, token: str, **kwargs: dict[str, Any]) -> None: 331 | """Run on new LLM token. Only available when streaming is enabled.""" 332 | if not self.streaming: 333 | self.streaming = True 334 | 335 | if self.llm_cache_used: # cache missed (or was never enabled) if we are here 336 | self.llm_cache_used = False 337 | 338 | message = self._construct_message( 339 | data=get_token_data(token, self.mode), event=Events.COMPLETION 340 | ) 341 | await self.websocket.send_json(message) 342 | 343 | async def on_chain_end( 344 | self, outputs: dict[str, Any], **kwargs: dict[str, Any] 345 | ) -> None: 346 | """Run when chain ends running. 347 | 348 | Final output is streamed only if LLM cache is enabled. 349 | """ 350 | if self.llm_cache_used or not self.streaming: 351 | if self.output_key in outputs: 352 | message = self._construct_message( 353 | data=get_token_data(outputs[self.output_key], self.mode), 354 | event=Events.COMPLETION, 355 | ) 356 | await self.websocket.send_json(message) 357 | else: 358 | raise KeyError(f"missing outputs key: {self.output_key}") 359 | 360 | 361 | class SourceDocumentsWebSocketCallbackHandler(WebSocketCallbackHandler): 362 | """Callback handler for sending source documents in websocket sessions.""" 363 | 364 | async def on_chain_end( 365 | self, outputs: dict[str, Any], **kwargs: dict[str, Any] 366 | ) -> None: 367 | """Run when chain ends running.""" 368 | if "source_documents" in outputs: 369 | if not isinstance(outputs["source_documents"], list): 370 | raise ValueError("source_documents must be a list") 371 | if not isinstance(outputs["source_documents"][0], Document): 372 | raise ValueError("source_documents must be a list of Document") 373 | 374 | # NOTE: langchain is using pydantic_v1 for `Document` 375 | source_documents: list[dict] = [ 376 | document.dict() for document in outputs["source_documents"] 377 | ] 378 | message = self._construct_message( 379 | data=model_dump_json( 380 | SourceDocumentsEventData(source_documents=source_documents) 381 | ), 382 | event=LangchainEvents.SOURCE_DOCUMENTS, 383 | ) 384 | await self.websocket.send_json(message) 385 | 386 | 387 | class FinalTokenWebSocketCallbackHandler( 388 | TokenWebSocketCallbackHandler, FinalStreamingStdOutCallbackHandler 389 | ): 390 | """Callback handler for sending final answer tokens in websocket sessions. 391 | 392 | Useful for streaming responses from Langchain Agents. 393 | """ 394 | 395 | def __init__( 396 | self, 397 | *, 398 | answer_prefix_tokens: Optional[list[str]] = None, 399 | strip_tokens: bool = True, 400 | stream_prefix: bool = False, 401 | **kwargs: dict[str, Any], 402 | ) -> None: 403 | """Constructor method. 404 | 405 | Args: 406 | answer_prefix_tokens: The answer prefix tokens to use. 407 | strip_tokens: Whether to strip tokens. 408 | stream_prefix: Whether to stream the answer prefix. 409 | **kwargs: Keyword arguments to pass to the parent constructor. 410 | """ 411 | super().__init__(output_key=None, **kwargs) 412 | 413 | FinalStreamingStdOutCallbackHandler.__init__( 414 | self, 415 | answer_prefix_tokens=answer_prefix_tokens, 416 | strip_tokens=strip_tokens, 417 | stream_prefix=stream_prefix, 418 | ) 419 | 420 | async def on_llm_start(self, *args, **kwargs) -> None: 421 | """Run when LLM starts running.""" 422 | self.answer_reached = False 423 | self.streaming = False 424 | 425 | async def on_llm_new_token(self, token: str, **kwargs: dict[str, Any]) -> None: 426 | """Run on new LLM token. Only available when streaming is enabled.""" 427 | if not self.streaming: 428 | self.streaming = True 429 | 430 | # Remember the last n tokens, where n = len(answer_prefix_tokens) 431 | self.append_to_last_tokens(token) 432 | 433 | # Check if the last n tokens match the answer_prefix_tokens list ... 434 | if self.check_if_answer_reached(): 435 | self.answer_reached = True 436 | if self.stream_prefix: 437 | message = self._construct_message( 438 | data=get_token_data("".join(self.last_tokens), self.mode), 439 | event=Events.COMPLETION, 440 | ) 441 | await self.websocket.send_json(message) 442 | 443 | # ... if yes, then print tokens from now on 444 | if self.answer_reached: 445 | message = self._construct_message( 446 | data=get_token_data(token, self.mode), event=Events.COMPLETION 447 | ) 448 | await self.websocket.send_json(message) 449 | -------------------------------------------------------------------------------- /lanarky/adapters/langchain/dependencies.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional 2 | 3 | from fastapi import params 4 | from langchain.chains.base import Chain 5 | 6 | from lanarky.adapters.langchain.utils import create_request_model, create_response_model 7 | from lanarky.utils import model_dump 8 | 9 | 10 | def Depends( 11 | dependency: Optional[Callable[..., Any]], 12 | *, 13 | dependency_kwargs: dict[str, Any] = {}, 14 | use_cache: bool = True 15 | ) -> params.Depends: 16 | """Dependency injection for LangChain. 17 | 18 | Args: 19 | dependency: a "dependable" chain factory callable. 20 | dependency_kwargs: kwargs to pass to chain dependency. 21 | use_cache: use_cache parameter of `fastapi.Depends`. 22 | """ 23 | try: 24 | chain = dependency() 25 | except TypeError: 26 | raise TypeError("set default values for all dependency parameters") 27 | 28 | if not isinstance(chain, Chain): 29 | raise TypeError("dependency must return a Chain instance") 30 | 31 | request_model = create_request_model(chain) 32 | response_model = create_response_model(chain) 33 | 34 | async def chain_dependency( 35 | request: request_model, 36 | chain: Chain = params.Depends(dependency, use_cache=use_cache), 37 | ) -> response_model: 38 | return await chain.acall(inputs=model_dump(request), **dependency_kwargs) 39 | 40 | return params.Depends(chain_dependency, use_cache=use_cache) 41 | -------------------------------------------------------------------------------- /lanarky/adapters/langchain/responses.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from functools import partial 3 | from typing import Any 4 | 5 | from fastapi import status 6 | from langchain.chains.base import Chain 7 | from starlette.types import Send 8 | 9 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 10 | from lanarky.logging import logger 11 | from lanarky.responses import HTTPStatusDetail 12 | from lanarky.responses import StreamingResponse as _StreamingResponse 13 | from lanarky.utils import StrEnum 14 | 15 | 16 | class ChainRunMode(StrEnum): 17 | """Enum for LangChain run modes.""" 18 | 19 | ASYNC = "async" 20 | SYNC = "sync" 21 | 22 | 23 | class StreamingResponse(_StreamingResponse): 24 | """StreamingResponse class for LangChain resources.""" 25 | 26 | def __init__( 27 | self, 28 | chain: Chain, 29 | config: dict[str, Any], 30 | run_mode: ChainRunMode = ChainRunMode.ASYNC, 31 | *args: Any, 32 | **kwargs: dict[str, Any], 33 | ) -> None: 34 | """Constructor method. 35 | 36 | Args: 37 | chain: A LangChain instance. 38 | config: A config dict. 39 | *args: Positional arguments to pass to the parent constructor. 40 | **kwargs: Keyword arguments to pass to the parent constructor. 41 | """ 42 | super().__init__(*args, **kwargs) 43 | 44 | self.chain = chain 45 | self.config = config 46 | 47 | if run_mode not in list(ChainRunMode): 48 | raise ValueError( 49 | f"Invalid run mode '{run_mode}'. Must be one of {list(ChainRunMode)}" 50 | ) 51 | 52 | self.run_mode = run_mode 53 | 54 | async def stream_response(self, send: Send) -> None: 55 | """Stream LangChain outputs. 56 | 57 | If an exception occurs while iterating over the LangChain, an 58 | internal server error is sent to the client. 59 | 60 | Args: 61 | send: The ASGI send callable. 62 | """ 63 | await send( 64 | { 65 | "type": "http.response.start", 66 | "status": self.status_code, 67 | "headers": self.raw_headers, 68 | } 69 | ) 70 | 71 | if "callbacks" in self.config: 72 | for callback in self.config["callbacks"]: 73 | if hasattr(callback, "send"): 74 | callback.send = send 75 | 76 | try: 77 | # TODO: migrate to `.ainvoke` when adding support 78 | # for LCEL 79 | if self.run_mode == ChainRunMode.ASYNC: 80 | outputs = await self.chain.acall(**self.config) 81 | else: 82 | loop = asyncio.get_event_loop() 83 | outputs = await loop.run_in_executor( 84 | None, partial(self.chain, **self.config) 85 | ) 86 | if self.background is not None: 87 | self.background.kwargs.update({"outputs": outputs}) 88 | except Exception as e: 89 | logger.error(f"chain runtime error: {e}") 90 | if self.background is not None: 91 | self.background.kwargs.update({"outputs": {}, "error": e}) 92 | chunk = ServerSentEvent( 93 | data=dict( 94 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 95 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 96 | ), 97 | event=Events.ERROR, 98 | ) 99 | await send( 100 | { 101 | "type": "http.response.body", 102 | "body": ensure_bytes(chunk, None), 103 | "more_body": True, 104 | } 105 | ) 106 | 107 | await send({"type": "http.response.body", "body": b"", "more_body": False}) 108 | -------------------------------------------------------------------------------- /lanarky/adapters/langchain/routing.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from typing import Any, Callable, Optional, Sequence 3 | 4 | from fastapi import params 5 | from fastapi.datastructures import Default 6 | from fastapi.routing import APIRoute, APIRouter, APIWebSocketRoute 7 | 8 | from .utils import build_factory_api_endpoint, build_factory_websocket_endpoint 9 | 10 | 11 | class LangchainAPIRoute(APIRoute): 12 | """APIRoute class for LangChain.""" 13 | 14 | def __init__( 15 | self, 16 | path: str, 17 | endpoint: Callable[..., Any], 18 | *, 19 | response_model: Any = Default(None), 20 | **kwargs: dict[str, Any], 21 | ) -> None: 22 | """Constructor method. 23 | 24 | Args: 25 | path: The path for the route. 26 | endpoint: The endpoint to call when the route is requested. 27 | response_model: The response model to use for the route. 28 | **kwargs: Keyword arguments to pass to the parent constructor. 29 | """ 30 | # NOTE: LangchainAPIRoute is initialised again when 31 | # router is included in app. This is a hack to 32 | # build the factory endpoint only once. 33 | if not inspect.iscoroutinefunction(endpoint): 34 | factory_endpoint = build_factory_api_endpoint(path, endpoint) 35 | super().__init__( 36 | path, factory_endpoint, response_model=response_model, **kwargs 37 | ) 38 | else: 39 | super().__init__(path, endpoint, response_model=response_model, **kwargs) 40 | 41 | 42 | class LangchainAPIWebSocketRoute(APIWebSocketRoute): 43 | """APIWebSocketRoute class for LangChain.""" 44 | 45 | def __init__( 46 | self, 47 | path: str, 48 | endpoint: Callable[..., Any], 49 | *, 50 | name: Optional[str] = None, 51 | **kwargs: dict[str, Any], 52 | ) -> None: 53 | """Constructor method. 54 | 55 | Args: 56 | path: The path for the route. 57 | endpoint: The endpoint to call when the route is requested. 58 | name: The name of the route. 59 | **kwargs: Keyword arguments to pass to the parent constructor. 60 | """ 61 | super().__init__(path, endpoint, name=name, **kwargs) 62 | # NOTE: LangchainAPIRoute is initialised again when 63 | # router is included in app. This is a hack to 64 | # build the factory endpoint only once. 65 | if not inspect.iscoroutinefunction(endpoint): 66 | factory_endpoint = build_factory_websocket_endpoint(path, endpoint) 67 | super().__init__(path, factory_endpoint, name=name, **kwargs) 68 | else: 69 | super().__init__(path, endpoint, name=name, **kwargs) 70 | 71 | 72 | class LangchainAPIRouter(APIRouter): 73 | """APIRouter class for LangChain.""" 74 | 75 | def __init__(self, *, route_class: type[APIRoute] = LangchainAPIRoute, **kwargs): 76 | super().__init__(route_class=route_class, **kwargs) 77 | 78 | def add_api_websocket_route( 79 | self, 80 | path: str, 81 | endpoint: Callable[..., Any], 82 | name: Optional[str] = None, 83 | *, 84 | dependencies: Optional[Sequence[params.Depends]] = None, 85 | ) -> None: 86 | current_dependencies = self.dependencies.copy() 87 | if dependencies: 88 | current_dependencies.extend(dependencies) 89 | 90 | route = LangchainAPIWebSocketRoute( 91 | self.prefix + path, 92 | endpoint=endpoint, 93 | name=name, 94 | dependencies=current_dependencies, 95 | dependency_overrides_provider=self.dependency_overrides_provider, 96 | ) 97 | self.routes.append(route) 98 | -------------------------------------------------------------------------------- /lanarky/adapters/langchain/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Any, Awaitable, Callable 3 | 4 | from fastapi import Depends 5 | from langchain.agents import AgentExecutor 6 | from langchain.chains.base import Chain 7 | from langchain.schema.document import Document 8 | from pydantic import BaseModel, create_model 9 | from starlette.routing import compile_path 10 | 11 | from lanarky.adapters.langchain.callbacks import ( 12 | FinalTokenStreamingCallbackHandler, 13 | FinalTokenWebSocketCallbackHandler, 14 | SourceDocumentsStreamingCallbackHandler, 15 | SourceDocumentsWebSocketCallbackHandler, 16 | TokenStreamingCallbackHandler, 17 | TokenWebSocketCallbackHandler, 18 | ) 19 | from lanarky.adapters.langchain.responses import HTTPStatusDetail, StreamingResponse 20 | from lanarky.events import Events 21 | from lanarky.logging import logger 22 | from lanarky.utils import model_dump 23 | from lanarky.websockets import WebSocket, WebsocketSession 24 | 25 | 26 | def build_factory_api_endpoint( 27 | path: str, endpoint: Callable[..., Any] 28 | ) -> Callable[..., Awaitable[Any]]: 29 | """Build a factory endpoint for API routes. 30 | 31 | Args: 32 | path: The path for the route. 33 | endpoint: LangChain instance factory function. 34 | """ 35 | chain = compile_chain_factory(endpoint) 36 | 37 | # index 1 of `compile_path` contains path_format output 38 | model_prefix = compile_model_prefix(compile_path(path)[1], chain) 39 | request_model = create_request_model(chain, model_prefix) 40 | 41 | callbacks = get_streaming_callbacks(chain) 42 | 43 | async def factory_endpoint( 44 | request: request_model, chain: Chain = Depends(endpoint) 45 | ): 46 | return StreamingResponse( 47 | chain=chain, config={"inputs": model_dump(request), "callbacks": callbacks} 48 | ) 49 | 50 | return factory_endpoint 51 | 52 | 53 | def build_factory_websocket_endpoint( 54 | path: str, endpoint: Callable[..., Any] 55 | ) -> Callable[..., Awaitable[Any]]: 56 | """Build a factory endpoint for WebSocket routes. 57 | 58 | Args: 59 | path: The path for the route. 60 | endpoint: LangChain instance factory function. 61 | """ 62 | chain = compile_chain_factory(endpoint) 63 | 64 | # index 1 of `compile_path` contains path_format output 65 | model_prefix = compile_model_prefix(compile_path(path)[1], chain) 66 | request_model = create_request_model(chain, model_prefix) 67 | 68 | async def factory_endpoint(websocket: WebSocket, chain: Chain = Depends(endpoint)): 69 | callbacks = get_websocket_callbacks(chain, websocket) 70 | async with WebsocketSession().connect(websocket) as session: 71 | async for data in session: 72 | try: 73 | await chain.acall( 74 | inputs=model_dump(request_model(**data)), 75 | callbacks=callbacks, 76 | ) 77 | except Exception as e: 78 | logger.error(f"langchain error: {e}") 79 | await websocket.send_json( 80 | dict( 81 | data=dict( 82 | status=500, 83 | detail=HTTPStatusDetail( 84 | code=500, 85 | message="Internal Server Error", 86 | ), 87 | ), 88 | event=Events.ERROR, 89 | ) 90 | ) 91 | await websocket.send_json(dict(data="", event=Events.END)) 92 | 93 | return factory_endpoint 94 | 95 | 96 | def compile_chain_factory(endpoint: Callable[..., Any]): 97 | """Compile a LangChain instance factory function. 98 | 99 | Args: 100 | endpoint: LangChain instance factory function. 101 | """ 102 | try: 103 | chain = endpoint() 104 | except TypeError: 105 | raise TypeError("set default values for all factory endpoint parameters") 106 | 107 | if not isinstance(chain, Chain): 108 | raise TypeError("factory endpoint must return a Chain instance") 109 | return chain 110 | 111 | 112 | def create_request_model(chain: Chain, prefix: str = "") -> BaseModel: 113 | """Create a pydantic request model for a LangChain instance. 114 | 115 | Args: 116 | chain: A LangChain instance. 117 | prefix: A prefix for the model name. 118 | """ 119 | request_fields = {} 120 | 121 | for key in chain.input_keys: 122 | # TODO: add support for other input key types 123 | # based on demand 124 | if key == "chat_history": 125 | request_fields[key] = (list[tuple[str, str]], ...) 126 | else: 127 | request_fields[key] = (str, ...) 128 | 129 | prefix = prefix or chain.__class__.__name__ 130 | 131 | return create_model(f"{prefix}Request", **request_fields) 132 | 133 | 134 | def create_response_model(chain: Chain, prefix: str = None) -> BaseModel: 135 | """Create a pydantic response model for a LangChain instance. 136 | 137 | Args: 138 | chain: A LangChain instance. 139 | prefix: A prefix for the model name. 140 | """ 141 | response_fields = {} 142 | 143 | for key in chain.output_keys: 144 | # TODO: add support for other output key types 145 | # based on demand 146 | if key == "source_documents": 147 | response_fields[key] = (list[Document], ...) 148 | else: 149 | response_fields[key] = (str, ...) 150 | 151 | prefix = prefix or chain.__class__.__name__ 152 | 153 | return create_model(f"{prefix}Response", **response_fields) 154 | 155 | 156 | def compile_model_prefix(path: str, chain: Chain) -> str: 157 | """Compile a prefix for pydantic models. 158 | 159 | Args: 160 | path: The path for the route. 161 | chain: A LangChain instance. 162 | """ 163 | # Remove placeholders like '{item}' using regex 164 | path_wo_params = re.sub(r"\{.*?\}", "", path) 165 | path_prefix = "".join([part.capitalize() for part in path_wo_params.split("/")]) 166 | 167 | chain_prefix = chain.__class__.__name__ 168 | 169 | return f"{path_prefix}{chain_prefix}" 170 | 171 | 172 | def get_streaming_callbacks(chain: Chain) -> list[Callable]: 173 | """Get streaming callbacks for a LangChain instance. 174 | 175 | Note: This function might not support all LangChain 176 | chain and agent types. Please open an issue on GitHub to 177 | request support for a specific type. 178 | 179 | Args: 180 | chain: A LangChain instance. 181 | """ 182 | callbacks = [] 183 | 184 | if "source_documents" in chain.output_keys: 185 | callbacks.append(SourceDocumentsStreamingCallbackHandler()) 186 | 187 | if len(set(chain.output_keys) - {"source_documents"}) > 1: 188 | logger.warning( 189 | f"""multiple output keys found: {set(chain.output_keys) - {'source_documents'}}. 190 | 191 | Only the first output key will be used for streaming tokens. For more complex API logic, define the endpoint function manually. 192 | """ 193 | ) 194 | 195 | if isinstance(chain, AgentExecutor): 196 | callbacks.append(FinalTokenStreamingCallbackHandler()) 197 | else: 198 | callbacks.extend( 199 | [ 200 | TokenStreamingCallbackHandler(output_key=chain.output_keys[0]), 201 | ] 202 | ) 203 | 204 | return callbacks 205 | 206 | 207 | def get_websocket_callbacks(chain: Chain, websocket: WebSocket) -> list[Callable]: 208 | """Get websocket callbacks for a LangChain instance. 209 | 210 | Note: This function might not support all LangChain 211 | chain and agent types. Please open an issue on GitHub to 212 | request support for a specific type. 213 | 214 | Args: 215 | chain: A LangChain instance. 216 | websocket: A WebSocket instance. 217 | """ 218 | callbacks = [] 219 | 220 | if "source_documents" in chain.output_keys: 221 | callbacks.append(SourceDocumentsWebSocketCallbackHandler(websocket=websocket)) 222 | 223 | if len(set(chain.output_keys) - {"source_documents"}) > 1: 224 | logger.warning( 225 | f"""multiple output keys found: {set(chain.output_keys) - {'source_documents'}}. 226 | 227 | Only the first output key will be used for sending tokens. For more complex websocket logic, define the endpoint function manually. 228 | """ 229 | ) 230 | 231 | if isinstance(chain, AgentExecutor): 232 | callbacks.append(FinalTokenWebSocketCallbackHandler(websocket=websocket)) 233 | else: 234 | callbacks.extend( 235 | [ 236 | TokenWebSocketCallbackHandler( 237 | output_key=chain.output_keys[0], websocket=websocket 238 | ), 239 | ] 240 | ) 241 | 242 | return callbacks 243 | -------------------------------------------------------------------------------- /lanarky/adapters/openai/__init__.py: -------------------------------------------------------------------------------- 1 | from importlib.util import find_spec 2 | 3 | if not find_spec("openai"): 4 | raise ImportError("run `pip install lanarky[openai]` to use the openai adapter") 5 | -------------------------------------------------------------------------------- /lanarky/adapters/openai/dependencies.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional 2 | 3 | from fastapi import params 4 | 5 | from lanarky.utils import model_dump 6 | 7 | from .resources import OpenAIResource 8 | from .utils import create_request_model, create_response_model 9 | 10 | 11 | def Depends( 12 | dependency: Optional[Callable[..., Any]], 13 | *, 14 | dependency_kwargs: dict[str, Any] = {}, 15 | use_cache: bool = True 16 | ) -> params.Depends: 17 | """Dependency injection for OpenAI. 18 | 19 | Args: 20 | dependency: a "dependable" resource factory callable. 21 | dependency_kwargs: kwargs to pass to resource dependency. 22 | use_cache: use_cache parameter of `fastapi.Depends`. 23 | """ 24 | try: 25 | resource = dependency() 26 | except TypeError: 27 | raise TypeError("set default values for all dependency parameters") 28 | 29 | if not isinstance(resource, OpenAIResource): 30 | raise TypeError("dependency must return a OpenAIResource instance") 31 | 32 | request_model = create_request_model(resource) 33 | response_model = create_response_model(resource) 34 | 35 | async def resource_dependency( 36 | request: request_model, 37 | resource: OpenAIResource = params.Depends(dependency, use_cache=use_cache), 38 | ) -> response_model: 39 | resource_kwargs = {**model_dump(request), **dependency_kwargs} 40 | 41 | return await resource(**resource_kwargs) 42 | 43 | return params.Depends(resource_dependency, use_cache=use_cache) 44 | -------------------------------------------------------------------------------- /lanarky/adapters/openai/resources.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | from typing import Any, Generator 3 | 4 | from openai import AsyncOpenAI 5 | from openai.types.chat import ChatCompletion, ChatCompletionChunk 6 | from pydantic import BaseModel, Field 7 | 8 | from lanarky.utils import model_dump 9 | 10 | 11 | class Message(BaseModel): 12 | role: str = Field(pattern=r"^(user|assistant)$") 13 | content: str 14 | 15 | 16 | class SystemMessage(BaseModel): 17 | role: str = "system" 18 | content: str 19 | 20 | 21 | class OpenAIResource: 22 | """Base class for OpenAI resources.""" 23 | 24 | def __init__(self, client: AsyncOpenAI = None): 25 | self._client = client or AsyncOpenAI() 26 | 27 | @abstractmethod 28 | async def stream_response( 29 | self, *args: Any, **kwargs: dict[str, Any] 30 | ) -> Generator[str, None, None]: ... 31 | 32 | 33 | class ChatCompletionResource(OpenAIResource): 34 | """OpenAIResource class for chat completions.""" 35 | 36 | def __init__( 37 | self, 38 | *, 39 | client: AsyncOpenAI = None, 40 | model: str = "gpt-3.5-turbo", 41 | stream: bool = False, 42 | system: str = None, 43 | **create_kwargs: dict[str, Any], 44 | ): 45 | """Constructor method. 46 | 47 | Args: 48 | client: An AsyncOpenAI instance. 49 | model: The model to use for completions. 50 | stream: Whether to stream completions. 51 | system: A system message to prepend to the messages. 52 | **create_kwargs: Keyword arguments to pass to the `chat.completions.create` method. 53 | """ 54 | super().__init__(client=client) 55 | 56 | self.model = model 57 | self.stream = stream 58 | self.system = SystemMessage(content=system) if system else None 59 | self.create_kwargs = create_kwargs 60 | 61 | async def stream_response(self, messages: list[dict]) -> Generator[str, None, None]: 62 | """Stream chat completions. 63 | 64 | If `stream` attribute is False, the generator will yield only one completion. 65 | Otherwise, it will yield chunk completions. 66 | 67 | Args: 68 | messages: A list of messages to use for the completion. 69 | message format: {"role": "user", "content": "Hello, world!"} 70 | """ 71 | messages = self._prepare_messages(messages) 72 | data = await self._client.chat.completions.create( 73 | messages=messages, 74 | model=self.model, 75 | stream=self.stream, 76 | **self.create_kwargs, 77 | ) 78 | 79 | if self.stream: 80 | async for chunk in data: 81 | if not isinstance(chunk, ChatCompletionChunk): 82 | raise TypeError(f"Unexpected data type: {type(data)}") 83 | if chunk.choices[0].delta.content is not None: 84 | yield chunk.choices[0].delta.content 85 | else: 86 | if not isinstance(data, ChatCompletion): 87 | raise TypeError(f"Unexpected data type: {type(data)}") 88 | yield data.choices[0].message.content 89 | 90 | async def __call__(self, messages: list[dict]) -> ChatCompletion: 91 | """Create a chat completion. 92 | 93 | Args: 94 | messages: A list of messages to use for the completion. 95 | message format: {"role": "user", "content": "Hello, world!"} 96 | 97 | Returns: 98 | A ChatCompletion instance. 99 | """ 100 | messages = self._prepare_messages(messages) 101 | return await self._client.chat.completions.create( 102 | messages=messages, 103 | model=self.model, 104 | **self.create_kwargs, 105 | ) 106 | 107 | def _prepare_messages(self, messages: list[dict]) -> list[dict]: 108 | if self.system is not None: 109 | messages = [model_dump(self.system)] + messages 110 | return messages 111 | -------------------------------------------------------------------------------- /lanarky/adapters/openai/responses.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from fastapi import status 4 | from starlette.types import Send 5 | 6 | from lanarky.adapters.openai.resources import Message 7 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 8 | from lanarky.logging import logger 9 | from lanarky.responses import HTTPStatusDetail 10 | from lanarky.responses import StreamingResponse as _StreamingResponse 11 | 12 | from .resources import OpenAIResource 13 | 14 | 15 | class StreamingResponse(_StreamingResponse): 16 | """StreamingResponse class for OpenAI resources.""" 17 | 18 | def __init__( 19 | self, 20 | resource: OpenAIResource, 21 | messages: list[Message], 22 | *args: Any, 23 | **kwargs: dict[str, Any], 24 | ) -> None: 25 | """Constructor method. 26 | 27 | Args: 28 | resource: An OpenAIResource instance. 29 | messages: A list of `Message` instances. 30 | *args: Positional arguments to pass to the parent constructor. 31 | **kwargs: Keyword arguments to pass to the parent constructor. 32 | """ 33 | super().__init__(*args, **kwargs) 34 | 35 | self.resource = resource 36 | self.messages = messages 37 | 38 | async def stream_response(self, send: Send) -> None: 39 | """Stream chat completions. 40 | 41 | If an exception occurs while iterating over the OpenAI resource, an 42 | internal server error is sent to the client. 43 | 44 | Args: 45 | send: The ASGI send callable. 46 | """ 47 | await send( 48 | { 49 | "type": "http.response.start", 50 | "status": self.status_code, 51 | "headers": self.raw_headers, 52 | } 53 | ) 54 | 55 | try: 56 | async for chunk in self.resource.stream_response(self.messages): 57 | event_body = ServerSentEvent( 58 | data=chunk, 59 | event=Events.COMPLETION, 60 | ) 61 | await send( 62 | { 63 | "type": "http.response.body", 64 | "body": ensure_bytes(event_body, None), 65 | "more_body": True, 66 | } 67 | ) 68 | except Exception as e: 69 | logger.error(f"openai error: {e}") 70 | error_event_body = ServerSentEvent( 71 | data=dict( 72 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 73 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 74 | ), 75 | event=Events.ERROR, 76 | ) 77 | await send( 78 | { 79 | "type": "http.response.body", 80 | "body": ensure_bytes(error_event_body, None), 81 | "more_body": True, 82 | } 83 | ) 84 | 85 | await send({"type": "http.response.body", "body": b"", "more_body": False}) 86 | -------------------------------------------------------------------------------- /lanarky/adapters/openai/routing.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from typing import Any, Callable, Optional, Sequence 3 | 4 | from fastapi import params 5 | from fastapi.datastructures import Default 6 | from fastapi.routing import APIRoute, APIRouter, APIWebSocketRoute 7 | 8 | from .utils import build_factory_api_endpoint, build_factory_websocket_endpoint 9 | 10 | 11 | class OpenAIAPIRoute(APIRoute): 12 | """APIRoute class for OpenAI resources.""" 13 | 14 | def __init__( 15 | self, 16 | path: str, 17 | endpoint: Callable[..., Any], 18 | *, 19 | response_model: Any = Default(None), 20 | **kwargs: dict[str, Any], 21 | ) -> None: 22 | """Constructor method. 23 | 24 | Args: 25 | path: The path for the route. 26 | endpoint: The endpoint to call when the route is requested. 27 | response_model: The response model to use for the route. 28 | **kwargs: Keyword arguments to pass to the parent constructor. 29 | """ 30 | # NOTE: OpenAIAPIRoute is initialised again when 31 | # router is included in app. This is a hack to 32 | # build the factory endpoint only once. 33 | if not inspect.iscoroutinefunction(endpoint): 34 | factory_endpoint = build_factory_api_endpoint(path, endpoint) 35 | super().__init__( 36 | path, factory_endpoint, response_model=response_model, **kwargs 37 | ) 38 | else: 39 | super().__init__(path, endpoint, response_model=response_model, **kwargs) 40 | 41 | 42 | class OpenAIAPIWebSocketRoute(APIWebSocketRoute): 43 | """APIWebSocketRoute class for OpenAI resources.""" 44 | 45 | def __init__( 46 | self, 47 | path: str, 48 | endpoint: Callable[..., Any], 49 | *, 50 | name: Optional[str] = None, 51 | **kwargs: dict[str, Any], 52 | ) -> None: 53 | """Constructor method. 54 | 55 | Args: 56 | path: The path for the route. 57 | endpoint: The endpoint to call when the route is requested. 58 | name: The name of the route. 59 | **kwargs: Keyword arguments to pass to the parent constructor. 60 | """ 61 | super().__init__(path, endpoint, name=name, **kwargs) 62 | # NOTE: OpenAIAPIRoute is initialised again when 63 | # router is included in app. This is a hack to 64 | # build the factory endpoint only once. 65 | if not inspect.iscoroutinefunction(endpoint): 66 | factory_endpoint = build_factory_websocket_endpoint(path, endpoint) 67 | super().__init__(path, factory_endpoint, name=name, **kwargs) 68 | else: 69 | super().__init__(path, endpoint, name=name, **kwargs) 70 | 71 | 72 | class OpenAIAPIRouter(APIRouter): 73 | """APIRouter class for OpenAI resources.""" 74 | 75 | def __init__( 76 | self, *, route_class: type[APIRoute] = OpenAIAPIRoute, **kwargs: dict[str, Any] 77 | ): 78 | super().__init__(route_class=route_class, **kwargs) 79 | 80 | def add_api_websocket_route( 81 | self, 82 | path: str, 83 | endpoint: Callable[..., Any], 84 | name: Optional[str] = None, 85 | *, 86 | dependencies: Optional[Sequence[params.Depends]] = None, 87 | ) -> None: 88 | current_dependencies = self.dependencies.copy() 89 | if dependencies: 90 | current_dependencies.extend(dependencies) 91 | 92 | route = OpenAIAPIWebSocketRoute( 93 | self.prefix + path, 94 | endpoint=endpoint, 95 | name=name, 96 | dependencies=current_dependencies, 97 | dependency_overrides_provider=self.dependency_overrides_provider, 98 | ) 99 | self.routes.append(route) 100 | -------------------------------------------------------------------------------- /lanarky/adapters/openai/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Any, Awaitable, Callable 3 | 4 | from fastapi import Depends 5 | from pydantic import BaseModel, create_model 6 | from starlette.routing import compile_path 7 | 8 | from lanarky.events import Events 9 | from lanarky.logging import logger 10 | from lanarky.utils import model_dump, model_fields 11 | from lanarky.websockets import WebSocket, WebsocketSession 12 | 13 | from .resources import ChatCompletion, ChatCompletionResource, Message, OpenAIResource 14 | from .responses import HTTPStatusDetail, StreamingResponse, status 15 | 16 | 17 | def build_factory_api_endpoint( 18 | path: str, endpoint: Callable[..., Any] 19 | ) -> Callable[..., Awaitable[Any]]: 20 | """Build a factory endpoint for API routes. 21 | 22 | Args: 23 | path: The path for the route. 24 | endpoint: openai resource factory function. 25 | """ 26 | resource = compile_openai_resource_factory(endpoint) 27 | 28 | # index 1 of `compile_path` contains path_format output 29 | model_prefix = compile_model_prefix(compile_path(path)[1], resource) 30 | request_model = create_request_model(resource, model_prefix) 31 | 32 | async def factory_endpoint( 33 | request: request_model, resource: OpenAIResource = Depends(endpoint) 34 | ): 35 | return StreamingResponse(resource=resource, **model_dump(request)) 36 | 37 | return factory_endpoint 38 | 39 | 40 | def build_factory_websocket_endpoint( 41 | path: str, endpoint: Callable[..., Any] 42 | ) -> Callable[..., Awaitable[Any]]: 43 | """Build a factory endpoint for WebSocket routes. 44 | 45 | Args: 46 | path: The path for the route. 47 | endpoint: openai resource factory function. 48 | """ 49 | resource = compile_openai_resource_factory(endpoint) 50 | 51 | # index 1 of `compile_path` contains path_format output 52 | model_prefix = compile_model_prefix(compile_path(path)[1], resource) 53 | request_model = create_request_model(resource, model_prefix) 54 | 55 | async def factory_endpoint( 56 | websocket: WebSocket, resource: OpenAIResource = Depends(endpoint) 57 | ): 58 | async with WebsocketSession().connect(websocket) as session: 59 | async for data in session: 60 | try: 61 | async for chunk in resource.stream_response( 62 | **model_dump(request_model(**data)) 63 | ): 64 | await websocket.send_json( 65 | dict( 66 | data=chunk, 67 | event=Events.COMPLETION, 68 | ) 69 | ) 70 | except Exception as e: 71 | logger.error(f"openai error: {e}") 72 | await websocket.send_json( 73 | dict( 74 | data=dict( 75 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 76 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 77 | ), 78 | event=Events.ERROR, 79 | ) 80 | ) 81 | await websocket.send_json(dict(data="", event=Events.END)) 82 | 83 | return factory_endpoint 84 | 85 | 86 | def compile_openai_resource_factory(endpoint: Callable[..., Any]) -> OpenAIResource: 87 | """Compile an OpenAI resource factory function. 88 | 89 | Args: 90 | endpoint: openai resource factory function. 91 | 92 | Returns: 93 | An OpenAIResource instance. 94 | """ 95 | try: 96 | resource = endpoint() 97 | except TypeError: 98 | raise TypeError("set default values for all factory endpoint parameters") 99 | 100 | if not isinstance(resource, OpenAIResource): 101 | raise TypeError("factory endpoint must return a LanarkyOpenAIResource instance") 102 | return resource 103 | 104 | 105 | def compile_model_prefix(path: str, resource: OpenAIResource) -> str: 106 | """Compile a prefix for pydantic models. 107 | 108 | Args: 109 | path: The path for the route. 110 | resource: An OpenAIResource instance. 111 | """ 112 | # Remove placeholders like '{item}' using regex 113 | path_wo_params = re.sub(r"\{.*?\}", "", path) 114 | path_prefix = "".join([part.capitalize() for part in path_wo_params.split("/")]) 115 | 116 | resource_prefix = resource.__class__.__name__ 117 | 118 | return f"{path_prefix}{resource_prefix}" 119 | 120 | 121 | def create_request_model( 122 | resource: ChatCompletionResource, prefix: str = "" 123 | ) -> BaseModel: 124 | """Create a pydantic model for incoming requests. 125 | 126 | Note: Support limited to ChatCompletion resource. 127 | 128 | Args: 129 | resource: An OpenAIResource instance. 130 | prefix: A prefix for the model name. 131 | """ 132 | if not isinstance(resource, ChatCompletionResource): 133 | raise TypeError("resource must be a ChatCompletion instance") 134 | 135 | request_fields = {"messages": (list[Message], ...)} 136 | 137 | prefix = prefix or resource.__class__.__name__ 138 | 139 | return create_model(f"{prefix}Request", **request_fields) 140 | 141 | 142 | def create_response_model( 143 | resource: ChatCompletionResource, prefix: str = None 144 | ) -> BaseModel: 145 | """Create a pydantic model for responses. 146 | 147 | Note: Support limited to ChatCompletion resource. 148 | 149 | Args: 150 | resource: An OpenAIResource instance. 151 | prefix: A prefix for the model name. 152 | """ 153 | if not isinstance(resource, ChatCompletionResource): 154 | raise TypeError("resource must be a ChatCompletion instance") 155 | 156 | response_fields = { 157 | k: (v.annotation, ...) for k, v in model_fields(ChatCompletion).items() 158 | } 159 | 160 | prefix = prefix or resource.__class__.__name__ 161 | 162 | return create_model(f"{prefix}Response", **response_fields) 163 | -------------------------------------------------------------------------------- /lanarky/applications.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from fastapi.applications import AppType, FastAPI 4 | from fastapi.openapi.docs import ( 5 | get_redoc_html, 6 | get_swagger_ui_html, 7 | get_swagger_ui_oauth2_redirect_html, 8 | ) 9 | from fastapi.requests import Request 10 | from fastapi.responses import HTMLResponse, JSONResponse 11 | 12 | 13 | class Lanarky(FastAPI): 14 | """The main application class. 15 | 16 | To know more about the FastAPI parameters, read the 17 | [FastAPI documentation](https://fastapi.tiangolo.com/reference/fastapi/). 18 | """ 19 | 20 | def __init__( 21 | self: AppType, *, title: str = "Lanarky", **kwargs: dict[str, Any] 22 | ) -> None: 23 | """Constructor method. 24 | 25 | Args: 26 | title: The title of the application. 27 | **kwargs: Additional arguments to pass to the FastAPI constructor. 28 | """ 29 | super().__init__(title=title, **kwargs) 30 | 31 | def setup(self) -> None: # pragma: no cover 32 | """Setup the application. 33 | 34 | Overrides the `setup` method of the FastAPI class. 35 | """ 36 | if self.openapi_url: 37 | urls = (server_data.get("url") for server_data in self.servers) 38 | server_urls = {url for url in urls if url} 39 | 40 | async def openapi(req: Request) -> JSONResponse: 41 | root_path = req.scope.get("root_path", "").rstrip("/") 42 | if root_path not in server_urls: 43 | if root_path and self.root_path_in_servers: 44 | self.servers.insert(0, {"url": root_path}) 45 | server_urls.add(root_path) 46 | return JSONResponse(self.openapi()) 47 | 48 | self.add_route(self.openapi_url, openapi, include_in_schema=False) 49 | if self.openapi_url and self.docs_url: 50 | 51 | async def swagger_ui_html(req: Request) -> HTMLResponse: 52 | root_path = req.scope.get("root_path", "").rstrip("/") 53 | openapi_url = root_path + self.openapi_url 54 | oauth2_redirect_url = self.swagger_ui_oauth2_redirect_url 55 | if oauth2_redirect_url: 56 | oauth2_redirect_url = root_path + oauth2_redirect_url 57 | return get_swagger_ui_html( 58 | openapi_url=openapi_url, 59 | title=self.title + " - Swagger UI", 60 | oauth2_redirect_url=oauth2_redirect_url, 61 | init_oauth=self.swagger_ui_init_oauth, 62 | swagger_ui_parameters=self.swagger_ui_parameters, 63 | swagger_favicon_url="https://lanarky.ajndkr.com/assets/icon.svg", 64 | ) 65 | 66 | self.add_route(self.docs_url, swagger_ui_html, include_in_schema=False) 67 | 68 | if self.swagger_ui_oauth2_redirect_url: 69 | 70 | async def swagger_ui_redirect(req: Request) -> HTMLResponse: 71 | return get_swagger_ui_oauth2_redirect_html() 72 | 73 | self.add_route( 74 | self.swagger_ui_oauth2_redirect_url, 75 | swagger_ui_redirect, 76 | include_in_schema=False, 77 | ) 78 | if self.openapi_url and self.redoc_url: 79 | 80 | async def redoc_html(req: Request) -> HTMLResponse: 81 | root_path = req.scope.get("root_path", "").rstrip("/") 82 | openapi_url = root_path + self.openapi_url 83 | return get_redoc_html( 84 | openapi_url=openapi_url, 85 | title=self.title + " - ReDoc", 86 | redoc_favicon_url="https://lanarky.ajndkr.com/assets/icon.svg", 87 | ) 88 | 89 | self.add_route(self.redoc_url, redoc_html, include_in_schema=False) 90 | -------------------------------------------------------------------------------- /lanarky/clients.py: -------------------------------------------------------------------------------- 1 | import json 2 | from contextlib import contextmanager 3 | from typing import Any, Iterator, Optional 4 | 5 | import httpx 6 | from httpx_sse import ServerSentEvent, connect_sse 7 | from websockets.sync.client import connect as websocket_connect 8 | 9 | from lanarky.websockets import DataMode 10 | 11 | 12 | class StreamingClient: 13 | """Test client for streaming server-sent events.""" 14 | 15 | def __init__( 16 | self, 17 | base_url: str = "http://localhost:8000", 18 | client: Optional[httpx.Client] = None, 19 | ): 20 | """Constructor method. 21 | 22 | Args: 23 | base_url: The base URL of the server. 24 | client: The HTTP client to use. 25 | """ 26 | self.base_url = base_url 27 | self.client = client or httpx.Client() 28 | 29 | def stream_response( 30 | self, method: str, path: str, **kwargs: dict[str, Any] 31 | ) -> Iterator[ServerSentEvent]: 32 | """Stream data from the server. 33 | 34 | Args: 35 | method: The HTTP method to use. 36 | path: The path to stream from. 37 | **kwargs: The keyword arguments to pass to the HTTP client. 38 | """ 39 | url = self.base_url + path 40 | with connect_sse(self.client, method, url, **kwargs) as event_source: 41 | for sse in event_source.iter_sse(): 42 | yield sse 43 | 44 | 45 | class WebSocketClient: 46 | """Test client for WebSockets. 47 | 48 | Supports 3 data modes: JSON, TEXT, and BYTES. 49 | """ 50 | 51 | def __init__( 52 | self, uri: str = "ws://localhost:8000/ws", mode: DataMode = DataMode.JSON 53 | ): 54 | """Constructor method. 55 | 56 | Args: 57 | uri: The URI of the websocket. 58 | mode: The data mode to use. 59 | """ 60 | self.uri = uri 61 | self.mode = mode 62 | self.websocket = None 63 | 64 | @contextmanager 65 | def connect(self): 66 | """Connect to a websocket and yield data from it.""" 67 | with websocket_connect(self.uri) as websocket: 68 | self.websocket = websocket 69 | yield self 70 | self.websocket = None 71 | 72 | def send(self, message: Any): 73 | """Send data to the websocket. 74 | 75 | Args: 76 | message: The data to send. 77 | """ 78 | if self.websocket: 79 | if self.mode == DataMode.JSON: 80 | message = json.dumps(message) 81 | elif self.mode == DataMode.TEXT: 82 | message = str(message) 83 | elif self.mode == DataMode.BYTES: 84 | message = message.encode("utf-8") 85 | self.websocket.send(message) 86 | 87 | def receive(self, mode: DataMode = None): 88 | """Receive data from the websocket. 89 | 90 | Args: 91 | mode: The data mode to use. 92 | """ 93 | mode = mode or self.mode 94 | if self.websocket: 95 | response = self.websocket.recv() 96 | if mode == DataMode.JSON: 97 | response = json.loads(response) 98 | elif mode == DataMode.TEXT: 99 | response = str(response) 100 | elif mode == DataMode.BYTES: 101 | response = response.decode("utf-8") 102 | return response 103 | 104 | def stream_response(self): 105 | """Stream data from the websocket. 106 | 107 | Streaming stops until the websocket is closed or 108 | the `end` event is received. 109 | """ 110 | if self.websocket: 111 | while True: 112 | response = self.receive(mode=DataMode.JSON) 113 | if response["event"] == "end": 114 | break 115 | yield response 116 | -------------------------------------------------------------------------------- /lanarky/events.py: -------------------------------------------------------------------------------- 1 | from sse_starlette.sse import ServerSentEvent as ServerSentEvent 2 | from sse_starlette.sse import ensure_bytes as ensure_bytes 3 | 4 | from lanarky.utils import StrEnum 5 | 6 | 7 | class Events(StrEnum): 8 | COMPLETION = "completion" 9 | ERROR = "error" 10 | END = "end" 11 | -------------------------------------------------------------------------------- /lanarky/logging.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from typing import Any 3 | 4 | import loguru 5 | 6 | 7 | def get_logger(handler: Any = sys.stderr, **kwargs: dict[str, Any]) -> Any: 8 | """Get a logger instance. 9 | 10 | Lanarky uses `loguru` for logging. 11 | 12 | Args: 13 | handler: The handler to use for the logger. 14 | 15 | Returns: 16 | A loguru logger instance. 17 | """ 18 | logger = loguru.logger 19 | logger.remove() 20 | logger.add(handler, **kwargs) 21 | return logger 22 | 23 | 24 | logger = get_logger() 25 | -------------------------------------------------------------------------------- /lanarky/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ajndkr/lanarky/43375b45f4e17cc0bd75752f2ef6b45829864cee/lanarky/py.typed -------------------------------------------------------------------------------- /lanarky/responses.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from fastapi import status 4 | from sse_starlette.sse import EventSourceResponse 5 | from starlette.types import Send 6 | 7 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 8 | from lanarky.logging import logger 9 | from lanarky.utils import StrEnum 10 | 11 | 12 | class HTTPStatusDetail(StrEnum): 13 | INTERNAL_SERVER_ERROR = "Internal Server Error" 14 | 15 | 16 | class StreamingResponse(EventSourceResponse): 17 | """`Response` class for streaming server-sent events. 18 | 19 | Follows the 20 | [EventSource protocol](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events#interfaces) 21 | """ 22 | 23 | def __init__( 24 | self, 25 | content: Any = iter(()), 26 | *args: Any, 27 | **kwargs: dict[str, Any], 28 | ) -> None: 29 | """Constructor method. 30 | 31 | Args: 32 | content: The content to stream. 33 | """ 34 | super().__init__(content=content, *args, **kwargs) 35 | 36 | async def stream_response(self, send: Send) -> None: 37 | """Streams data chunks to client by iterating over `content`. 38 | 39 | If an exception occurs while iterating over `content`, an 40 | internal server error is sent to the client. 41 | 42 | Args: 43 | send: The send function from the ASGI framework. 44 | """ 45 | await send( 46 | { 47 | "type": "http.response.start", 48 | "status": self.status_code, 49 | "headers": self.raw_headers, 50 | } 51 | ) 52 | 53 | try: 54 | async for data in self.body_iterator: 55 | chunk = ensure_bytes(data, self.sep) 56 | logger.debug(f"chunk: {chunk.decode()}") 57 | await send( 58 | {"type": "http.response.body", "body": chunk, "more_body": True} 59 | ) 60 | except Exception as e: 61 | logger.error(f"body iterator error: {e}") 62 | chunk = ServerSentEvent( 63 | data=dict( 64 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 65 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 66 | ), 67 | event=Events.ERROR, 68 | ) 69 | await send( 70 | { 71 | "type": "http.response.body", 72 | "body": ensure_bytes(chunk, None), 73 | "more_body": True, 74 | } 75 | ) 76 | 77 | await send({"type": "http.response.body", "body": b"", "more_body": False}) 78 | -------------------------------------------------------------------------------- /lanarky/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import pydantic 4 | from pydantic.fields import FieldInfo 5 | 6 | try: 7 | from enum import StrEnum # type: ignore 8 | except ImportError: 9 | from enum import Enum 10 | 11 | class StrEnum(str, Enum): ... 12 | 13 | 14 | PYDANTIC_V2 = pydantic.VERSION.startswith("2.") 15 | 16 | 17 | def model_dump(model: pydantic.BaseModel, **kwargs) -> dict[str, Any]: 18 | """Dump a pydantic model to a dictionary. 19 | 20 | Args: 21 | model: A pydantic model. 22 | """ 23 | if PYDANTIC_V2: 24 | return model.model_dump(**kwargs) 25 | else: 26 | return model.dict(**kwargs) 27 | 28 | 29 | def model_dump_json(model: pydantic.BaseModel, **kwargs) -> str: 30 | """Dump a pydantic model to a JSON string. 31 | 32 | Args: 33 | model: A pydantic model. 34 | """ 35 | if PYDANTIC_V2: 36 | return model.model_dump_json(**kwargs) 37 | else: 38 | return model.json(**kwargs) 39 | 40 | 41 | def model_fields(model: pydantic.BaseModel) -> dict[str, FieldInfo]: 42 | """Get the fields of a pydantic model. 43 | 44 | Args: 45 | model: A pydantic model. 46 | """ 47 | if PYDANTIC_V2: 48 | return model.model_fields 49 | else: 50 | return model.__fields__ 51 | -------------------------------------------------------------------------------- /lanarky/websockets.py: -------------------------------------------------------------------------------- 1 | from contextlib import asynccontextmanager 2 | from typing import Generator 3 | 4 | from fastapi.websockets import WebSocket, WebSocketDisconnect 5 | 6 | from lanarky.logging import logger 7 | from lanarky.utils import StrEnum 8 | 9 | 10 | class DataMode(StrEnum): 11 | JSON = "json" 12 | TEXT = "text" 13 | BYTES = "bytes" 14 | 15 | 16 | class WebsocketSession: 17 | """Class to handle websocket connections. 18 | 19 | Supports 3 data modes: JSON, TEXT, and BYTES. 20 | 21 | To know more about WebSockets, read the 22 | [FastAPI documentation](https://fastapi.tiangolo.com/advanced/websockets/). 23 | """ 24 | 25 | @asynccontextmanager 26 | async def connect( 27 | self, websocket: WebSocket, mode: DataMode = DataMode.JSON 28 | ) -> Generator: 29 | """Connect to a websocket and yield data from it. 30 | 31 | Args: 32 | websocket: The websocket to connect to. 33 | mode: The data mode to use. Defaults to DataMode.JSON. 34 | 35 | Yields: 36 | Any: data from client side. 37 | """ 38 | await websocket.accept() 39 | try: 40 | if mode == DataMode.JSON: 41 | yield self.iter_json(websocket) 42 | elif mode == DataMode.TEXT: 43 | yield self.iter_text(websocket) 44 | elif mode == DataMode.BYTES: 45 | yield self.iter_bytes(websocket) 46 | else: 47 | raise ValueError(f"Invalid DataMode: {mode}") 48 | except WebSocketDisconnect: 49 | logger.info("Websocket disconnected") 50 | 51 | async def iter_text(self, websocket: WebSocket): 52 | while True: 53 | data = await websocket.receive_text() 54 | yield data 55 | 56 | async def iter_bytes(self, websocket: WebSocket): 57 | while True: 58 | data = await websocket.receive_bytes() 59 | yield data 60 | 61 | async def iter_json(self, websocket: WebSocket): 62 | while True: 63 | data = await websocket.receive_json() 64 | yield data 65 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Lanarky 2 | site_url: https://lanarky.ajndkr.com/ 3 | site_author: Ajinkya Indulkar 4 | site_description: The web framework for building LLM microservices 5 | repo_name: ajndkr/lanarky 6 | repo_url: https://github.com/ajndkr/lanarky 7 | copyright: Copyright © 2023 Ajinkya Indulkar 8 | 9 | edit_uri: "" 10 | 11 | theme: 12 | name: material 13 | palette: 14 | - media: "(prefers-color-scheme: light)" 15 | scheme: lanarky 16 | toggle: 17 | icon: material/toggle-switch 18 | name: Switch to dark mode 19 | - media: "(prefers-color-scheme: dark)" 20 | scheme: slate 21 | toggle: 22 | icon: material/toggle-switch-off-outline 23 | name: Switch to light mode 24 | features: 25 | - search.suggest 26 | - search.highlight 27 | - navigation.path 28 | - navigation.sections 29 | - navigation.tabs 30 | - navigation.top 31 | - navigation.footer 32 | - navigation.indexes 33 | - navigation.tracking 34 | - content.tabs.link 35 | - content.tooltips 36 | - content.code.annotate 37 | - content.code.copy 38 | - content.code.select 39 | icon: 40 | repo: fontawesome/brands/github-alt 41 | logo: assets/icon.svg 42 | favicon: assets/favicon.png 43 | language: en 44 | font: 45 | text: Roboto 46 | code: Roboto Mono 47 | 48 | nav: 49 | - Lanarky: index.md 50 | - Getting Started: getting-started.md 51 | - Learn: 52 | - learn/index.md 53 | - Streaming: learn/streaming.md 54 | - WebSockets: learn/websockets.md 55 | - Adapters: 56 | - learn/adapters/index.md 57 | - OpenAI: 58 | - learn/adapters/openai/index.md 59 | - Router: learn/adapters/openai/router.md 60 | - Advanced: 61 | - Dependency Injection: learn/adapters/openai/dependency.md 62 | - FastAPI Backport: learn/adapters/openai/fastapi.md 63 | - LangChain: 64 | - learn/adapters/langchain/index.md 65 | - Router: learn/adapters/langchain/router.md 66 | - learn/adapters/langchain/callbacks.md 67 | - Advanced: 68 | - Dependency Injection: learn/adapters/langchain/dependency.md 69 | - FastAPI Backport: learn/adapters/langchain/fastapi.md 70 | - API Reference: 71 | - reference/index.md 72 | - Lanarky: reference/lanarky.md 73 | - Streaming: reference/streaming.md 74 | - WebSockets: reference/websockets.md 75 | - Adapters: 76 | - OpenAI: reference/adapters/openai.md 77 | - LangChain: reference/adapters/langchain.md 78 | - Miscellaneous: reference/misc.md 79 | 80 | markdown_extensions: 81 | - attr_list 82 | - md_in_html 83 | - toc: 84 | permalink: true 85 | - markdown.extensions.codehilite: 86 | guess_lang: false 87 | - mdx_include: 88 | base_path: docs 89 | - admonition 90 | - codehilite 91 | - extra 92 | - pymdownx.superfences 93 | - pymdownx.tabbed: 94 | alternate_style: true 95 | 96 | extra: 97 | social: 98 | - icon: fontawesome/brands/github 99 | link: https://github.com/ajndkr 100 | - icon: fontawesome/brands/python 101 | link: https://pypi.org/project/lanarky/ 102 | - icon: fontawesome/brands/twitter 103 | link: https://twitter.com/LanarkyAPI 104 | 105 | extra_css: 106 | - stylesheets/extra.css 107 | 108 | plugins: 109 | - social 110 | - termynal 111 | - mkdocstrings: 112 | handlers: 113 | python: 114 | options: 115 | show_root_heading: false 116 | inherited_members: true 117 | members_order: source 118 | separate_signature: true 119 | filters: ["!^_"] 120 | merge_init_into_class: true 121 | show_source: false 122 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "lanarky" 3 | version = "0.8.8" 4 | description = "The web framework for building LLM microservices" 5 | authors = ["Ajinkya Indulkar "] 6 | readme = "README.pypi.md" 7 | homepage = "https://lanarky.ajndkr.com/" 8 | repository = "https://github.com/ajndkr/lanarky" 9 | documentation = "https://lanarky.ajndkr.com/" 10 | license = "MIT" 11 | packages = [{include = "lanarky"}] 12 | 13 | [tool.poetry.dependencies] 14 | python = ">=3.9,<3.12" 15 | fastapi = ">=0.97.0" 16 | pydantic = ">=1,<3" 17 | sse-starlette = "^1.6.5" 18 | loguru = "^0.7.2" 19 | httpx-sse = "^0.3.1" 20 | websockets = "<12.0" 21 | openai = {version = "^1", optional = true} 22 | tiktoken = {version = "^0.4.0", optional = true} 23 | langchain = {version = "<0.3", optional = true} 24 | langchain-community = {version = "<0.3", optional = true} 25 | 26 | [tool.poetry.group.dev.dependencies] 27 | pre-commit = "^3.3.3" 28 | ipykernel = "^6.26.0" 29 | pyclean = "^2.7.5" 30 | uvicorn = {extras = ["standard"], version = "<1"} 31 | 32 | [tool.poetry.group.tests.dependencies] 33 | pytest = "^7.3.2" 34 | pytest-cov = "^4.1.0" 35 | pytest-asyncio = "^0.21.0" 36 | coveralls = "^3.3.1" 37 | 38 | [tool.poetry.group.docs.dependencies] 39 | mkdocs = "^1.5.3" 40 | mkdocs-material = {extras = ["imaging"], version = "^9.4.14"} 41 | mdx-include = "^1.4.2" 42 | termynal = "^0.11.1" 43 | mkdocstrings = {extras = ["python"], version = "^0.24.0"} 44 | 45 | [tool.poetry.group.examples.dependencies] 46 | faiss-cpu = "<1.8" 47 | gradio = "^4.7.1" 48 | 49 | [tool.poetry.extras] 50 | openai = ["openai", "tiktoken"] 51 | langchain = ["langchain", "langchain-community"] 52 | 53 | [build-system] 54 | requires = ["poetry-core"] 55 | build-backend = "poetry.core.masonry.api" 56 | 57 | [tool.ruff.lint] 58 | ignore = ["E501"] 59 | -------------------------------------------------------------------------------- /tests/adapters/langchain/test_langchain_callbacks.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, create_autospec, patch 2 | 3 | import pytest 4 | from langchain.agents import AgentExecutor 5 | from langchain.chains import ConversationChain 6 | from langchain.schema.document import Document 7 | from starlette.types import Send 8 | 9 | from lanarky.adapters.langchain import callbacks 10 | from lanarky.adapters.langchain.utils import ( 11 | get_streaming_callbacks, 12 | get_websocket_callbacks, 13 | ) 14 | from lanarky.websockets import WebSocket 15 | 16 | 17 | def test_always_verbose(): 18 | callback = callbacks.LanarkyCallbackHandler() 19 | assert callback.always_verbose is True 20 | 21 | 22 | def test_llm_cache_used(): 23 | callback = callbacks.LanarkyCallbackHandler() 24 | assert callback.llm_cache_used is False 25 | 26 | with patch("lanarky.adapters.langchain.callbacks.get_llm_cache") as get_llm_cache: 27 | from langchain_community.cache import InMemoryCache 28 | 29 | get_llm_cache.return_value = InMemoryCache() 30 | 31 | callback = callbacks.LanarkyCallbackHandler() 32 | assert callback.llm_cache_used is True 33 | 34 | 35 | def test_streaming_callback(send: Send): 36 | callback = callbacks.StreamingCallbackHandler() 37 | 38 | assert callback.send is None 39 | 40 | callback.send = send 41 | assert callback.send == send 42 | 43 | with pytest.raises(ValueError): 44 | callback.send = "non_callable_value" 45 | 46 | 47 | def test_websocket_callback(websocket: WebSocket): 48 | callback = callbacks.WebSocketCallbackHandler() 49 | 50 | assert callback.websocket is None 51 | 52 | callback.websocket = websocket 53 | assert callback.websocket == websocket 54 | 55 | with pytest.raises(ValueError): 56 | callback.websocket = "non_websocket_value" 57 | 58 | 59 | def test_callbacks_construct_message(): 60 | callback = callbacks.StreamingCallbackHandler() 61 | 62 | with patch("lanarky.adapters.langchain.callbacks.ServerSentEvent") as sse, patch( 63 | "lanarky.adapters.langchain.callbacks.ensure_bytes" 64 | ) as ensure_bytes: 65 | data = "test_data" 66 | event = "test_event" 67 | expected_return_value = { 68 | "type": "http.response.body", 69 | "body": ensure_bytes.return_value, 70 | "more_body": True, 71 | } 72 | 73 | result = callback._construct_message(data, event) 74 | 75 | sse.assert_called_with(data=data, event=event) 76 | ensure_bytes.assert_called_with(sse.return_value, None) 77 | 78 | assert result == expected_return_value 79 | 80 | callback = callbacks.WebSocketCallbackHandler() 81 | 82 | data = "test_data" 83 | event = "test_event" 84 | 85 | expected_return_value = dict(data=data, event=event) 86 | 87 | assert callback._construct_message(data, event) == expected_return_value 88 | 89 | 90 | def test_get_token(): 91 | with pytest.raises(ValueError): 92 | callbacks.get_token_data(token="test_token", mode="wrong_mode") 93 | 94 | assert callbacks.get_token_data(token="test_token", mode="text") == "test_token" 95 | assert ( 96 | callbacks.get_token_data(token="test_token", mode="json") 97 | == '{"token":"test_token"}' 98 | ) 99 | 100 | 101 | @pytest.mark.asyncio 102 | async def test_token_callbacks(send: Send, websocket: WebSocket): 103 | with pytest.raises(ValueError): 104 | callbacks.TokenStreamingCallbackHandler(mode="wrong_mode", output_key="dummy") 105 | 106 | callback = callbacks.TokenStreamingCallbackHandler(send=send, output_key="dummy") 107 | 108 | await callback.on_chain_start() 109 | assert not callback.streaming 110 | 111 | await callback.on_llm_new_token("test_token") 112 | assert callback.streaming 113 | assert not callback.llm_cache_used 114 | callback.send.assert_awaited() 115 | 116 | callback.llm_cache_used = True 117 | await callback.on_llm_new_token("test_token") 118 | assert not callback.llm_cache_used 119 | 120 | send.reset_mock() 121 | callback = callbacks.TokenStreamingCallbackHandler(send=send, output_key="dummy") 122 | outputs = {"dummy": "output_data"} 123 | await callback.on_chain_end(outputs) 124 | callback.send.assert_awaited() 125 | 126 | send.reset_mock() 127 | callback = callbacks.TokenStreamingCallbackHandler(send=send, output_key="dummy") 128 | callback.llm_cache_used = False 129 | callback.streaming = True 130 | await callback.on_chain_end(outputs) 131 | callback.send.assert_not_awaited() 132 | 133 | send.reset_mock() 134 | callback = callbacks.TokenStreamingCallbackHandler(send=send, output_key="dummy") 135 | callback.streaming = False 136 | with pytest.raises(KeyError): 137 | await callback.on_chain_end({"wrong_key": "output_data"}) 138 | 139 | with pytest.raises(ValueError): 140 | callbacks.TokenWebSocketCallbackHandler(mode="wrong_mode", output_key="dummy") 141 | 142 | callback = callbacks.TokenWebSocketCallbackHandler( 143 | websocket=websocket, output_key="dummy" 144 | ) 145 | 146 | await callback.on_chain_start() 147 | assert not callback.streaming 148 | 149 | await callback.on_llm_new_token("test_token") 150 | assert callback.streaming 151 | assert not callback.llm_cache_used 152 | callback.websocket.send_json.assert_awaited() 153 | 154 | callback.llm_cache_used = True 155 | await callback.on_llm_new_token("test_token") 156 | assert not callback.llm_cache_used 157 | 158 | websocket.send_json.reset_mock() 159 | callback = callbacks.TokenWebSocketCallbackHandler( 160 | websocket=websocket, output_key="dummy" 161 | ) 162 | outputs = {"dummy": "output_data"} 163 | await callback.on_chain_end(outputs) 164 | callback.websocket.send_json.assert_awaited() 165 | 166 | websocket.send_json.reset_mock() 167 | callback = callbacks.TokenWebSocketCallbackHandler( 168 | websocket=websocket, output_key="dummy" 169 | ) 170 | callback.llm_cache_used = False 171 | callback.streaming = True 172 | await callback.on_chain_end(outputs) 173 | callback.websocket.send_json.assert_not_awaited() 174 | 175 | websocket.send_json.reset_mock() 176 | callback = callbacks.TokenWebSocketCallbackHandler( 177 | websocket=websocket, output_key="dummy" 178 | ) 179 | callback.streaming = False 180 | with pytest.raises(KeyError): 181 | await callback.on_chain_end({"wrong_key": "output_data"}) 182 | 183 | 184 | @pytest.mark.asyncio 185 | async def test_source_documents_callbacks(send: Send, websocket: WebSocket): 186 | callback = callbacks.SourceDocumentsStreamingCallbackHandler(send=send) 187 | 188 | outputs = {"source_documents": "output_data"} 189 | with pytest.raises(ValueError): 190 | await callback.on_chain_end(outputs) 191 | 192 | outputs = {"source_documents": ["output_data"]} 193 | with pytest.raises(ValueError): 194 | await callback.on_chain_end(outputs) 195 | 196 | outputs = {"source_documents": [Document(page_content="test_content")]} 197 | await callback.on_chain_end(outputs) 198 | callback.send.assert_awaited() 199 | 200 | send.reset_mock() 201 | outputs = {"dummy": "output_data"} 202 | await callback.on_chain_end(outputs) 203 | callback.send.assert_not_awaited() 204 | 205 | callback = callbacks.SourceDocumentsWebSocketCallbackHandler(websocket=websocket) 206 | 207 | outputs = {"source_documents": "output_data"} 208 | with pytest.raises(ValueError): 209 | await callback.on_chain_end(outputs) 210 | 211 | outputs = {"source_documents": ["output_data"]} 212 | with pytest.raises(ValueError): 213 | await callback.on_chain_end(outputs) 214 | 215 | outputs = {"source_documents": [Document(page_content="test_content")]} 216 | await callback.on_chain_end(outputs) 217 | callback.websocket.send_json.assert_awaited() 218 | 219 | websocket.send_json.reset_mock() 220 | outputs = {"dummy": "output_data"} 221 | await callback.on_chain_end(outputs) 222 | callback.websocket.send_json.assert_not_awaited() 223 | 224 | 225 | @pytest.mark.asyncio 226 | async def test_final_token_callbacks(send: Send, websocket: WebSocket): 227 | callback = callbacks.FinalTokenStreamingCallbackHandler( 228 | send=send, stream_prefix=True 229 | ) 230 | 231 | await callback.on_llm_start() 232 | assert not callback.answer_reached 233 | assert not callback.streaming 234 | 235 | await callback.on_llm_new_token("test_token") 236 | assert callback.streaming 237 | 238 | callback.check_if_answer_reached = MagicMock(return_value=True) 239 | await callback.on_llm_new_token("test_token") 240 | assert callback.answer_reached 241 | callback.send.assert_awaited() 242 | 243 | send.reset_mock() 244 | callback = callbacks.FinalTokenStreamingCallbackHandler( 245 | send=send, stream_prefix=True 246 | ) 247 | callback.check_if_answer_reached = MagicMock(return_value=False) 248 | await callback.on_llm_new_token("test_token") 249 | assert not callback.answer_reached 250 | callback.send.assert_not_awaited() 251 | 252 | callback = callbacks.FinalTokenWebSocketCallbackHandler( 253 | websocket=websocket, stream_prefix=True 254 | ) 255 | 256 | await callback.on_llm_start() 257 | assert not callback.answer_reached 258 | assert not callback.streaming 259 | 260 | await callback.on_llm_new_token("test_token") 261 | assert callback.streaming 262 | 263 | callback.check_if_answer_reached = MagicMock(return_value=True) 264 | await callback.on_llm_new_token("test_token") 265 | assert callback.answer_reached 266 | callback.websocket.send_json.assert_awaited() 267 | 268 | websocket.send_json.reset_mock() 269 | callback = callbacks.FinalTokenWebSocketCallbackHandler( 270 | websocket=websocket, stream_prefix=True 271 | ) 272 | callback.check_if_answer_reached = MagicMock(return_value=False) 273 | await callback.on_llm_new_token("test_token") 274 | assert not callback.answer_reached 275 | callback.websocket.send_json.assert_not_awaited() 276 | 277 | 278 | @pytest.mark.asyncio 279 | async def test_get_callbacks(websocket: WebSocket): 280 | def chain_factory(): 281 | chain: ConversationChain = create_autospec(ConversationChain) 282 | chain.input_keys = ["input"] 283 | chain.output_keys = ["response", "source_documents"] 284 | return chain 285 | 286 | streaming_callbacks = get_streaming_callbacks(chain_factory()) 287 | assert len(streaming_callbacks) == 2 288 | 289 | def agent_factory(): 290 | agent: AgentExecutor = create_autospec(AgentExecutor) 291 | return agent 292 | 293 | websocket_callbacks = get_streaming_callbacks(agent_factory()) 294 | assert len(websocket_callbacks) == 1 295 | 296 | websocket_callbacks = get_websocket_callbacks(chain_factory(), websocket) 297 | assert len(websocket_callbacks) == 2 298 | 299 | websocket_callbacks = get_websocket_callbacks(agent_factory(), websocket) 300 | assert len(websocket_callbacks) == 1 301 | -------------------------------------------------------------------------------- /tests/adapters/langchain/test_langchain_dependencies.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, create_autospec 2 | 3 | import pytest 4 | from fastapi import params 5 | from langchain.chains import ConversationChain 6 | 7 | from lanarky.adapters.langchain.dependencies import Depends 8 | 9 | 10 | def test_depends_success(): 11 | def mock_dependency(): 12 | chain: ConversationChain = create_autospec(ConversationChain) 13 | chain.input_keys = ["input"] 14 | chain.output_keys = ["response"] 15 | return chain 16 | 17 | dependency = Depends(mock_dependency) 18 | assert isinstance(dependency, params.Depends) 19 | 20 | 21 | def test_depends_invalid_dependency(): 22 | def dependency_wo_defaults(arg1, arg2="default_value"): 23 | pass 24 | 25 | with pytest.raises(TypeError): 26 | Depends(dependency_wo_defaults) 27 | 28 | with pytest.raises(TypeError): 29 | Depends(lambda: MagicMock()) 30 | -------------------------------------------------------------------------------- /tests/adapters/langchain/test_langchain_responses.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | from unittest.mock import AsyncMock, MagicMock, call 3 | 4 | import pytest 5 | from langchain.chains.base import Chain 6 | from starlette.background import BackgroundTask 7 | from starlette.types import Send 8 | 9 | from lanarky.adapters.langchain.callbacks import TokenStreamingCallbackHandler 10 | from lanarky.adapters.langchain.responses import ( 11 | HTTPStatusDetail, 12 | StreamingResponse, 13 | status, 14 | ) 15 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 16 | 17 | 18 | @pytest.fixture 19 | def chain() -> Type[Chain]: 20 | return MagicMock(spec=Chain) 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_stream_response_successful(send: Send, chain: Type[Chain]): 25 | response = StreamingResponse( 26 | chain=chain, 27 | config={"callbacks": [TokenStreamingCallbackHandler(output_key="dummy")]}, 28 | background=BackgroundTask(lambda: None), 29 | ) 30 | 31 | chain.acall = AsyncMock(return_value={}) 32 | 33 | await response.stream_response(send) 34 | 35 | chain.acall.assert_called_once() 36 | 37 | expected_calls = [ 38 | call( 39 | { 40 | "type": "http.response.start", 41 | "status": response.status_code, 42 | "headers": response.raw_headers, 43 | } 44 | ), 45 | call( 46 | { 47 | "type": "http.response.body", 48 | "body": b"", 49 | "more_body": False, 50 | } 51 | ), 52 | ] 53 | 54 | send.assert_has_calls(expected_calls, any_order=False) 55 | 56 | assert "outputs" in response.background.kwargs 57 | 58 | 59 | @pytest.mark.asyncio 60 | async def test_stream_response_error(send: Send, chain: Type[Chain]): 61 | response = StreamingResponse( 62 | chain=chain, 63 | config={"callbacks": []}, 64 | background=BackgroundTask(lambda: None), 65 | ) 66 | 67 | chain.acall = AsyncMock(side_effect=Exception("Some error occurred")) 68 | 69 | await response.stream_response(send) 70 | 71 | chain.acall.assert_called_once() 72 | 73 | expected_calls = [ 74 | call( 75 | { 76 | "type": "http.response.start", 77 | "status": response.status_code, 78 | "headers": response.raw_headers, 79 | } 80 | ), 81 | call( 82 | { 83 | "type": "http.response.body", 84 | "body": ensure_bytes( 85 | ServerSentEvent( 86 | data=dict( 87 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 88 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 89 | ), 90 | event=Events.ERROR, 91 | ), 92 | None, 93 | ), 94 | "more_body": True, 95 | } 96 | ), 97 | call( 98 | { 99 | "type": "http.response.body", 100 | "body": b"", 101 | "more_body": False, 102 | } 103 | ), 104 | ] 105 | 106 | send.assert_has_calls(expected_calls, any_order=False) 107 | 108 | assert "error" in response.background.kwargs 109 | -------------------------------------------------------------------------------- /tests/adapters/langchain/test_langchain_routing.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import AsyncMock, MagicMock, create_autospec, patch 2 | 3 | import pytest 4 | from fastapi import Depends 5 | from langchain.chains import ConversationChain 6 | 7 | from lanarky.adapters.langchain.routing import ( 8 | LangchainAPIRoute, 9 | LangchainAPIRouter, 10 | LangchainAPIWebSocketRoute, 11 | ) 12 | 13 | 14 | def test_langchain_api_router(): 15 | router = LangchainAPIRouter() 16 | 17 | assert isinstance(router, LangchainAPIRouter) 18 | assert isinstance(router.routes, list) 19 | assert router.route_class == LangchainAPIRoute 20 | 21 | with pytest.raises(TypeError): 22 | router.add_api_websocket_route( 23 | "/test", 24 | endpoint=lambda: None, 25 | name="test_ws_route", 26 | dependencies=[Depends(lambda: None)], 27 | ) 28 | 29 | def mock_chain_factory(): 30 | return create_autospec(ConversationChain) 31 | 32 | router.add_api_websocket_route( 33 | "/test", 34 | endpoint=mock_chain_factory, 35 | name="test_ws_route", 36 | ) 37 | 38 | assert len(router.routes) == 1 39 | assert router.routes[0].path == "/test" 40 | assert router.routes[0].name == "test_ws_route" 41 | 42 | 43 | def test_langchain_api_route(): 44 | with pytest.raises(TypeError): 45 | route = LangchainAPIRoute( 46 | "/test", 47 | endpoint=lambda: None, 48 | ) 49 | 50 | route = LangchainAPIRoute( 51 | "/test", 52 | endpoint=lambda: create_autospec(ConversationChain), 53 | ) 54 | 55 | assert isinstance(route, LangchainAPIRoute) 56 | assert route.path == "/test" 57 | 58 | with patch( 59 | "lanarky.adapters.langchain.routing.build_factory_api_endpoint" 60 | ) as endpoint_mock: 61 | endpoint_mock.return_value = AsyncMock() 62 | LangchainAPIRoute( 63 | "/test", 64 | endpoint=lambda: None, 65 | ) 66 | endpoint_mock.assert_called() 67 | 68 | async def factory_endpoint(): 69 | return MagicMock() 70 | 71 | endpoint_mock.reset_mock() 72 | LangchainAPIRoute( 73 | "/test", 74 | endpoint=factory_endpoint, 75 | ) 76 | endpoint_mock.assert_not_called() 77 | 78 | 79 | def test_langchain_websocket_route(): 80 | with patch( 81 | "lanarky.adapters.langchain.routing.build_factory_websocket_endpoint" 82 | ) as endpoint_mock: 83 | endpoint_mock.return_value = AsyncMock() 84 | LangchainAPIWebSocketRoute( 85 | "/test", 86 | endpoint=lambda: None, 87 | ) 88 | endpoint_mock.assert_called() 89 | 90 | async def factory_endpoint(): 91 | return MagicMock() 92 | 93 | endpoint_mock.reset_mock() 94 | LangchainAPIWebSocketRoute( 95 | "/test", 96 | endpoint=factory_endpoint, 97 | ) 98 | endpoint_mock.assert_not_called() 99 | -------------------------------------------------------------------------------- /tests/adapters/openai/test_openai_dependencies.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, create_autospec 2 | 3 | import pytest 4 | from fastapi import params 5 | 6 | from lanarky.adapters.openai.dependencies import Depends 7 | from lanarky.adapters.openai.resources import ChatCompletionResource 8 | 9 | 10 | def test_depends_success(): 11 | def mock_dependency() -> ChatCompletionResource: 12 | return create_autospec(ChatCompletionResource) 13 | 14 | dependency = Depends(mock_dependency) 15 | assert isinstance(dependency, params.Depends) 16 | 17 | 18 | def test_depends_invalid_dependency(): 19 | def dependency_wo_defaults(arg1, arg2="default_value"): 20 | pass 21 | 22 | with pytest.raises(TypeError): 23 | Depends(dependency_wo_defaults) 24 | 25 | with pytest.raises(TypeError): 26 | Depends(lambda: MagicMock()) 27 | -------------------------------------------------------------------------------- /tests/adapters/openai/test_openai_resources.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import AsyncMock, MagicMock 2 | 3 | import pytest 4 | from openai.types.chat import chat_completion, chat_completion_chunk 5 | 6 | from lanarky.adapters.openai.resources import ( 7 | AsyncOpenAI, 8 | ChatCompletion, 9 | ChatCompletionChunk, 10 | ChatCompletionResource, 11 | Message, 12 | ) 13 | 14 | 15 | @pytest.mark.asyncio 16 | async def test_chat_completion_resource_stream_response(): 17 | async def mock_completion_chunk_stream(): 18 | yield ChatCompletionChunk( 19 | id="chat-completion-id", 20 | created=1700936386, 21 | model="gpt-3.5-turbo-0613", 22 | object="chat.completion.chunk", 23 | choices=[ 24 | chat_completion_chunk.Choice( 25 | index=0, 26 | finish_reason=None, 27 | delta=chat_completion_chunk.ChoiceDelta( 28 | content="Hello! How can I assist you today?" 29 | ), 30 | ) 31 | ], 32 | ) 33 | 34 | chat_completion_resource = ChatCompletionResource( 35 | client=MagicMock(spec=AsyncOpenAI), stream=True 36 | ) 37 | chat_completion_resource._client.chat = MagicMock() 38 | chat_completion_resource._client.chat.completions = MagicMock() 39 | chat_completion_resource._client.chat.completions.create = AsyncMock( 40 | return_value=mock_completion_chunk_stream() 41 | ) 42 | 43 | messages = [Message(role="user", content="Hello")] 44 | async for response in chat_completion_resource.stream_response(messages): 45 | assert response == "Hello! How can I assist you today?" 46 | break 47 | 48 | chat_completion_resource = ChatCompletionResource( 49 | client=MagicMock(spec=AsyncOpenAI) 50 | ) 51 | chat_completion_resource._client.chat = MagicMock() 52 | chat_completion_resource._client.chat.completions = MagicMock() 53 | chat_completion_resource._client.chat.completions.create = AsyncMock() 54 | chat_completion_resource._client.chat.completions.create.return_value = ( 55 | ChatCompletion( 56 | id="chat-completion-id", 57 | created=1700936386, 58 | model="gpt-3.5-turbo-0613", 59 | object="chat.completion", 60 | choices=[ 61 | chat_completion.Choice( 62 | finish_reason="stop", 63 | index=0, 64 | message=chat_completion.ChatCompletionMessage( 65 | content="Hello! How can I assist you today?", role="assistant" 66 | ), 67 | ) 68 | ], 69 | ) 70 | ) 71 | 72 | messages = [Message(role="user", content="Hello")] 73 | async for response in chat_completion_resource.stream_response(messages): 74 | assert response == "Hello! How can I assist you today?" 75 | break 76 | 77 | 78 | @pytest.mark.asyncio 79 | async def test_chat_completion_resource_call(): 80 | mocked_completion = ChatCompletion( 81 | id="chat-completion-id", 82 | created=1700936386, 83 | model="gpt-3.5-turbo-0613", 84 | object="chat.completion", 85 | choices=[ 86 | chat_completion.Choice( 87 | finish_reason="stop", 88 | index=0, 89 | message=chat_completion.ChatCompletionMessage( 90 | content="Hello! How can I assist you today?", role="assistant" 91 | ), 92 | ) 93 | ], 94 | ) 95 | 96 | chat_completion_resource = ChatCompletionResource( 97 | client=MagicMock(spec=AsyncOpenAI) 98 | ) 99 | chat_completion_resource._client.chat = MagicMock() 100 | chat_completion_resource._client.chat.completions = MagicMock() 101 | chat_completion_resource._client.chat.completions.create = AsyncMock( 102 | return_value=mocked_completion 103 | ) 104 | 105 | messages = [Message(role="user", content="Hello")] 106 | response = await chat_completion_resource(messages) 107 | assert response == mocked_completion 108 | -------------------------------------------------------------------------------- /tests/adapters/openai/test_openai_responses.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, call 2 | 3 | import pytest 4 | from starlette.types import Send 5 | 6 | from lanarky.adapters.openai.resources import ChatCompletionResource 7 | from lanarky.adapters.openai.responses import ( 8 | HTTPStatusDetail, 9 | StreamingResponse, 10 | status, 11 | ) 12 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 13 | 14 | 15 | @pytest.mark.asyncio 16 | async def test_stream_response_successful(send: Send): 17 | async def async_generator(): 18 | yield "" 19 | 20 | resource = MagicMock(spec=ChatCompletionResource) 21 | resource.stream_response.__aiter__ = MagicMock(return_value=async_generator()) 22 | 23 | response = StreamingResponse( 24 | resource=resource, 25 | messages=[], 26 | ) 27 | 28 | await response.stream_response(send) 29 | 30 | resource.stream_response.assert_called_once() 31 | 32 | expected_calls = [ 33 | call( 34 | { 35 | "type": "http.response.start", 36 | "status": response.status_code, 37 | "headers": response.raw_headers, 38 | } 39 | ), 40 | call( 41 | { 42 | "type": "http.response.body", 43 | "body": b"", 44 | "more_body": False, 45 | } 46 | ), 47 | ] 48 | 49 | send.assert_has_calls(expected_calls, any_order=False) 50 | 51 | 52 | @pytest.mark.asyncio 53 | async def test_stream_response_error(send: Send): 54 | resource = MagicMock(spec=ChatCompletionResource) 55 | resource.stream_response = MagicMock(side_effect=Exception("Some error occurred")) 56 | 57 | response = StreamingResponse( 58 | resource=resource, 59 | messages=[], 60 | ) 61 | 62 | await response.stream_response(send) 63 | 64 | resource.stream_response.assert_called_once() 65 | 66 | expected_calls = [ 67 | call( 68 | { 69 | "type": "http.response.start", 70 | "status": response.status_code, 71 | "headers": response.raw_headers, 72 | } 73 | ), 74 | call( 75 | { 76 | "type": "http.response.body", 77 | "body": ensure_bytes( 78 | ServerSentEvent( 79 | data=dict( 80 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 81 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 82 | ), 83 | event=Events.ERROR, 84 | ), 85 | None, 86 | ), 87 | "more_body": True, 88 | } 89 | ), 90 | call( 91 | { 92 | "type": "http.response.body", 93 | "body": b"", 94 | "more_body": False, 95 | } 96 | ), 97 | ] 98 | 99 | send.assert_has_calls(expected_calls, any_order=False) 100 | -------------------------------------------------------------------------------- /tests/adapters/openai/test_openai_routing.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import create_autospec 2 | 3 | import pytest 4 | 5 | from lanarky.adapters.openai.resources import ChatCompletionResource 6 | from lanarky.adapters.openai.routing import OpenAIAPIRoute, OpenAIAPIRouter 7 | 8 | 9 | def test_langchain_api_router(): 10 | router = OpenAIAPIRouter() 11 | 12 | assert isinstance(router, OpenAIAPIRouter) 13 | assert isinstance(router.routes, list) 14 | assert router.route_class == OpenAIAPIRoute 15 | 16 | def mock_endpoint(): 17 | pass 18 | 19 | with pytest.raises(TypeError): 20 | router.add_api_websocket_route( 21 | "/test", 22 | endpoint=mock_endpoint, 23 | name="test_ws_route", 24 | ) 25 | 26 | def mock_chain_factory(): 27 | return create_autospec(ChatCompletionResource) 28 | 29 | router.add_api_websocket_route( 30 | "/test", 31 | endpoint=mock_chain_factory, 32 | name="test_ws_route", 33 | ) 34 | 35 | assert len(router.routes) == 1 36 | assert router.routes[0].path == "/test" 37 | assert router.routes[0].name == "test_ws_route" 38 | 39 | 40 | def test_langchain_api_route(): 41 | def mock_endpoint(): 42 | pass 43 | 44 | with pytest.raises(TypeError): 45 | route = OpenAIAPIRoute( 46 | "/test", 47 | endpoint=mock_endpoint, 48 | ) 49 | 50 | def mock_chain_factory(): 51 | return create_autospec(ChatCompletionResource) 52 | 53 | route = OpenAIAPIRoute( 54 | "/test", 55 | endpoint=mock_chain_factory, 56 | ) 57 | 58 | assert isinstance(route, OpenAIAPIRoute) 59 | assert route.path == "/test" 60 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from typing import Iterator, Type 2 | from unittest.mock import AsyncMock, create_autospec 3 | 4 | import pytest 5 | from starlette.types import Send 6 | 7 | from lanarky.websockets import WebSocket 8 | 9 | 10 | @pytest.fixture(scope="function") 11 | def send() -> Send: 12 | return AsyncMock(spec=Send) 13 | 14 | 15 | @pytest.fixture(scope="function") 16 | def body_iterator() -> Iterator[bytes]: 17 | async def iterator(): 18 | yield b"Chunk 1" 19 | yield b"Chunk 2" 20 | 21 | return iterator() 22 | 23 | 24 | @pytest.fixture(scope="function") 25 | def websocket() -> Type[WebSocket]: 26 | websocket: Type[WebSocket] = create_autospec(WebSocket) 27 | websocket.send_json = AsyncMock() 28 | return websocket 29 | -------------------------------------------------------------------------------- /tests/test_applications.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from fastapi.testclient import TestClient 3 | 4 | from lanarky import Lanarky 5 | 6 | 7 | @pytest.fixture 8 | def app() -> Lanarky: 9 | return Lanarky() 10 | 11 | 12 | def test_app_instance(app: Lanarky): 13 | assert isinstance(app, Lanarky) 14 | assert app.title == "Lanarky" 15 | 16 | 17 | def test_custom_title(): 18 | custom_title = "My Custom Title" 19 | app = Lanarky(title=custom_title) 20 | assert app.title == custom_title 21 | 22 | 23 | def test_app_routes(app: Lanarky): 24 | client = TestClient(app) 25 | 26 | response = client.get("/") 27 | assert response.status_code == 404 28 | -------------------------------------------------------------------------------- /tests/test_logging.py: -------------------------------------------------------------------------------- 1 | import loguru 2 | 3 | from lanarky.logging import get_logger 4 | 5 | 6 | def test_get_logger_default(): 7 | logger = get_logger() 8 | 9 | assert logger is not None 10 | assert isinstance(logger, loguru._logger.Logger) 11 | 12 | 13 | def test_get_logger_custom_handler(tmpdir): 14 | custom_handler = tmpdir.join("test.log") 15 | logger = get_logger(handler=custom_handler, format="{time} - {message}") 16 | 17 | assert logger is not None 18 | assert isinstance(logger, loguru._logger.Logger) 19 | assert len(logger._core.handlers) == 1 20 | for handler in logger._core.handlers.values(): 21 | assert handler._name == f"'{custom_handler}'" 22 | 23 | 24 | def test_get_logger_kwargs(tmpdir): 25 | import logging 26 | 27 | custom_handler = tmpdir.join("test.log") 28 | level = logging.WARNING 29 | logger = get_logger(handler=custom_handler, level=level) 30 | 31 | assert logger is not None 32 | assert isinstance(logger, loguru._logger.Logger) 33 | assert len(logger._core.handlers) == 1 34 | for handler in logger._core.handlers.values(): 35 | assert handler._name == f"'{custom_handler}'" 36 | assert handler._levelno == level 37 | -------------------------------------------------------------------------------- /tests/test_responses.py: -------------------------------------------------------------------------------- 1 | from typing import Iterator, Type 2 | from unittest.mock import MagicMock, call 3 | 4 | import pytest 5 | from fastapi import status 6 | from starlette.types import Send 7 | 8 | from lanarky.events import Events, ServerSentEvent, ensure_bytes 9 | from lanarky.responses import HTTPStatusDetail, StreamingResponse 10 | 11 | 12 | @pytest.fixture 13 | def streaming_response(body_iterator: Iterator[bytes]) -> Type[StreamingResponse]: 14 | return StreamingResponse(content=body_iterator) 15 | 16 | 17 | @pytest.mark.asyncio 18 | async def test_stream_response_successful( 19 | send: Send, streaming_response: Type[StreamingResponse] 20 | ): 21 | await streaming_response.stream_response(send) 22 | 23 | expected_calls = [ 24 | call( 25 | { 26 | "type": "http.response.start", 27 | "status": streaming_response.status_code, 28 | "headers": streaming_response.raw_headers, 29 | } 30 | ), 31 | call( 32 | { 33 | "type": "http.response.body", 34 | "body": b"Chunk 1", 35 | "more_body": True, 36 | } 37 | ), 38 | call( 39 | { 40 | "type": "http.response.body", 41 | "body": b"Chunk 2", 42 | "more_body": True, 43 | } 44 | ), 45 | call( 46 | { 47 | "type": "http.response.body", 48 | "body": b"", 49 | "more_body": False, 50 | } 51 | ), 52 | ] 53 | 54 | send.assert_has_calls(expected_calls, any_order=False) 55 | 56 | 57 | @pytest.mark.asyncio 58 | async def test_stream_response_exception( 59 | send: Send, streaming_response: Type[StreamingResponse] 60 | ): 61 | streaming_response.body_iterator = MagicMock() 62 | streaming_response.body_iterator.__aiter__.side_effect = Exception("Some error") 63 | 64 | await streaming_response.stream_response(send) 65 | 66 | expected_calls = [ 67 | call( 68 | { 69 | "type": "http.response.start", 70 | "status": streaming_response.status_code, 71 | "headers": streaming_response.raw_headers, 72 | } 73 | ), 74 | call( 75 | { 76 | "type": "http.response.body", 77 | "body": ensure_bytes( 78 | ServerSentEvent( 79 | data=dict( 80 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 81 | detail=HTTPStatusDetail.INTERNAL_SERVER_ERROR, 82 | ), 83 | event=Events.ERROR, 84 | ), 85 | None, 86 | ), 87 | "more_body": True, 88 | } 89 | ), 90 | call( 91 | { 92 | "type": "http.response.body", 93 | "body": b"", 94 | "more_body": False, 95 | } 96 | ), 97 | ] 98 | 99 | send.assert_has_calls(expected_calls, any_order=False) 100 | -------------------------------------------------------------------------------- /tests/test_websockets.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | from unittest.mock import AsyncMock, patch 3 | 4 | import pytest 5 | 6 | from lanarky.websockets import ( 7 | DataMode, 8 | WebSocket, 9 | WebSocketDisconnect, 10 | WebsocketSession, 11 | ) 12 | 13 | 14 | @pytest.mark.asyncio 15 | async def test_connect_json(websocket: Type[WebSocket]): 16 | session = WebsocketSession() 17 | data_to_send = ["Hello", "World", "!"] 18 | websocket.receive_json = AsyncMock(side_effect=data_to_send) 19 | 20 | async with session.connect(websocket, mode=DataMode.JSON) as data_stream: 21 | for expected_data in data_to_send: 22 | received_data = await data_stream.__anext__() 23 | assert received_data == expected_data 24 | 25 | 26 | @pytest.mark.asyncio 27 | async def test_connect_text(websocket: Type[WebSocket]): 28 | session = WebsocketSession() 29 | data_to_send = ["Text", "Messages", "Here"] 30 | websocket.receive_text = AsyncMock(side_effect=data_to_send) 31 | 32 | async with session.connect(websocket, mode=DataMode.TEXT) as data_stream: 33 | for expected_data in data_to_send: 34 | received_data = await data_stream.__anext__() 35 | assert received_data == expected_data 36 | 37 | 38 | @pytest.mark.asyncio 39 | async def test_connect_bytes(websocket: Type[WebSocket]): 40 | session = WebsocketSession() 41 | data_to_send = [b"Bytes", b"Data", b"Test"] 42 | websocket.receive_bytes = AsyncMock(side_effect=data_to_send) 43 | 44 | async with session.connect(websocket, mode=DataMode.BYTES) as data_stream: 45 | for expected_data in data_to_send: 46 | received_data = await data_stream.__anext__() 47 | assert received_data == expected_data 48 | 49 | 50 | @pytest.mark.asyncio 51 | async def test_invalid_data_mode(websocket: Type[WebSocket]): 52 | session = WebsocketSession() 53 | 54 | with pytest.raises(ValueError): 55 | async with session.connect(websocket, mode="invalid_mode"): 56 | pass 57 | 58 | 59 | @pytest.mark.asyncio 60 | async def test_websocket_disconnect(websocket: Type[WebSocket]): 61 | websocket.receive_text = AsyncMock(side_effect=WebSocketDisconnect()) 62 | 63 | with patch("lanarky.websockets.logger") as logger: 64 | async with WebsocketSession().connect( 65 | websocket, mode=DataMode.TEXT 66 | ) as data_stream: 67 | async for _ in data_stream: 68 | pass 69 | 70 | logger.info.assert_called_once_with("Websocket disconnected") 71 | --------------------------------------------------------------------------------