├── .github ├── renovate.json5 └── workflows │ ├── codeflash.yml │ ├── publish.yml │ └── tests.yml ├── .gitignore ├── .mailmap ├── .pre-commit-config.yaml ├── .python-version ├── .secrets.allowlist ├── CITATION.cff ├── LICENSE ├── README.md ├── citation.bib ├── docs └── assets │ ├── Aviary.png │ ├── ldp_chessboard.png │ └── ldp_definition.png ├── packages └── lmi │ ├── LICENSE │ ├── README.md │ ├── pyproject.toml │ ├── src │ └── lmi │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── cost_tracker.py │ │ ├── embeddings.py │ │ ├── exceptions.py │ │ ├── llms.py │ │ ├── py.typed │ │ ├── rate_limiter.py │ │ ├── types.py │ │ └── utils.py │ └── tests │ ├── __init__.py │ ├── cassettes │ ├── TestLiteLLMEmbeddingModel.test_caching.yaml │ ├── TestLiteLLMModel.test_call[Anthropic-model].yaml │ ├── TestLiteLLMModel.test_call[OpenAI-model].yaml │ ├── TestLiteLLMModel.test_call[chat-model].yaml │ ├── TestLiteLLMModel.test_call[completion-model].yaml │ ├── TestLiteLLMModel.test_call_single[with-router].yaml │ ├── TestLiteLLMModel.test_call_single[without-router].yaml │ ├── TestLiteLLMModel.test_call_w_figure.yaml │ ├── TestLiteLLMModel.test_call_w_multiple_models[multiple-models].yaml │ ├── TestLiteLLMModel.test_call_with_prompt[with-router].yaml │ ├── TestLiteLLMModel.test_call_with_prompt[without-router].yaml │ ├── TestLiteLLMModel.test_cost_call[Anthropic-model].yaml │ ├── TestLiteLLMModel.test_cost_call[OpenAI-model].yaml │ ├── TestLiteLLMModel.test_cost_call_single[with-router].yaml │ ├── TestLiteLLMModel.test_cost_call_single[without-router].yaml │ ├── TestLiteLLMModel.test_max_token_truncation[with-router].yaml │ ├── TestLiteLLMModel.test_max_token_truncation[without-router].yaml │ ├── TestLiteLLMModel.test_run_prompt[with-router].yaml │ ├── TestLiteLLMModel.test_run_prompt[without-router].yaml │ ├── TestMultipleCompletion.test_model[gpt-3.5-turbo-0125].yaml │ ├── TestMultipleCompletion.test_model[gpt-3.5-turbo].yaml │ ├── TestMultipleCompletion.test_model[gpt-4o-mini-2024-07-18].yaml │ ├── TestMultipleCompletion.test_model[gpt-4o-mini].yaml │ ├── TestMultipleCompletion.test_model[gpt-4o].yaml │ ├── TestMultipleCompletion.test_multiple_completion[openai].yaml │ ├── TestMultipleCompletion.test_output_schema[json-mode-base-model].yaml │ ├── TestMultipleCompletion.test_output_schema[json-mode-type-adapter].yaml │ ├── TestMultipleCompletion.test_output_schema[structured-outputs].yaml │ ├── TestMultipleCompletion.test_parameterizing_tool_from_arg_union.yaml │ ├── TestMultipleCompletion.test_single_completion[claude-3-5-haiku-20241022].yaml │ ├── TestMultipleCompletion.test_single_completion[claude-3-haiku-20240307].yaml │ ├── TestMultipleCompletion.test_single_completion[gpt-3.5-turbo-0125].yaml │ ├── TestMultipleCompletion.test_single_completion[gpt-3.5-turbo].yaml │ ├── TestMultipleCompletion.test_single_completion[gpt-4o-mini-2024-07-18].yaml │ ├── TestMultipleCompletion.test_text_image_message[gpt-4o-mini-2024-07-18].yaml │ ├── TestReasoning.test_deepseek_model[deepseek-reasoner].yaml │ ├── TestReasoning.test_deepseek_model[openrouter-deepseek].yaml │ ├── TestTooling.test_empty_tools[Anthropic-empty-tools].yaml │ ├── TestTooling.test_empty_tools[Anthropic-no-tools].yaml │ ├── TestTooling.test_empty_tools[OpenAI-empty-tools].yaml │ ├── TestTooling.test_empty_tools[OpenAI-no-tools].yaml │ └── TestTooling.test_tool_selection.yaml │ ├── conftest.py │ ├── test_cost_tracking.py │ ├── test_embeddings.py │ ├── test_llms.py │ └── test_rate_limiter.py ├── pyproject.toml ├── src └── ldp │ ├── __init__.py │ ├── agent │ ├── __init__.py │ ├── agent.py │ ├── agent_client.py │ ├── interactive_agent.py │ ├── memory_agent.py │ ├── react_agent.py │ ├── simple_agent.py │ └── tree_of_thoughts_agent.py │ ├── alg │ ├── __init__.py │ ├── algorithms.py │ ├── beam_search.py │ ├── callbacks.py │ ├── datasets.py │ ├── optimizer │ │ ├── __init__.py │ │ ├── ape.py │ │ ├── memory.py │ │ ├── opt.py │ │ └── replay_buffers.py │ ├── rollout.py │ ├── runners.py │ └── tree_search.py │ ├── data_structures.py │ ├── graph │ ├── __init__.py │ ├── async_torch.py │ ├── common_ops.py │ ├── gradient_estimators.py │ ├── loss_ops.py │ ├── memory.py │ ├── modules │ │ ├── __init__.py │ │ ├── llm_call.py │ │ ├── react.py │ │ ├── reflect.py │ │ └── thought.py │ ├── op_utils.py │ ├── ops.py │ └── torch_ops.py │ ├── llms │ ├── __init__.py │ └── prompts.py │ ├── main.py │ ├── nn │ ├── __init__.py │ ├── agent │ │ └── simple_local_agent.py │ ├── chat_templates │ │ ├── README.md │ │ ├── llama2_chat_template_ori.jinja │ │ ├── llama3.1_chat_template_hf.jinja │ │ ├── llama3.1_chat_template_nothought.jinja │ │ ├── llama3.1_chat_template_thought.jinja │ │ ├── llama3.1_chat_template_vllm.jinja │ │ └── llama3_chat_template_ori.jinja │ ├── generation │ │ ├── __init__.py │ │ └── base.py │ ├── graph │ │ └── llm_call_op.py │ ├── handlers │ │ ├── chunking.py │ │ ├── module_handler.py │ │ └── transformer_handler.py │ ├── lm_config.py │ └── utils.py │ ├── py.typed │ ├── shims.py │ └── utils.py ├── tests ├── .gitignore ├── __init__.py ├── cassettes │ ├── TestAgentState.test_no_state_mutation[agent0].yaml │ ├── TestAgentState.test_no_state_mutation[agent1].yaml │ ├── TestAgentState.test_no_state_mutation[agent2].yaml │ ├── TestHTTPAgentClient.test_lifecycle.yaml │ ├── TestLLMCallOp.test_compute_logprob[0.0].yaml │ ├── TestLLMCallOp.test_compute_logprob[0.5].yaml │ ├── TestLLMCallOp.test_compute_logprob[1.0].yaml │ ├── TestLLMCallOp.test_cost_tracking.yaml │ ├── TestLLMCallOp.test_empty_tools.yaml │ ├── TestLLMCallOp.test_validation.yaml │ ├── TestLLMModel.test_model[claude-3-haiku-20240307].yaml │ ├── TestLLMModel.test_model[gpt-3.5-turbo].yaml │ ├── TestLLMModel.test_output_schema[json-mode].yaml │ ├── TestLLMModel.test_output_schema[structured-outputs].yaml │ ├── TestLLMModel.test_output_type_rejected_validation.yaml │ ├── TestLLMModel.test_parameterizing_tool_from_arg_union.yaml │ ├── TestLLMModel.test_streaming[claude-3-haiku-20240307].yaml │ ├── TestLLMModel.test_streaming[gpt-3.5-turbo].yaml │ ├── TestLLMModel.test_text_image_message[claude-3-haiku-20240307].yaml │ ├── TestLLMModel.test_text_image_message[gpt-4-turbo].yaml │ ├── TestLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml │ ├── TestLiteEmbeddingModel.test_caching.yaml │ ├── TestMemoryAgent.test_agent_grad.yaml │ ├── TestMemoryAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml │ ├── TestMemoryOpt.test_lessons_memory_optimizer.yaml │ ├── TestMultipleCompletionLLMModel.test_model[gpt-3.5-turbo].yaml │ ├── TestMultipleCompletionLLMModel.test_output_schema[json-mode].yaml │ ├── TestMultipleCompletionLLMModel.test_output_schema[structured-outputs].yaml │ ├── TestMultipleCompletionLLMModel.test_parameterizing_tool_from_arg_union.yaml │ ├── TestMultipleCompletionLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml │ ├── TestNoToolsSimpleAgent.test_dummyenv[claude-3-5-haiku-20241022].yaml │ ├── TestNoToolsSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml │ ├── TestNoToolsSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml │ ├── TestParallelism.test_SimpleAgent_can_parallel_call.yaml │ ├── TestReActAgent.test_agent_grad[False-claude-3-5-haiku-20241022].yaml │ ├── TestReActAgent.test_agent_grad[False-claude-3-haiku-20240307].yaml │ ├── TestReActAgent.test_agent_grad[False-gpt-4o].yaml │ ├── TestReActAgent.test_agent_grad[True-claude-3-5-haiku-20241022].yaml │ ├── TestReActAgent.test_agent_grad[True-claude-3-haiku-20240307].yaml │ ├── TestReActAgent.test_agent_grad[True-gpt-4-turbo].yaml │ ├── TestReActAgent.test_multi_step[False].yaml │ ├── TestReActAgent.test_multi_step[True].yaml │ ├── TestReActAgent.test_react_dummyenv[False-claude-3-5-haiku-20241022].yaml │ ├── TestReActAgent.test_react_dummyenv[False-claude-3-haiku-20240307].yaml │ ├── TestReActAgent.test_react_dummyenv[False-gpt-4o].yaml │ ├── TestReActAgent.test_react_dummyenv[True-claude-3-5-haiku-20241022].yaml │ ├── TestReActAgent.test_react_dummyenv[True-claude-3-haiku-20240307].yaml │ ├── TestReActAgent.test_react_dummyenv[True-gpt-4-turbo].yaml │ ├── TestSimpleAgent.test_agent_grad[claude-3-5-haiku-20241022].yaml │ ├── TestSimpleAgent.test_agent_grad[claude-3-haiku-20240307].yaml │ ├── TestSimpleAgent.test_agent_grad[gpt-4o-mini-2024-07-18].yaml │ ├── TestSimpleAgent.test_dummyenv[claude-3-5-haiku-20241022].yaml │ ├── TestSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml │ ├── TestSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml │ ├── TestSimpleAgent.test_hide_old_action_content.yaml │ ├── TestSimpleAgent.test_hide_old_env_states.yaml │ ├── test_beam_search.yaml │ ├── test_embedding_op[text-embedding-3-large-0-256].yaml │ ├── test_embedding_op[text-embedding-3-large-0-512].yaml │ ├── test_embedding_op[text-embedding-3-large-32-256].yaml │ ├── test_embedding_op[text-embedding-3-large-32-512].yaml │ ├── test_embedding_op[text-embedding-3-large-64-256].yaml │ ├── test_embedding_op[text-embedding-3-large-64-512].yaml │ ├── test_embedding_op[text-embedding-3-small-0-256].yaml │ ├── test_embedding_op[text-embedding-3-small-0-512].yaml │ ├── test_embedding_op[text-embedding-3-small-32-256].yaml │ ├── test_embedding_op[text-embedding-3-small-32-512].yaml │ ├── test_embedding_op[text-embedding-3-small-64-256].yaml │ ├── test_embedding_op[text-embedding-3-small-64-512].yaml │ ├── test_fallbacks_working[False].yaml │ ├── test_fallbacks_working[True].yaml │ ├── test_llm_call_graph.yaml │ ├── test_offline_trainer[False].yaml │ ├── test_offline_trainer[True].yaml │ ├── test_online_trainer[False].yaml │ ├── test_online_trainer[True].yaml │ ├── test_reflect_module.yaml │ ├── test_rollout[False].yaml │ └── test_rollout[True].yaml ├── conftest.py ├── test_agents.py ├── test_algorithms.py ├── test_buffers.py ├── test_context_managers.py ├── test_data_structures.py ├── test_envs.py ├── test_gradients.py ├── test_loss_ops.py ├── test_memory.py ├── test_modules.py ├── test_nn_models.py ├── test_nn_ops.py ├── test_ops.py ├── test_optimizer.py ├── test_prompts.py ├── test_rollouts.py ├── test_runners.py ├── test_shims.py ├── test_torch_ops.py └── test_utils.py ├── tutorials ├── creating_a_language_agent.ipynb └── evaluating_a_llama_agent.ipynb └── uv.lock /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | $schema: "https://docs.renovatebot.com/renovate-schema.json", 3 | extends: ["config:recommended"], 4 | schedule: ["* 2 1-7 * 1"], 5 | prHourlyLimit: 4, 6 | timezone: "America/Los_Angeles", 7 | rangeStrategy: "widen", 8 | lockFileMaintenance: { 9 | enabled: true, 10 | schedule: ["* 2 1-7 * 1"], // Work around https://github.com/renovatebot/renovate/discussions/33152 11 | }, 12 | minimumReleaseAge: "2 weeks", 13 | "pre-commit": { 14 | enabled: true, 15 | }, 16 | packageRules: [ 17 | { 18 | matchUpdateTypes: ["lockFileMaintenance"], 19 | automerge: true, 20 | }, 21 | { 22 | // group:allNonMajor, with automerge 23 | groupName: "all non-major dependencies", 24 | groupSlug: "all-minor-patch", 25 | matchPackageNames: ["*"], 26 | matchUpdateTypes: ["minor", "patch"], 27 | automerge: true, 28 | }, 29 | { 30 | // TODO: remove after torch supports Python 3.13 31 | matchPackageNames: ["python"], 32 | allowedVersions: "<=3.12", 33 | }, 34 | ], 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/codeflash.yml: -------------------------------------------------------------------------------- 1 | name: CodeFlash 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "ldp/**" 7 | - "src/**" 8 | - "packages/**" 9 | workflow_dispatch: 10 | 11 | concurrency: # Cancel prior if new push, SEE: https://stackoverflow.com/a/72408109 12 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | optimize: # SEE: https://docs.codeflash.ai/getting-started/codeflash-github-actions 17 | runs-on: ubuntu-latest 18 | env: 19 | CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }} 20 | CODEFLASH_PR_NUMBER: ${{ github.event.number }} 21 | steps: 22 | - name: Check if PR is from CodeFlash bot 23 | id: bot_check 24 | working-directory: . 25 | run: | 26 | echo "Checking if this PR is created by CodeFlash bot..." 27 | if [ "${{ github.event.pull_request.user.login }}" == "codeflash-ai[bot]" ]; then 28 | echo "PR created by Codeflash bot. Skipping optimization." 29 | echo "skip_remaining_steps=yes" >> $GITHUB_OUTPUT 30 | else 31 | echo "skip_remaining_steps=no" >> $GITHUB_OUTPUT 32 | echo "It's not. Proceeding with the optimization." 33 | fi 34 | - if: steps.bot_check.outputs.skip_remaining_steps == 'no' 35 | uses: actions/checkout@v4 36 | with: 37 | fetch-depth: 0 38 | - if: steps.bot_check.outputs.skip_remaining_steps == 'no' 39 | uses: astral-sh/setup-uv@v5 40 | with: 41 | enable-cache: true 42 | - if: steps.bot_check.outputs.skip_remaining_steps == 'no' 43 | run: uv sync --group=codeflash 44 | - if: steps.bot_check.outputs.skip_remaining_steps == 'no' 45 | name: Run CodeFlash on ldp 46 | run: uv run codeflash --module-root=src/ldp --tests-root=tests 47 | - if: steps.bot_check.outputs.skip_remaining_steps == 'no' 48 | name: Run CodeFlash on lmi 49 | run: uv run codeflash --module-root=packages/lmi/src/lmi --tests-root=packages/lmi/tests 50 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | release: 5 | types: [created] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - id: build-ldp 14 | uses: hynek/build-and-inspect-python-package@v2 15 | with: 16 | upload-name-suffix: -ldp 17 | - name: Download built artifact to dist/ 18 | uses: actions/download-artifact@v4 19 | with: 20 | name: ${{ steps.build-ldp.outputs.artifact-name }} 21 | path: dist_ldp 22 | - id: build-fhlmi 23 | uses: hynek/build-and-inspect-python-package@v2 24 | with: 25 | path: packages/lmi 26 | upload-name-suffix: -fhlmi 27 | - name: Download built artifact to dist/ 28 | uses: actions/download-artifact@v4 29 | with: 30 | name: ${{ steps.build-fhlmi.outputs.artifact-name }} 31 | path: dist_fhlmi 32 | - uses: pypa/gh-action-pypi-publish@release/v1 33 | with: 34 | password: ${{ secrets.PYPI_API_LDP_TOKEN }} 35 | packages-dir: dist_ldp 36 | - uses: pypa/gh-action-pypi-publish@release/v1 37 | with: 38 | password: ${{ secrets.PYPI_API_LMI_TOKEN }} 39 | packages-dir: dist_fhlmi 40 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Lint and Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - renovate/* 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | jobs: 12 | pre-commit: 13 | runs-on: ubuntu-latest 14 | if: github.event_name == 'pull_request' # pre-commit-ci/lite-action only runs here 15 | strategy: 16 | matrix: 17 | python-version: [3.11, 3.13] # Our min and max supported Python versions 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 0 # For setuptools-scm, replace with fetch-tags after https://github.com/actions/checkout/issues/1471 22 | - uses: actions/setup-python@v5 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - uses: astral-sh/setup-uv@v5 26 | with: 27 | enable-cache: true 28 | - run: echo "UV_PROJECT_ENVIRONMENT=$(python -c "import sysconfig; print(sysconfig.get_config_var('prefix'))")" >> $GITHUB_ENV 29 | - run: uv python pin ${{ matrix.python-version }} # uv requires .python-version to match OS Python: https://github.com/astral-sh/uv/issues/11389 30 | - run: uv sync --python-preference only-system 31 | - run: git checkout .python-version # For clean git diff given `pre-commit run --show-diff-on-failure` 32 | - uses: pre-commit/action@v3.0.1 33 | - uses: pre-commit-ci/lite-action@v1.1.0 34 | if: always() 35 | lint: 36 | runs-on: ubuntu-latest 37 | strategy: 38 | matrix: 39 | python-version: [3.11, 3.13] # Our min and max supported Python versions 40 | steps: 41 | - uses: actions/checkout@v4 42 | - uses: astral-sh/setup-uv@v5 43 | with: 44 | enable-cache: true 45 | - run: uv python pin ${{ matrix.python-version }} 46 | - run: uv sync --python-preference=only-managed 47 | - run: uv run refurb . 48 | - if: matrix.python-version == '3.11' # Only need to run this on one version 49 | uses: suzuki-shunsuke/github-action-renovate-config-validator@v1.1.1 50 | test-ldp: 51 | runs-on: ubuntu-latest 52 | steps: 53 | - uses: actions/checkout@v4 54 | - uses: astral-sh/setup-uv@v5 55 | with: 56 | enable-cache: true 57 | - run: uv sync 58 | - run: uv run pytest -n 16 --dist=loadfile tests 59 | env: 60 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} 61 | ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} 62 | test-lmi: 63 | runs-on: ubuntu-latest 64 | steps: 65 | - uses: actions/checkout@v4 66 | - uses: astral-sh/setup-uv@v5 67 | with: 68 | enable-cache: true 69 | - run: uv sync 70 | - run: | 71 | uv run pytest -n 16 --dist=loadfile packages/lmi/tests 72 | env: 73 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} 74 | ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} 75 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Andrew White 2 | James Braza 3 | Mayk Caldas maykcaldas 4 | Michael Skarlinski mskarlin <12701035+mskarlin@users.noreply.github.com> 5 | Ryan-Rhys Griffiths 6 | Siddharth Narayanan 7 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | default_language_version: 3 | python: python3 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v5.0.0 7 | hooks: 8 | - id: check-added-large-files 9 | - id: check-byte-order-marker 10 | - id: check-case-conflict 11 | - id: check-merge-conflict 12 | - id: check-shebang-scripts-are-executable 13 | - id: check-symlinks 14 | - id: check-toml 15 | - id: check-yaml 16 | - id: debug-statements 17 | - id: detect-private-key 18 | - id: end-of-file-fixer 19 | - id: mixed-line-ending 20 | - id: trailing-whitespace 21 | - repo: https://github.com/jsh9/markdown-toc-creator 22 | rev: 0.0.10 23 | hooks: 24 | - id: markdown-toc-creator 25 | - repo: https://github.com/astral-sh/ruff-pre-commit 26 | rev: v0.9.9 27 | hooks: 28 | - id: ruff 29 | args: [--fix, --exit-non-zero-on-fix] 30 | - id: ruff-format 31 | - repo: https://github.com/rbubley/mirrors-prettier 32 | rev: v3.5.3 33 | hooks: 34 | - id: prettier 35 | - repo: https://github.com/Yelp/detect-secrets 36 | rev: v1.5.0 37 | hooks: 38 | - id: detect-secrets 39 | additional_dependencies: [".[word_list]"] 40 | args: 41 | - --word-list=.secrets.allowlist 42 | - --exclude-files=.secrets.baseline$ 43 | exclude: tests/cassettes 44 | - repo: https://github.com/adamchainz/blacken-docs 45 | rev: 1.19.1 46 | hooks: 47 | - id: blacken-docs 48 | - repo: https://github.com/jsh9/markdown-toc-creator 49 | rev: 0.0.10 50 | hooks: 51 | - id: markdown-toc-creator 52 | - repo: https://github.com/jumanjihouse/pre-commit-hooks 53 | rev: 3.0.0 54 | hooks: 55 | - id: check-mailmap 56 | - repo: https://github.com/codespell-project/codespell 57 | rev: v2.4.1 58 | hooks: 59 | - id: codespell 60 | additional_dependencies: [".[toml]"] 61 | exclude_types: [jupyter] 62 | - repo: https://github.com/pappasam/toml-sort 63 | rev: v0.24.2 64 | hooks: 65 | - id: toml-sort-fix 66 | - repo: https://github.com/srstevenson/nb-clean 67 | rev: 4.0.1 68 | hooks: 69 | - id: nb-clean 70 | args: [--preserve-cell-outputs, --remove-empty-cells] 71 | - repo: https://github.com/henryiii/validate-pyproject-schema-store 72 | rev: 2025.02.24 73 | hooks: 74 | - id: validate-pyproject 75 | - repo: https://github.com/astral-sh/uv-pre-commit 76 | rev: 0.6.3 77 | hooks: 78 | - id: uv-lock 79 | - repo: local 80 | hooks: 81 | - id: mypy 82 | name: mypy 83 | entry: mypy 84 | language: system 85 | types_or: [python, pyi] 86 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /.secrets.allowlist: -------------------------------------------------------------------------------- 1 | authorization 2 | x-api-key 3 | abc123 4 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | --- 2 | cff-version: 1.2.0 3 | title: "Aviary: training language agents on challenging scientific tasks" 4 | message: >- 5 | If you use this software, please cite it using the 6 | metadata from this file. 7 | authors: 8 | - given-names: Siddharth 9 | family-names: Narayanan 10 | - given-names: James D. 11 | family-names: Braza 12 | - given-names: Ryan-Rhys 13 | family-names: Griffiths 14 | - family-names: Ponnapati 15 | given-names: Manvitha 16 | - given-names: Albert 17 | family-names: Bou 18 | - given-names: Jon 19 | family-names: Laurent 20 | - given-names: Ori 21 | family-names: Kabeli 22 | - given-names: Geemi 23 | family-names: Wellawatte 24 | - given-names: Sam 25 | family-names: Cox 26 | - given-names: Samuel G. 27 | family-names: Rodriques 28 | - given-names: Andrew D. 29 | family-names: White 30 | identifiers: 31 | - type: doi 32 | value: 10.48550/arXiv.2412.21154 33 | description: ArXiv DOI 34 | - type: url 35 | value: https://arxiv.org/abs/2412.21154 36 | description: ArXiv abstract 37 | repository-code: https://github.com/Future-House/ldp 38 | keywords: 39 | - Artificial Intelligence 40 | - Computation and Language 41 | - Machine Learning 42 | license: Apache-2.0 43 | preferred-citation: 44 | authors: 45 | - given-names: Siddharth 46 | family-names: Narayanan 47 | - given-names: James D. 48 | family-names: Braza 49 | - given-names: Ryan-Rhys 50 | family-names: Griffiths 51 | - family-names: Ponnapati 52 | given-names: Manvitha 53 | - given-names: Albert 54 | family-names: Bou 55 | - given-names: Jon 56 | family-names: Laurent 57 | - given-names: Ori 58 | family-names: Kabeli 59 | - given-names: Geemi 60 | family-names: Wellawatte 61 | - given-names: Sam 62 | family-names: Cox 63 | - given-names: Samuel G. 64 | family-names: Rodriques 65 | - given-names: Andrew D. 66 | family-names: White 67 | date-published: 2024-12-30 68 | doi: 10.48550/arXiv.2412.21154 69 | journal: preprint 70 | title: "Aviary: training language agents on challenging scientific tasks" 71 | type: article 72 | url: https://arxiv.org/abs/2412.21154 73 | -------------------------------------------------------------------------------- /citation.bib: -------------------------------------------------------------------------------- 1 | @article{narayanan2024aviary, 2 | title={Aviary: training language agents on challenging scientific tasks}, 3 | author={Narayanan, Siddharth and Braza, James D and Griffiths, Ryan-Rhys and Ponnapati, Manu and Bou, Albert and Laurent, Jon and Kabeli, Ori and Wellawatte, Geemi and Cox, Sam and Rodriques, Samuel G and others}, 4 | journal={arXiv preprint arXiv:2412.21154}, 5 | year={2024} 6 | } 7 | -------------------------------------------------------------------------------- /docs/assets/Aviary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/docs/assets/Aviary.png -------------------------------------------------------------------------------- /docs/assets/ldp_chessboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/docs/assets/ldp_chessboard.png -------------------------------------------------------------------------------- /docs/assets/ldp_definition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/docs/assets/ldp_definition.png -------------------------------------------------------------------------------- /packages/lmi/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | build-backend = "setuptools.build_meta" 3 | requires = ["setuptools>=64", "setuptools_scm>=8"] 4 | 5 | [dependency-groups] 6 | codeflash = [ 7 | "codeflash>=0.8", # Pin for --verify-setup checking formatter-cmds 8 | "fhlmi[dev]", 9 | ] 10 | dev = ["fhlmi[dev]"] 11 | 12 | [project] 13 | authors = [ 14 | {email = "hello@futurehouse.org", name = "FutureHouse technical staff"}, 15 | ] 16 | # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 17 | classifiers = [ 18 | "Operating System :: OS Independent", 19 | "Programming Language :: Python :: 3 :: Only", 20 | "Programming Language :: Python :: 3.11", 21 | "Programming Language :: Python :: 3.12", 22 | "Programming Language :: Python :: 3.13", 23 | "Programming Language :: Python", 24 | ] 25 | dependencies = [ 26 | "coredis", 27 | "fhaviary>=0.14.0", # For multi-image support 28 | "limits>=4.8", # Pin for RedisBridge.key_prefix 29 | "litellm>=1.63.5", # Pin lower for reasoning + streaming fix 30 | "pydantic~=2.0,>=2.10.1", 31 | "tiktoken>=0.4.0", 32 | "typing-extensions; python_version <= '3.11'", # for typing.override 33 | ] 34 | description = "A client to provide LLM responses for FutureHouse applications." 35 | dynamic = ["version"] 36 | license = {file = "LICENSE"} 37 | name = "fhlmi" 38 | readme = "README.md" 39 | requires-python = ">=3.11" 40 | 41 | [project.optional-dependencies] 42 | dev = [ 43 | "fhaviary[xml]", 44 | "fhlmi[local,progress,typing,vcr]", 45 | "ipython>=8", # Pin to keep recent 46 | "litellm>=1.68", # Pin for PydanticDeprecatedSince20 fixes 47 | "mypy>=1.8", # Pin for mutable-override 48 | "pre-commit>=3.4", # Pin to keep recent 49 | "pylint-pydantic", 50 | "pytest-asyncio", 51 | "pytest-recording", 52 | "pytest-rerunfailures", 53 | "pytest-subtests", 54 | "pytest-sugar", 55 | "pytest-timer[colorama]", 56 | "pytest-xdist", 57 | "pytest>=8", # Pin to keep recent 58 | "python-dotenv", 59 | "refurb>=2", # Pin to keep recent 60 | "typeguard", 61 | ] 62 | local = [ 63 | "numpy", 64 | "sentence-transformers", 65 | ] 66 | progress = ["tqdm"] 67 | typing = ["types-tqdm"] 68 | vcr = [ 69 | "vcrpy>=6", # Pin for https://github.com/kevin1024/vcrpy/issues/884 70 | ] 71 | 72 | [project.urls] 73 | issues = "https://github.com/Future-House/ldp/packages/lmi/issues" 74 | repository = "https://github.com/Future-House/ldp/packages/lmi" 75 | 76 | [tool.ruff] 77 | extend = "../../pyproject.toml" 78 | 79 | [tool.setuptools.packages.find] 80 | where = ["src"] 81 | 82 | [tool.setuptools_scm] 83 | root = "../.." 84 | version_file = "src/lmi/version.py" 85 | -------------------------------------------------------------------------------- /packages/lmi/src/lmi/__init__.py: -------------------------------------------------------------------------------- 1 | from .constants import ( 2 | CHARACTERS_PER_TOKEN_ASSUMPTION, 3 | EXTRA_TOKENS_FROM_USER_ROLE, 4 | MODEL_COST_MAP, 5 | ) 6 | from .cost_tracker import GLOBAL_COST_TRACKER, cost_tracking_ctx, enable_cost_tracking 7 | from .embeddings import ( 8 | EmbeddingModel, 9 | EmbeddingModes, 10 | HybridEmbeddingModel, 11 | LiteLLMEmbeddingModel, 12 | SentenceTransformerEmbeddingModel, 13 | SparseEmbeddingModel, 14 | embedding_model_factory, 15 | ) 16 | from .exceptions import ( 17 | JSONSchemaValidationError, 18 | ) 19 | from .llms import ( 20 | CommonLLMNames, 21 | LiteLLMModel, 22 | LLMModel, 23 | sum_logprobs, 24 | validate_json_completion, 25 | ) 26 | from .types import ( 27 | Embeddable, 28 | LLMResult, 29 | ) 30 | from .utils import ( 31 | configure_llm_logs, 32 | ) 33 | 34 | __all__ = [ 35 | "CHARACTERS_PER_TOKEN_ASSUMPTION", 36 | "EXTRA_TOKENS_FROM_USER_ROLE", 37 | "GLOBAL_COST_TRACKER", 38 | "MODEL_COST_MAP", 39 | "CommonLLMNames", 40 | "Embeddable", 41 | "EmbeddingModel", 42 | "EmbeddingModes", 43 | "HybridEmbeddingModel", 44 | "JSONSchemaValidationError", 45 | "LLMModel", 46 | "LLMResult", 47 | "LiteLLMEmbeddingModel", 48 | "LiteLLMModel", 49 | "SentenceTransformerEmbeddingModel", 50 | "SparseEmbeddingModel", 51 | "configure_llm_logs", 52 | "cost_tracking_ctx", 53 | "embedding_model_factory", 54 | "enable_cost_tracking", 55 | "sum_logprobs", 56 | "validate_json_completion", 57 | ] 58 | -------------------------------------------------------------------------------- /packages/lmi/src/lmi/constants.py: -------------------------------------------------------------------------------- 1 | from sys import version_info 2 | 3 | import litellm 4 | 5 | # Estimate from OpenAI's FAQ 6 | # https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them 7 | CHARACTERS_PER_TOKEN_ASSUMPTION: float = 4.0 8 | # Added tokens from user/role message 9 | # Need to add while doing rate limits 10 | # Taken from empirical counts in tests 11 | EXTRA_TOKENS_FROM_USER_ROLE: int = 7 12 | 13 | MODEL_COST_MAP = litellm.get_model_cost_map("") 14 | 15 | DEFAULT_VERTEX_SAFETY_SETTINGS: list[dict[str, str]] = [ 16 | { 17 | "category": "HARM_CATEGORY_HARASSMENT", 18 | "threshold": "BLOCK_ONLY_HIGH", 19 | }, 20 | { 21 | "category": "HARM_CATEGORY_HATE_SPEECH", 22 | "threshold": "BLOCK_ONLY_HIGH", 23 | }, 24 | { 25 | "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", 26 | "threshold": "BLOCK_ONLY_HIGH", 27 | }, 28 | { 29 | "category": "HARM_CATEGORY_DANGEROUS_CONTENT", 30 | "threshold": "BLOCK_ONLY_HIGH", 31 | }, 32 | ] 33 | 34 | IS_PYTHON_BELOW_312 = version_info < (3, 12) 35 | -------------------------------------------------------------------------------- /packages/lmi/src/lmi/exceptions.py: -------------------------------------------------------------------------------- 1 | class JSONSchemaValidationError(ValueError): 2 | """Raised when the completion does not match the specified schema.""" 3 | -------------------------------------------------------------------------------- /packages/lmi/src/lmi/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/packages/lmi/src/lmi/py.typed -------------------------------------------------------------------------------- /packages/lmi/src/lmi/types.py: -------------------------------------------------------------------------------- 1 | import contextvars 2 | import logging 3 | from contextlib import contextmanager 4 | from datetime import datetime 5 | from uuid import UUID, uuid4 6 | 7 | import litellm 8 | from aviary.core import Message 9 | from pydantic import ( 10 | BaseModel, 11 | ConfigDict, 12 | Field, 13 | computed_field, 14 | ) 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | # A context var that will be unique to threads/processes 19 | cvar_session_id = contextvars.ContextVar[UUID | None]("session_id", default=None) 20 | 21 | 22 | @contextmanager 23 | def set_llm_session_ids(session_id: UUID): 24 | token = cvar_session_id.set(session_id) 25 | try: 26 | yield 27 | finally: 28 | cvar_session_id.reset(token) 29 | 30 | 31 | class Embeddable(BaseModel): 32 | embedding: list[float] | None = Field(default=None, repr=False) 33 | 34 | 35 | class LLMResult(BaseModel): 36 | """A class to hold the result of a LLM completion. 37 | 38 | To associate a group of LLMResults, you can use the `set_llm_session_ids` context manager: 39 | 40 | ```python 41 | my_session_id = uuid4() 42 | with set_llm_session_ids(my_session_id): 43 | # code that generates LLMResults 44 | pass 45 | ``` 46 | 47 | and all the LLMResults generated within the context will have the same `session_id`. 48 | This can be combined with LLMModels `llm_result_callback` to store all LLMResults. 49 | """ 50 | 51 | model_config = ConfigDict(populate_by_name=True) 52 | 53 | id: UUID = Field(default_factory=uuid4) 54 | session_id: UUID | None = Field( 55 | default_factory=cvar_session_id.get, # type: ignore[arg-type] 56 | description="A persistent ID to associate a group of LLMResults", 57 | alias="answer_id", 58 | ) 59 | name: str | None = None 60 | config: dict | None = None 61 | prompt: str | list[dict] | Message | list[Message] | None = Field( 62 | default=None, 63 | description="Optional prompt or list of serialized prompts.", 64 | ) 65 | text: str | None = None 66 | messages: list[Message] | None = Field( 67 | default=None, description="Messages received from the LLM." 68 | ) 69 | prompt_count: int = 0 70 | completion_count: int = 0 71 | model: str 72 | date: str = Field(default_factory=datetime.now().isoformat) 73 | seconds_to_first_token: float = Field( 74 | default=0.0, description="Delta time (sec) to first response token's arrival." 75 | ) 76 | seconds_to_last_token: float = Field( 77 | default=0.0, description="Delta time (sec) to last response token's arrival." 78 | ) 79 | logprob: float | None = Field( 80 | default=None, description="Sum of logprobs in the completion." 81 | ) 82 | reasoning_content: str | None = Field( 83 | default=None, description="Reasoning content from LLMs such as DeepSeek-R1." 84 | ) 85 | 86 | def __str__(self) -> str: 87 | return self.text or "" 88 | 89 | @computed_field # type: ignore[prop-decorator] 90 | @property 91 | def cost(self) -> float: 92 | """Return the cost of the result in dollars.""" 93 | if self.prompt_count and self.completion_count: 94 | try: 95 | pc = litellm.model_cost[self.model]["input_cost_per_token"] 96 | oc = litellm.model_cost[self.model]["output_cost_per_token"] 97 | return pc * self.prompt_count + oc * self.completion_count 98 | except KeyError: 99 | logger.warning(f"Could not find cost for model {self.model}.") 100 | return 0.0 101 | 102 | # TODO: These two methods were implemented in ldp, but not in pqa. 103 | # TODO: Check if they're necessary 104 | @property 105 | def provider(self) -> str: 106 | """Get the model provider's name (e.g. "openai", "mistral").""" 107 | return litellm.get_llm_provider(self.model)[1] 108 | 109 | def get_supported_openai_params(self) -> list[str] | None: 110 | """Get the supported OpenAI parameters for the model.""" 111 | return litellm.get_supported_openai_params(self.model) 112 | -------------------------------------------------------------------------------- /packages/lmi/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/packages/lmi/tests/__init__.py -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestLiteLLMModel.test_cost_call[Anthropic-model].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-5-haiku-20241022", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "What is the meaning of the universe?"}]}], "temperature": 6 | 0, "max_tokens": 56, "system": [{"type": "text", "text": "Respond with single 7 | words."}]}' 8 | headers: 9 | accept: 10 | - application/json 11 | accept-encoding: 12 | - gzip, deflate 13 | anthropic-version: 14 | - "2023-06-01" 15 | connection: 16 | - keep-alive 17 | content-length: 18 | - "253" 19 | content-type: 20 | - application/json 21 | host: 22 | - api.anthropic.com 23 | user-agent: 24 | - litellm/1.59.5 25 | method: POST 26 | uri: https://api.anthropic.com/v1/messages 27 | response: 28 | body: 29 | string: !!binary | 30 | H4sIAAAAAAAAA2SOMWvDMBSE/8vNcrDVplCNXbpkrOkQilDkRyLbeXKkJ9Ji/N+LQwMtnQ7uuztu 31 | RuhgcM5HWzdtv7vu0uHSvz21L8+U3g/9KzdQkK+J1hTl7I4EhRTH1XA5hyyOBQrn2NEIAz+60lH1 32 | UG2rkwtDqXStH5taayj4yEIsMPv5vin0ubZvYtDywPHKGywfClniZBO5HBkGxJ2Vkhg/INOlEHuC 33 | 4TKOCuV2zcwIPBWxEgfiDKMbBe/8iaxP5CREtn8D9Z0nct1/Fov8drbL8g0AAP//AwD1M9oFMwEA 34 | AA== 35 | headers: 36 | CF-RAY: 37 | - 909370809f0f15e3-SJC 38 | Connection: 39 | - keep-alive 40 | Content-Encoding: 41 | - gzip 42 | Content-Type: 43 | - application/json 44 | Date: 45 | - Tue, 28 Jan 2025 19:37:51 GMT 46 | Server: 47 | - cloudflare 48 | Transfer-Encoding: 49 | - chunked 50 | X-Robots-Tag: 51 | - none 52 | anthropic-ratelimit-input-tokens-limit: 53 | - "5000000" 54 | anthropic-ratelimit-input-tokens-remaining: 55 | - "5000000" 56 | anthropic-ratelimit-input-tokens-reset: 57 | - "2025-01-28T19:37:51Z" 58 | anthropic-ratelimit-output-tokens-limit: 59 | - "1000000" 60 | anthropic-ratelimit-output-tokens-remaining: 61 | - "1000000" 62 | anthropic-ratelimit-output-tokens-reset: 63 | - "2025-01-28T19:37:51Z" 64 | anthropic-ratelimit-requests-limit: 65 | - "5000" 66 | anthropic-ratelimit-requests-remaining: 67 | - "4999" 68 | anthropic-ratelimit-requests-reset: 69 | - "2025-01-28T19:37:51Z" 70 | anthropic-ratelimit-tokens-limit: 71 | - "6000000" 72 | anthropic-ratelimit-tokens-remaining: 73 | - "6000000" 74 | anthropic-ratelimit-tokens-reset: 75 | - "2025-01-28T19:37:51Z" 76 | cf-cache-status: 77 | - DYNAMIC 78 | request-id: 79 | - req_01Eppf1xbA8q5A3TkRYsr68g 80 | via: 81 | - 1.1 google 82 | status: 83 | code: 200 84 | message: OK 85 | version: 1 86 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestLiteLLMModel.test_cost_call[OpenAI-model].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "What is the meaning of the universe?"}], "model": 6 | "gpt-4o-mini-2024-07-18", "max_tokens": 56, "temperature": 0}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - "215" 16 | content-type: 17 | - application/json 18 | host: 19 | - api.openai.com 20 | user-agent: 21 | - AsyncOpenAI/Python 1.60.0 22 | x-stainless-arch: 23 | - arm64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - MacOS 30 | x-stainless-package-version: 31 | - 1.60.0 32 | x-stainless-raw-response: 33 | - "true" 34 | x-stainless-retry-count: 35 | - "0" 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.11.10 40 | method: POST 41 | uri: https://api.openai.com/v1/chat/completions 42 | response: 43 | body: 44 | string: !!binary | 45 | H4sIAAAAAAAAAwAAAP//jJJBT+MwEIXv+RXWnBuUlqKU3jhwQHBgkRAHhCLXniSmjseyJ7usUP87 46 | clKaIFhpLznMN+/lzXjeMyHAaNgKUK1k1XmbX/W23fx6etjvLx4e63b55zEEurm7vLnt6B4WSUG7 47 | V1T8qTpT1HmLbMiNWAWUjMl1WZ5visvzoiwG0JFGm2SN53xNeWecyVfFap0XZb7cHNUtGYURtuI5 48 | E0KI9+GbcjqNb7AVg9dQ6TBG2SBsT01CQCCbKiBjNJGlY1hMUJFjdEP06zcTGZ3Cs3lDwLqPMoV0 49 | vbXH+uH0R0uND7SLR36q18aZ2FYBZSSX3COTh4EeMiFehsn6L2HBB+o8V0x7dMlwtR7tYNrnBD8Z 50 | E0s702wWP5hVGlkaG2eLASVVi3pSTluUvTY0A9ls5O9ZfvIexzau+R/7CSiFnlFXPqA26uu8U1vA 51 | dGz/ajuteAgMEcNvo7BigyE9g8Za9nY8AYh/I2NX1cY1GHww4x3UvipXqEu5u1gryA7ZBwAAAP// 52 | AwC1HS6hFQMAAA== 53 | headers: 54 | CF-Cache-Status: 55 | - DYNAMIC 56 | CF-RAY: 57 | - 9093707c3c60174a-SJC 58 | Connection: 59 | - keep-alive 60 | Content-Encoding: 61 | - gzip 62 | Content-Type: 63 | - application/json 64 | Date: 65 | - Tue, 28 Jan 2025 19:37:51 GMT 66 | Server: 67 | - cloudflare 68 | Set-Cookie: 69 | - __cf_bm=dqIilPhCKBMWrfzBIKv4Z773uZfIHyMVkz9CwPgIUtA-1738093071-1.0.1.1-eLZ4cnMA8fb5u55VPgPzLBwaJH8MbcL3FaS_WIZ7m1Y2PPk00cpUZHpTcwEErfVXeYRqTRcDOGjb4niE.byJ3A; 70 | path=/; expires=Tue, 28-Jan-25 20:07:51 GMT; domain=.api.openai.com; HttpOnly; 71 | Secure; SameSite=None 72 | - _cfuvid=VubHV4jc3NAG1.KYK.9ezrasy_FR9RQDSt8gSdEubnM-1738093071389-0.0.1.1-604800000; 73 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 74 | Transfer-Encoding: 75 | - chunked 76 | X-Content-Type-Options: 77 | - nosniff 78 | access-control-expose-headers: 79 | - X-Request-ID 80 | alt-svc: 81 | - h3=":443"; ma=86400 82 | openai-organization: 83 | - future-house-xr4tdh 84 | openai-processing-ms: 85 | - "490" 86 | openai-version: 87 | - "2020-10-01" 88 | strict-transport-security: 89 | - max-age=31536000; includeSubDomains; preload 90 | x-ratelimit-limit-requests: 91 | - "30000" 92 | x-ratelimit-limit-tokens: 93 | - "150000000" 94 | x-ratelimit-remaining-requests: 95 | - "29999" 96 | x-ratelimit-remaining-tokens: 97 | - "149999926" 98 | x-ratelimit-reset-requests: 99 | - 2ms 100 | x-ratelimit-reset-tokens: 101 | - 0s 102 | x-request-id: 103 | - req_e0ce487241223e824cc737d821b5495c 104 | status: 105 | code: 200 106 | message: OK 107 | version: 1 108 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[with-router].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Please tell me a story"}], "model": 5 | "gpt-4o-mini", "max_tokens": 3}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "110" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.46.1 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.46.1 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-runtime: 34 | - CPython 35 | x-stainless-runtime-version: 36 | - 3.12.7 37 | method: POST 38 | uri: https://api.openai.com/v1/chat/completions 39 | response: 40 | body: 41 | string: !!binary | 42 | H4sIAAAAAAAAA4ySUUvDMBSF3/srQp5b6bp13fomE3wRRBEUREqW3LaZaRKTFBxj/13SdmuHE3zJ 43 | w/3uOTn3JocAIcwZzhGmNXG00SK6fSNfyw1/TWdsV64fnjYZZC/ts7in+/oOh16htjug7qS6oarR 44 | AhxXssfUAHHgXWfZPEmX62y96kCjGAgvq7SLFipquORREieLKM6i2WpQ14pTsDhH7wFCCB260+eU 45 | DL5xjuLwVGnAWlIBzs9NCGGjhK9gYi23jkiHwxFSJR3ILvqjpIBarSQi0w4DZWuJTylbIYb68Xyl 46 | UJU2amsHfq6XXHJbFwaIVdLbC5CVq3HHjwFCH91w7UVerI1qtCuc+gTpLWdJb4jHlY5wPjCnHBET 47 | TRpeMSsYOMKFnewGU0JrYKNyXCRpGVcTEEyG/p3lmnc/OJfVf+xHQCloB6zQBhinl/OObQb8f/ur 48 | 7bziLjC2e+ugKUouKzDa8P61S13EWZxuy1VGYxwcgx8AAAD//wMAIAzc+vsCAAA= 49 | headers: 50 | CF-Cache-Status: 51 | - DYNAMIC 52 | CF-RAY: 53 | - 8e84b2f90c81230e-SJC 54 | Connection: 55 | - keep-alive 56 | Content-Encoding: 57 | - gzip 58 | Content-Type: 59 | - application/json 60 | Date: 61 | - Mon, 25 Nov 2024 21:23:18 GMT 62 | Server: 63 | - cloudflare 64 | Set-Cookie: 65 | - __cf_bm=KqKw89zgaG32GNn3lg4IvjG2X2zLmPKRiY1oedcDUVM-1732569798-1.0.1.1-y_oblt_Jp3n1T.HtHFHrxbRegDqoC8gojQPBSV52IMBH.bx8c0QNAUrWotLzzQGqqbIDjdhl0AUutzvWk20psg; 66 | path=/; expires=Mon, 25-Nov-24 21:53:18 GMT; domain=.api.openai.com; HttpOnly; 67 | Secure; SameSite=None 68 | - _cfuvid=F_vLQWJJbY8GvEB4YIomOCy2NMswE7Ex8TL0Z4OIxgg-1732569798934-0.0.1.1-604800000; 69 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 70 | Transfer-Encoding: 71 | - chunked 72 | X-Content-Type-Options: 73 | - nosniff 74 | access-control-expose-headers: 75 | - X-Request-ID 76 | alt-svc: 77 | - h3=":443"; ma=86400 78 | openai-organization: 79 | - future-house-xr4tdh 80 | openai-processing-ms: 81 | - "256" 82 | openai-version: 83 | - "2020-10-01" 84 | strict-transport-security: 85 | - max-age=31536000; includeSubDomains; preload 86 | x-ratelimit-limit-requests: 87 | - "30000" 88 | x-ratelimit-limit-tokens: 89 | - "150000000" 90 | x-ratelimit-remaining-requests: 91 | - "29997" 92 | x-ratelimit-remaining-tokens: 93 | - "149998170" 94 | x-ratelimit-reset-requests: 95 | - 4ms 96 | x-ratelimit-reset-tokens: 97 | - 0s 98 | x-request-id: 99 | - req_77977a66fa96e40ffe5c3bc7840c0948 100 | status: 101 | code: 200 102 | message: OK 103 | version: 1 104 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[without-router].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Please tell me a story"}], "model": 5 | "gpt-4o-mini", "max_tokens": 3}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "110" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.46.1 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.46.1 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-runtime: 34 | - CPython 35 | x-stainless-runtime-version: 36 | - 3.12.7 37 | method: POST 38 | uri: https://api.openai.com/v1/chat/completions 39 | response: 40 | body: 41 | string: !!binary | 42 | H4sIAAAAAAAAA4ySMW+DMBCFd36F5RkqQkJI2DpnqBSpUtqqQo59gFtju7aRkkb575UhAaKmUhcP 43 | 9917fnf2KUAIc4ZzhGlNHG20iB535GvewmZ53MR8szhu29ftLnmx3/T5UOHQK9T+A6i7qh6oarQA 44 | x5XsMTVAHHjXWTZP0uU6W6cdaBQD4WWVdtFCRQ2XPEriZBHFWTRbXdS14hQsztFbgBBCp+70OSWD 45 | A85RHF4rDVhLKsD50IQQNkr4CibWcuuIdDgcIVXSgeyiP0kKqNVKIjLtMFC2lviUshXiUj8PVwpV 46 | aaP29sKHesklt3VhgFglvb0AWbkad/wcIPTeDdfe5MXaqEa7wqlPkN5ylvSGeFzpCOcX5pQjYqJJ 47 | wztmBQNHuLCT3WBKaA1sVI6LJC3jagKCydC/s9zz7gfnsvqP/QgoBe2AFdoA4/R23rHNgP9vf7UN 48 | K+4CY3u0Dpqi5LICow3vX7vURZzF6b5cZTTGwTn4AQAA//8DAGJIF8n7AgAA 49 | headers: 50 | CF-Cache-Status: 51 | - DYNAMIC 52 | CF-RAY: 53 | - 8e84b2e46a3e5c18-SJC 54 | Connection: 55 | - keep-alive 56 | Content-Encoding: 57 | - gzip 58 | Content-Type: 59 | - application/json 60 | Date: 61 | - Mon, 25 Nov 2024 21:23:15 GMT 62 | Server: 63 | - cloudflare 64 | Set-Cookie: 65 | - __cf_bm=pmbS4O0SdzCjzvZjensXpq5w1I1GUEOUOh_2ExJ8_Rc-1732569795-1.0.1.1-RsW0ExCXu..OFPcHXSvL3vh7_PqZu9gX0DgJI0BGjr2oborEPzdC6ZSsqZTfP9zf3YOigH1hcfDePksbYyIO8A; 66 | path=/; expires=Mon, 25-Nov-24 21:53:15 GMT; domain=.api.openai.com; HttpOnly; 67 | Secure; SameSite=None 68 | - _cfuvid=v_wT.VKOzzmIot1JtDgHglmHPmgOB.YvZxQznjpUEiA-1732569795663-0.0.1.1-604800000; 69 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 70 | Transfer-Encoding: 71 | - chunked 72 | X-Content-Type-Options: 73 | - nosniff 74 | access-control-expose-headers: 75 | - X-Request-ID 76 | alt-svc: 77 | - h3=":443"; ma=86400 78 | openai-organization: 79 | - future-house-xr4tdh 80 | openai-processing-ms: 81 | - "150" 82 | openai-version: 83 | - "2020-10-01" 84 | strict-transport-security: 85 | - max-age=31536000; includeSubDomains; preload 86 | x-ratelimit-limit-requests: 87 | - "30000" 88 | x-ratelimit-limit-tokens: 89 | - "150000000" 90 | x-ratelimit-remaining-requests: 91 | - "29999" 92 | x-ratelimit-remaining-tokens: 93 | - "149999990" 94 | x-ratelimit-reset-requests: 95 | - 2ms 96 | x-ratelimit-reset-tokens: 97 | - 0s 98 | x-request-id: 99 | - req_817b41328aed9a1e4a84bd7ec79e22f7 100 | status: 101 | code: 200 102 | message: OK 103 | version: 1 104 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestMultipleCompletion.test_model[gpt-3.5-turbo-0125].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-3.5-turbo-0125", 6 | "max_tokens": 4096, "n": 2, "temperature": 0.1}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - "206" 16 | content-type: 17 | - application/json 18 | host: 19 | - api.openai.com 20 | user-agent: 21 | - AsyncOpenAI/Python 1.60.0 22 | x-stainless-arch: 23 | - arm64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - MacOS 30 | x-stainless-package-version: 31 | - 1.60.0 32 | x-stainless-raw-response: 33 | - "true" 34 | x-stainless-retry-count: 35 | - "0" 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.12.4 40 | method: POST 41 | uri: https://api.openai.com/v1/chat/completions 42 | response: 43 | body: 44 | string: !!binary | 45 | H4sIAAAAAAAAAwAAAP//1FPLTsMwELznKyyfm6oPSqE3JCQOcELihFDk2NvUxfEa7wYBFf+OnLRN 46 | UEHiyiWHmZ3J7MO7TAhpjVwJqTeKdR1cfkXb8rq2lb29O79Xdx+3b01J6uHhZQsXUY6SAsstaD6o 47 | xhrr4IAt+o7WERRDcp0u58vzycV8cdkSNRpwSVYFzufjRc5NLDGfTGeLvXKDVgPJlXjMhBBi135T 48 | Rm/gTa7EZHRAaiBSFcjVsUgIGdElRCoiS6w8y1FPavQMvo19g2iGVIR1QypF841ze/zz+C+HVYhY 49 | 0p4/4mvrLW2KCIrQJ19iDDIbiE8amP6bBjIhntqlNN9iyhCxDlwwPoNPhrNZZyf7MxiQe46RlRvA 50 | Z6MfzAoDrKyjwUikVnoDplf2B6AaY3FADMd+muUn765t66u/2PeE1hAYTBEiGKu/99uXRUhv5Ley 51 | 44jbwJIgvloNBVuIaQ0G1qpx3fIlvRNDXaytryCGaNsLaNf8mX0BAAD//wMAWjXvTcEDAAA= 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 906536b37c63679b-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Thu, 23 Jan 2025 04:59:19 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=bbe91RL8pynoc8FwJWONXTz9iPs24xtiuVm.KiZccgg-1737608359-1.0.1.1-4MY0muOtVyYr8hyafTjKQEnS6CzShHD_gr1zV4jE0BX8Q7GKl0jg.CfuuJAejDm7ssESqqnEFdi90KdjJqX0iA; 69 | path=/; expires=Thu, 23-Jan-25 05:29:19 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=fVO2Os4JLfnO.SjLcpo.a17BzAYLtUPKLCrieYjvhQo-1737608359268-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "212" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "12000" 91 | x-ratelimit-limit-tokens: 92 | - "1000000" 93 | x-ratelimit-remaining-requests: 94 | - "11999" 95 | x-ratelimit-remaining-tokens: 96 | - "991822" 97 | x-ratelimit-reset-requests: 98 | - 5ms 99 | x-ratelimit-reset-tokens: 100 | - 490ms 101 | x-request-id: 102 | - req_d5da645d9bde91ef8bbcd32b1d58cb33 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestMultipleCompletion.test_model[gpt-3.5-turbo].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-3.5-turbo"}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "153" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.59.7 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.59.7 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAAwAAAP//jJJPT8MwDMXv/RRRzt20v0zaDXEYQmgHJMQBoSpLvDYsjUPibqBp3x2l 45 | LWsnhsSlh/ezX5/tHBPGuFZ8ybgsBMnSmcHtx+7uxdyv5Wo3Wz/A0wGfZyu3f8z3CzrwNHbg5h0k 46 | /XQNJZbOAGm0DZYeBEF0HS+mi/F0Nhrf1KBEBSa25Y4G0+F8QJXf4GA0nszbzgK1hMCX7DVhjLFj 47 | /Y0ZrYJPvmSj9EcpIQSRA1+eixjjHk1UuAhBBxKWeNpBiZbA1rFXiKqPPGyrIGI0WxnT6qfzvwzm 48 | zuMmtPysb7XVocg8iIA2+gZCx2t6Shh7q2eqLmJy57F0lBHuwEbDyaSx490We7BlhCRMT56lV8wy 49 | BSS0Cb2VcClkAarr7PYnKqWxB5LeyL+zXPNuxtY2/499B6QER6Ay50FpeTlvV+YhPrG/ys4rrgPz 50 | AH6vJWSkwcczKNiKyjTH5+ErEJTZVtscvPO6fgH1mU/JNwAAAP//AwAmgFaGAAMAAA== 51 | headers: 52 | CF-Cache-Status: 53 | - DYNAMIC 54 | CF-RAY: 55 | - 9037fa12aa64ebe5-SJC 56 | Connection: 57 | - keep-alive 58 | Content-Encoding: 59 | - gzip 60 | Content-Type: 61 | - application/json 62 | Date: 63 | - Fri, 17 Jan 2025 17:13:36 GMT 64 | Server: 65 | - cloudflare 66 | Set-Cookie: 67 | - __cf_bm=sBEhET_ucmEw1fj49k97O6lYPFKpa8FC7g85fCwHQdA-1737134016-1.0.1.1-Y2pqd.8untxxrd4CV3MVcvrzkxtXf7VBF717.ygynQmR3B_Hl_YvH2o4zsYUQFj75ENY35qzY8hjkF3qGvyZOw; 68 | path=/; expires=Fri, 17-Jan-25 17:43:36 GMT; domain=.api.openai.com; HttpOnly; 69 | Secure; SameSite=None 70 | - _cfuvid=.OXPSLgaNOtRCwyCdctf7AFOBazBNO8OQRpV77U6rCM-1737134016656-0.0.1.1-604800000; 71 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 72 | Transfer-Encoding: 73 | - chunked 74 | X-Content-Type-Options: 75 | - nosniff 76 | access-control-expose-headers: 77 | - X-Request-ID 78 | alt-svc: 79 | - h3=":443"; ma=86400 80 | openai-organization: 81 | - future-house-xr4tdh 82 | openai-processing-ms: 83 | - "133" 84 | openai-version: 85 | - "2020-10-01" 86 | strict-transport-security: 87 | - max-age=31536000; includeSubDomains; preload 88 | x-ratelimit-limit-requests: 89 | - "12000" 90 | x-ratelimit-limit-tokens: 91 | - "1000000" 92 | x-ratelimit-remaining-requests: 93 | - "11999" 94 | x-ratelimit-remaining-tokens: 95 | - "999969" 96 | x-ratelimit-reset-requests: 97 | - 5ms 98 | x-ratelimit-reset-tokens: 99 | - 1ms 100 | x-request-id: 101 | - req_e1b9df957943f220f9b008b98d8d4ad3 102 | status: 103 | code: 200 104 | message: OK 105 | version: 1 106 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestMultipleCompletion.test_model[gpt-4o-mini-2024-07-18].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-4o-mini-2024-07-18", 6 | "max_tokens": 4096, "n": 2, "temperature": 0.1}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - "210" 16 | content-type: 17 | - application/json 18 | host: 19 | - api.openai.com 20 | user-agent: 21 | - AsyncOpenAI/Python 1.59.4 22 | x-stainless-arch: 23 | - arm64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - MacOS 30 | x-stainless-package-version: 31 | - 1.59.4 32 | x-stainless-raw-response: 33 | - "true" 34 | x-stainless-retry-count: 35 | - "0" 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.12.7 40 | method: POST 41 | uri: https://api.openai.com/v1/chat/completions 42 | response: 43 | body: 44 | string: !!binary | 45 | H4sIAAAAAAAAAwAAAP//1FPBatwwEL37K9Q5r8va3bXD3paE5tDeWgqlFKOVxrZSWSOkcUkJ+fci 46 | e7N2SAq95uLDvHnP782MHjIhwGg4CFC9ZDV4mx/j9693lf/YX8fjpxs1fj59+cb37W11LMMNbBKD 47 | Tneo+In1XtHgLbIhN8MqoGRMqkX9od5X+6ooJmAgjTbROs/5jvLBOJOX23KXb+u8uDqzezIKIxzE 48 | j0wIIR6mb/LpNN7DQWw3T5UBY5QdwuHSJAQEsqkCMkYTWTqGzQIqcoxusn5LpN+tsYDtGGXy50Zr 49 | z/XHy88sdT7QKZ7xS701zsS+CSgjuSQcmTxkK/KLBMXbSZAJ8XNay/jMJ/hAg+eG6Re6JFiWsxws 50 | x7CA1RljYmlXnKvNK2KNRpbGxtVMQEnVo16YywnIURtaAeu5v/TymvYc27juf+QXQCn0jLrxAbVR 51 | z/MubQHTS/lX22XEk2GIGH4bhQ0bDGkNGls52nn7EP9ExqFpjesw+GDmE2h9U5eoa3na7xRkj9lf 52 | AAAA//8DAL+Guu3SAwAA 53 | headers: 54 | CF-RAY: 55 | - 906123104eef6804-SJC 56 | Connection: 57 | - keep-alive 58 | Content-Encoding: 59 | - gzip 60 | Content-Type: 61 | - application/json 62 | Date: 63 | - Wed, 22 Jan 2025 17:06:52 GMT 64 | Server: 65 | - cloudflare 66 | Set-Cookie: 67 | - __cf_bm=0AU5UBKcyJz8Is5N3AXFGiVfs40PPUx5w39vHFzp8ZE-1737565612-1.0.1.1-i3ipRUtk3POiwcBFVN_BwKvHxpOBDsWu0rNxwUC2FDSvaO0r5Ye.s9_FdZkNSKdiC.dRJRy5LbqjCCll.G26oQ; 68 | path=/; expires=Wed, 22-Jan-25 17:36:52 GMT; domain=.api.openai.com; HttpOnly; 69 | Secure; SameSite=None 70 | - _cfuvid=_hKYeboiOAsgsNVboFQJcjY4DMG.3STQtZ6iRMz1QOU-1737565612020-0.0.1.1-604800000; 71 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 72 | Transfer-Encoding: 73 | - chunked 74 | X-Content-Type-Options: 75 | - nosniff 76 | access-control-expose-headers: 77 | - X-Request-ID 78 | alt-svc: 79 | - h3=":443"; ma=86400 80 | cf-cache-status: 81 | - DYNAMIC 82 | openai-organization: 83 | - future-house-xr4tdh 84 | openai-processing-ms: 85 | - "338" 86 | openai-version: 87 | - "2020-10-01" 88 | strict-transport-security: 89 | - max-age=31536000; includeSubDomains; preload 90 | x-ratelimit-limit-requests: 91 | - "30000" 92 | x-ratelimit-limit-tokens: 93 | - "150000000" 94 | x-ratelimit-remaining-requests: 95 | - "29999" 96 | x-ratelimit-remaining-tokens: 97 | - "149991793" 98 | x-ratelimit-reset-requests: 99 | - 2ms 100 | x-ratelimit-reset-tokens: 101 | - 3ms 102 | x-request-id: 103 | - req_af628db22df56361b4e04d5bf342574d 104 | status: 105 | code: 200 106 | message: OK 107 | version: 1 108 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestMultipleCompletion.test_model[gpt-4o-mini].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-4o-mini"}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "151" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.59.7 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.59.7 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA4ySQW/UMBCF7/kVZs4blE2Wbrs3pAq4IcQBJIQirz1J3NoeY09QUbX/HTnZblK1 45 | SFx8mG/e85uxHwshwGg4CFCDZOWCLd//ur+lB3d0Oz987VMlu+/WfPl2++HT5zsNm6yg4x0qflK9 46 | VeSCRTbkZ6wiSsbsut03+22zq26uJuBIo82yPnC5o9IZb8q6qndltS+312f1QEZhgoP4UQghxON0 47 | 5pxe4wMcRLV5qjhMSfYIh0uTEBDJ5grIlExi6Rk2C1TkGf0U/SORfrNmEbsxyZzPj9ae66fLZZb6 48 | EOmYzvxS74w3aWgjykQ+GyemABM9FUL8nIYan+WEEMkFbpnu0WfDup7tYFnlApszY2JpV5p3m1fM 49 | Wo0sjU2rnYCSakC9KJcFylEbWoFiNfLLLK95z2Mb3/+P/QKUwsCo2xBRG/V83qUtYv5n/2q7rHgK 50 | DAnjb6OwZYMxP4PGTo52fn1IfxKjazvje4whmvkLdKE96uumqW+6qwaKU/EXAAD//wMAQaS4thAD 51 | AAA= 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 9037fc09580bd045-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Fri, 17 Jan 2025 17:14:57 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=HlSNS.gT._y7cHrNqXSekKB0MjIrTa0AaS0nE.welTg-1737134097-1.0.1.1-o2GwXAUWpjygAFlkSQqTOJk9PIvrzfkL5hcC4C_ebboFSTvSb8e0018Z42ilOx18M.vaCSWqF.oh861HSvq85Q; 69 | path=/; expires=Fri, 17-Jan-25 17:44:57 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=KHvEM4IFbWYj2GdNChMj5sEdEejFcK_8z8SFW.hDqvY-1737134097138-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "166" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "30000" 91 | x-ratelimit-limit-tokens: 92 | - "150000000" 93 | x-ratelimit-remaining-requests: 94 | - "29999" 95 | x-ratelimit-remaining-tokens: 96 | - "149999970" 97 | x-ratelimit-reset-requests: 98 | - 2ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_a304ff40cdb7bf3c789644e502d1651d 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestTooling.test_empty_tools[Anthropic-empty-tools].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-5-haiku-20241022", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "What does 42 mean?"}]}], "temperature": 0.1, "max_tokens": 6 | 56}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | anthropic-version: 13 | - "2023-06-01" 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "169" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.anthropic.com 22 | user-agent: 23 | - litellm/1.59.5 24 | method: POST 25 | uri: https://api.anthropic.com/v1/messages 26 | response: 27 | body: 28 | string: !!binary | 29 | H4sIAAAAAAAAAwAAAP//ZJFNj9NADIb/iuXLXqaoDW0FuVXa1YLgggQHRFHknTjJkMk4jGe6zVb9 30 | 7yiBSiBOlv28/r6gq7HEQdtqvfnAstu//Hjzss+br/eH3L49dZOgwTSNPKtYlVpGg1H8HCBVp4lC 31 | QoOD1OyxROsp17x6vdqtOnJ9XhXrYrtZFwUatBISh4Tlt8utZuLznL2YErcFOIWGBsnqJ+iDPAdo 32 | ogxwL7n1pHCoadA7UOs4WIbG2eQkgJWB6wmeRHo44ueO4Z1Ltutcz/FO4TG7miEJpI7hkTydpyMC 33 | 6eIf8RD0meONf/HJDZQYPmXWpbo08NE1bH7j4E4clQ1QqOHhxHFKnQvtqyPC+7BI5jEMEGgeOVoZ 34 | xpw4giVvs6fEc1unQEtXvH43qEnGKjKphPnOdK6S9BwU/yDln3neF8uQvTeYlz+UF3RhzOkmLjdb 35 | g5Zsx5WNTPPo1b+C9Y1Hpvp/Jjn9Hdntr9dfAAAA//8DAPISxfshAgAA 36 | headers: 37 | CF-RAY: 38 | - 910ad9db995ded38-SJC 39 | Connection: 40 | - keep-alive 41 | Content-Encoding: 42 | - gzip 43 | Content-Type: 44 | - application/json 45 | Date: 46 | - Wed, 12 Feb 2025 07:26:33 GMT 47 | Server: 48 | - cloudflare 49 | Transfer-Encoding: 50 | - chunked 51 | X-Robots-Tag: 52 | - none 53 | anthropic-ratelimit-input-tokens-limit: 54 | - "5000000" 55 | anthropic-ratelimit-input-tokens-remaining: 56 | - "5000000" 57 | anthropic-ratelimit-input-tokens-reset: 58 | - "2025-02-12T07:26:32Z" 59 | anthropic-ratelimit-output-tokens-limit: 60 | - "1000000" 61 | anthropic-ratelimit-output-tokens-remaining: 62 | - "1000000" 63 | anthropic-ratelimit-output-tokens-reset: 64 | - "2025-02-12T07:26:33Z" 65 | anthropic-ratelimit-requests-limit: 66 | - "5000" 67 | anthropic-ratelimit-requests-remaining: 68 | - "4999" 69 | anthropic-ratelimit-requests-reset: 70 | - "2025-02-12T07:26:32Z" 71 | anthropic-ratelimit-tokens-limit: 72 | - "6000000" 73 | anthropic-ratelimit-tokens-remaining: 74 | - "6000000" 75 | anthropic-ratelimit-tokens-reset: 76 | - "2025-02-12T07:26:32Z" 77 | cf-cache-status: 78 | - DYNAMIC 79 | request-id: 80 | - req_014o74duDsmRJrXujBWdcNDt 81 | via: 82 | - 1.1 google 83 | status: 84 | code: 200 85 | message: OK 86 | version: 1 87 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestTooling.test_empty_tools[Anthropic-no-tools].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-5-haiku-20241022", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "What does 42 mean?"}]}], "temperature": 0.1, "max_tokens": 6 | 56}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | anthropic-version: 13 | - "2023-06-01" 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "169" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.anthropic.com 22 | user-agent: 23 | - litellm/1.59.5 24 | method: POST 25 | uri: https://api.anthropic.com/v1/messages 26 | response: 27 | body: 28 | string: !!binary | 29 | H4sIAAAAAAAAA2SRwYrbQAyGX0XoshenJG42Bd8WUjYtSyGwPTXFTMeyZ/B45I40u3FD3r3YbWBL 30 | T0L6folf0gV9gxUO0tXrzdOB7fFYfvm8a3dx82F7yOr2v7BAnUaaVSRiOsICE4e5YES8qImKBQ7c 31 | UMAKbTC5odX71f3KGd/nVbkut5t1WWKBlqNSVKy+XW4zlc5z9xIq3JbgBVozcJYwQR/5NUKbeIA9 32 | 5y4YgYfGDHIHYj1FS9B6q54jWB6omeAHcw8nfHYEB6/WOd9TuhN4zL4hUAZ1BI8mmPN0QjCy5Cd8 33 | iPJK6ca/BvWDUYJjJlmmcwtPvqXiD47+hZJQASY28PGF0qTOx+7dCeFTXCSzjQIMSB4pWR7GrJTA 34 | mhCogT3RCM+Oc+cUr98LFOWxTmSE43xkc66Ve4qCf5HQzzwvi1XMIRSYlydUF/RxzHoTV5ttgdZY 35 | R7VNZGbf9b+C9Y0nMs3/jLO+rdzvrtffAAAA//8DAEH9A5MeAgAA 36 | headers: 37 | CF-RAY: 38 | - 910ada053e7f689a-SJC 39 | Connection: 40 | - keep-alive 41 | Content-Encoding: 42 | - gzip 43 | Content-Type: 44 | - application/json 45 | Date: 46 | - Wed, 12 Feb 2025 07:26:40 GMT 47 | Server: 48 | - cloudflare 49 | Transfer-Encoding: 50 | - chunked 51 | X-Robots-Tag: 52 | - none 53 | anthropic-ratelimit-input-tokens-limit: 54 | - "5000000" 55 | anthropic-ratelimit-input-tokens-remaining: 56 | - "5000000" 57 | anthropic-ratelimit-input-tokens-reset: 58 | - "2025-02-12T07:26:39Z" 59 | anthropic-ratelimit-output-tokens-limit: 60 | - "1000000" 61 | anthropic-ratelimit-output-tokens-remaining: 62 | - "1000000" 63 | anthropic-ratelimit-output-tokens-reset: 64 | - "2025-02-12T07:26:40Z" 65 | anthropic-ratelimit-requests-limit: 66 | - "5000" 67 | anthropic-ratelimit-requests-remaining: 68 | - "4999" 69 | anthropic-ratelimit-requests-reset: 70 | - "2025-02-12T07:26:39Z" 71 | anthropic-ratelimit-tokens-limit: 72 | - "6000000" 73 | anthropic-ratelimit-tokens-remaining: 74 | - "6000000" 75 | anthropic-ratelimit-tokens-reset: 76 | - "2025-02-12T07:26:39Z" 77 | cf-cache-status: 78 | - DYNAMIC 79 | request-id: 80 | - req_014N65NM1873o5sdT25QAzx2 81 | via: 82 | - 1.1 google 83 | status: 84 | code: 200 85 | message: OK 86 | version: 1 87 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestTooling.test_empty_tools[OpenAI-empty-tools].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "What does 42 mean?"}], "model": 5 | "gpt-4o-mini-2024-07-18", "max_tokens": 56, "n": 1, "temperature": 0.1}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "146" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.60.0 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.60.0 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.4 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAAwAAAP//jFNNb9swDL3nVxC69GIX+WzW3Dp0yzbssALbaRkMRaJtNrJkiHRbo+h/ 45 | H2yncYZ1wC4G9B758MhHP08AFFm1AWVKLaaqXfp+Orsz23zbsKzu2y+8Xcy3324/PpkqHA4q6TrC 46 | /h6NvHZdmlDVDoWCH2gTUQt2qrP14nqxXM2upz1RBYuuaytqSZchrchTOp/Ol+l0nc7eHbvLQAZZ 47 | beDnBADguf92Pr3FJ7WBXqtHKmTWBarNqQhAxeA6RGlmYtFeVDKSJnhB31v/XiL4ptpjhOUciCHX 48 | VWjYtXDw4dGDZtgpKRFuPD9iBAnQvX44oUoLwl2D3M0MIYevlGMy0J4eMDImoL2FDw8YWynJFzsF 49 | 5OE2NIXTDDdWV3wBbAi9QcjJ9EqMkZAT2PXmPpGYsqQDxguGbUMWXz1stdNP7eVOwWffAywhtglo 50 | KGJo6s5R2dYYU/KCzlGBXmCP5AuGfUPOMujzrUTMG9ZdMr5x7oi/nNbsQlHHsOcjf8Jz8sRlFlFz 51 | 8N1KHfpCStXzLxOAX32gzR8ZqTqGqpZMwgF9JzlbDIJqPKORXF0dSQmi3YhfXSdvqGUWRZPjs4NQ 52 | RpsS7dg5Xo9uLIUzYnI29d9m3tIeJidf/I/8SBiDtaDN6oh2SP6tsojdT/avstOOe8OKMT6QwUwI 53 | Y5eExVw3bjh9xS0LVllOvsBYRxruP6+z9RztWu9XS6MmL5PfAAAA//8DAIQI5eENBAAA 54 | headers: 55 | CF-RAY: 56 | - 910ad9cd0d9dfae3-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Wed, 12 Feb 2025 07:26:30 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=TSiOYmPilz2y7Jm0I0xOnHulR5vDLiJjM_Tk_6ibsb0-1739345190-1.0.1.1-XaU3LvYSVCset.TzV8TpZunDbHgKU02_JUjyYrd1cn8K4eSx2UYKCGbZis4LZejgM5LMmHAkUcbIsTRTeXrgyQ; 69 | path=/; expires=Wed, 12-Feb-25 07:56:30 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=U0EwGnOG.JseWfrnCZ1q.Sl0s6BhsNoFqTwgRWuYAJs-1739345190935-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | cf-cache-status: 82 | - DYNAMIC 83 | openai-organization: 84 | - future-house-xr4tdh 85 | openai-processing-ms: 86 | - "905" 87 | openai-version: 88 | - "2020-10-01" 89 | strict-transport-security: 90 | - max-age=31536000; includeSubDomains; preload 91 | x-ratelimit-limit-requests: 92 | - "30000" 93 | x-ratelimit-limit-tokens: 94 | - "150000000" 95 | x-ratelimit-remaining-requests: 96 | - "29999" 97 | x-ratelimit-remaining-tokens: 98 | - "149999937" 99 | x-ratelimit-reset-requests: 100 | - 2ms 101 | x-ratelimit-reset-tokens: 102 | - 0s 103 | x-request-id: 104 | - req_ca2b9465ba11da2901a8230e9fec93ba 105 | status: 106 | code: 200 107 | message: OK 108 | version: 1 109 | -------------------------------------------------------------------------------- /packages/lmi/tests/cassettes/TestTooling.test_empty_tools[OpenAI-no-tools].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "What does 42 mean?"}], "model": 5 | "gpt-4o-mini-2024-07-18", "max_tokens": 56, "n": 1, "temperature": 0.1}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "146" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.60.0 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.60.0 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.4 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAAwAAAP//jFNNb9swDL3nVxC69GIXcZI2bW5bC7Q7bd3HaRkMRaJtLrLkiFJao+h/ 45 | L2yncYZ1wC4G9B758MhHP08ABGmxAqEqGVTdmPTjNHvQN/uHOcer28/hS/Z41377cfP1flfNdiLp 46 | OtzmN6rw1nWuXN0YDOTsQCuPMmCnmi3n1/PFRXad9UTtNJqurWxCunBpTZbS2XS2SKfLNLs6dFeO 47 | FLJYwc8JAMBz/+18Wo1PYgXT5A2pkVmWKFbHIgDhnekQIZmJg7RBJCOpnA1oe+vfKwQb6w16WMyA 48 | GApZu8imha11jxYkw1qECkFafkQPwUH3iiZQLQPCLiJ3M4MrwFCByUBb2qNnTEBaDbhH34aKbLkW 49 | QBZuXSyNZPigZc1nwIrQKoSCVK/E6Ak5gXVv7p6Cqiraoj9juIuk8c3DnTTyqT1fC/hke4CD820C 50 | EkrvYtM5qtoGfUo2oDFUog2wQbIlwyaS0QzydCsei8iyS8ZGYw74y3HNxpWNdxs+8Ee8IEtc5R4l 51 | O9ut1KAtQyV6/mUC8KsPNP6RkWi8q5uQB7dF20lm80FQjGc0kheXBzK4IM2IX14n76jlGoMkwycH 52 | IZRUFeqxc7weGTW5E2JyMvXfZt7THiYnW/6P/EgohU1AnTce9ZD8e2Ueu5/sX2XHHfeGBaPfk8I8 53 | EPouCY2FjGY4fcEtB6zzgmyJvvE03H/R5MsZ6qXcXCyUmLxMXgEAAP//AwCu/+jJDQQAAA== 54 | headers: 55 | CF-Cache-Status: 56 | - DYNAMIC 57 | CF-RAY: 58 | - 910ad9d41ce87ab2-SJC 59 | Connection: 60 | - keep-alive 61 | Content-Encoding: 62 | - gzip 63 | Content-Type: 64 | - application/json 65 | Date: 66 | - Wed, 12 Feb 2025 07:26:32 GMT 67 | Server: 68 | - cloudflare 69 | Set-Cookie: 70 | - __cf_bm=wgGARbubXUu4RCa7O7vqQbGRDDz7T2S6.0lDcAZaWZg-1739345192-1.0.1.1-gi4_gPOvtYOcQgL_U._xS5wMuwepL3zIP3VenwWVZZffP8ANs3vyVFM07I38bKIOXgSi83Bq4Ed8ZUwobaVejA; 71 | path=/; expires=Wed, 12-Feb-25 07:56:32 GMT; domain=.api.openai.com; HttpOnly; 72 | Secure; SameSite=None 73 | - _cfuvid=h0OafY1hscLy_jhACoWa4TkXDoRejaRg0hf_tOisUEU-1739345192160-0.0.1.1-604800000; 74 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 75 | Transfer-Encoding: 76 | - chunked 77 | X-Content-Type-Options: 78 | - nosniff 79 | access-control-expose-headers: 80 | - X-Request-ID 81 | alt-svc: 82 | - h3=":443"; ma=86400 83 | openai-organization: 84 | - future-house-xr4tdh 85 | openai-processing-ms: 86 | - "997" 87 | openai-version: 88 | - "2020-10-01" 89 | strict-transport-security: 90 | - max-age=31536000; includeSubDomains; preload 91 | x-ratelimit-limit-requests: 92 | - "30000" 93 | x-ratelimit-limit-tokens: 94 | - "150000000" 95 | x-ratelimit-remaining-requests: 96 | - "29999" 97 | x-ratelimit-remaining-tokens: 98 | - "149999937" 99 | x-ratelimit-reset-requests: 100 | - 2ms 101 | x-ratelimit-reset-tokens: 102 | - 0s 103 | x-request-id: 104 | - req_0311ae3de81850199f4bfe95ab85a1e2 105 | status: 106 | code: 200 107 | message: OK 108 | version: 1 109 | -------------------------------------------------------------------------------- /packages/lmi/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import shutil 5 | from collections.abc import Iterator 6 | from pathlib import Path 7 | from typing import Any 8 | 9 | import pytest 10 | from dotenv import load_dotenv 11 | 12 | from lmi.utils import ( 13 | ANTHROPIC_API_KEY_HEADER, 14 | CROSSREF_KEY_HEADER, 15 | OPENAI_API_KEY_HEADER, 16 | SEMANTIC_SCHOLAR_KEY_HEADER, 17 | filter_api_keys, 18 | ) 19 | 20 | TESTS_DIR = Path(__file__).parent 21 | CASSETTES_DIR = TESTS_DIR / "cassettes" 22 | 23 | 24 | @pytest.fixture(autouse=True, scope="session") 25 | def _load_env() -> None: 26 | load_dotenv() 27 | 28 | 29 | @pytest.fixture(scope="session", name="vcr_config") 30 | def fixture_vcr_config() -> dict[str, Any]: 31 | return { 32 | "filter_headers": [ 33 | CROSSREF_KEY_HEADER, 34 | SEMANTIC_SCHOLAR_KEY_HEADER, 35 | OPENAI_API_KEY_HEADER, 36 | ANTHROPIC_API_KEY_HEADER, 37 | "cookie", 38 | ], 39 | "before_record_request": filter_api_keys, 40 | "record_mode": "once", 41 | "allow_playback_repeats": True, 42 | "cassette_library_dir": str(CASSETTES_DIR), 43 | } 44 | 45 | 46 | @pytest.fixture 47 | def tmp_path_cleanup(tmp_path: Path) -> Iterator[Path]: 48 | yield tmp_path 49 | # Cleanup after the test 50 | if tmp_path.exists(): 51 | shutil.rmtree(tmp_path, ignore_errors=True) 52 | 53 | 54 | @pytest.fixture(scope="session", name="stub_data_dir") 55 | def fixture_stub_data_dir() -> Path: 56 | return Path(__file__).parent / "stub_data" 57 | 58 | 59 | @pytest.fixture(name="reset_log_levels") 60 | def fixture_reset_log_levels(caplog) -> Iterator[None]: 61 | logging.getLogger().setLevel(logging.DEBUG) 62 | 63 | for name in logging.root.manager.loggerDict: 64 | logger = logging.getLogger(name) 65 | logger.setLevel(logging.DEBUG) 66 | logger.propagate = True 67 | 68 | caplog.set_level(logging.DEBUG) 69 | 70 | yield 71 | 72 | for name in logging.root.manager.loggerDict: 73 | logger = logging.getLogger(name) 74 | logger.setLevel(logging.NOTSET) 75 | logger.propagate = True 76 | -------------------------------------------------------------------------------- /src/ldp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/src/ldp/__init__.py -------------------------------------------------------------------------------- /src/ldp/agent/__init__.py: -------------------------------------------------------------------------------- 1 | # Lower than LiteLLM's 10-min default: https://github.com/BerriAI/litellm/blob/v1.48.10/litellm/main.py#L859 2 | DEFAULT_LLM_COMPLETION_TIMEOUT = 120 # seconds 3 | 4 | # ruff: noqa: E402 # Avoid circular imports 5 | 6 | from .agent import Agent, AgentConfig 7 | from .agent_client import HTTPAgentClient, make_simple_agent_server 8 | from .memory_agent import MemoryAgent 9 | from .react_agent import ReActAgent 10 | from .simple_agent import NoToolsSimpleAgent, SimpleAgent, SimpleAgentState 11 | from .tree_of_thoughts_agent import TreeofThoughtsAgent 12 | 13 | __all__ = [ 14 | "DEFAULT_LLM_COMPLETION_TIMEOUT", 15 | "Agent", 16 | "AgentConfig", 17 | "HTTPAgentClient", 18 | "MemoryAgent", 19 | "NoToolsSimpleAgent", 20 | "ReActAgent", 21 | "SimpleAgent", 22 | "SimpleAgentState", 23 | "TreeofThoughtsAgent", 24 | "make_simple_agent_server", 25 | ] 26 | -------------------------------------------------------------------------------- /src/ldp/alg/__init__.py: -------------------------------------------------------------------------------- 1 | from .algorithms import ( 2 | bulk_evaluate_consensus, 3 | compute_pass_at_k, 4 | evaluate_consensus, 5 | to_network, 6 | ) 7 | from .beam_search import Beam, BeamSearchRollout 8 | from .callbacks import ( 9 | Callback, 10 | ClearContextCallback, 11 | ClearOptimizerBuffersCallback, 12 | ComputeTrajectoryMetricsMixin, 13 | LoggingCallback, 14 | MeanMetricsCallback, 15 | RolloutDebugDumpCallback, 16 | StoreTrajectoriesCallback, 17 | TrajectoryFileCallback, 18 | TrajectoryMetricsCallback, 19 | WandBLoggingCallback, 20 | ) 21 | from .rollout import RolloutManager 22 | from .runners import ( 23 | Evaluator, 24 | EvaluatorConfig, 25 | OfflineTrainer, 26 | OfflineTrainerConfig, 27 | OnlineTrainer, 28 | OnlineTrainerConfig, 29 | ) 30 | from .tree_search import TEnvCloneFn, TreeSearchRollout 31 | 32 | __all__ = [ 33 | "Beam", 34 | "BeamSearchRollout", 35 | "Callback", 36 | "ClearContextCallback", 37 | "ClearOptimizerBuffersCallback", 38 | "ComputeTrajectoryMetricsMixin", 39 | "Evaluator", 40 | "EvaluatorConfig", 41 | "LoggingCallback", 42 | "MeanMetricsCallback", 43 | "OfflineTrainer", 44 | "OfflineTrainerConfig", 45 | "OnlineTrainer", 46 | "OnlineTrainerConfig", 47 | "RolloutDebugDumpCallback", 48 | "RolloutManager", 49 | "StoreTrajectoriesCallback", 50 | "TEnvCloneFn", 51 | "TrajectoryFileCallback", 52 | "TrajectoryMetricsCallback", 53 | "TreeSearchRollout", 54 | "WandBLoggingCallback", 55 | "bulk_evaluate_consensus", 56 | "compute_pass_at_k", 57 | "evaluate_consensus", 58 | "to_network", 59 | ] 60 | -------------------------------------------------------------------------------- /src/ldp/alg/datasets.py: -------------------------------------------------------------------------------- 1 | from aviary.core import TASK_DATASET_REGISTRY 2 | from aviary.core import DummyTaskDataset as _DummyTaskDataset 3 | 4 | from .callbacks import ComputeTrajectoryMetricsMixin 5 | 6 | 7 | class DummyTaskDataset(_DummyTaskDataset, ComputeTrajectoryMetricsMixin): 8 | pass 9 | 10 | 11 | TASK_DATASET_REGISTRY["dummy"] = "ldp.alg.datasets", "DummyTaskDataset" 12 | -------------------------------------------------------------------------------- /src/ldp/alg/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, cast 2 | 3 | from pydantic import BaseModel, ConfigDict, Field 4 | 5 | from ldp.agent import Agent, MemoryAgent, ReActAgent 6 | from ldp.alg.optimizer.ape import APEOpt, APEScoreFn 7 | from ldp.alg.optimizer.memory import MemoryFactory, MemoryOpt, PositiveMemoryOpt 8 | from ldp.alg.optimizer.opt import _OPTIMIZER_REGISTRY, ChainedOptimizer, Optimizer 9 | 10 | _DEFAULT_OPTIMIZER_ERROR_MSG = ( 11 | "Didn't yet implement an optimizer of type {opt_type} for {agent_type}." 12 | ) 13 | AUTOSELECT_BEST_OPTIMIZER = None 14 | 15 | 16 | class OptimizerConfig(BaseModel): 17 | model_config = ConfigDict(extra="forbid") 18 | 19 | optimizer_type: str | None = AUTOSELECT_BEST_OPTIMIZER 20 | optimizer_kwargs: dict[str, Any] = Field(default_factory=dict) 21 | 22 | 23 | _DEFAULT_OPTIMIZER_MAP: dict[type[Agent], type[Optimizer]] = { 24 | MemoryAgent: MemoryOpt, 25 | ReActAgent: APEOpt, 26 | } 27 | 28 | 29 | def default_optimizer_factory( 30 | agent: Agent, 31 | optimizer_cls: str | type[Optimizer] | None = AUTOSELECT_BEST_OPTIMIZER, 32 | **optimizer_kwargs, 33 | ) -> Optimizer: 34 | """A method that constructs a default optimizer for commonly-used agents. 35 | 36 | Args: 37 | agent: Agent to construct the optimizer for. 38 | optimizer_cls: The optimizer class to use. If not specified, we will try a 39 | default based on the provided agent. 40 | optimizer_kwargs: Arguments forwarded to optimizer_cls. 41 | 42 | Returns: 43 | Instantiated optimizer. 44 | """ 45 | if isinstance(optimizer_cls, str): 46 | try: 47 | optimizer_cls = _OPTIMIZER_REGISTRY[optimizer_cls] 48 | except KeyError: 49 | raise TypeError( 50 | "Optimizer class not supported by default_optimizer_factory:" 51 | f" {optimizer_cls}" 52 | ) from None 53 | 54 | if optimizer_cls is None: 55 | optimizer_cls = _DEFAULT_OPTIMIZER_MAP.get(agent.__class__) 56 | 57 | # convince mypy that optimizer_cls is a type from here on 58 | optimizer_cls = cast("type", optimizer_cls) 59 | 60 | if isinstance(agent, MemoryAgent): 61 | if optimizer_cls != MemoryOpt: 62 | raise NotImplementedError( 63 | _DEFAULT_OPTIMIZER_ERROR_MSG.format( 64 | opt_type=optimizer_cls.__name__, agent_type=MemoryAgent.__name__ 65 | ) 66 | ) 67 | return MemoryOpt.from_agent(agent, **optimizer_kwargs) 68 | if isinstance(agent, ReActAgent): 69 | if optimizer_cls != APEOpt: 70 | raise NotImplementedError( 71 | _DEFAULT_OPTIMIZER_ERROR_MSG.format( 72 | opt_type=optimizer_cls.__name__, agent_type=ReActAgent.__name__ 73 | ) 74 | ) 75 | return APEOpt.from_agent(agent, **optimizer_kwargs) 76 | raise TypeError(f"Unsupported agent type: {agent.__class__.__name__}") 77 | 78 | 79 | __all__ = [ 80 | "APEOpt", 81 | "APEScoreFn", 82 | "ChainedOptimizer", 83 | "MemoryFactory", 84 | "MemoryOpt", 85 | "Optimizer", 86 | "OptimizerConfig", 87 | "PositiveMemoryOpt", 88 | "default_optimizer_factory", 89 | ] 90 | -------------------------------------------------------------------------------- /src/ldp/alg/optimizer/opt.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from abc import ABC, abstractmethod 5 | from collections.abc import Iterable 6 | 7 | from ldp.data_structures import Trajectory 8 | from ldp.shims import tqdm 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | # Registry for all optimizers 14 | _OPTIMIZER_REGISTRY: dict[str, type[Optimizer]] = {} 15 | 16 | 17 | class Optimizer(ABC): 18 | """Base class for all optimizers.""" 19 | 20 | def __init_subclass__(cls) -> None: 21 | # Register each optimizer subclass 22 | _OPTIMIZER_REGISTRY[cls.__name__] = cls 23 | return super().__init_subclass__() 24 | 25 | def aggregate( 26 | self, trajectories: Iterable[Trajectory], show_pbar: bool = False, **kwargs 27 | ) -> None: 28 | """Aggregate trajectories to construct training samples.""" 29 | trajectories_with_pbar = tqdm( 30 | trajectories, 31 | desc="Aggregating trajectories", 32 | ncols=0, 33 | mininterval=1, 34 | disable=not show_pbar, 35 | ) 36 | for trajectory in trajectories_with_pbar: 37 | self.aggregate_trajectory(trajectory, **kwargs) 38 | 39 | @abstractmethod 40 | def aggregate_trajectory(self, trajectory: Trajectory) -> None: 41 | """Aggregate transitions from a single trajectory to construct training samples.""" 42 | 43 | @abstractmethod 44 | async def update(self) -> None: 45 | """Update the model based on the aggregated samples.""" 46 | 47 | 48 | class ChainedOptimizer(Optimizer): 49 | """An optimizer that runs a sequence of sub-optimizers in the order they are provided.""" 50 | 51 | def __init__(self, *optimizers: Optimizer): 52 | self.optimizers = optimizers 53 | 54 | def aggregate( 55 | self, trajectories: Iterable[Trajectory], show_pbar: bool = False, **kwargs 56 | ) -> None: 57 | for optimizer in self.optimizers: 58 | optimizer.aggregate(trajectories, show_pbar=show_pbar, **kwargs) 59 | 60 | async def update(self) -> None: 61 | for optimizer in self.optimizers: 62 | await optimizer.update() 63 | -------------------------------------------------------------------------------- /src/ldp/graph/__init__.py: -------------------------------------------------------------------------------- 1 | from .common_ops import ( 2 | ConfigOp, 3 | EmbeddingOp, 4 | FxnOp, 5 | IdentityOp, 6 | LLMCallOp, 7 | MemoryOp, 8 | PromptOp, 9 | ) 10 | from .loss_ops import MSELossOp 11 | from .memory import Memory, MemoryModel 12 | from .op_utils import ( 13 | CallID, 14 | compute_graph, 15 | eval_mode, 16 | get_call_id, 17 | get_run_id, 18 | get_training_mode, 19 | op_call, 20 | set_training_mode, 21 | train_mode, 22 | ) 23 | from .ops import Op, OpCtx, OpResult 24 | 25 | __all__ = [ 26 | "CallID", 27 | "ConfigOp", 28 | "EmbeddingOp", 29 | "FxnOp", 30 | "IdentityOp", 31 | "LLMCallOp", 32 | "MSELossOp", 33 | "Memory", 34 | "MemoryModel", 35 | "MemoryOp", 36 | "Op", 37 | "OpCtx", 38 | "OpResult", 39 | "PromptOp", 40 | "compute_graph", 41 | "eval_mode", 42 | "get_call_id", 43 | "get_run_id", 44 | "get_training_mode", 45 | "op_call", 46 | "set_training_mode", 47 | "train_mode", 48 | ] 49 | -------------------------------------------------------------------------------- /src/ldp/graph/loss_ops.py: -------------------------------------------------------------------------------- 1 | """This module contains loss Op implementations.""" 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from .op_utils import CallID 6 | from .ops import GradInType, Op, OpCtx 7 | 8 | if TYPE_CHECKING: 9 | import numpy.typing as npt 10 | import torch 11 | import tree 12 | 13 | 14 | class MSELossOp(Op): 15 | async def forward( 16 | self, 17 | prediction: "npt.NDArray | torch.Tensor", 18 | target: "npt.NDArray | torch.Tensor", 19 | ) -> "float | torch.Tensor": 20 | return ((prediction - target) ** 2).mean() 21 | 22 | @classmethod 23 | def backward( 24 | cls, 25 | ctx: OpCtx, 26 | input_args, 27 | input_kwargs, 28 | grad_output: "tree.Structure", 29 | call_id: CallID, 30 | ) -> GradInType: 31 | prediction = input_kwargs["prediction"] 32 | target = input_kwargs["target"] 33 | grad = 2 * (prediction - target) 34 | return [], {"prediction": grad, "target": None} 35 | -------------------------------------------------------------------------------- /src/ldp/graph/modules/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A module is a subgraph of a compute graph that can be exposed like a single node/op. 3 | 4 | An analogous entity in PyTorch is torch.nn.Module. 5 | """ 6 | 7 | from .llm_call import ParsedLLMCallModule 8 | from .react import ( 9 | ReActModule, 10 | ReActModuleSinglePrompt, 11 | ToolDescriptionMethods, 12 | parse_message, 13 | ) 14 | from .reflect import ReflectModule, ReflectModuleConfig 15 | from .thought import ThoughtModule 16 | 17 | __all__ = [ 18 | "ParsedLLMCallModule", 19 | "ReActModule", 20 | "ReActModuleSinglePrompt", 21 | "ReflectModule", 22 | "ReflectModuleConfig", 23 | "ThoughtModule", 24 | "ToolDescriptionMethods", 25 | "parse_message", 26 | ] 27 | -------------------------------------------------------------------------------- /src/ldp/graph/modules/llm_call.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable, Iterable 2 | from typing import Any, Generic, TypeVar 3 | 4 | from aviary.core import Message 5 | 6 | from ldp.graph import ConfigOp, FxnOp, LLMCallOp, OpResult, compute_graph 7 | 8 | TParsedMessage = TypeVar("TParsedMessage", bound=Message) 9 | 10 | 11 | class ParsedLLMCallModule(Generic[TParsedMessage]): 12 | """Module for a processing-based tool selection, with a learnable configuration.""" 13 | 14 | def __init__( 15 | self, llm_model: dict[str, Any], parser: Callable[..., TParsedMessage] 16 | ): 17 | self.config_op = ConfigOp[dict](config=llm_model) 18 | self.llm_call_op = LLMCallOp() 19 | self.parse_msg_op = FxnOp(parser) 20 | 21 | @compute_graph() 22 | async def __call__( 23 | self, messages: Iterable[Message], *parse_args, **parse_kwargs 24 | ) -> tuple[OpResult[TParsedMessage], Message]: 25 | raw_result = await self.llm_call_op(await self.config_op(), msgs=messages) 26 | return ( 27 | await self.parse_msg_op(raw_result, *parse_args, **parse_kwargs), 28 | raw_result.value, 29 | ) 30 | -------------------------------------------------------------------------------- /src/ldp/graph/modules/reflect.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from aviary.core import Message 4 | from pydantic import BaseModel, Field 5 | 6 | from ldp.graph import ConfigOp, FxnOp, LLMCallOp, PromptOp, compute_graph 7 | from ldp.graph.ops import ResultOrValue 8 | from ldp.llms import append_to_sys, indent_xml 9 | 10 | 11 | class ReflectModuleConfig(BaseModel): 12 | """Configuration for the ReflectModuleConfig.""" 13 | 14 | llm_model: dict[str, Any] = Field( 15 | default={"name": "gpt-3.5-turbo"}, 16 | description="Starting configuration for the LLM model.", 17 | ) 18 | 19 | 20 | class ReflectModule: 21 | """A module that simply gives an LLM to reflect on an input.""" 22 | 23 | def __init__(self, start_config: ReflectModuleConfig): 24 | self.llm_call_op = LLMCallOp() 25 | self.prompt_op = PromptOp( 26 | "Consider a proposed response based on context. Reflect on the response" 27 | " within tags then conclude with a possibly revised response" 28 | " within tags." 29 | ) 30 | self.config_op = ConfigOp[ReflectModuleConfig](config=start_config) 31 | self.llm_config_op = FxnOp[dict](lambda c: c.llm_model) 32 | self.package_fxn = FxnOp(append_to_sys) 33 | 34 | def extract_msg(msg: Message, backup_response: str) -> str: 35 | msg_str = msg.content 36 | if msg_str and "" in msg_str: 37 | return msg_str.split("")[1].split("")[ 38 | 0 39 | ] 40 | if msg_str and "" in msg_str: 41 | return msg_str.split("")[1].split("")[0] 42 | return backup_response 43 | 44 | self.extract_msg = FxnOp(extract_msg) 45 | 46 | @compute_graph() 47 | async def __call__( 48 | self, context: ResultOrValue[str], response: ResultOrValue[str] 49 | ) -> ResultOrValue[str]: 50 | llm_config = await self.llm_config_op(await self.config_op()) 51 | sys_str = await self.prompt_op() 52 | user_str = indent_xml( 53 | f"{context}{response}" 54 | ) 55 | msg = await self.package_fxn(user_str, sys_str) 56 | llm_result = await self.llm_call_op(llm_config, msg) 57 | return await self.extract_msg(llm_result, response) 58 | -------------------------------------------------------------------------------- /src/ldp/graph/modules/thought.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable, Mapping 2 | from typing import Any 3 | 4 | from aviary.core import Message, ToolRequestMessage 5 | 6 | from ldp.graph import FxnOp, OpResult, PromptOp, compute_graph 7 | from ldp.llms import prepend_sys_and_append_sys 8 | 9 | from .llm_call import ParsedLLMCallModule 10 | 11 | 12 | class ThoughtModule: 13 | @staticmethod 14 | def _downcast_to_message(message: Message | ToolRequestMessage) -> Message: 15 | if isinstance(message, ToolRequestMessage): 16 | # Downcast into a normal Message if the LLM tried to call tools 17 | return Message(role=message.role, content=message.content) 18 | return message 19 | 20 | def __init__( 21 | self, llm_model: dict[str, Any], first_sys_prompt: str, second_sys_prompt: str 22 | ): 23 | self.first_sys_prompt_op = PromptOp(first_sys_prompt) 24 | self.second_sys_prompt_op = PromptOp(second_sys_prompt) 25 | self.package_msg_op = FxnOp(prepend_sys_and_append_sys) 26 | self.llm_call = ParsedLLMCallModule[Message]( 27 | llm_model, parser=self._downcast_to_message 28 | ) 29 | 30 | @compute_graph() 31 | async def __call__( 32 | self, 33 | messages: Iterable[Message], 34 | first_prompt_kwargs: Mapping[str, Any], 35 | second_prompt_kwargs: Mapping[str, Any], 36 | ) -> OpResult[Message]: 37 | packaged_msgs = await self.package_msg_op( 38 | messages, 39 | initial_sys_content=await self.first_sys_prompt_op(**first_prompt_kwargs), 40 | final_sys_content=await self.second_sys_prompt_op(**second_prompt_kwargs), 41 | ) 42 | return (await self.llm_call(packaged_msgs))[0] # type: ignore[arg-type] 43 | -------------------------------------------------------------------------------- /src/ldp/llms/__init__.py: -------------------------------------------------------------------------------- 1 | from lmi import LiteLLMModel as LLMModel 2 | from lmi import ( 3 | LLMResult, 4 | sum_logprobs, 5 | validate_json_completion, 6 | ) 7 | from lmi.embeddings import ( 8 | EmbeddingModel, 9 | EmbeddingModes, 10 | HybridEmbeddingModel, 11 | LiteLLMEmbeddingModel, 12 | SparseEmbeddingModel, 13 | ) 14 | from lmi.exceptions import ( 15 | JSONSchemaValidationError, 16 | ) 17 | 18 | from .prompts import ( 19 | append_to_messages, 20 | append_to_sys, 21 | indent_xml, 22 | prepend_sys, 23 | prepend_sys_and_append_sys, 24 | ) 25 | 26 | __all__ = [ 27 | "EmbeddingModel", 28 | "EmbeddingModes", 29 | "HybridEmbeddingModel", 30 | "JSONSchemaValidationError", 31 | "LLMModel", 32 | "LLMResult", 33 | "LiteLLMEmbeddingModel", 34 | "SparseEmbeddingModel", 35 | "append_to_messages", 36 | "append_to_sys", 37 | "indent_xml", 38 | "prepend_sys", 39 | "prepend_sys_and_append_sys", 40 | "sum_logprobs", 41 | "validate_json_completion", 42 | ] 43 | -------------------------------------------------------------------------------- /src/ldp/llms/prompts.py: -------------------------------------------------------------------------------- 1 | """This module provides utility functions for appending and prepending system messages.""" 2 | 3 | from collections.abc import Collection, Iterable 4 | 5 | from aviary.core import Message 6 | 7 | 8 | def append_to_messages(messages: list[Message], new_message: Message) -> list[Message]: 9 | """Appends a message to a list of messages, returning that in-place modified list. 10 | 11 | Examples: 12 | >>> messages = [Message(content="Hello")] 13 | >>> modified_messages = append_to_messages(messages, Message(content="New")) 14 | >>> modified_messages 15 | [Message(role='user', content='Hello'), Message(role='user', content='New')] 16 | >>> id(messages) == id(modified_messages) 17 | True 18 | """ 19 | messages.append(new_message) 20 | return messages 21 | 22 | 23 | def append_to_sys(user_content: str, sys_content: str | None = None) -> list[Message]: 24 | """Appends a user message to a list of messages, optionally including a system message. 25 | 26 | Args: 27 | user_content: The content of the user message. 28 | sys_content: Optional content for the system message. Defaults to None. 29 | 30 | Returns: 31 | A list of messages including the optional system message and the user message. 32 | 33 | Examples: 34 | >>> append_to_sys("Hello, world!") 35 | [Message(role='user', content='Hello, world!')] 36 | 37 | >>> append_to_sys("Hello, world!", "System initialized.") 38 | [Message(role='system', content='System initialized.'), Message(role='user', content='Hello, world!')] 39 | """ 40 | sys = [Message(role="system", content=sys_content)] if sys_content else [] 41 | return [*sys, Message(content=user_content)] 42 | 43 | 44 | def prepend_sys(messages: Collection, sys_content: str) -> list[Message]: 45 | """Prepends a system message to a list of messages. 46 | 47 | Args: 48 | messages: The list of existing messages. 49 | sys_content: The content of the system message to be prepended. 50 | 51 | Returns: 52 | A new list of messages with the system message prepended. 53 | 54 | Examples: 55 | >>> messages = [Message(role="user", content="Hello!")] 56 | >>> prepend_sys(messages, "System initialized.") 57 | [Message(role='system', content='System initialized.'), Message(role='user', content='Hello!')] 58 | """ 59 | return [Message(role="system", content=sys_content), *messages] 60 | 61 | 62 | def indent_xml(xml_string, indent_size=2): 63 | output = [] 64 | indent_level = 0 65 | 66 | # Split the input XML into parts by tags 67 | parts = xml_string.replace(">", ">\n").replace("<", "\n<").split("\n") 68 | parts = [part for part in parts if part.strip()] # Remove empty parts 69 | 70 | for part in parts: 71 | if part.startswith("") and ">" in part: 76 | # Opening tag, maintain then increase indent 77 | output.append(" " * indent_level + part) 78 | indent_level += indent_size 79 | elif part.endswith("/>"): 80 | # Self-closing tag, just append 81 | output.append(" " * indent_level + part) 82 | else: 83 | # Text or other data, maintain current indent 84 | # Handle multiple lines within text nodes 85 | text_lines = part.split("\n") 86 | output.extend([ 87 | " " * indent_level + line.strip() for line in text_lines if line.strip() 88 | ]) 89 | 90 | return "\n".join(output) 91 | 92 | 93 | def prepend_sys_and_append_sys( 94 | messages: Iterable[Message], initial_sys_content: str, final_sys_content: str 95 | ) -> list[Message]: 96 | return [ 97 | Message(role="system", content=initial_sys_content), 98 | *messages, 99 | Message(role="system", content=final_sys_content), 100 | ] 101 | -------------------------------------------------------------------------------- /src/ldp/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import pickle 4 | from contextlib import suppress 5 | from os import PathLike 6 | from pathlib import Path 7 | 8 | from aviary.core import Environment 9 | 10 | from ldp.agent import Agent 11 | from ldp.alg.callbacks import TerminalPrintingCallback 12 | from ldp.alg.rollout import RolloutManager 13 | 14 | 15 | def get_or_make_agent(agent: Agent | str | PathLike) -> Agent: 16 | if isinstance(agent, Agent): 17 | return agent 18 | 19 | if isinstance(agent, str): 20 | with suppress(KeyError): 21 | return Agent.from_name(agent) 22 | 23 | path = Path(agent) 24 | if not path.exists(): 25 | raise ValueError(f"Could not resolve agent: {agent}") 26 | 27 | with path.open("rb") as f: 28 | return pickle.load(f) # noqa: S301 29 | 30 | 31 | def get_or_make_environment(environment: Environment | str, task: str) -> Environment: 32 | if isinstance(environment, Environment): 33 | return environment 34 | 35 | if isinstance(environment, str): 36 | with suppress(KeyError): 37 | return Environment.from_name(environment, task=task) 38 | 39 | raise ValueError( 40 | f"Could not resolve environment: {environment}. Available environments:" 41 | f" {Environment.available()}" 42 | ) 43 | 44 | 45 | async def main( 46 | task: str, 47 | environment: Environment | str, 48 | agent: Agent | str | PathLike = "SimpleAgent", 49 | ): 50 | agent = get_or_make_agent(agent) 51 | 52 | callback = TerminalPrintingCallback() 53 | rollout_manager = RolloutManager(agent=agent, callbacks=[callback]) 54 | 55 | _ = await rollout_manager.sample_trajectories( 56 | environment_factory=lambda: get_or_make_environment(environment, task) 57 | ) 58 | 59 | 60 | if __name__ == "__main__": 61 | parser = argparse.ArgumentParser() 62 | parser.add_argument("task", help="Task to prompt environment with.") 63 | parser.add_argument( 64 | "--env", required=True, help="Environment to sample trajectories from." 65 | ) 66 | parser.add_argument( 67 | "--agent", default="SimpleAgent", help="Agent to sample trajectories with." 68 | ) 69 | args = parser.parse_args() 70 | 71 | asyncio.run(main(args.task, args.env, args.agent)) 72 | -------------------------------------------------------------------------------- /src/ldp/nn/__init__.py: -------------------------------------------------------------------------------- 1 | from ldp.nn.handlers.chunking import TensorChunker 2 | 3 | from .agent.simple_local_agent import AgentLMConfig, SimpleLocalLLMAgent 4 | from .graph.llm_call_op import LocalLLMCallOp 5 | from .handlers.transformer_handler import ( 6 | AsyncTransformer, 7 | AsyncTransformerInterface, 8 | ExecutionMode, 9 | LMType, 10 | ParallelAsyncTransformer, 11 | ParallelModeConfig, 12 | ParallelTransformerHandler, 13 | TransformerHandler, 14 | TransformerHandlerConfig, 15 | collate_fn_transformer_left_pad, 16 | collate_fn_transformer_right_pad, 17 | decollate_fn_transformer_decoder, 18 | ) 19 | from .lm_config import LMConfig, TorchDType 20 | from .utils import set_seed 21 | 22 | __all__ = [ 23 | "AgentLMConfig", 24 | "AsyncTransformer", 25 | "AsyncTransformerInterface", 26 | "ExecutionMode", 27 | "LMConfig", 28 | "LMType", 29 | "LocalLLMCallOp", 30 | "ParallelAsyncTransformer", 31 | "ParallelModeConfig", 32 | "ParallelTransformerHandler", 33 | "SimpleLocalLLMAgent", 34 | "TensorChunker", 35 | "TorchDType", 36 | "TransformerHandler", 37 | "TransformerHandlerConfig", 38 | "collate_fn_transformer_left_pad", 39 | "collate_fn_transformer_right_pad", 40 | "decollate_fn_transformer_decoder", 41 | "set_seed", 42 | ] 43 | -------------------------------------------------------------------------------- /src/ldp/nn/chat_templates/README.md: -------------------------------------------------------------------------------- 1 | - llama3.1_chat_template_vllm.jinja: https://github.com/vllm-project/vllm/blob/4fb8e329fd6f51d576bcf4b7e8907e0d83c4b5cf/examples/tool_chat_template_llama3.1_json.jinja 2 | - llama3.1_chat_template_hf.jinja: https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/blob/main/tokenizer_config.json#L2053 3 | - llama3.1_chat_template_thought.jinja: Fixes a typo in llama3.1_chat_template_ori.jinja 4 | - llama3.1_chat_template_nothought.jinja: Derived from llama3.1_chat_template_ori.jinja, but removes thoughts 5 | - llama\*ori.jinja: TODOC - these were written by @kwanUm 6 | -------------------------------------------------------------------------------- /src/ldp/nn/chat_templates/llama2_chat_template_ori.jinja: -------------------------------------------------------------------------------- 1 | {% if messages[0]['role'] == 'system' %} 2 | {% set loop_messages = messages[1:] %} 3 | {% set system_message = messages[0]['content'] %} 4 | {% else %} 5 | {% set loop_messages = messages %} 6 | {% set system_message = false %} 7 | {% endif %} 8 | {% for message in loop_messages %} 9 | {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} 10 | {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} 11 | {% endif %} 12 | {% if loop.index0 == 0 and system_message != false %} 13 | {% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %} 14 | {% else %} 15 | {% set content = message['content'] %} 16 | {% endif %} 17 | {% if message['role'] == 'user' %} 18 | {{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }} 19 | {% elif message['role'] == 'assistant' %} 20 | {% generation %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endgeneration %} 21 | {% endif %} 22 | {% endfor %} 23 | -------------------------------------------------------------------------------- /src/ldp/nn/generation/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import LogitsProcessorWithFinalize 2 | 3 | __all__ = [ 4 | "LogitsProcessorWithFinalize", 5 | ] 6 | -------------------------------------------------------------------------------- /src/ldp/nn/generation/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | import torch 4 | from transformers import LogitsProcessor 5 | 6 | 7 | class LogitsProcessorWithFinalize(LogitsProcessor, ABC): 8 | @abstractmethod 9 | def finalize(self, token_ids: torch.Tensor) -> None: 10 | """A method for subclasses to inject arbitrary finalization logic after sampling finishes. 11 | 12 | TransformerHandler will invoke logit_processor.finalize(token_ids), where token_ids are 13 | the sampled tokens. 14 | """ 15 | -------------------------------------------------------------------------------- /src/ldp/nn/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | from pathlib import Path 3 | 4 | import numpy as np 5 | import torch 6 | 7 | 8 | def set_seed(seed: int | None) -> None: 9 | if seed is None: 10 | return 11 | 12 | random.seed(seed) 13 | np.random.seed(seed) # noqa: NPY002 14 | torch.manual_seed(seed) 15 | torch.cuda.manual_seed(seed) 16 | 17 | 18 | REPO_ROOT = Path(__file__).parent.parent.parent 19 | -------------------------------------------------------------------------------- /src/ldp/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Future-House/ldp/2e9557233dc0e4c8abbb9f1f065059108a0dcfcd/src/ldp/py.typed -------------------------------------------------------------------------------- /src/ldp/shims.py: -------------------------------------------------------------------------------- 1 | __all__ = ["tqdm", "trange"] 2 | 3 | import logging 4 | import os 5 | 6 | from tqdm import tqdm, trange 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | if os.environ.get("LDP_TQDM_USE_RICH", "").lower() in {"1", "true", "yes"}: 11 | # TODO: remove after https://github.com/tqdm/tqdm/issues/1618 12 | try: 13 | # pylint: disable-next=reimported 14 | from tqdm.rich import tqdm, trange # type: ignore[no-redef] 15 | except ModuleNotFoundError: 16 | logger.warning( 17 | "User opted into rich progress via the environment variable" 18 | " LDP_TQDM_USE_RICH, but did not have 'rich' installed." 19 | " Please run `pip install ldp[rich]`.", 20 | exc_info=True, 21 | ) 22 | -------------------------------------------------------------------------------- /src/ldp/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.config 3 | from typing import Any 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | def configure_stdout_logs( 9 | name: str = "root", 10 | level: int | str = logging.INFO, 11 | fmt: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s", 12 | ) -> None: 13 | """Configure root logger to log to stdout. 14 | 15 | Args: 16 | name: Optional logger name, if unspecified the 'root' logger is configured. 17 | level: Log level to be emitted to stdout. 18 | fmt: Optional format string. 19 | """ 20 | config: dict[str, Any] = {name: {"level": level, "handlers": ["stdout"]}} 21 | if name != "root": # Non-root loggers need to be in a "loggers" key 22 | config["loggers"] = config 23 | logging.config.dictConfig( 24 | { 25 | "version": 1, 26 | "disable_existing_loggers": False, 27 | "formatters": {"standard": {"format": fmt}}, 28 | "handlers": { 29 | "stdout": { 30 | "level": "INFO", 31 | "formatter": "standard", 32 | "class": "logging.StreamHandler", 33 | "stream": "ext://sys.stdout", 34 | }, 35 | }, 36 | } 37 | | config 38 | ) 39 | 40 | 41 | def discounted_returns( 42 | rewards: list[float], terminated: list[bool], discount: float = 1.0 43 | ) -> list[float]: 44 | r""" 45 | Calculate the discounted returns for a list of rewards, considering termination flags and a discount factor. 46 | 47 | The discounted return represents the future discounted rewards from each time step onwards, taking into account 48 | whether an episode has terminated at each step. 49 | 50 | The discounted return \( G_t \) is given by: 51 | 52 | .. math:: 53 | G_t = \sum_{k=1}^{\infty} \gamma^{k-1} R_{t+k} 54 | 55 | where: 56 | - \( G_t \) is the discounted return starting from time step \( t \). 57 | - \( \gamma \) is the discount factor. 58 | - \( R_{t+k} \) is the reward received at time step \( t+k \). 59 | 60 | NOTE: this could live in ldp.alg, but it's here to avoid circular imports. 61 | 62 | Args: 63 | rewards: A list of rewards at each time step. 64 | terminated: A list of boolean flags indicating whether the episode terminated at each time step. 65 | discount: Discount factor to apply to future rewards. Defaults to 1.0 which means no discounting is applied. 66 | 67 | Returns: 68 | A list of discounted returns (rewards to go), with each element representing the 69 | total discounted reward from that step onwards. 70 | 71 | Example: 72 | >>> rewards = [1.0, 2.0, 3.0] 73 | >>> terminated = [False, False, True] 74 | >>> discounted_returns(rewards, terminated, discount=0.9) 75 | [5.23, 4.7, 3.0] 76 | """ 77 | returns = [] 78 | r = 0.0 79 | for reward, term in zip(reversed(rewards), reversed(terminated), strict=False): 80 | # 1 - term is 0 if the episode has terminated 81 | r = reward + discount * r * (1 - term) 82 | returns.append(r) 83 | returns.reverse() 84 | return returns 85 | 86 | 87 | def format_error_details(error: Exception) -> str: 88 | """Format detailed error information from an exception. 89 | 90 | Specially handles HTTP errors that have response attributes with status codes 91 | and JSON details, but works with any exception type. 92 | 93 | Args: 94 | error: The exception to format 95 | 96 | Returns: 97 | A formatted error string with available details 98 | """ 99 | error_details = f"{error!s}" 100 | 101 | if hasattr(error, "response"): 102 | error_details += f"\nStatus code: {error.response.status_code}" 103 | try: 104 | response_data = error.response.json() 105 | if "detail" in response_data: 106 | error_details += "\nServer Traceback:\n" 107 | for line in response_data["detail"].split("\n"): 108 | error_details += f" {line}\n" 109 | except Exception: 110 | error_details += f"\nResponse body: {error.response.text}" 111 | 112 | return error_details 113 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | test_outputs/ 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | TESTS_DIR = pathlib.Path(__file__).parent 4 | CASSETTES_DIR = TESTS_DIR / "cassettes" 5 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMCallOp.test_cost_tracking.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Generate a story"}], "model": 5 | "gpt-3.5-turbo", "temperature": 0.1, "tool_choice": "required", "tools": [{"type": 6 | "function", "function": {"name": "generate_story", "description": "Generate 7 | a story.", "parameters": {"type": "object", "properties": {}, "required": []}}}]}' 8 | headers: 9 | accept: 10 | - application/json 11 | accept-encoding: 12 | - gzip, deflate 13 | connection: 14 | - keep-alive 15 | content-length: 16 | - "313" 17 | content-type: 18 | - application/json 19 | host: 20 | - api.openai.com 21 | user-agent: 22 | - AsyncOpenAI/Python 1.54.3 23 | x-stainless-arch: 24 | - arm64 25 | x-stainless-async: 26 | - async:asyncio 27 | x-stainless-lang: 28 | - python 29 | x-stainless-os: 30 | - MacOS 31 | x-stainless-package-version: 32 | - 1.54.3 33 | x-stainless-raw-response: 34 | - "true" 35 | x-stainless-retry-count: 36 | - "0" 37 | x-stainless-runtime: 38 | - CPython 39 | x-stainless-runtime-version: 40 | - 3.12.7 41 | method: POST 42 | uri: https://api.openai.com/v1/chat/completions 43 | response: 44 | body: 45 | string: !!binary | 46 | H4sIAAAAAAAAA4xT70/bMBD9nr/Cus8talJ+lHxjElQgmCaYmGCaIte5JgbH9uwLW1X1f5+clCQt 47 | RVo+WNa9e+/enS/riDGQOaQMRMlJVFaNL+4f8vj58vnH9Vx9eTp/LZOvZXH75+14fhWfwigwzOIF 48 | Bb2zjoSprEKSRrewcMgJg2p8No3jySw5PWuAyuSoAq2wNJ4enYypdgsznsTJyZZZGinQQ8p+Rowx 49 | tm7O4FHn+BdSNhm9Ryr0nhcIaZfEGDijQgS499IT1wSjHhRGE+pgW9dKDQAyRmWCK9UXbr/14N4P 50 | iiuV2SS5eNA35uXpbjp/u53MH3/ffPs+ux/Ua6VXtjG0rLXoBjTAu3i6V4wx0LxquAVqdJww82Tc 51 | ak+BMeCuqCvUFNzDegM7+CY6dP816N7hsvZcbceyjW+6OStTWGcWfm9ssJRa+jJzyH1jHzwZ29YO 52 | dZoKUO88EVhnKksZmVfUQfA4buWg36AenG0xMsTVgHM+OiCW5UhcNg/Y7YzgosS8Z/a7w+tcmgEQ 53 | DVr+6OWQdtu21MX/yPeAEGgJ88w6zKXY7bdPcxh+r8/SuhE3hsGvPGGVLaUu0FknuwWPNtE/AAAA 54 | //8DAKbQ9nDfAwAA 55 | headers: 56 | CF-Cache-Status: 57 | - DYNAMIC 58 | CF-RAY: 59 | - 8df951106c0fce74-SJC 60 | Connection: 61 | - keep-alive 62 | Content-Encoding: 63 | - gzip 64 | Content-Type: 65 | - application/json 66 | Date: 67 | - Fri, 08 Nov 2024 23:24:28 GMT 68 | Server: 69 | - cloudflare 70 | Set-Cookie: 71 | - __cf_bm=rCyb7ZMvV5wiHKaeBdiNZlwHnkJK6tg2ANDGf7CLuDQ-1731108268-1.0.1.1-TV4yUO3TR983XeJzGOzeANJ9oow4wDmcxbZ8oaqgrJOQPEhkWv2F4Z.ZGNjzmCoXE.kIh6uVXz.pbGFeFvPSOQ; 72 | path=/; expires=Fri, 08-Nov-24 23:54:28 GMT; domain=.api.openai.com; HttpOnly; 73 | Secure; SameSite=None 74 | - _cfuvid=mB2ccx6IlyDMQkGU81Vj.KiHjbPKz88kyBcj9Q5UQCk-1731108268004-0.0.1.1-604800000; 75 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 76 | Transfer-Encoding: 77 | - chunked 78 | X-Content-Type-Options: 79 | - nosniff 80 | access-control-expose-headers: 81 | - X-Request-ID 82 | alt-svc: 83 | - h3=":443"; ma=86400 84 | openai-organization: 85 | - future-house-xr4tdh 86 | openai-processing-ms: 87 | - "281" 88 | openai-version: 89 | - "2020-10-01" 90 | strict-transport-security: 91 | - max-age=31536000; includeSubDomains; preload 92 | x-ratelimit-limit-requests: 93 | - "12000" 94 | x-ratelimit-limit-tokens: 95 | - "1000000" 96 | x-ratelimit-remaining-requests: 97 | - "11999" 98 | x-ratelimit-remaining-tokens: 99 | - "999978" 100 | x-ratelimit-reset-requests: 101 | - 5ms 102 | x-ratelimit-reset-tokens: 103 | - 1ms 104 | x-request-id: 105 | - req_448537f2622e7777bbf6f0e7ff4bb17e 106 | status: 107 | code: 200 108 | message: OK 109 | version: 1 110 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMCallOp.test_empty_tools.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini-2024-07-18", 5 | "max_tokens": 4096, "n": 1, "temperature": 0.1}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "135" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.59.9 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.59.9 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAAwAAAP//jFJBbtswELzrFds9W4UtB43sS5A2QdLm2gYBikKgyZXMhOISJNXWCPz3 45 | gpJtyWgL5KLDzM5oZrmvGQBqhWtAuRVRts7k15392BWrTw+Pj/Z29c09FdeKHr7cfe2eixucJQVv 46 | nknGo+q95NYZiprtQEtPIlJyXVwuy/mqLIuiJ1pWZJKscTG/4LzVVufFvLjI55f5ojyot6wlBVzD 47 | 9wwA4LX/ppxW0W9cw3x2RFoKQTSE69MQAHo2CUERgg5R2IizkZRsI9k++j0Zw+/gnn+BFBY+wyCA 48 | HXcQWYnd1VToqe6CSOFtZ8wB35+SGG6c50048Ce81laHbeVJBLbpryGyw57dZwA/+sbdWQl0nlsX 49 | q8gvZJNhObjhuOaRWxyWgZGjMBP8KDozqxRFoU2YLAylkFtSo3LcruiU5gmRTSr/HeZf3kNtbZu3 50 | 2I+ElOQiqcp5UlqeFx7HPKUj/N/YacV9YAzkf2pJVdTk0zMoqkVnhtPAsAuR2qrWtiHvvB7uo3bV 51 | RpXLZbGqPywx22d/AAAA//8DAGvr9u8tAwAA 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 9093fce7094e642c-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Tue, 28 Jan 2025 21:13:42 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=mxF_vDWaXSGuf9hKQnemjKvROt3mU3dFb69K_cFGmWA-1738098822-1.0.1.1-U9A9Wj1HMiApuDMzvl0Sz6xbEsOPlP3NiSxSQ1wVHhepccJyasTXm021aYIJ4kf5bGdc6KT_qLtB0wqQNwMTkg; 69 | path=/; expires=Tue, 28-Jan-25 21:43:42 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=tS3l16tuNKSemon6BRJsWhoLLS9VzvAut45DhtCV.Bk-1738098822778-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "399" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "30000" 91 | x-ratelimit-limit-tokens: 92 | - "150000000" 93 | x-ratelimit-remaining-requests: 94 | - "29999" 95 | x-ratelimit-remaining-tokens: 96 | - "149995900" 97 | x-ratelimit-reset-requests: 98 | - 2ms 99 | x-ratelimit-reset-tokens: 100 | - 1ms 101 | x-request-id: 102 | - req_e420e89656c3bae9b35eae4dbd97d277 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMCallOp.test_validation.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o-mini-2024-07-18", 5 | "n": 1}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "95" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.59.7 21 | x-stainless-arch: 22 | - x64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - Linux 29 | x-stainless-package-version: 30 | - 1.59.7 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.4 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA4xSwY7TMBC95yuGOTcoLV2l7QUhkOhKe4ADEssKRa49SQ2Ox9jOQrXqvyMnbZMK 45 | VtpLDu/Ne3lvPE8ZAGqFG0C5F1G2zuTvfr0v17vV/debg/h2t+WPj59UWX6+X3+4+3KLs6Tg3Q+S 46 | 8ax6Lbl1hqJmO9DSk4iUXOflm7IoFutV2RMtKzJJ1riYLzlvtdX5olgs86LM56uTes9aUsANPGQA 47 | AE/9N+W0iv7gBorZGWkpBNEQbi5DAOjZJARFCDpEYSPORlKyjWT76Fsyhl/Bln+DFBZuYRDAgTuI 48 | rMTh7VToqe6CSOFtZ8wJP16SGG6c51048Re81laHfeVJBLbpryGyw549ZgDf+8bdVQl0nlsXq8g/ 49 | ySbD1eCG45pHbn5aBkaOwkzws+jKrFIUhTZhsjCUQu5Jjcpxu6JTmidENqn8b5j/eQ+1tW1eYj8S 50 | UpKLpCrnSWl5XXgc85SO8Lmxy4r7wBjIP2pJVdTk0zMoqkVnhtPAcAiR2qrWtiHvvB7uo3ZVuSBV 51 | it3NUmJ2zP4CAAD//wMAc2QAgS0DAAA= 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 902b7b21ef9cba21-SEA 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Thu, 16 Jan 2025 04:49:48 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=940efh6IXrRBGzKpiScnpcSFSq.Qq5h.tZsG.tFRuXE-1737002988-1.0.1.1-glrn9jp0jJubbYQm1IpaQ0F7.hBwZxBO7pjIgct3Hry182dHr7EMmLLb4RleHEPiSBfV0OrZD33VvlywHYIHwg; 69 | path=/; expires=Thu, 16-Jan-25 05:19:48 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=fAwazJ9ZSOYywFdRAWoUeWZsgr45nzOyAHTcLlcUSP0-1737002988356-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "404" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "30000" 91 | x-ratelimit-limit-tokens: 92 | - "150000000" 93 | x-ratelimit-remaining-requests: 94 | - "29999" 95 | x-ratelimit-remaining-tokens: 96 | - "149999981" 97 | x-ratelimit-reset-requests: 98 | - 2ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_0ee9e043e2ad94fb0c69ad10bdbfa73e 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_model[claude-3-haiku-20240307].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello, 5 | how are you?"}]}], "system": [{"type": "text", "text": "Respond with single 6 | words."}], "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | anthropic-version: 13 | - "2023-06-01" 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "218" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.anthropic.com 22 | user-agent: 23 | - litellm/1.52.3 24 | method: POST 25 | uri: https://api.anthropic.com/v1/messages 26 | response: 27 | body: 28 | string: !!binary | 29 | H4sIAAAAAAAAA1SOzWrCQBSF3+WsJzJqsHW2LXSjoLRQipQwJBcdMrkTc+9AJeTdS8QuujrwnR/O 30 | iNDAoZNzZZfl7eO4fXl7v7yGzXN92Hfhuuu+YKC3nuYUifgzwWBIcQZeJIh6Vhh0qaEIhzr63FCx 31 | Li4+tLlY2VVp1/YJBnViJVa40/i3qPQzd+/i8EkxLjB9G4imvhrIS2I4EDeV5oHxMISumbgmOM4x 32 | GuT7KzcicJ+10tQSC9xya5Cy/kN2mn4BAAD//wMA53MTs/MAAAA= 33 | headers: 34 | CF-Cache-Status: 35 | - DYNAMIC 36 | CF-RAY: 37 | - 8df951213bb996c9-SJC 38 | Connection: 39 | - keep-alive 40 | Content-Encoding: 41 | - gzip 42 | Content-Type: 43 | - application/json 44 | Date: 45 | - Fri, 08 Nov 2024 23:24:30 GMT 46 | Server: 47 | - cloudflare 48 | Transfer-Encoding: 49 | - chunked 50 | X-Robots-Tag: 51 | - none 52 | anthropic-ratelimit-requests-limit: 53 | - "5000" 54 | anthropic-ratelimit-requests-remaining: 55 | - "4999" 56 | anthropic-ratelimit-requests-reset: 57 | - "2024-11-08T23:25:12Z" 58 | anthropic-ratelimit-tokens-limit: 59 | - "5000000" 60 | anthropic-ratelimit-tokens-remaining: 61 | - "5000000" 62 | anthropic-ratelimit-tokens-reset: 63 | - "2024-11-08T23:24:30Z" 64 | request-id: 65 | - req_01Ri7jk6wtRpYL45Y9DA7ubB 66 | via: 67 | - 1.1 google 68 | status: 69 | code: 200 70 | message: OK 71 | version: 1 72 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_model[gpt-3.5-turbo].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-3.5-turbo"}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "153" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.54.3 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.54.3 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "1" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA4ySMWvDMBCFd/8KodkOttOkwVspFEqXknQrxcjSxVYjS0I6l5SQ/17kJLZDU+ii 45 | 4b57T+9OOkSEUCloQShvGPLWquRhvRHpy93q0b3m+qvp9vgM2bp629mNzWkcFKb6BI4X1Yyb1ipA 46 | afQJcwcMIbhm9/MsS1f5ctmD1ghQQVZbTOazRYKdq0ySZvnirGyM5OBpQd4jQgg59GfIqAXsaUHS 47 | +FJpwXtWAy2GJkKoMypUKPNeemQaaTxCbjSC7mM/SQ1T5GDbeRai6U6pc/043KVMbZ2p/JkP9a3U 48 | 0jelA+aNDr4ejaU9PUaEfPQzdVcxqXWmtVii2YEOhnl+sqPjFkeYnRkaZGqimcc3zEoByKTyk5VQ 49 | zngDYlSO+2OdkGYCosnIv7Pc8j6NLXX9H/sRcA4WQZTWgZD8et6xzUH4Yn+1DSvuA1P/7RHacit1 50 | Dc462T9y/5LH6AcAAP//AwCdh9cz4wIAAA== 51 | headers: 52 | CF-Cache-Status: 53 | - DYNAMIC 54 | CF-RAY: 55 | - 8df9510b69f62716-SJC 56 | Connection: 57 | - keep-alive 58 | Content-Encoding: 59 | - gzip 60 | Content-Type: 61 | - application/json 62 | Date: 63 | - Fri, 08 Nov 2024 23:24:27 GMT 64 | Server: 65 | - cloudflare 66 | Transfer-Encoding: 67 | - chunked 68 | X-Content-Type-Options: 69 | - nosniff 70 | access-control-expose-headers: 71 | - X-Request-ID 72 | alt-svc: 73 | - h3=":443"; ma=86400 74 | openai-organization: 75 | - future-house-xr4tdh 76 | openai-processing-ms: 77 | - "182" 78 | openai-version: 79 | - "2020-10-01" 80 | strict-transport-security: 81 | - max-age=31536000; includeSubDomains; preload 82 | x-ratelimit-limit-requests: 83 | - "12000" 84 | x-ratelimit-limit-tokens: 85 | - "1000000" 86 | x-ratelimit-remaining-requests: 87 | - "11999" 88 | x-ratelimit-remaining-tokens: 89 | - "999969" 90 | x-ratelimit-reset-requests: 91 | - 5ms 92 | x-ratelimit-reset-tokens: 93 | - 1ms 94 | x-request-id: 95 | - req_c281bfa51f535d6093a3b049c4f0ca93 96 | status: 97 | code: 200 98 | message: OK 99 | version: 1 100 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_output_schema[json-mode].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages":[{"role":"system","content":"Respond following this JSON schema:\n\n{\"properties\": 5 | {\"name\": {\"title\": \"Name\", \"type\": \"string\"}, \"age\": {\"title\": 6 | \"Age\", \"type\": \"integer\"}}, \"required\": [\"name\", \"age\"], \"title\": 7 | \"DummyOutputSchema\", \"type\": \"object\"}"},{"role":"user","content":"My 8 | name is Claude and I am 1 year old. What is my name and age?"}],"model":"gpt-3.5-turbo","response_format":{"type":"json_object"}}' 9 | headers: 10 | accept: 11 | - application/json 12 | accept-encoding: 13 | - gzip, deflate 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "459" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.openai.com 22 | user-agent: 23 | - AsyncOpenAI/Python 1.55.3 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - async:asyncio 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 1.55.3 34 | x-stainless-raw-response: 35 | - "true" 36 | x-stainless-retry-count: 37 | - "0" 38 | x-stainless-runtime: 39 | - CPython 40 | x-stainless-runtime-version: 41 | - 3.12.7 42 | method: POST 43 | uri: https://api.openai.com/v1/chat/completions 44 | response: 45 | body: 46 | string: !!binary | 47 | H4sIAAAAAAAAA4ySQWvjMBCF7/4VYs5JqR1Ktr5ld1lKC720lJa6GEWaOOrKkpBGdJeQ/75IcWKH 48 | 7sJefHjfzOO9sXYFY6Ak1AzElpPonZ6v+PPdU/d++/0+fPmIq+opPt483/zAh/j1pYJZ2rDrdxR0 49 | 3LoQtncaSVlzwMIjJ0yu5XKxqJZldV1m0FuJOq11juaLi6s5Rb+288uyuho2t1YJDFCz14Ixxnb5 50 | mzIaib+gZpezo9JjCLxDqE9DjIG3OinAQ1CBuCGYjVBYQ2hy7F1jktSA4T02ULMGvmkeJTYwOyLe 51 | ZVI2Zj918biJgacWJmo96PtTLG075+06DPykb5RRYdt65MGaFCGQdZDpvmDsLdePZ43Aeds7asn+ 52 | RJMMrxcHOxgPPsJyOUCyxPVEL4ebndu1EokrHSb3A8HFFuW4Oh6bR6nsBBST0p/T/M37UFyZ7n/s 53 | RyAEOkLZOo9SifPG45jH9B7/NXY6cg4M4Xcg7NuNMh1651V+Eflf7os/AAAA//8DAOtt/IcQAwAA 54 | headers: 55 | CF-Cache-Status: 56 | - DYNAMIC 57 | CF-RAY: 58 | - 8ec79940487acf0a-SJC 59 | Connection: 60 | - keep-alive 61 | Content-Encoding: 62 | - gzip 63 | Content-Type: 64 | - application/json 65 | Date: 66 | - Wed, 04 Dec 2024 00:14:51 GMT 67 | Server: 68 | - cloudflare 69 | Set-Cookie: 70 | - __cf_bm=Jt74XtlzTIX4qJ2I6JWts_1PUt6239EzTEKob0gbMYI-1733271291-1.0.1.1-Q2lRwwtee9d00lyYx_vjY4jpuRYySoyrIe_R4yj2LqAaxIDQLNQ99FxRmToXDEeMUPk5tUL1416FgdY7MIAURA; 71 | path=/; expires=Wed, 04-Dec-24 00:44:51 GMT; domain=.api.openai.com; HttpOnly; 72 | Secure; SameSite=None 73 | - _cfuvid=LgL7hRr_QJbzVlAiPhNhtvybE1Xb67FkF.lr0Cryt2I-1733271291893-0.0.1.1-604800000; 74 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 75 | Transfer-Encoding: 76 | - chunked 77 | X-Content-Type-Options: 78 | - nosniff 79 | access-control-expose-headers: 80 | - X-Request-ID 81 | alt-svc: 82 | - h3=":443"; ma=86400 83 | openai-organization: 84 | - future-house-xr4tdh 85 | openai-processing-ms: 86 | - "255" 87 | openai-version: 88 | - "2020-10-01" 89 | strict-transport-security: 90 | - max-age=31536000; includeSubDomains; preload 91 | x-ratelimit-limit-requests: 92 | - "12000" 93 | x-ratelimit-limit-tokens: 94 | - "1000000" 95 | x-ratelimit-remaining-requests: 96 | - "11999" 97 | x-ratelimit-remaining-tokens: 98 | - "999910" 99 | x-ratelimit-reset-requests: 100 | - 5ms 101 | x-ratelimit-reset-tokens: 102 | - 5ms 103 | x-request-id: 104 | - req_9f36e635cd12593e29892e9b24310fc6 105 | status: 106 | code: 200 107 | message: OK 108 | version: 1 109 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_output_schema[structured-outputs].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages":[{"role":"user","content":"My name is Claude and I am 1 year 5 | old. What is my name and age?"}],"model":"gpt-4o","response_format":{"type":"json_schema","json_schema":{"strict":true,"schema":{"properties":{"name":{"title":"Name","type":"string"},"age":{"title":"Age","type":"integer"}},"required":["name","age"],"title":"DummyOutputSchema","type":"object","additionalProperties":false},"name":"DummyOutputSchema"}}}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "425" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.55.3 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.55.3 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA4xSu07EMBDs8xXW1gkKCeKOdAgqKCgoeAVFPnuTMzhey3YE6HT/jpx75E6ARONi 45 | Zmc8s/YqYQyUhIqBWPIgequzS/748PQh6AKL+5uebOvubuXl1/XzQKWFNCpo8YYi7FQngnqrMSgy 46 | G1o45AGj6+msLItZkZfzkehJoo6yzobsjLIiL86yfJ7l51vhkpRADxV7SRhjbDWeMaKR+AkVy9Md 47 | 0qP3vEOo9kOMgSMdEeDeKx+4CZBOpCAT0IypVzUY3mMNVQ1Xmg8Sa0hr4F2ETteHKoft4HkMbQat 48 | t/h6H0NTZx0t/Jbf460yyi8bh9yTiVf6QBZGdp0w9jrWHY4agHXU29AEekcTDWcXGzuY9juROy5Q 49 | 4HqC5/P0F7NGYuBK+4NtgeBiiXJSTqvlg1R0QCQHlX9m+c17U1uZ7j/2EyEE2oCysQ6lEsd9pzGH 50 | 8fP9NbZf8RgY/JcP2DetMh0669Tm/VvbzNrzBZbYLnJI1sk3AAAA//8DAEganQUIAwAA 51 | headers: 52 | CF-Cache-Status: 53 | - DYNAMIC 54 | CF-RAY: 55 | - 8ec7ab7ffe551679-SJC 56 | Connection: 57 | - keep-alive 58 | Content-Encoding: 59 | - gzip 60 | Content-Type: 61 | - application/json 62 | Date: 63 | - Wed, 04 Dec 2024 00:27:18 GMT 64 | Server: 65 | - cloudflare 66 | Transfer-Encoding: 67 | - chunked 68 | X-Content-Type-Options: 69 | - nosniff 70 | access-control-expose-headers: 71 | - X-Request-ID 72 | alt-svc: 73 | - h3=":443"; ma=86400 74 | openai-organization: 75 | - future-house-xr4tdh 76 | openai-processing-ms: 77 | - "367" 78 | openai-version: 79 | - "2020-10-01" 80 | strict-transport-security: 81 | - max-age=31536000; includeSubDomains; preload 82 | x-ratelimit-limit-requests: 83 | - "10000" 84 | x-ratelimit-limit-tokens: 85 | - "30000000" 86 | x-ratelimit-remaining-requests: 87 | - "9998" 88 | x-ratelimit-remaining-tokens: 89 | - "29999967" 90 | x-ratelimit-reset-requests: 91 | - 6ms 92 | x-ratelimit-reset-tokens: 93 | - 0s 94 | x-request-id: 95 | - req_b1f66bf72beec8f42091a30db5131d27 96 | status: 97 | code: 200 98 | message: OK 99 | version: 1 100 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_output_type_rejected_validation.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "text", "text": "What 5 | are three things I should do today?"}]}], "temperature": 0.1, "tools": [{"name": 6 | "json_tool_call", "input_schema": {"type": "object", "properties": {"values": 7 | null}}}], "tool_choice": {"name": "json_tool_call", "type": "tool"}, "system": 8 | [{"type": "text", "text": "Respond following this JSON schema:\n\n{\"properties\": 9 | {\"instructions\": {\"description\": \"list of instructions\", \"items\": {\"type\": 10 | \"string\"}, \"title\": \"Instructions\", \"type\": \"array\"}}, \"required\": 11 | [\"instructions\"], \"title\": \"InstructionList\", \"type\": \"object\"}"}], 12 | "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' 13 | headers: 14 | accept: 15 | - application/json 16 | accept-encoding: 17 | - gzip, deflate 18 | anthropic-version: 19 | - "2023-06-01" 20 | connection: 21 | - keep-alive 22 | content-length: 23 | - "692" 24 | content-type: 25 | - application/json 26 | host: 27 | - api.anthropic.com 28 | user-agent: 29 | - litellm/1.52.5 30 | method: POST 31 | uri: https://api.anthropic.com/v1/messages 32 | response: 33 | body: 34 | string: 35 | '{"type":"error","error":{"type":"invalid_request_error","message":"tools.0.input_schema: 36 | JSON schema is invalid - please consult https://json-schema.org or our documentation 37 | at https://docs.anthropic.com/en/docs/tool-use"}}' 38 | headers: 39 | CF-Cache-Status: 40 | - DYNAMIC 41 | CF-RAY: 42 | - 8e20db182b2d644d-SJC 43 | Connection: 44 | - keep-alive 45 | Content-Length: 46 | - "223" 47 | Content-Type: 48 | - application/json 49 | Date: 50 | - Wed, 13 Nov 2024 18:34:26 GMT 51 | Server: 52 | - cloudflare 53 | X-Robots-Tag: 54 | - none 55 | request-id: 56 | - req_01Ma1cK27HCaupfNBiwTjUFo 57 | via: 58 | - 1.1 google 59 | x-should-retry: 60 | - "false" 61 | status: 62 | code: 400 63 | message: Bad Request 64 | version: 1 65 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_streaming[gpt-3.5-turbo].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-3.5-turbo", 6 | "stream": true, "stream_options": {"include_usage": true}, "temperature": 0.1}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - "232" 16 | content-type: 17 | - application/json 18 | host: 19 | - api.openai.com 20 | user-agent: 21 | - AsyncOpenAI/Python 1.54.3 22 | x-stainless-arch: 23 | - arm64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - MacOS 30 | x-stainless-package-version: 31 | - 1.54.3 32 | x-stainless-raw-response: 33 | - "true" 34 | x-stainless-retry-count: 35 | - "1" 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.12.7 40 | method: POST 41 | uri: https://api.openai.com/v1/chat/completions 42 | response: 43 | body: 44 | string: 45 | 'data: {"id":"chatcmpl-ARSd2K2L3Lg87wmLEtOKlJ3tLCQem","object":"chat.completion.chunk","created":1731108268,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} 46 | 47 | 48 | data: {"id":"chatcmpl-ARSd2K2L3Lg87wmLEtOKlJ3tLCQem","object":"chat.completion.chunk","created":1731108268,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Good"},"logprobs":null,"finish_reason":null}],"usage":null} 49 | 50 | 51 | data: {"id":"chatcmpl-ARSd2K2L3Lg87wmLEtOKlJ3tLCQem","object":"chat.completion.chunk","created":1731108268,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} 52 | 53 | 54 | data: {"id":"chatcmpl-ARSd2K2L3Lg87wmLEtOKlJ3tLCQem","object":"chat.completion.chunk","created":1731108268,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":22,"completion_tokens":1,"total_tokens":23,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} 55 | 56 | 57 | data: [DONE] 58 | 59 | 60 | ' 61 | headers: 62 | CF-Cache-Status: 63 | - DYNAMIC 64 | CF-RAY: 65 | - 8df951133c95cf4d-SJC 66 | Connection: 67 | - keep-alive 68 | Content-Type: 69 | - text/event-stream; charset=utf-8 70 | Date: 71 | - Fri, 08 Nov 2024 23:24:28 GMT 72 | Server: 73 | - cloudflare 74 | Transfer-Encoding: 75 | - chunked 76 | X-Content-Type-Options: 77 | - nosniff 78 | access-control-expose-headers: 79 | - X-Request-ID 80 | alt-svc: 81 | - h3=":443"; ma=86400 82 | openai-organization: 83 | - future-house-xr4tdh 84 | openai-processing-ms: 85 | - "197" 86 | openai-version: 87 | - "2020-10-01" 88 | strict-transport-security: 89 | - max-age=31536000; includeSubDomains; preload 90 | x-ratelimit-limit-requests: 91 | - "12000" 92 | x-ratelimit-limit-tokens: 93 | - "1000000" 94 | x-ratelimit-remaining-requests: 95 | - "11999" 96 | x-ratelimit-remaining-tokens: 97 | - "999969" 98 | x-ratelimit-reset-requests: 99 | - 5ms 100 | x-ratelimit-reset-tokens: 101 | - 1ms 102 | x-request-id: 103 | - req_378b8fdfe73f1f24710faf4cdc08d994 104 | status: 105 | code: 200 106 | message: OK 107 | version: 1 108 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_text_image_message[claude-3-haiku-20240307].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "image", "source": 5 | {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, 6 | {"type": "text", "text": "What color is this square? Respond only with the color 7 | name."}]}], "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' 8 | headers: 9 | accept: 10 | - application/json 11 | accept-encoding: 12 | - gzip, deflate 13 | anthropic-version: 14 | - "2023-06-01" 15 | connection: 16 | - keep-alive 17 | content-length: 18 | - "411" 19 | content-type: 20 | - application/json 21 | host: 22 | - api.anthropic.com 23 | user-agent: 24 | - litellm/1.52.3 25 | method: POST 26 | uri: https://api.anthropic.com/v1/messages 27 | response: 28 | body: 29 | string: !!binary | 30 | H4sIAAAAAAAAA0yOzWrDMBCE32XOcnHtQIjOuRZKrqUYYW1cEXnleHdbF+N3Lw4t9DTwzQ+zIkV4 31 | jDJ09fNpGD+vsnxdDq/ntIzty3mw6wwH/Z5oT5FIGAgOc8k7CCJJNLDCYSyRMjz6HCxS1VYfId2s 32 | aurmULf1EQ59YSVW+Lf1b1Fp2bsP8bhQfML27iBapm6mIIXhQRw7tZnxawjdjbgneLacHexxyq9I 33 | PJl2Wm7EAt+cHIrpf3Tcth8AAAD//wMASKMtgPEAAAA= 34 | headers: 35 | CF-Cache-Status: 36 | - DYNAMIC 37 | CF-RAY: 38 | - 8df951169f4d1563-SJC 39 | Connection: 40 | - keep-alive 41 | Content-Encoding: 42 | - gzip 43 | Content-Type: 44 | - application/json 45 | Date: 46 | - Fri, 08 Nov 2024 23:24:28 GMT 47 | Server: 48 | - cloudflare 49 | Transfer-Encoding: 50 | - chunked 51 | X-Robots-Tag: 52 | - none 53 | anthropic-ratelimit-requests-limit: 54 | - "5000" 55 | anthropic-ratelimit-requests-remaining: 56 | - "4999" 57 | anthropic-ratelimit-requests-reset: 58 | - "2024-11-08T23:25:12Z" 59 | anthropic-ratelimit-tokens-limit: 60 | - "5000000" 61 | anthropic-ratelimit-tokens-remaining: 62 | - "5000000" 63 | anthropic-ratelimit-tokens-reset: 64 | - "2024-11-08T23:24:28Z" 65 | request-id: 66 | - req_01MCaZRsMu3nB5nTfMA4ahTw 67 | via: 68 | - 1.1 google 69 | status: 70 | code: 200 71 | message: OK 72 | version: 1 73 | -------------------------------------------------------------------------------- /tests/cassettes/TestLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": 5 | {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, 6 | {"type": "text", "text": "What color is this square? Respond only with the color 7 | name."}]}], "model": "gpt-4o-mini-2024-07-18"}' 8 | headers: 9 | accept: 10 | - application/json 11 | accept-encoding: 12 | - gzip, deflate 13 | connection: 14 | - keep-alive 15 | content-length: 16 | - "373" 17 | content-type: 18 | - application/json 19 | host: 20 | - api.openai.com 21 | user-agent: 22 | - AsyncOpenAI/Python 1.54.3 23 | x-stainless-arch: 24 | - arm64 25 | x-stainless-async: 26 | - async:asyncio 27 | x-stainless-lang: 28 | - python 29 | x-stainless-os: 30 | - MacOS 31 | x-stainless-package-version: 32 | - 1.54.3 33 | x-stainless-raw-response: 34 | - "true" 35 | x-stainless-retry-count: 36 | - "1" 37 | x-stainless-runtime: 38 | - CPython 39 | x-stainless-runtime-version: 40 | - 3.12.7 41 | method: POST 42 | uri: https://api.openai.com/v1/chat/completions 43 | response: 44 | body: 45 | string: !!binary | 46 | H4sIAAAAAAAAA4ySzWrDMBCE734KoXNcYjs/Tm6FEuilLUnppRQjSxtHjawVkgxtQ969yHFih7bQ 47 | iw/77Yxn1j5EhFAp6JJQvmOe10bFt+uNyPTdepUvnh6zJtunm9WXNfgsX+4f6CgosHwH7s+qG461 48 | UeAl6hPmFpiH4JrMsyQZ5+ls0YIaBaggq4yPJxjXUss4HaeTeDyPk7xT71BycHRJXiNCCDm0z5BT 49 | C/igSzIenSc1OMcqoMvLEiHUogoTypyTzjPt6aiHHLUH3UZfgxgSC9vGsZBON0p18+PlVQorY7F0 50 | Hb/Mt1JLtyssMIc62DqPhrb0GBHy1lZqrlJSY7E2vvC4Bx0M82nadaL9KXucdMyjZ+pKdSZXhoUA 51 | z6Ryg6tQzvgORK/tT8gaIXEAokHtn2l+8z5Vl7r6j30POAfjQRTGgpD8unG/ZiH8aX+tXc7cBqbu 52 | 03moi63UFVhj5ek7b02xKOd5OUv4NKXRMfoGAAD//wMAHblqFfUCAAA= 53 | headers: 54 | CF-Cache-Status: 55 | - DYNAMIC 56 | CF-RAY: 57 | - 8df951157da42350-SJC 58 | Connection: 59 | - keep-alive 60 | Content-Encoding: 61 | - gzip 62 | Content-Type: 63 | - application/json 64 | Date: 65 | - Fri, 08 Nov 2024 23:24:29 GMT 66 | Server: 67 | - cloudflare 68 | Transfer-Encoding: 69 | - chunked 70 | X-Content-Type-Options: 71 | - nosniff 72 | access-control-expose-headers: 73 | - X-Request-ID 74 | alt-svc: 75 | - h3=":443"; ma=86400 76 | openai-organization: 77 | - future-house-xr4tdh 78 | openai-processing-ms: 79 | - "1481" 80 | openai-version: 81 | - "2020-10-01" 82 | strict-transport-security: 83 | - max-age=31536000; includeSubDomains; preload 84 | x-ratelimit-limit-input-images: 85 | - "50000" 86 | x-ratelimit-limit-requests: 87 | - "30000" 88 | x-ratelimit-limit-tokens: 89 | - "150000000" 90 | x-ratelimit-remaining-input-images: 91 | - "49999" 92 | x-ratelimit-remaining-requests: 93 | - "29999" 94 | x-ratelimit-remaining-tokens: 95 | - "149999202" 96 | x-ratelimit-reset-input-images: 97 | - 1ms 98 | x-ratelimit-reset-requests: 99 | - 2ms 100 | x-ratelimit-reset-tokens: 101 | - 0s 102 | x-request-id: 103 | - req_d6479069b3a902d72775895cd95ea324 104 | status: 105 | code: 200 106 | message: OK 107 | version: 1 108 | -------------------------------------------------------------------------------- /tests/cassettes/TestLiteEmbeddingModel.test_caching.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"input": ["test1"], "model": "text-embedding-3-small", "dimensions": 8, 5 | "encoding_format": "float"}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "100" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.54.3 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.54.3 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "1" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/embeddings 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA1yQXWvDIBSG7/MrDue6KX4cP+JfGWXYKSWbiWVxUCj97yMu6+K8UHh88PW89w4A 45 | 8/k9vhV0gGlcCh5WFnzx6OClAwC4170x43SOIYzzper1cpxDvKED9iR/0u9L6+rZkRMJ4ooODdSa 46 | Bst2kB0tESPijSesUpq4bSAzSpI0TP9XmTFGtDlysFoKvs8RQlmyetjQqZ6PDuBU25hyiGkdu8Rb 47 | 6Z9j9bJfJp/ST2Vfi79EdFtbeP3M07W8lvwR5wUdbJ/AkotPO9ytQY/uGwAA//8DAOb7reqKAQAA 48 | headers: 49 | CF-Cache-Status: 50 | - DYNAMIC 51 | CF-RAY: 52 | - 8df950fdab81cf65-SJC 53 | Connection: 54 | - keep-alive 55 | Content-Encoding: 56 | - gzip 57 | Content-Type: 58 | - application/json 59 | Date: 60 | - Fri, 08 Nov 2024 23:24:24 GMT 61 | Server: 62 | - cloudflare 63 | Transfer-Encoding: 64 | - chunked 65 | X-Content-Type-Options: 66 | - nosniff 67 | access-control-allow-origin: 68 | - "*" 69 | access-control-expose-headers: 70 | - X-Request-ID 71 | alt-svc: 72 | - h3=":443"; ma=86400 73 | openai-model: 74 | - text-embedding-3-small 75 | openai-organization: 76 | - future-house-xr4tdh 77 | openai-processing-ms: 78 | - "64" 79 | openai-version: 80 | - "2020-10-01" 81 | strict-transport-security: 82 | - max-age=31536000; includeSubDomains; preload 83 | x-ratelimit-limit-requests: 84 | - "10000" 85 | x-ratelimit-limit-tokens: 86 | - "10000000" 87 | x-ratelimit-remaining-requests: 88 | - "9999" 89 | x-ratelimit-remaining-tokens: 90 | - "9991004" 91 | x-ratelimit-reset-requests: 92 | - 6ms 93 | x-ratelimit-reset-tokens: 94 | - 53ms 95 | x-request-id: 96 | - req_f154c6c0d1f72d1496189b1dc8372122 97 | status: 98 | code: 200 99 | message: OK 100 | version: 1 101 | -------------------------------------------------------------------------------- /tests/cassettes/TestMultipleCompletionLLMModel.test_model[gpt-3.5-turbo].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "system", "content": "Respond with single words."}, 5 | {"role": "user", "content": "Hello, how are you?"}], "model": "gpt-3.5-turbo", 6 | "n": 2}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - "161" 16 | content-type: 17 | - application/json 18 | host: 19 | - api.openai.com 20 | user-agent: 21 | - AsyncOpenAI/Python 1.54.3 22 | x-stainless-arch: 23 | - arm64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - MacOS 30 | x-stainless-package-version: 31 | - 1.54.3 32 | x-stainless-raw-response: 33 | - "true" 34 | x-stainless-retry-count: 35 | - "1" 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.12.7 40 | method: POST 41 | uri: https://api.openai.com/v1/chat/completions 42 | response: 43 | body: 44 | string: !!binary | 45 | H4sIAAAAAAAAA7RTsW7CMBDd8xWWZ0CEFNGysbSIpVIrsVRVZOwjGByfa18qEOLfKwdKgqBSh3bx 46 | cO/e83t39j5hjGvFx4zLlSBZOtOdvLyq7Sb78NNl9vn88Ojms918odczv72b8E5k4GINkr5ZPYml 47 | M0Aa7RGWHgRBVE1HWZr277PBsAZKVGAirXDUzXrDLlV+gd1+OhiemCvUEgIfs7eEMcb29Rk9WgVb 48 | Pmb9znelhBBEAXx8bmKMezSxwkUIOpCwxDsNKNES2Nr2E6JqQx6WVRDRmq2MOdUP57sMFs7jIpzw 49 | c32prQ6r3IMIaKNuIHQ8aZGvAqR/E2AKxmCPxRy9fw6SMPZeL6e6sMudx9JRTrgBGwUHg6Mcb55D 50 | Aw5PGCEJ0+KMOjfEcgUktAmt0XAp5ApUw2wegqiUxhbQHv+1l1vax9jaFr+RbwApwRGo3HlQWl7m 51 | bdo8xL/yU9t5xLVhHnaBoMyX2hbgndf1sutNHpIvAAAA//8DADNNhHOsAwAA 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 8df9527aaebbcfa8-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Fri, 08 Nov 2024 23:25:25 GMT 65 | Server: 66 | - cloudflare 67 | Transfer-Encoding: 68 | - chunked 69 | X-Content-Type-Options: 70 | - nosniff 71 | access-control-expose-headers: 72 | - X-Request-ID 73 | alt-svc: 74 | - h3=":443"; ma=86400 75 | openai-organization: 76 | - future-house-xr4tdh 77 | openai-processing-ms: 78 | - "188" 79 | openai-version: 80 | - "2020-10-01" 81 | strict-transport-security: 82 | - max-age=31536000; includeSubDomains; preload 83 | x-ratelimit-limit-requests: 84 | - "12000" 85 | x-ratelimit-limit-tokens: 86 | - "1000000" 87 | x-ratelimit-remaining-requests: 88 | - "11999" 89 | x-ratelimit-remaining-tokens: 90 | - "999953" 91 | x-ratelimit-reset-requests: 92 | - 5ms 93 | x-ratelimit-reset-tokens: 94 | - 2ms 95 | x-request-id: 96 | - req_bcb3d0f3b0c9612ae3e97088211a9b9b 97 | status: 98 | code: 200 99 | message: OK 100 | version: 1 101 | -------------------------------------------------------------------------------- /tests/cassettes/TestMultipleCompletionLLMModel.test_output_schema[json-mode].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages":[{"role":"system","content":"Respond following this JSON schema:\n\n{\"properties\": 5 | {\"name\": {\"title\": \"Name\", \"type\": \"string\"}, \"age\": {\"title\": 6 | \"Age\", \"type\": \"integer\"}}, \"required\": [\"name\", \"age\"], \"title\": 7 | \"DummyOutputSchema\", \"type\": \"object\"}"},{"role":"user","content":"My 8 | name is Claude and I am 1 year old. What is my name and age?"}],"model":"gpt-3.5-turbo","n":2,"response_format":{"type":"json_object"}}' 9 | headers: 10 | accept: 11 | - application/json 12 | accept-encoding: 13 | - gzip, deflate 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "465" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.openai.com 22 | user-agent: 23 | - AsyncOpenAI/Python 1.55.3 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - async:asyncio 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 1.55.3 34 | x-stainless-raw-response: 35 | - "true" 36 | x-stainless-retry-count: 37 | - "0" 38 | x-stainless-runtime: 39 | - CPython 40 | x-stainless-runtime-version: 41 | - 3.12.7 42 | method: POST 43 | uri: https://api.openai.com/v1/chat/completions 44 | response: 45 | body: 46 | string: !!binary | 47 | H4sIAAAAAAAAA7RTwYrbMBS8+yvEOyfL2ma7xLel9NRDW1rKQl2MIr3Y2sp6QnqG7Ib8e5ETxw67 48 | hR7aiw8zb8YzetIhEwKMhkqA6iSr3tv1g3z8+P3xc/fu5Ss9Df2nb1+GZ958kPt98RJglRS0fULF 49 | k+pGUe8tsiF3olVAyZhc8/uyLO7zYpOPRE8abZK1ntflzd2ah7Cl9W1e3J2VHRmFESrxIxNCiMP4 50 | TRmdxj1U4nY1IT3GKFuE6jIkBASyCQEZo4ksHcNqJhU5RjfGPtTgZI81VKKG91YOGmtYiRpkO4L5 51 | cSkMuBuiTMHdYO0ZP16SWGp9oG088xd8Z5yJXRNQRnLpr5HJQ7YQv6qX/6t6LkFvlpyoqWnt/nfX 52 | TIif43aHq0bgA/WeG6Zf6JLhpjzZwXyfZrI8bx6YWNoZz4tJdWXXaGRpbFycHyipOtSzdL5LctCG 53 | FsRyR6/TvOV9Km5c+zf2M6EUekbd+IDaqOvG81jA9Nz+NHY55DEwxOfI2Dc741oMPpjxRoy7PGa/ 54 | AQAA//8DAL4bR47vAwAA 55 | headers: 56 | CF-Cache-Status: 57 | - DYNAMIC 58 | CF-RAY: 59 | - 8ec799405a7ecfac-SJC 60 | Connection: 61 | - keep-alive 62 | Content-Encoding: 63 | - gzip 64 | Content-Type: 65 | - application/json 66 | Date: 67 | - Wed, 04 Dec 2024 00:14:51 GMT 68 | Server: 69 | - cloudflare 70 | Set-Cookie: 71 | - __cf_bm=br87AQInkCnSV_I9MXX5tYGv6_ZP_82Htl2IQun7srQ-1733271291-1.0.1.1-.r_ewngzuQkAkcKmG5atHV1LISuUrs2yVGfkZp0pxbuZa00Mqp.aWv.QYORcyr6VriqkQoJnjfJY99tvrd7rKw; 72 | path=/; expires=Wed, 04-Dec-24 00:44:51 GMT; domain=.api.openai.com; HttpOnly; 73 | Secure; SameSite=None 74 | - _cfuvid=_yXi1Uk8mQKkHMlntTwGtMye1ID9VJRq.3psfLHacuU-1733271291921-0.0.1.1-604800000; 75 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 76 | Transfer-Encoding: 77 | - chunked 78 | X-Content-Type-Options: 79 | - nosniff 80 | access-control-expose-headers: 81 | - X-Request-ID 82 | alt-svc: 83 | - h3=":443"; ma=86400 84 | openai-organization: 85 | - future-house-xr4tdh 86 | openai-processing-ms: 87 | - "268" 88 | openai-version: 89 | - "2020-10-01" 90 | strict-transport-security: 91 | - max-age=31536000; includeSubDomains; preload 92 | x-ratelimit-limit-requests: 93 | - "12000" 94 | x-ratelimit-limit-tokens: 95 | - "1000000" 96 | x-ratelimit-remaining-requests: 97 | - "11999" 98 | x-ratelimit-remaining-tokens: 99 | - "999894" 100 | x-ratelimit-reset-requests: 101 | - 5ms 102 | x-ratelimit-reset-tokens: 103 | - 6ms 104 | x-request-id: 105 | - req_2d64ba6207682b9086d527ac85f85959 106 | status: 107 | code: 200 108 | message: OK 109 | version: 1 110 | -------------------------------------------------------------------------------- /tests/cassettes/TestMultipleCompletionLLMModel.test_output_schema[structured-outputs].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages":[{"role":"user","content":"My name is Claude and I am 1 year 5 | old. What is my name and age?"}],"model":"gpt-4o","n":2,"response_format":{"type":"json_schema","json_schema":{"strict":true,"schema":{"properties":{"name":{"title":"Name","type":"string"},"age":{"title":"Age","type":"integer"}},"required":["name","age"],"title":"DummyOutputSchema","type":"object","additionalProperties":false},"name":"DummyOutputSchema"}}}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "431" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.55.3 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.55.3 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA9xTTW+cMBS88yusd4YK2CgQbv26VZWiHpqoVOit/Zb1rrEt20iNVvvfK7NkIUoi 45 | 9dwLh5k3w4z9fEoYAymgYcD3GPhgVfYRH34+Fsd79+U7Hp5+0P2nb1usvx6OD/SoIY0Ksz0QD8+q 46 | D9wMVlGQZqa5IwwUXYtqsymrMt/UEzEYQSrKehuyG5OVeXmT5XWW387CvZGcPDTsV8IYY6fpGyNq 47 | QX+gYXn6jAzkPfYEzXWIMXBGRQTQe+kD6gDpQnKjA+kp9akFjQO10LTwWeEoqIW0BewjVJzXKke7 48 | 0WMMrUelZvx8jaFMb53Z+pm/4juppd93jtAbHX/pg7GQrMSvuhX/Q7eEsd/TVY4vGoB1ZrChC+ZI 49 | OhpWdxc7WHZnIYt6JoMJqBb8rkrfcOsEBZTKr44LOPI9iUW57A2OQpoVsb6S12He8r70lrr/F/uF 50 | 4JxsINFZR0Lyl4WXMUfxZb03dj3jKTD4Jx9o6HZS9+Ssk5cF2Nmu3hSU31aiLiE5J38BAAD//wMA 51 | vVnbbuUDAAA= 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 8ec7ab7fe966aaa5-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Wed, 04 Dec 2024 00:27:19 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=5AuGimsg2FH0ntBI.jKIdG9P._9c78WMhN9jaiVQGeM-1733272039-1.0.1.1-F92PupP7R_e1Usesjv3KlZUrgLh73lB5LXJjilxG5ZwkFDrI1tOZsZp0zhC5gY.WjprQpIckFtJxWsiUNww0kw; 69 | path=/; expires=Wed, 04-Dec-24 00:57:19 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=.1wBj_f8n7gG1Go8qdTIwKV7Pk8XaH9yfTumfoAGccU-1733272039104-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "610" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "10000" 91 | x-ratelimit-limit-tokens: 92 | - "30000000" 93 | x-ratelimit-remaining-requests: 94 | - "9999" 95 | x-ratelimit-remaining-tokens: 96 | - "29999951" 97 | x-ratelimit-reset-requests: 98 | - 6ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_86d20ee0ac20474675168c7b0c693f02 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/cassettes/TestMultipleCompletionLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": 5 | {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, 6 | {"type": "text", "text": "What color is this square? Respond only with the color 7 | name."}]}], "model": "gpt-4o-mini-2024-07-18", "n": 2}' 8 | headers: 9 | accept: 10 | - application/json 11 | accept-encoding: 12 | - gzip, deflate 13 | connection: 14 | - keep-alive 15 | content-length: 16 | - "381" 17 | content-type: 18 | - application/json 19 | host: 20 | - api.openai.com 21 | user-agent: 22 | - AsyncOpenAI/Python 1.54.3 23 | x-stainless-arch: 24 | - arm64 25 | x-stainless-async: 26 | - async:asyncio 27 | x-stainless-lang: 28 | - python 29 | x-stainless-os: 30 | - MacOS 31 | x-stainless-package-version: 32 | - 1.54.3 33 | x-stainless-raw-response: 34 | - "true" 35 | x-stainless-retry-count: 36 | - "1" 37 | x-stainless-runtime: 38 | - CPython 39 | x-stainless-runtime-version: 40 | - 3.12.7 41 | method: POST 42 | uri: https://api.openai.com/v1/chat/completions 43 | response: 44 | body: 45 | string: !!binary | 46 | H4sIAAAAAAAAA9STP0/DMBDF93wKy3ODmtA/IRszWxkQQihy7GticHyWz5WKqn535DRtUhUkVpYM 47 | 97v38nxnHxLGuFa8ZFy2IsjOmfRx86xyp572+FL715pq7ZCKtqOVssRnUYH1B8hwVt1J7JyBoNGe 48 | sPQgAkTXbH2fZfMiXxU96FCBibLGhXSBaaetTvN5vkjn6zQrBnWLWgLxkr0ljDF26L8xp1Ww5yWb 49 | z86VDohEA7y8NDHGPZpY4YJIUxA28NkIJdoAto++ATUlHrY7EjGd3Rkz1I+XXxlsnMeaBn6pb7XV 50 | 1FYeBKGNthTQ8WQivsmf/Zf8CWPv/Up2Vym589i5UAX8BBsNi2U+7ISPV2HE+cACBmGuVGdyZVgp 51 | CEIbmkyFSyFbUKN2vAJipzROwHTyt2l+8j4dXdvmL/YjkBJcAFU5D0rL6xOPbR7iS/mt7TLmPjCn 52 | LwrQVVttG/DO69Oet656qNdFvcrkMufJMfkGAAD//wMAhyjYPLUDAAA= 53 | headers: 54 | CF-Cache-Status: 55 | - DYNAMIC 56 | CF-RAY: 57 | - 8df9510e7fa3172e-SJC 58 | Connection: 59 | - keep-alive 60 | Content-Encoding: 61 | - gzip 62 | Content-Type: 63 | - application/json 64 | Date: 65 | - Fri, 08 Nov 2024 23:24:29 GMT 66 | Server: 67 | - cloudflare 68 | Transfer-Encoding: 69 | - chunked 70 | X-Content-Type-Options: 71 | - nosniff 72 | access-control-expose-headers: 73 | - X-Request-ID 74 | alt-svc: 75 | - h3=":443"; ma=86400 76 | openai-organization: 77 | - future-house-xr4tdh 78 | openai-processing-ms: 79 | - "1964" 80 | openai-version: 81 | - "2020-10-01" 82 | strict-transport-security: 83 | - max-age=31536000; includeSubDomains; preload 84 | x-ratelimit-limit-input-images: 85 | - "50000" 86 | x-ratelimit-limit-requests: 87 | - "30000" 88 | x-ratelimit-limit-tokens: 89 | - "150000000" 90 | x-ratelimit-remaining-input-images: 91 | - "49999" 92 | x-ratelimit-remaining-requests: 93 | - "29999" 94 | x-ratelimit-remaining-tokens: 95 | - "149999187" 96 | x-ratelimit-reset-input-images: 97 | - 1ms 98 | x-ratelimit-reset-requests: 99 | - 2ms 100 | x-ratelimit-reset-tokens: 101 | - 0s 102 | x-request-id: 103 | - req_7823d27f0ca9fc3fefbc0261333d26e3 104 | status: 105 | code: 200 106 | message: OK 107 | version: 1 108 | -------------------------------------------------------------------------------- /tests/cassettes/TestNoToolsSimpleAgent.test_dummyenv[claude-3-5-haiku-20241022].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-5-haiku-20241022", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "Write a 5 word story via print_story"}]}], "temperature": 6 | 0.1, "max_tokens": 4096}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | anthropic-version: 13 | - "2023-06-01" 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "189" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.anthropic.com 22 | user-agent: 23 | - litellm/1.59.5 24 | method: POST 25 | uri: https://api.anthropic.com/v1/messages 26 | response: 27 | body: 28 | string: !!binary | 29 | H4sIAAAAAAAAAwAAAP//ZJAxb9wwDIX/CsGlCeAL7twc0GprpgwZMnRqXdiqRNtqHNIRqbTG4f57 30 | YTcBEnQi8N6nx0edMEV0+KhDuz/cfAv3862PdT/yz+V64frL/U3CCm2ZaaVI1Q+EFWaZVsGrJjXP 31 | hhU+SqQJHYbJl0i7j7vjbvTpoezqfX192Nc1VhiEjdjQfT+9Zhr9WV9vw+EtZfqg4OG4+y05gprk 32 | BYomHsBGgm7Oia3d5A76wsGSsGu44a7r5sVG4YYj9fAGvLh0DQPAP+2iwTtRg0meCXopHMEPPjGY 33 | RL9cNXi5Za2RX8ekEIRDUnqpomUYSG2tyD5nb+mZQHrIFISZtjrgOcIoM0Fi+FXUoF+p9SC9wvOP 34 | CtVkbjN5FUaHxLG1khlfDKWnQhwIHZdpqrBsn+5OmHgu1po8ECu6w6cKgw8jtSGTXxe374H9q5/J 35 | x/89KfZWOX4+n/8CAAD//wMA4xURPQ4CAAA= 36 | headers: 37 | CF-RAY: 38 | - 9093f64fdef02302-SJC 39 | Connection: 40 | - keep-alive 41 | Content-Encoding: 42 | - gzip 43 | Content-Type: 44 | - application/json 45 | Date: 46 | - Tue, 28 Jan 2025 21:09:13 GMT 47 | Server: 48 | - cloudflare 49 | Transfer-Encoding: 50 | - chunked 51 | X-Robots-Tag: 52 | - none 53 | anthropic-ratelimit-input-tokens-limit: 54 | - "5000000" 55 | anthropic-ratelimit-input-tokens-remaining: 56 | - "5000000" 57 | anthropic-ratelimit-input-tokens-reset: 58 | - "2025-01-28T21:09:12Z" 59 | anthropic-ratelimit-output-tokens-limit: 60 | - "1000000" 61 | anthropic-ratelimit-output-tokens-remaining: 62 | - "1000000" 63 | anthropic-ratelimit-output-tokens-reset: 64 | - "2025-01-28T21:09:13Z" 65 | anthropic-ratelimit-requests-limit: 66 | - "5000" 67 | anthropic-ratelimit-requests-remaining: 68 | - "4999" 69 | anthropic-ratelimit-requests-reset: 70 | - "2025-01-28T21:09:12Z" 71 | anthropic-ratelimit-tokens-limit: 72 | - "6000000" 73 | anthropic-ratelimit-tokens-remaining: 74 | - "6000000" 75 | anthropic-ratelimit-tokens-reset: 76 | - "2025-01-28T21:09:12Z" 77 | cf-cache-status: 78 | - DYNAMIC 79 | request-id: 80 | - req_015KajCSj3bs81Latrp75Lv6 81 | via: 82 | - 1.1 google 83 | status: 84 | code: 200 85 | message: OK 86 | version: 1 87 | -------------------------------------------------------------------------------- /tests/cassettes/TestNoToolsSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-haiku-20240307", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "Write a 5 word story via print_story"}]}], "temperature": 6 | 0.1, "max_tokens": 4096}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | anthropic-version: 13 | - "2023-06-01" 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "187" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.anthropic.com 22 | user-agent: 23 | - litellm/1.57.0 24 | method: POST 25 | uri: https://api.anthropic.com/v1/messages 26 | response: 27 | body: 28 | string: !!binary | 29 | H4sIAAAAAAAAA2SPT0sDMRDFv0qYk0Iq238oObZFevIg6kFXQtxMt2mzkzUzoS2l311aFBRPj3m/ 30 | Nw/eEYIHAx23thrOeLZ9bR+e+vki7Cbzx+XLYnq/AQ1y6PGcQmbXImjIKZ4NxxxYHAlo6JLHCAaa 31 | 6IrHwXiwdmFbBqNqNKnG1S1oaBIJkoB5O/40Cu7PvxcxsMSMKrByaqp2KXvFkvLB1FRTnwOJvdxX 32 | NTwT7ntsBL3apJIJD1oVWqXcooj7iKg67FIOyFrFsMKYqFWrHJA8r0N/U8M1nN41sKTeZnScCAwg 33 | eSslE3wDxs+C1CAYKjFqKJfx5giB+iJW0haJwQzvNDSuWaNtMjoJiezfQPXDMzr/n6Uiv53x8HT6 34 | AgAA//8DADKg2IOWAQAA 35 | headers: 36 | CF-Cache-Status: 37 | - DYNAMIC 38 | CF-RAY: 39 | - 905a90659e5cf9d4-SJC 40 | Connection: 41 | - keep-alive 42 | Content-Encoding: 43 | - gzip 44 | Content-Type: 45 | - application/json 46 | Date: 47 | - Tue, 21 Jan 2025 21:58:10 GMT 48 | Server: 49 | - cloudflare 50 | Transfer-Encoding: 51 | - chunked 52 | X-Robots-Tag: 53 | - none 54 | anthropic-ratelimit-input-tokens-limit: 55 | - "5000000" 56 | anthropic-ratelimit-input-tokens-remaining: 57 | - "5000000" 58 | anthropic-ratelimit-input-tokens-reset: 59 | - "2025-01-21T21:58:09Z" 60 | anthropic-ratelimit-output-tokens-limit: 61 | - "1000000" 62 | anthropic-ratelimit-output-tokens-remaining: 63 | - "1000000" 64 | anthropic-ratelimit-output-tokens-reset: 65 | - "2025-01-21T21:58:09Z" 66 | anthropic-ratelimit-requests-limit: 67 | - "5000" 68 | anthropic-ratelimit-requests-remaining: 69 | - "4999" 70 | anthropic-ratelimit-requests-reset: 71 | - "2025-01-21T21:58:09Z" 72 | anthropic-ratelimit-tokens-limit: 73 | - "6000000" 74 | anthropic-ratelimit-tokens-remaining: 75 | - "6000000" 76 | anthropic-ratelimit-tokens-reset: 77 | - "2025-01-21T21:58:09Z" 78 | request-id: 79 | - req_01WUTufxQjbhVVuJMS56CsLC 80 | via: 81 | - 1.1 google 82 | status: 83 | code: 200 84 | message: OK 85 | version: 1 86 | -------------------------------------------------------------------------------- /tests/cassettes/TestNoToolsSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Write a 5 word story via print_story"}], 5 | "model": "gpt-4o-mini-2024-07-18", "n": 1, "temperature": 0.1}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "146" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.59.3 21 | x-stainless-arch: 22 | - arm64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - MacOS 29 | x-stainless-package-version: 30 | - 1.59.3 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.12.7 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAAwAAAP//jFJNT+MwEL3nV1g+J6gNpYHedi+LgOsuWhCKXHuSenE8lmfCh1D/+8pJ 45 | 2xQBEpcc3pv38t543jIhpDVyJaTeKNZdcMUP+vX3/C68PBt1dnd57dwfvo1XvzH8bCnIPClw/Q80 46 | 71UnGrvggC36kdYRFENynVen1eJiubwoB6JDAy7J2sDFAovOeluUs3JRzKpifr5Tb9BqILkS95kQ 47 | QrwN35TTG3iRKzHL90gHRKoFuToMCSEjuoRIRWSJlWeZT6RGz+CH6DdILB7hNRcGMYoGIzxBFNoh 48 | gTk5FkVoelIpuO+d2+HbQwqHbYi4ph1/wBvrLW3qCIrQpz8SY5ADu82EeBja9u8KyBCxC1wzPoJP 49 | hvPlaCenHU9kteMYWbkJLk/zT8xqA6yso6NlSa30BsyknDaremPxiMiOKn/M8pn3WNv69jv2E6E1 50 | BAZThwjG6vd9p7EI6QC/GjuseAgsCeKT1VCzhZiewUCjejeehaRXYujqxvoWYoh2vI0m1FUJplLr 51 | s4WW2Tb7DwAA//8DALVhDRgpAwAA 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 905a9069eaafebe9-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Tue, 21 Jan 2025 21:58:12 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=2dYW.ZjJiarII_MAmJyoaMr3FRfPWBOFDGojtrktaCY-1737496692-1.0.1.1-XP4CYYKfgXRJujlYOUGIPtWWH6aJ0jLgiPgiYT46.vTw0mR_5bWClcm4Hw_M9oTWb8KKJVyoyLfto3RLRGJMug; 69 | path=/; expires=Tue, 21-Jan-25 22:28:12 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=cKqI45QiSqB042Gew1XZbNwSC3TvNkhGer1N8m9zLbc-1737496692594-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "940" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "30000" 91 | x-ratelimit-limit-tokens: 92 | - "150000000" 93 | x-ratelimit-remaining-requests: 94 | - "29999" 95 | x-ratelimit-remaining-tokens: 96 | - "149999973" 97 | x-ratelimit-reset-requests: 98 | - 2ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_19dbe433a04699f46315a7af746c0386 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/cassettes/TestSimpleAgent.test_agent_grad[claude-3-5-haiku-20241022].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-5-haiku-20241022", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "Write a 5 word story via print_story"}]}], "temperature": 6 | 0.1, "max_tokens": 4096, "tools": [{"name": "print_story", "input_schema": {"type": 7 | "object", "properties": {"story": {"description": "Story to print.", "title": 8 | "Story", "type": "string"}}, "required": ["story"]}, "description": "Print a 9 | story."}, {"name": "cast_float", "input_schema": {"type": "object", "properties": 10 | {"x": {"title": "X", "type": "string"}}, "required": ["x"]}, "description": 11 | "Cast the input argument x to a float."}, {"name": "cast_int", "input_schema": 12 | {"type": "object", "properties": {"x": {"title": "X", "type": "number"}}, "required": 13 | ["x"]}, "description": "Cast the input argument x to an integer."}], "tool_choice": 14 | {"type": "any"}}' 15 | headers: 16 | accept: 17 | - application/json 18 | accept-encoding: 19 | - gzip, deflate 20 | anthropic-version: 21 | - "2023-06-01" 22 | connection: 23 | - keep-alive 24 | content-length: 25 | - "827" 26 | content-type: 27 | - application/json 28 | host: 29 | - api.anthropic.com 30 | user-agent: 31 | - litellm/1.59.5 32 | method: POST 33 | uri: https://api.anthropic.com/v1/messages 34 | response: 35 | body: 36 | string: !!binary | 37 | H4sIAAAAAAAAA2RPy2rDMBD8FbNnu9jK46BjS2kJOTSFUHApQpG3joisda1Vm2D870UJgYYed147 38 | M4JtQEIXWlVWC2Fi9bp5PD5Xb5t6tez1Z7UmyIFPPSYVhqBbhBwGcgnQIdjA2jPk0FGDDiQYp2OD 39 | xaxYFHttD7EQpZhXpRCQgyHP6Bnk+3jNZCKnYkih5ybpjqqsnnY/dc223m5f7vWhFctu9TBLGV53 40 | ydcP1rMKTMMpWX0fGeQIF0DCmr4x2zmiDpvM+ix6PPZoGJssWIfe4B1M00eeDL0aUAfyt23ORMCv 41 | mMQgfXQuh3jeL8fLQ8V0QB9ALpYiB6PNHpUZULMlr24V5ZUfUDf/OYr8F5nPpukXAAD//wMANKfE 42 | mZoBAAA= 43 | headers: 44 | CF-RAY: 45 | - 9093f64eef9f7e2d-SJC 46 | Connection: 47 | - keep-alive 48 | Content-Encoding: 49 | - gzip 50 | Content-Type: 51 | - application/json 52 | Date: 53 | - Tue, 28 Jan 2025 21:09:14 GMT 54 | Server: 55 | - cloudflare 56 | Transfer-Encoding: 57 | - chunked 58 | X-Robots-Tag: 59 | - none 60 | anthropic-ratelimit-input-tokens-limit: 61 | - "5000000" 62 | anthropic-ratelimit-input-tokens-remaining: 63 | - "5000000" 64 | anthropic-ratelimit-input-tokens-reset: 65 | - "2025-01-28T21:09:13Z" 66 | anthropic-ratelimit-output-tokens-limit: 67 | - "1000000" 68 | anthropic-ratelimit-output-tokens-remaining: 69 | - "1000000" 70 | anthropic-ratelimit-output-tokens-reset: 71 | - "2025-01-28T21:09:14Z" 72 | anthropic-ratelimit-requests-limit: 73 | - "5000" 74 | anthropic-ratelimit-requests-remaining: 75 | - "4997" 76 | anthropic-ratelimit-requests-reset: 77 | - "2025-01-28T21:09:12Z" 78 | anthropic-ratelimit-tokens-limit: 79 | - "6000000" 80 | anthropic-ratelimit-tokens-remaining: 81 | - "6000000" 82 | anthropic-ratelimit-tokens-reset: 83 | - "2025-01-28T21:09:13Z" 84 | cf-cache-status: 85 | - DYNAMIC 86 | request-id: 87 | - req_01ACKBwLEvWvcDqXyBCXrFBj 88 | via: 89 | - 1.1 google 90 | status: 91 | code: 200 92 | message: OK 93 | version: 1 94 | -------------------------------------------------------------------------------- /tests/cassettes/TestSimpleAgent.test_agent_grad[claude-3-haiku-20240307].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Write 5 | a 5 word story via print_story"}]}], "temperature": 0.1, "tools": [{"name": 6 | "print_story", "input_schema": {"type": "object", "properties": {"story": {"description": 7 | "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}, 8 | "description": "Print a story."}, {"name": "cast_float", "input_schema": {"type": 9 | "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": 10 | ["x"]}, "description": "Cast the input argument x to a float."}, {"name": "cast_int", 11 | "input_schema": {"type": "object", "properties": {"x": {"title": "X", "type": 12 | "number"}}, "required": ["x"]}, "description": "Cast the input argument x to 13 | an integer."}], "tool_choice": {"type": "any"}, "max_tokens": 4096, "model": 14 | "claude-3-haiku-20240307"}' 15 | headers: 16 | accept: 17 | - application/json 18 | accept-encoding: 19 | - gzip, deflate 20 | anthropic-version: 21 | - "2023-06-01" 22 | connection: 23 | - keep-alive 24 | content-length: 25 | - "825" 26 | content-type: 27 | - application/json 28 | host: 29 | - api.anthropic.com 30 | user-agent: 31 | - litellm/1.52.3 32 | method: POST 33 | uri: https://api.anthropic.com/v1/messages 34 | response: 35 | body: 36 | string: !!binary | 37 | H4sIAAAAAAAAA1SQT0vDQBDFv0qY81bSRKvsTaV4qIh/KIoiYUnGNjaZjTsz0Rjy3WVbPPQ4b957 38 | /Hgj1BVYaHlTpPPn/GG4UL7q85vVS/nUf+d1e9eBARk6jC5kdhsEA8E3UXDMNYsjAQOtr7ABC2Xj 39 | tMJZPtu6eqezLM1O0zw9BwOlJ0ESsG/jf6N43xTKsXLPEW8t0vlj/UputbxdyC9ur5dc3vfrdg0G 40 | yLUx14WapGDxYYhR6lTAjnAQLFxWPZJowJAw4o4TCehYA5rko6aKEyX86bAUrJJPr4FwOIFpejex 41 | oSui2dMx3v7B+KVIJYIlbRoDup/DjgeCQvwOicGeLTIDXuVIm0/THwAAAP//AwAD26uSbQEAAA== 42 | headers: 43 | CF-Cache-Status: 44 | - DYNAMIC 45 | CF-RAY: 46 | - 8df952260dedcfed-SJC 47 | Connection: 48 | - keep-alive 49 | Content-Encoding: 50 | - gzip 51 | Content-Type: 52 | - application/json 53 | Date: 54 | - Fri, 08 Nov 2024 23:25:13 GMT 55 | Server: 56 | - cloudflare 57 | Transfer-Encoding: 58 | - chunked 59 | X-Robots-Tag: 60 | - none 61 | anthropic-ratelimit-requests-limit: 62 | - "5000" 63 | anthropic-ratelimit-requests-remaining: 64 | - "4999" 65 | anthropic-ratelimit-requests-reset: 66 | - "2024-11-08T23:26:12Z" 67 | anthropic-ratelimit-tokens-limit: 68 | - "5000000" 69 | anthropic-ratelimit-tokens-remaining: 70 | - "5000000" 71 | anthropic-ratelimit-tokens-reset: 72 | - "2024-11-08T23:25:13Z" 73 | request-id: 74 | - req_01NDnvrz2u3MaKcf1cbQULuY 75 | via: 76 | - 1.1 google 77 | status: 78 | code: 200 79 | message: OK 80 | version: 1 81 | -------------------------------------------------------------------------------- /tests/cassettes/TestSimpleAgent.test_dummyenv[claude-3-5-haiku-20241022].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"model": "claude-3-5-haiku-20241022", "messages": [{"role": "user", "content": 5 | [{"type": "text", "text": "Write a 5 word story via print_story"}]}], "temperature": 6 | 0.1, "max_tokens": 4096, "tools": [{"name": "print_story", "input_schema": {"type": 7 | "object", "properties": {"story": {"description": "Story to print.", "title": 8 | "Story", "type": "string"}}, "required": ["story"]}, "description": "Print a 9 | story."}, {"name": "cast_float", "input_schema": {"type": "object", "properties": 10 | {"x": {"title": "X", "type": "string"}}, "required": ["x"]}, "description": 11 | "Cast the input argument x to a float."}, {"name": "cast_int", "input_schema": 12 | {"type": "object", "properties": {"x": {"title": "X", "type": "number"}}, "required": 13 | ["x"]}, "description": "Cast the input argument x to an integer."}], "tool_choice": 14 | {"type": "any"}}' 15 | headers: 16 | accept: 17 | - application/json 18 | accept-encoding: 19 | - gzip, deflate 20 | anthropic-version: 21 | - "2023-06-01" 22 | connection: 23 | - keep-alive 24 | content-length: 25 | - "827" 26 | content-type: 27 | - application/json 28 | host: 29 | - api.anthropic.com 30 | user-agent: 31 | - litellm/1.59.5 32 | method: POST 33 | uri: https://api.anthropic.com/v1/messages 34 | response: 35 | body: 36 | string: !!binary | 37 | H4sIAAAAAAAAA2RPTWvCQBD9K2HOmxJjUmVvaqEXBW+FlrKsu4NGN7PJftjakP9eVhEqPc77mvcG 38 | aDRwaP1eFJPVbPn2szi/XnZ6tXwJbrOpFvsNMAiXDpMKvZd7BAbOmgRI7xsfJAVg0FqNBjgoI6PG 39 | fJrX+UE2p5iXRVlNirIEBspSQArAP4Z7ZrDWiOhT6LVJuqMoJvO6n/bv2yPSYa5mW9lX66/aAgOS 40 | bfJ1rqEgfLDukqzUxQB8gBvAYW3PmO2MtS3qrKEsEn53qALqzDcGSeETjOMnS4ZOOJTe0mObK+Gx 41 | j0kMnKIxDOJ1Px9uD0WwJyQPvH4uGSipDiiUQxkaS+JRUdx5h1L/52wMf5FqOo6/AAAA//8DAEbz 42 | 44eaAQAA 43 | headers: 44 | CF-Cache-Status: 45 | - DYNAMIC 46 | CF-RAY: 47 | - 9093f64eed7ffa3e-SJC 48 | Connection: 49 | - keep-alive 50 | Content-Encoding: 51 | - gzip 52 | Content-Type: 53 | - application/json 54 | Date: 55 | - Tue, 28 Jan 2025 21:09:13 GMT 56 | Server: 57 | - cloudflare 58 | Transfer-Encoding: 59 | - chunked 60 | X-Robots-Tag: 61 | - none 62 | anthropic-ratelimit-input-tokens-limit: 63 | - "5000000" 64 | anthropic-ratelimit-input-tokens-remaining: 65 | - "5000000" 66 | anthropic-ratelimit-input-tokens-reset: 67 | - "2025-01-28T21:09:13Z" 68 | anthropic-ratelimit-output-tokens-limit: 69 | - "1000000" 70 | anthropic-ratelimit-output-tokens-remaining: 71 | - "1000000" 72 | anthropic-ratelimit-output-tokens-reset: 73 | - "2025-01-28T21:09:13Z" 74 | anthropic-ratelimit-requests-limit: 75 | - "5000" 76 | anthropic-ratelimit-requests-remaining: 77 | - "4998" 78 | anthropic-ratelimit-requests-reset: 79 | - "2025-01-28T21:09:12Z" 80 | anthropic-ratelimit-tokens-limit: 81 | - "6000000" 82 | anthropic-ratelimit-tokens-remaining: 83 | - "6000000" 84 | anthropic-ratelimit-tokens-reset: 85 | - "2025-01-28T21:09:13Z" 86 | request-id: 87 | - req_01RDaN9hSeKAcGKfk39icQK8 88 | via: 89 | - 1.1 google 90 | status: 91 | code: 200 92 | message: OK 93 | version: 1 94 | -------------------------------------------------------------------------------- /tests/cassettes/TestSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Write 5 | a 5 word story via print_story"}]}], "temperature": 0.1, "tools": [{"name": 6 | "print_story", "input_schema": {"type": "object", "properties": {"story": {"description": 7 | "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}, 8 | "description": "Print a story."}, {"name": "cast_float", "input_schema": {"type": 9 | "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": 10 | ["x"]}, "description": "Cast the input argument x to a float."}, {"name": "cast_int", 11 | "input_schema": {"type": "object", "properties": {"x": {"title": "X", "type": 12 | "number"}}, "required": ["x"]}, "description": "Cast the input argument x to 13 | an integer."}], "tool_choice": {"type": "any"}, "max_tokens": 4096, "model": 14 | "claude-3-haiku-20240307"}' 15 | headers: 16 | accept: 17 | - application/json 18 | accept-encoding: 19 | - gzip, deflate 20 | anthropic-version: 21 | - "2023-06-01" 22 | connection: 23 | - keep-alive 24 | content-length: 25 | - "825" 26 | content-type: 27 | - application/json 28 | host: 29 | - api.anthropic.com 30 | user-agent: 31 | - litellm/1.52.3 32 | method: POST 33 | uri: https://api.anthropic.com/v1/messages 34 | response: 35 | body: 36 | string: !!binary | 37 | H4sIAAAAAAAAA1SQW2vCQBCF/0qY57XES9u4b7UIUmihlF6klBCyo6aJs3FnRtSQ/15W6YOPc+ac 38 | w8fpoHJgYcvrPB0ugmwW7275Os3aOpuc5ruv+mMKBuTYYnQhc7FGMBB8E4WCuWIpSMDA1jtswELZ 39 | FOpwMB5siqrWwSgdTdJxeg8GSk+CJGC/u/9G8b7JlWPlmSPemqfD4dPuU1dvzy+zx3km+9Msmx+y 40 | JRigYhtzbahIchYfjjFKrQrYDi6ChQe3RxINGBJGrDmRgAVrQJOsKnKcKOGhxVLQJb9eA+HxBvr+ 41 | x8SGNo9mT9d45wfjTpFKBEvaNAb0PIftLgS5+BqJwd7ejQx4lSst7fs/AAAA//8DAAk5Fe5tAQAA 42 | headers: 43 | CF-Cache-Status: 44 | - DYNAMIC 45 | CF-RAY: 46 | - 8df950f8e8a49452-SJC 47 | Connection: 48 | - keep-alive 49 | Content-Encoding: 50 | - gzip 51 | Content-Type: 52 | - application/json 53 | Date: 54 | - Fri, 08 Nov 2024 23:24:25 GMT 55 | Server: 56 | - cloudflare 57 | Transfer-Encoding: 58 | - chunked 59 | X-Robots-Tag: 60 | - none 61 | anthropic-ratelimit-requests-limit: 62 | - "5000" 63 | anthropic-ratelimit-requests-remaining: 64 | - "4998" 65 | anthropic-ratelimit-requests-reset: 66 | - "2024-11-08T23:25:12Z" 67 | anthropic-ratelimit-tokens-limit: 68 | - "5000000" 69 | anthropic-ratelimit-tokens-remaining: 70 | - "5000000" 71 | anthropic-ratelimit-tokens-reset: 72 | - "2024-11-08T23:24:25Z" 73 | request-id: 74 | - req_014oiuUyG6RydFy6cCAKTi8Z 75 | via: 76 | - 1.1 google 77 | status: 78 | code: 200 79 | message: OK 80 | version: 1 81 | -------------------------------------------------------------------------------- /tests/cassettes/TestSimpleAgent.test_hide_old_env_states.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": ""}], "model": "gpt-4o-2024-08-06", 5 | "temperature": 0.1}' 6 | headers: 7 | accept: 8 | - application/json 9 | accept-encoding: 10 | - gzip, deflate 11 | connection: 12 | - keep-alive 13 | content-length: 14 | - "97" 15 | content-type: 16 | - application/json 17 | host: 18 | - api.openai.com 19 | user-agent: 20 | - AsyncOpenAI/Python 1.52.2 21 | x-stainless-arch: 22 | - x64 23 | x-stainless-async: 24 | - async:asyncio 25 | x-stainless-lang: 26 | - python 27 | x-stainless-os: 28 | - Linux 29 | x-stainless-package-version: 30 | - 1.52.2 31 | x-stainless-raw-response: 32 | - "true" 33 | x-stainless-retry-count: 34 | - "0" 35 | x-stainless-runtime: 36 | - CPython 37 | x-stainless-runtime-version: 38 | - 3.11.0rc1 39 | method: POST 40 | uri: https://api.openai.com/v1/chat/completions 41 | response: 42 | body: 43 | string: !!binary | 44 | H4sIAAAAAAAAA4xSwWrcMBC9+yumOq+Ld+PsdvdSQhNIQgkEAjmUYhRpbKuVNUIat92E/fcge3e9 45 | oSn0osN7857eG+YlAxBGiw0I1UpWnbf5xT1+/XJ59/ywmi/Kurs4C4+3Uf9qG3OlpZglBT39QMUH 46 | 1UdFnbfIhtxIq4CSMbnOV2fFel6ul4uB6EijTbLGc15SvigWZV58yovlXtiSURjFBr5lAAAvw5si 47 | Oo1/xAaK2QHpMEbZoNgchwBEIJsQIWM0kaVjMZtIRY7RDamv0Vr6ANf0G5R0cAOjALbUA5OW28+n 48 | woB1H2XK7Xpr9/jumMRS4wM9xT1/xGvjTGyrgDKSS79GJi8GdpcBfB8a929KCB+o81wx/USXDFej 49 | m5g2PHHrPcfE0k7wfDl7x6vSyNLYeLIvoaRqUU/Kabmy14ZOiOyk8d9Z3vMeWxvX/I/9RCiFnlFX 50 | PqA26m3faSxgOr9/jR03PAQWcRsZu6o2rsHggxkvoPZVea7q81KjRJHtslcAAAD//wMAhkW/KQoD 51 | AAA= 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 8de6e1adf9fbfb40-SJC 57 | Connection: 58 | - keep-alive 59 | Content-Encoding: 60 | - gzip 61 | Content-Type: 62 | - application/json 63 | Date: 64 | - Wed, 06 Nov 2024 17:42:42 GMT 65 | Server: 66 | - cloudflare 67 | Set-Cookie: 68 | - __cf_bm=IV1Bcm.OiOhx5Fueebsi7W6vfkykRRtCudT2Nx2Si.8-1730914962-1.0.1.1-gOxkIRQ2IVoWcd1Zz84mSVVwBHV5.OKb837EPZuheau5Qz5mzTUKUZbirerOuWHYGvTe3mf9h3oXoV6SISWa2Q; 69 | path=/; expires=Wed, 06-Nov-24 18:12:42 GMT; domain=.api.openai.com; HttpOnly; 70 | Secure; SameSite=None 71 | - _cfuvid=Cu15wqjKcy0EqDJ_H8EAhxyN9YVdxEDkGLA6vmynwxo-1730914962519-0.0.1.1-604800000; 72 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 73 | Transfer-Encoding: 74 | - chunked 75 | X-Content-Type-Options: 76 | - nosniff 77 | access-control-expose-headers: 78 | - X-Request-ID 79 | alt-svc: 80 | - h3=":443"; ma=86400 81 | openai-organization: 82 | - future-house-xr4tdh 83 | openai-processing-ms: 84 | - "350" 85 | openai-version: 86 | - "2020-10-01" 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - "10000" 91 | x-ratelimit-limit-tokens: 92 | - "30000000" 93 | x-ratelimit-remaining-requests: 94 | - "9999" 95 | x-ratelimit-remaining-tokens: 96 | - "29999983" 97 | x-ratelimit-reset-requests: 98 | - 6ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_654554dd6c32e59d7f3e4e0ff5dafa77 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/cassettes/test_fallbacks_working[False].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages":[{"role":"user","content":"Hello!"}],"model":"gpt-4o-mini","n":1,"tool_choice":"required","tools":[{"type":"function","function":{"name":"talk","description":"Say 5 | something to me.","parameters":{"type":"object","properties":{"message":{"description":"what 6 | you want to say","title":"Message","type":"string"}},"required":["message"]}}}]}' 7 | headers: 8 | accept: 9 | - application/json 10 | accept-encoding: 11 | - gzip, deflate 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - "348" 16 | content-type: 17 | - application/json 18 | host: 19 | - api.openai.com 20 | user-agent: 21 | - AsyncOpenAI/Python 1.65.2 22 | x-stainless-arch: 23 | - arm64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - MacOS 30 | x-stainless-package-version: 31 | - 1.65.2 32 | x-stainless-raw-response: 33 | - "true" 34 | x-stainless-read-timeout: 35 | - "60.0" 36 | x-stainless-retry-count: 37 | - "0" 38 | x-stainless-runtime: 39 | - CPython 40 | x-stainless-runtime-version: 41 | - 3.12.4 42 | method: POST 43 | uri: https://api.openai.com/v1/chat/completions 44 | response: 45 | body: 46 | string: 47 | "{\n \"error\": {\n \"message\": \"Incorrect API key provided: 48 | abc123. You can find your API key at https://platform.openai.com/account/api-keys.\",\n 49 | \ \"type\": \"invalid_request_error\",\n \"param\": null,\n \"code\": 50 | \"invalid_api_key\"\n }\n}\n" 51 | headers: 52 | CF-RAY: 53 | - 93106ee58a25e9e1-LAX 54 | Connection: 55 | - keep-alive 56 | Content-Length: 57 | - "256" 58 | Content-Type: 59 | - application/json; charset=utf-8 60 | Date: 61 | - Wed, 16 Apr 2025 03:00:34 GMT 62 | Server: 63 | - cloudflare 64 | Set-Cookie: 65 | - __cf_bm=fno7x_jxANnm27vgnNYNxHE09oaBaF..5rUNTsRQ1WI-1744772434-1.0.1.1-rsmcL1TWF3LBc0flL7Kmdpy1VXvWhNk7QMMykdrKSZJ.bnFPZCzILfUcIZc6xZDGKuaUQwHRadfN6xtPvj4gkVo_4mM6lTZZPyrOWWDdmCg; 66 | path=/; expires=Wed, 16-Apr-25 03:30:34 GMT; domain=.api.openai.com; HttpOnly; 67 | Secure; SameSite=None 68 | - _cfuvid=S6PXUFvHj99UvUnTDkwFAy45z3FyWDJESdzHk5S0Hy8-1744772434881-0.0.1.1-604800000; 69 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 70 | X-Content-Type-Options: 71 | - nosniff 72 | alt-svc: 73 | - h3=":443"; ma=86400 74 | cf-cache-status: 75 | - DYNAMIC 76 | strict-transport-security: 77 | - max-age=31536000; includeSubDomains; preload 78 | vary: 79 | - Origin 80 | x-request-id: 81 | - req_aedfefa7d8f530b37b53effeaa97d3a0 82 | status: 83 | code: 401 84 | message: Unauthorized 85 | version: 1 86 | -------------------------------------------------------------------------------- /tests/cassettes/test_reflect_module.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": [{"type": "text", "text": "\n I 5 | am happy. How do I feel?\n\n\n You are sad.\n"}]}], 6 | "temperature": 0, "system": [{"type": "text", "text": "Consider a proposed response 7 | based on context. Reflect on the response within tags then conclude 8 | with a possibly revised response within tags."}], "max_tokens": 9 | 4096, "model": "claude-3-haiku-20240307"}' 10 | headers: 11 | accept: 12 | - application/json 13 | accept-encoding: 14 | - gzip, deflate 15 | anthropic-version: 16 | - "2023-06-01" 17 | connection: 18 | - keep-alive 19 | content-length: 20 | - "452" 21 | content-type: 22 | - application/json 23 | host: 24 | - api.anthropic.com 25 | user-agent: 26 | - litellm/1.52.3 27 | method: POST 28 | uri: https://api.anthropic.com/v1/messages 29 | response: 30 | body: 31 | string: !!binary | 32 | H4sIAAAAAAAAA3RRS2vcQAz+K0Ln2cTZFAJmyaU9lRxaWkJKXJaJR94ZYkuTkSabzbL/vdgk9AE9 33 | Cb6nhI6YArY46W7bXNzdfN2/2sXNF3+In/np9tun10E+okM7ZJpVpOp3hA6LjDPgVZOaZ0OHkwQa 34 | scV+9DXQ6nIVfXqsq3Wz/tBcNlfosBc2YsP2/vieaPQye5fR4sai1F20646/R4JCmoWVoMMfUsEX 35 | AvWhQwhCCiwGk7c+gkWCJfvFIBd5ToGCg31MfQQ1b6Rg0duiy1RUGJJC9DkfzmAu8jkXySV5+6N0 36 | L3UM8EBgMqcPqUz/jXHw4JUCCC9s4kHK5C0Jwy49E591vDn/fVzHmyGxH1fvZdcdzxcOROPbXrPh 37 | Hw2efjpUk7wt5FUYWyQOW6uF8Y1QeqrEPWHLdRwd1uVd7RET52pbk0dixfZq7VCq/QU1p9MvAAAA 38 | //8DAOX3hPEMAgAA 39 | headers: 40 | CF-Cache-Status: 41 | - DYNAMIC 42 | CF-RAY: 43 | - 8df9510c597cceb1-SJC 44 | Connection: 45 | - keep-alive 46 | Content-Encoding: 47 | - gzip 48 | Content-Type: 49 | - application/json 50 | Date: 51 | - Fri, 08 Nov 2024 23:24:27 GMT 52 | Server: 53 | - cloudflare 54 | Transfer-Encoding: 55 | - chunked 56 | X-Robots-Tag: 57 | - none 58 | anthropic-ratelimit-requests-limit: 59 | - "5000" 60 | anthropic-ratelimit-requests-remaining: 61 | - "4999" 62 | anthropic-ratelimit-requests-reset: 63 | - "2024-11-08T23:25:12Z" 64 | anthropic-ratelimit-tokens-limit: 65 | - "5000000" 66 | anthropic-ratelimit-tokens-remaining: 67 | - "5000000" 68 | anthropic-ratelimit-tokens-reset: 69 | - "2024-11-08T23:24:27Z" 70 | request-id: 71 | - req_012qo7D67oQLZLHC1k3jXiiN 72 | via: 73 | - 1.1 google 74 | status: 75 | code: 200 76 | message: OK 77 | version: 1 78 | -------------------------------------------------------------------------------- /tests/cassettes/test_rollout[False].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: 4 | '{"messages": [{"role": "user", "content": "Hello!"}], "model": "gpt-4o-2024-08-06", 5 | "temperature": 0.1, "tool_choice": "required", "tools": [{"type": "function", 6 | "function": {"name": "talk", "description": "Say something to me.", "parameters": 7 | {"type": "object", "properties": {"message": {"description": "what you want 8 | to say", "title": "Message", "type": "string"}}, "required": ["message"]}}}]}' 9 | headers: 10 | accept: 11 | - application/json 12 | accept-encoding: 13 | - gzip, deflate 14 | connection: 15 | - keep-alive 16 | content-length: 17 | - "397" 18 | content-type: 19 | - application/json 20 | host: 21 | - api.openai.com 22 | user-agent: 23 | - AsyncOpenAI/Python 1.54.3 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - async:asyncio 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 1.54.3 34 | x-stainless-raw-response: 35 | - "true" 36 | x-stainless-retry-count: 37 | - "1" 38 | x-stainless-runtime: 39 | - CPython 40 | x-stainless-runtime-version: 41 | - 3.12.7 42 | method: POST 43 | uri: https://api.openai.com/v1/chat/completions 44 | response: 45 | body: 46 | string: !!binary | 47 | H4sIAAAAAAAAA4xTTY/aMBC951e4c4YqCbBALlUP1W4rdfupdndLFRl7EgyO7dpOF4T471USSAKl 48 | UnOIrHnz3sw8j/cBISA4JATYinpWGDl8/fkLn6x/bXfG3nx9eshvn/zv7M06pxt2/xEGFUMv18j8 49 | ifWS6cJI9EKrBmYWqcdKNZqOoiicxdOoBgrNUVa03PjhWA/jMB4Pw9kwvDkSV1owdJCQHwEhhOzr 50 | f9Wi4riFhISDU6RA52iOkLRJhIDVsooAdU44T5WHQQcyrTyqqmtVStkDvNYyZVTKrnDz7Xvnzicq 51 | ZTpePW/eL5fzcP59++72w6e1uo/y8vFbr14jvTN1Q1mpWOtPD2/jyUUxQkDRouZ6KjcXPEKA2rws 52 | UPmqZ9gvTn4sIFnAHUqpX5A7/UwYVeQtafwgO10SrzndvVrAAc4UD8G188+eSxaz0lF5tO8YP7T3 53 | IXVurF66C3shE0q4VWqRunrMvtvBqVpdB8qzCwVjdWF86vUGVSU7GTWi0K1bB8bxEfTaU9nFp5PB 54 | FbWUo6eivu92xRhlK+Qds1s1WnKhe0DQm/zvZq5pN9MLlf+PfAcwhsYjT41FLtj5wF2axeox/iut 55 | 9bhuGNzOeSzSTKgcrbGifg+QmTSazPlsNI4Yg+AQ/AEAAP//AwAghIdTGAQAAA== 56 | headers: 57 | CF-Cache-Status: 58 | - DYNAMIC 59 | CF-RAY: 60 | - 8df95128e8da2386-SJC 61 | Connection: 62 | - keep-alive 63 | Content-Encoding: 64 | - gzip 65 | Content-Type: 66 | - application/json 67 | Date: 68 | - Fri, 08 Nov 2024 23:24:32 GMT 69 | Server: 70 | - cloudflare 71 | Transfer-Encoding: 72 | - chunked 73 | X-Content-Type-Options: 74 | - nosniff 75 | access-control-expose-headers: 76 | - X-Request-ID 77 | alt-svc: 78 | - h3=":443"; ma=86400 79 | openai-organization: 80 | - future-house-xr4tdh 81 | openai-processing-ms: 82 | - "423" 83 | openai-version: 84 | - "2020-10-01" 85 | strict-transport-security: 86 | - max-age=31536000; includeSubDomains; preload 87 | x-ratelimit-limit-requests: 88 | - "10000" 89 | x-ratelimit-limit-tokens: 90 | - "30000000" 91 | x-ratelimit-remaining-requests: 92 | - "9999" 93 | x-ratelimit-remaining-tokens: 94 | - "29999980" 95 | x-ratelimit-reset-requests: 96 | - 6ms 97 | x-ratelimit-reset-tokens: 98 | - 0s 99 | x-request-id: 100 | - req_173cc061bfeb21f2b6fc27acf7f90100 101 | status: 102 | code: 200 103 | message: OK 104 | version: 1 105 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | from typing import Any 4 | 5 | import numpy as np 6 | import pytest 7 | import torch 8 | from aviary.core import DummyEnv 9 | from lmi import configure_llm_logs 10 | 11 | from ldp.nn.handlers.transformer_handler import ExecutionMode, ParallelModeConfig 12 | 13 | from . import CASSETTES_DIR 14 | 15 | IN_GITHUB_ACTIONS: bool = os.getenv("GITHUB_ACTIONS") == "true" 16 | 17 | 18 | @pytest.fixture(name="dummy_env") 19 | def fixture_dummy_env() -> DummyEnv: 20 | return DummyEnv() 21 | 22 | 23 | @pytest.fixture(scope="session", autouse=True) 24 | def _fixture_set_up_environment() -> None: 25 | configure_llm_logs() 26 | 27 | 28 | def set_seed(seed: int | None) -> None: 29 | if seed is None: 30 | return 31 | 32 | random.seed(seed) 33 | np.random.seed(seed) # noqa: NPY002 34 | torch.manual_seed(seed) 35 | torch.cuda.manual_seed(seed) 36 | 37 | 38 | @pytest.fixture(name="seed_zero") 39 | def fixture_seed_zero() -> None: 40 | """Set a 0 seed to minimize the chances of test flakiness.""" 41 | set_seed(0) 42 | 43 | 44 | OPENAI_API_KEY_HEADER = "authorization" 45 | ANTHROPIC_API_KEY_HEADER = "x-api-key" 46 | # SEE: https://github.com/kevin1024/vcrpy/blob/v6.0.1/vcr/config.py#L43 47 | VCR_DEFAULT_MATCH_ON = "method", "scheme", "host", "port", "path", "query" 48 | 49 | 50 | @pytest.fixture(scope="session", name="vcr_config") 51 | def fixture_vcr_config() -> dict[str, Any]: 52 | return { 53 | "filter_headers": [OPENAI_API_KEY_HEADER, ANTHROPIC_API_KEY_HEADER, "cookie"], 54 | "record_mode": "once", 55 | "match_on": ["method", "host", "path", "query"], 56 | "allow_playback_repeats": True, 57 | "cassette_library_dir": str(CASSETTES_DIR), 58 | } 59 | 60 | 61 | ENABLED = {"true", "1", "yes"} 62 | TEST_GPUS: bool = os.getenv("TEST_GPUS", "").lower() in ENABLED 63 | TEST_SLURM: bool = os.getenv("TEST_SLURM", "").lower() in ENABLED 64 | 65 | PARALLEL_MODE_CONFIGS = [ 66 | pytest.param(None, id="cpu-only"), 67 | pytest.param( 68 | ParallelModeConfig(num_workers=2, num_cpus_per_worker=1), 69 | id="two-gpu", 70 | marks=pytest.mark.skipif(not TEST_GPUS, reason="Requires GPUs"), 71 | ), 72 | pytest.param( 73 | ParallelModeConfig(num_workers=2, num_cpus_per_worker=1, offload_cpu=True), 74 | id="two-gpu-offload", 75 | marks=pytest.mark.skipif(not TEST_GPUS, reason="Requires GPUs"), 76 | ), 77 | pytest.param( 78 | ParallelModeConfig( 79 | num_workers=2, 80 | num_cpus_per_worker=1, 81 | offload_cpu=True, 82 | activation_checkpointing=True, 83 | cpu_ram_efficient_loading=True, 84 | ), 85 | id="two-gpu-all-enabled", 86 | marks=pytest.mark.skipif(not TEST_GPUS, reason="Requires GPUs"), 87 | ), 88 | pytest.param( 89 | ParallelModeConfig( 90 | num_workers=2, 91 | num_cpus_per_worker=1, 92 | execution_mode=ExecutionMode.SLURM_CLUSTER, 93 | ), 94 | id="two-gpus-slurm", 95 | marks=pytest.mark.skipif( 96 | not TEST_GPUS or not TEST_SLURM, reason="Requires GPUs and SLURM" 97 | ), 98 | ), 99 | pytest.param( 100 | ParallelModeConfig( 101 | num_workers=2, 102 | num_cpus_per_worker=1, 103 | execution_mode=ExecutionMode.SLURM_CLUSTER, 104 | offload_cpu=True, 105 | ), 106 | id="two-gpus-slurm-offload", 107 | marks=pytest.mark.skipif( 108 | not TEST_GPUS or not TEST_SLURM, reason="Requires GPUs and SLURM" 109 | ), 110 | ), 111 | ] 112 | -------------------------------------------------------------------------------- /tests/test_buffers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from ldp.alg.optimizer.replay_buffers import ( 4 | CircularReplayBuffer, 5 | PrioritizedReplayBuffer, 6 | ) 7 | 8 | 9 | @pytest.mark.asyncio 10 | async def test_circular_buffer() -> None: 11 | buf = CircularReplayBuffer() 12 | 13 | samples = [{"state": 1, "action": 2, "reward": 3, "t": t} for t in range(5)] 14 | buf += samples 15 | buf.resize(3) # should eject t=0, 1 16 | assert {sample["t"] for sample in buf} == {2, 3, 4} 17 | 18 | await buf.prepare_for_sampling() 19 | 20 | # check we can iterate 21 | next(buf.batched_iter(batch_size=3)) 22 | 23 | # add a bad sample 24 | buf.append({}) 25 | with pytest.raises( 26 | RuntimeError, match="Found buffer element with inconsistent keys" 27 | ): 28 | next(buf.batched_iter(batch_size=4)) 29 | 30 | buf.clear() 31 | assert not buf, "Failed to clear data" 32 | 33 | 34 | async def _dummy_q_function(*args, **kwargs) -> float: # noqa: ARG001, RUF029 35 | return 1.0 36 | 37 | 38 | @pytest.mark.asyncio 39 | async def test_prioritized_buffer(): 40 | buf = PrioritizedReplayBuffer(alpha=1, ranked=False, q_function=_dummy_q_function) 41 | 42 | buf += [ 43 | { 44 | "input_args": (), 45 | "input_kwargs": {}, 46 | "discounted_return": -1.0 if t % 2 else 1.0, 47 | "t": t, 48 | } 49 | for t in range(6) 50 | ] 51 | buf.resize(3) 52 | 53 | with pytest.raises(RuntimeError, match="TD errors not available"): 54 | next(buf.batched_iter(batch_size=3)) 55 | 56 | await buf.prepare_for_sampling() 57 | 58 | # check we can iterate 59 | batch = next(buf.batched_iter(batch_size=3)) 60 | 61 | # The odd timesteps should have priority because they have higher error 62 | assert all(t % 2 for t in batch["t"]) 63 | -------------------------------------------------------------------------------- /tests/test_loss_ops.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | import torch 4 | 5 | from ldp.graph import MSELossOp, compute_graph 6 | 7 | 8 | @pytest.mark.asyncio 9 | @pytest.mark.parametrize("input_size", [4, 10]) 10 | @pytest.mark.parametrize("dtype", ["numpy", "torch"]) 11 | async def test_embedding_op(input_size, dtype) -> None: 12 | op = MSELossOp() 13 | 14 | # Generate data based on dtype 15 | if dtype == "numpy": 16 | rng = np.random.default_rng(12345) 17 | prediction = rng.random(input_size) 18 | target = rng.random(input_size) 19 | else: 20 | prediction = torch.rand(input_size) 21 | target = torch.rand(input_size) 22 | async with compute_graph(): 23 | op_result = await op( 24 | prediction=prediction, 25 | target=target, 26 | ) 27 | 28 | # Validate the output and grads 29 | op_result.compute_grads() 30 | grads = op.get_input_grads(op_result.call_id) 31 | assert grads[0] == [] 32 | assert grads[1].keys() == {"prediction", "target"} 33 | assert grads[1].get("target") is None 34 | pred = grads[1].get("prediction") 35 | if dtype == "numpy": 36 | assert isinstance(pred, np.ndarray) 37 | assert isinstance(op_result.value, float) 38 | else: 39 | assert isinstance(pred, torch.Tensor) 40 | assert isinstance(op_result.value, torch.Tensor) 41 | assert pred.shape == (input_size,) 42 | -------------------------------------------------------------------------------- /tests/test_memory.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lmi import EmbeddingModel 3 | from pytest_subtests import SubTests 4 | 5 | from ldp.graph import Memory 6 | from ldp.graph.memory import UIndexMemoryModel 7 | 8 | 9 | @pytest.fixture(name="sample_memory") 10 | def fixture_sample_memory() -> Memory: 11 | return Memory( 12 | query="sample string representation", output="observation", value=42.0 13 | ) 14 | 15 | 16 | class TestUIndexMemoryModel: 17 | def test_initialization_serialization(self, subtests: SubTests) -> None: 18 | with subtests.test(msg="default-model-specified"): 19 | model = UIndexMemoryModel() 20 | assert isinstance(model.embedding_model, EmbeddingModel), ( 21 | "Default embedding model should be set" 22 | ) 23 | model.model_dump() # Check we can serialize 24 | 25 | with subtests.test(msg="nondefault-model-specified"): 26 | model_custom = UIndexMemoryModel( 27 | embedding_model=EmbeddingModel.from_name("text-embedding-3-small") 28 | ) 29 | model_custom.model_dump() # Check we can serialize 30 | 31 | @pytest.mark.asyncio 32 | async def test_add_then_get_memory(self, sample_memory: Memory) -> None: 33 | memory_model = UIndexMemoryModel( 34 | embedding_model=EmbeddingModel.from_name("text-embedding-3-small") 35 | ) 36 | async with memory_model.safe_access_index() as index: 37 | assert len(index) == 0, "Should have no memories" 38 | await memory_model.add_memory(sample_memory) 39 | async with memory_model.safe_access_index() as index: 40 | assert len(index) == 1, "Should have one memory" 41 | assert memory_model.memories[0] == sample_memory 42 | result = await memory_model.get_memory("sample query", matches=1) 43 | assert len(result) == 1 44 | assert result[0] == sample_memory 45 | -------------------------------------------------------------------------------- /tests/test_nn_ops.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | from aviary.core import Message 5 | 6 | import ldp.nn 7 | from ldp.graph import OpResult 8 | 9 | 10 | class TestLocalLLMCallOp: 11 | @pytest.mark.asyncio 12 | async def test_batching_consistent_results(self): 13 | """Tests that padding in batched calls does not affect the results (attention_mask is set correctly).""" 14 | model_config = ldp.nn.LMConfig( 15 | model="gpt2", device="cpu", dtype=ldp.nn.TorchDType.fp32 16 | ) 17 | local_llm_call_op = ldp.nn.LocalLLMCallOp( 18 | model_config, batch_size=2, max_wait_interval=1.0 19 | ) 20 | 21 | messages = [ 22 | Message(content=text) 23 | for text in ("Hello, how are you?", "Hello, how are you?") 24 | ] 25 | 26 | async def forward_batch() -> list[OpResult[Message]]: 27 | return await asyncio.gather(*[ 28 | local_llm_call_op([msg], temperature=1.0, max_new_tokens=10) 29 | for msg in messages 30 | ]) 31 | 32 | # First forward batch with seed set to 0 33 | ldp.nn.set_seed(0) 34 | results_first_call = await forward_batch() 35 | 36 | # Re-seed and forward batch again with padding to check for consistency 37 | ldp.nn.set_seed(0) 38 | messages[ 39 | -1 40 | ].content = ( 41 | "Some very long text that would create lots of padding in the batch." 42 | ) 43 | results_second_call = await forward_batch() 44 | 45 | assert len(results_first_call) == len(results_second_call), ( 46 | "Expected the number of results to match between the two calls." 47 | ) 48 | assert results_first_call[0].value == results_second_call[0].value, ( 49 | "Expected the results to match between the two calls, but got differing" 50 | " results." 51 | ) 52 | -------------------------------------------------------------------------------- /tests/test_prompts.py: -------------------------------------------------------------------------------- 1 | import textwrap 2 | 3 | from ldp.llms.prompts import indent_xml 4 | 5 | 6 | def test_indent_xml(): 7 | xml = "fooline1\nline2\nline3" 8 | expected = textwrap.dedent( 9 | """\ 10 | 11 | 12 | foo 13 | 14 | line1 15 | line2 16 | line3 17 | 18 | 19 | """ 20 | ) 21 | assert indent_xml(xml) == expected 22 | -------------------------------------------------------------------------------- /tests/test_shims.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | from unittest.mock import patch 4 | 5 | import ldp.shims 6 | 7 | 8 | def test_tqdm_import() -> None: 9 | assert ldp.shims.tqdm.__module__ == "tqdm.std" 10 | with patch.dict(os.environ, {"LDP_TQDM_USE_RICH": "1"}): 11 | importlib.reload(ldp.shims) 12 | assert ldp.shims.tqdm.__module__ == "tqdm.rich" 13 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Any 3 | 4 | from ldp.utils import format_error_details 5 | 6 | 7 | @dataclass 8 | class MockResponse: 9 | status_code: int 10 | _json: dict | None = None 11 | _text: str = "" 12 | 13 | def json(self) -> dict[str, Any]: 14 | if self._json is None: 15 | raise ValueError("No JSON") 16 | return self._json 17 | 18 | @property 19 | def text(self) -> str: 20 | return self._text 21 | 22 | 23 | class MockHTTPError(Exception): 24 | def __init__(self, status_code: int, detail: str | None = None, text: str = ""): 25 | self.response = MockResponse( 26 | status_code=status_code, 27 | _json={"detail": detail} if detail else None, 28 | _text=text, 29 | ) 30 | super().__init__(f"HTTP {status_code}") 31 | 32 | 33 | def test_format_basic_error(): 34 | error = ValueError("something went wrong") 35 | details = format_error_details(error) 36 | assert details == "something went wrong" 37 | 38 | 39 | def test_format_http_error_with_json(): 40 | error = MockHTTPError( 41 | status_code=500, 42 | detail="Traceback:\n File 'app.py', line 123\n raise ValueError('oops')", 43 | ) 44 | details = format_error_details(error) 45 | assert "Status code: 500" in details 46 | assert "Server Traceback:" in details 47 | assert "File 'app.py'" in details 48 | 49 | 50 | def test_format_http_error_with_text(): 51 | error = MockHTTPError(status_code=404, text="Not found") 52 | details = format_error_details(error) 53 | assert "Status code: 404" in details 54 | assert "Response body: Not found" in details 55 | -------------------------------------------------------------------------------- /tutorials/evaluating_a_llama_agent.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Llama Agent Inference (GPU)\n", 8 | "\n", 9 | "A notebook demonstrating how to use an agent powered by a local model (Llama 3.1 8B) to solve problems in the GSM8k test set using a calculator. **The tutorial assumes access to a GPU machine**.\n", 10 | "\n", 11 | "Note that the model struggles to output valid tool call syntax consistently, causing many failures.\n", 12 | "\n", 13 | "TODO: add EI notebook demonstrating how to address this\n", 14 | "\n", 15 | "NB: To run this notebook you need to install ldp with the `nn` dependency as well as the aviary dependencies\n", 16 | "\n", 17 | "```bash\n", 18 | "pip install \"ldp[nn]\" \"fhaviary[gsm8k]\"\n", 19 | "```" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "from aviary.envs.gsm8k import GSM8kDataset\n", 29 | "\n", 30 | "from ldp.alg import Evaluator, EvaluatorConfig\n", 31 | "from ldp.alg.callbacks import Callback\n", 32 | "from ldp.data_structures import Trajectory\n", 33 | "from ldp.nn import AgentLMConfig, SimpleLocalLLMAgent, TorchDType" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "class AccuracyCallback(Callback):\n", 43 | " \"\"\"Simple callback that logs accuracy of each batch.\"\"\"\n", 44 | "\n", 45 | " async def after_eval_step(self, trajectories: list[Trajectory]):\n", 46 | " # CalculatorEnvironment returns a terminal reward of 1 if\n", 47 | " # the agent solved the problem correctly.\n", 48 | " pass_rate = sum(t.steps[-1].reward == 1 for t in trajectories) / len(\n", 49 | " trajectories\n", 50 | " )\n", 51 | " print(f\"Pass rate: {100 * pass_rate:.2f}%\")" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "lm_config = AgentLMConfig(\n", 61 | " model=\"meta-llama/Llama-3.1-8B-Instruct\",\n", 62 | " dtype=TorchDType.bf16,\n", 63 | " chat_template=\"llama3.1_chat_template_thought.jinja\",\n", 64 | " max_new_tokens=100,\n", 65 | " # Parameters for async inference\n", 66 | " batch_size=8, # fits onto a single 4090 with these params\n", 67 | " max_wait_interval=10.0,\n", 68 | ")\n", 69 | "agent = SimpleLocalLLMAgent(lm_config)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "dataset = GSM8kDataset(split=\"test\")\n", 79 | "callback = AccuracyCallback()\n", 80 | "evaluator = Evaluator(\n", 81 | " config=EvaluatorConfig(\n", 82 | " batch_size=64,\n", 83 | " num_eval_iterations=1, # Only run one batch, then exit\n", 84 | " max_rollout_steps=10,\n", 85 | " ),\n", 86 | " agent=agent,\n", 87 | " dataset=dataset,\n", 88 | " callbacks=[callback],\n", 89 | ")\n", 90 | "\n", 91 | "# Note that Llama 3.1 8B does not always reliably follow the tool-calling\n", 92 | "# syntax, so we will see several (caught) errors. The pass rate will be <10%.\n", 93 | "await evaluator.evaluate()" 94 | ] 95 | } 96 | ], 97 | "metadata": { 98 | "kernelspec": { 99 | "display_name": "Python 3 (ipykernel)", 100 | "language": "python", 101 | "name": "python3" 102 | }, 103 | "language_info": { 104 | "codemirror_mode": { 105 | "name": "ipython", 106 | "version": 3 107 | }, 108 | "file_extension": ".py", 109 | "mimetype": "text/x-python", 110 | "name": "python", 111 | "nbconvert_exporter": "python", 112 | "pygments_lexer": "ipython3" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 4 117 | } 118 | --------------------------------------------------------------------------------